From b6d0cda3878c031d9c1a241b7c3f171e947d6574 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Tue, 23 Nov 2021 12:10:28 -0500 Subject: [PATCH] [+] Filter by language, reselect sample --- src/main.py | 4 ++-- src/process/twitter_process.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main.py b/src/main.py index 8808b9f..e9f9b04 100644 --- a/src/main.py +++ b/src/main.py @@ -22,13 +22,13 @@ if __name__ == '__main__': # Data processing - Step P1 # (After step C1) Process the downloaded twitter users, extract screen name, popularity, and # number of tweets data. - process_users() + # process_users() ##################### # Data processing - Step P2 # (After step P1) Select 500 most popular users and 500 random users who meet a particular # criteria as our sample. - # select_user_sample() + select_user_sample() ##################### # Data collection - Step C2.1 diff --git a/src/process/twitter_process.py b/src/process/twitter_process.py index 28e9189..9cf90f1 100644 --- a/src/process/twitter_process.py +++ b/src/process/twitter_process.py @@ -120,6 +120,9 @@ def select_user_sample(user_dir: str = './data/twitter/user/') -> None: # Load users users = load_users(user_dir) + # Filter by language first + users = [u for u in users if any(lang in u.lang for lang in {'en', 'zh', 'ja'})] + # Find most popular, and exclude them from the random sample most_popular = users[:500] users = users[500:]