[+] Filter by language, reselect sample

2021-11-23 12:10:28 -05:00
parent 3333c5377b
commit b6d0cda387
2 changed files with 5 additions and 2 deletions
@@ -120,6 +120,9 @@ def select_user_sample(user_dir: str = './data/twitter/user/') -> None:
    # Load users
    users = load_users(user_dir)

+    # Filter by language first
+    users = [u for u in users if any(lang in u.lang for lang in {'en', 'zh', 'ja'})]
+
    # Find most popular, and exclude them from the random sample
    most_popular = users[:500]
    users = users[500:]