From 46b8f80b70297741708b23f883d8305397c94494 Mon Sep 17 00:00:00 2001
From: Hykilpikonna <me@hydev.org>
Date: Mon, 22 Nov 2021 10:31:47 -0500
Subject: [PATCH] [F] Fix load_users_popularity

---
 src/process/twitter_process.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/process/twitter_process.py b/src/process/twitter_process.py
index 966240f..650889e 100644
--- a/src/process/twitter_process.py
+++ b/src/process/twitter_process.py
@@ -3,7 +3,7 @@ import os
 from dataclasses import dataclass
 from datetime import datetime
 
-from utils import normalize_directory
+from utils import normalize_directory, debug
 
 
 @dataclass
@@ -43,20 +43,24 @@ def load_users_popularity(user_dir: str = './data/twitter/user/') -> list[Genera
      download_user_start. (Default: "./data/twitter/user/")
     :return: List of users' screen names and popularity, sorted descending by popularity.
     """
-    user_dir = normalize_directory(user_dir)
+    user_dir = normalize_directory(user_dir) + '/users'
     users = []
 
     # Loop through all the files
-    for filename in os.listdir(user_dir + ''):
+    for filename in os.listdir(user_dir):
         # Only check json files and ignore macos dot files
         if filename.endswith('.json') and not filename.startswith('.'):
             # Read
-            with open(filename, 'r', encoding='utf-8') as f:
+            with open(f'{user_dir}/{filename}', 'r', encoding='utf-8') as f:
                 user = json.load(f)
                 users.append(GeneralUser(user['screen_name'], user['followers_count'], 'twitter'))
 
+            # Log progress
+            if len(users) % 2000 == 0:
+                debug(f'load_users_popularity: Loaded {len(users)} users.')
+
     # Sort by followers count, descending
-    users.sort(key=lambda x: x[1])
+    users.sort(key=lambda x: x.popularity, reverse=True)
 
     # Return
     return users