[O] Use named tuple instead of data class

requirements.txt

@@ -8,6 +8,9 @@ tweepy==4.4.0
 # Pytz is for timezone manipulations
 pytz~=2021.1
 
+# Print table data
+tabulate==0.8.9
+
 # Testing and code checking
 pytest
 python-ta

src/process/twitter_process.py

@@ -2,18 +2,21 @@ import json
 import os
 from dataclasses import dataclass
 from datetime import datetime
+from typing import NamedTuple
 
-from utils import normalize_directory, debug
+from utils import normalize_directory, debug, json_stringify
 
 
-@dataclass
+class UserPopularity(NamedTuple):
-class GeneralUser:
+    """
+    User and popularity (we use NamedTuple instead of dataclass because named tuples are easier to
+    serialize in JSON and they require much less space in the stored json format because no key info
+    is stored).
+    """
     # Username
     username: str
     # A measurement of a user's popularity, such as followers count
     popularity: int
-    # Which platform is this user from
-    platform: str
 
 
 @dataclass
@@ -32,12 +35,13 @@ class Posting:
     date: datetime
 
 
-def load_users_popularity(user_dir: str = './data/twitter/user/') -> list[GeneralUser]:
+def process_users_popularity(user_dir: str = './data/twitter/user/') -> list[UserPopularity]:
     """
     After downloading a wide range of users using download_users_start in raw_collect/twitter.py,
     this function will read the user files and rank the users by popularity.
 
-    The return format will consist of a list of users' screen names and popularity.
+    The return format will consist of a list of users' screen names and popularity, which will be
+    saved to <user_dir>/processed_popularity.json
 
     :param user_dir: Download directory of users data, should be the same as the downloads dir in
      download_user_start. (Default: "./data/twitter/user/")
@@ -53,7 +57,7 @@ def load_users_popularity(user_dir: str = './data/twitter/user/') -> list[Genera
             # Read
             with open(f'{user_dir}/{filename}', 'r', encoding='utf-8') as f:
                 user = json.load(f)
-                users.append(GeneralUser(user['screen_name'], user['followers_count'], 'twitter'))
+                users.append(UserPopularity(user['screen_name'], user['followers_count']))
 
             # Log progress
             if len(users) % 2000 == 0:
@@ -62,5 +66,22 @@ def load_users_popularity(user_dir: str = './data/twitter/user/') -> list[Genera
     # Sort by followers count, descending
     users.sort(key=lambda x: x.popularity, reverse=True)
 
+    # Save data
+    with open(f'{user_dir}/../processed_popularity.json', 'w', encoding='utf-8') as f:
+        f.write(json_stringify(users, indent=None))
+
     # Return
     return users
+
+
+def load_users_popularity(user_dir: str = './data/twitter/user/') -> list[UserPopularity]:
+    """
+    Load user popularity data after processing in process_users_popularity
+
+    :param user_dir: Download directory of users data, should be the same as the downloads dir in
+     download_user_start. (Default: "./data/twitter/user/")
+    :return: List of users' screen names and popularity, sorted descending by popularity.
+    """
+    user_dir = normalize_directory(user_dir) + '/users'
+    with open(f'{user_dir}/processed_popularity.json', 'r', encoding='utf-8') as f:
+        return json.load(f)

src/raw_collect/twitter.py

@@ -309,12 +309,12 @@ def convert_to_generic(username: str, tweet: Tweet) -> Posting:
 
 
 if __name__ == '__main__':
-    python_ta.check_all(config={
+    # python_ta.check_all(config={
-        'max-line-length': 100,
+    #     'max-line-length': 100,
-        'disable': ['R1705', 'C0200', 'E9998', 'E9999']
+    #     'disable': ['R1705', 'C0200', 'E9998', 'E9999']
-    })
+    # })
 
     config = load_config('config.json5')
     tweepy_api = tweepy_login(config)
     # download_users_start(tweepy_api, 'sauricat')
-    # download_users_resume_progress(tweepy_api)
+    download_users_resume_progress(tweepy_api)