[-] Remove custom tweet model, optimize imports

src/main.py

@@ -1,7 +1,7 @@
 from tabulate import tabulate
 
 from process.twitter_process import load_users_popularity, process_users_popularity
-from raw_collect.twitter import tweepy_login
+from raw_collect.twitter import tweepy_login, download_all_tweets
 from utils import load_config
 
 if __name__ == '__main__':
@@ -33,7 +33,7 @@ if __name__ == '__main__':
     print(tabulate(((u.username, u.popularity) for u in users[:20]), headers=['Name', 'Followers']))
 
     # Start download
-
+    download_all_tweets(api, 'sauricat')
 
     #####################
     # Data collection - Step C2

src/process/twitter_process.py

@@ -82,6 +82,6 @@ def load_users_popularity(user_dir: str = './data/twitter/user/') -> list[UserPo
      download_user_start. (Default: "./data/twitter/user/")
     :return: List of users' screen names and popularity, sorted descending by popularity.
     """
-    user_dir = normalize_directory(user_dir) + '/users'
+    user_dir = normalize_directory(user_dir)
     with open(f'{user_dir}/processed_popularity.json', 'r', encoding='utf-8') as f:
-        return json.load(f)
+        return [UserPopularity(*u) for u in json.load(f)]

src/raw_collect/twitter.py

@@ -5,12 +5,9 @@ import json
 import math
 import random
 import time
-from dataclasses import dataclass
-from datetime import datetime
 from pathlib import Path
 from typing import Union, List
 
-import python_ta
 import tweepy
 from tweepy import API, TooManyRequests, User, Tweet
 
@@ -19,34 +16,6 @@ from utils import Config, debug, json_stringify, load_config, normalize_director
     calculate_rate_delay
 
 
-@dataclass
-class Tweet:
-    created_at: datetime
-    id: int
-    id_str: str
-    full_text: str
-    truncated: bool
-    display_text_range: list[int]
-    entities: dict[str: list]
-    source: str
-
-    in_reply_to_status_id: int
-    in_reply_to_status_id_str: str
-    in_reply_to_user_id: int
-    in_reply_to_user_id_str: str
-    in_reply_to_screen_name: str
-
-    geo: str
-    coordinates: str
-    user: User
-
-    is_quote_status: bool
-
-    retweeted_status: Union[dict, None]
-    retweet_count: int
-    favorite_count: int
-
-
 def tweepy_login(conf: Config) -> tweepy.API:
     """
     Login to tweepy
@@ -127,8 +96,9 @@ def download_all_tweets(api: API, screen_name: str,
     # Store in file
     with open(f'{base_dir}/{screen_name}.json', 'w', encoding='utf-8') as f:
         # Even though we are not supposed to use internal fields, there aren't any efficient way of
-        # obtaining the json without the field.
-        f.write(json_stringify([t.__dict__ for t in tweets]))
+        # obtaining the json without the field. Using t.__dict__ will include the API object, which
+        # is not serializable.
+        f.write(json_stringify([t._json for t in tweets]))
 
 
 def download_users_start(api: API, start_point: str, n: float = math.inf,