[O] Optimize imports

src/main.py

@@ -12,7 +12,7 @@ if __name__ == '__main__':
     #####################
     # Data collection - Step C1
     # Download a wide range of users from Twitter using follow-chaining starting from a single user.
-    # download_users_start(api, 'sauricat')
+    # download_users_start(api, 'voxdotcom')
 
     # This task will run for a very very long time to obtain a large dataset of twitter users. If
     # you want to stop the process, you can resume it later using the following line:

src/process/twitter_process.py

@@ -1,10 +1,9 @@
 import json
 import os
-from dataclasses import dataclass
 from datetime import datetime
 from typing import NamedTuple
 
-from utils import normalize_directory, debug, json_stringify
+from utils import *
 
 
 class UserPopularity(NamedTuple):

src/raw_collect/twitter.py

@@ -1,19 +1,15 @@
 """
 TODO: Module docstring
 """
-import json
 import math
-import os
 import random
 import time
-from typing import Union, List
+from typing import List
 
 import tweepy
 from tweepy import API, TooManyRequests, User, Tweet
 
-from process.twitter_process import Posting
-from utils import Config, debug, json_stringify, load_config, normalize_directory, \
-    calculate_rate_delay, write, read
+from utils import *
 
 
 def tweepy_login(conf: Config) -> tweepy.API:
@@ -292,22 +288,6 @@ def download_users_execute(api: API, n: float, base_dir: str,
         time.sleep(rate_delay)
 
 
-def convert_to_generic(username: str, tweet: Tweet) -> Posting:
-    """
-    Convert a twitter's tweet to a generic posting
-
-    :param username: Username (for optimization, because including a user object in every tweet
-    slows computation significantly.)
-    :param tweet: Tweet data
-    :return: Generic posting
-    """
-    return Posting('twitter', username,
-                   text=tweet.full_text,
-                   popularity=tweet.favorite_count + tweet.retweet_count,
-                   repost=hasattr(tweet, 'retweeted_status'),
-                   date=tweet.created_at)
-
-
 if __name__ == '__main__':
     # python_ta.check_all(config={
     #     'max-line-length': 100,

src/raw_collect/twitter_individual.py

@@ -1,4 +1,7 @@
-from raw_collect.twitter import tweepy_login, download_user_tweets
+from tabulate import tabulate
+
+from process.twitter_process import *
+from raw_collect.twitter import *
 from utils import *
 
 
@@ -6,5 +9,11 @@ if __name__ == '__main__':
     conf = load_config()
     api = tweepy_login(conf)
 
-    download_user_tweets(api, 'sauricat')
+    users = load_users_popularity()
 
+    # Just curious, who are the 20 most popular individuals on twitter?
+    print(tabulate(((u.username, u.popularity) for u in users[:20]), headers=['Name', 'Followers']))
+
+    # Start download
+    for u in users:
+        download_all_tweets(api, u.username)