From a73a7921891f4c1458fbf936a5bcdb6a8879b8b3 Mon Sep 17 00:00:00 2001
From: Hykilpikonna <me@hydev.org>
Date: Mon, 22 Nov 2021 13:59:55 -0500
Subject: [PATCH] [O] Optimize imports

---
 src/main.py                           |  2 +-
 src/process/twitter_process.py        |  3 +--
 src/raw_collect/twitter.py            | 24 ++----------------------
 src/raw_collect/twitter_individual.py | 13 +++++++++++--
 4 files changed, 15 insertions(+), 27 deletions(-)

diff --git a/src/main.py b/src/main.py
index 275caee..c86bd87 100644
--- a/src/main.py
+++ b/src/main.py
@@ -12,7 +12,7 @@ if __name__ == '__main__':
     #####################
     # Data collection - Step C1
     # Download a wide range of users from Twitter using follow-chaining starting from a single user.
-    # download_users_start(api, 'sauricat')
+    # download_users_start(api, 'voxdotcom')
 
     # This task will run for a very very long time to obtain a large dataset of twitter users. If
     # you want to stop the process, you can resume it later using the following line:
diff --git a/src/process/twitter_process.py b/src/process/twitter_process.py
index 62c2b01..c6c899a 100644
--- a/src/process/twitter_process.py
+++ b/src/process/twitter_process.py
@@ -1,10 +1,9 @@
 import json
 import os
-from dataclasses import dataclass
 from datetime import datetime
 from typing import NamedTuple
 
-from utils import normalize_directory, debug, json_stringify
+from utils import *
 
 
 class UserPopularity(NamedTuple):
diff --git a/src/raw_collect/twitter.py b/src/raw_collect/twitter.py
index 9c3c02f..2eddded 100644
--- a/src/raw_collect/twitter.py
+++ b/src/raw_collect/twitter.py
@@ -1,19 +1,15 @@
 """
 TODO: Module docstring
 """
-import json
 import math
-import os
 import random
 import time
-from typing import Union, List
+from typing import List
 
 import tweepy
 from tweepy import API, TooManyRequests, User, Tweet
 
-from process.twitter_process import Posting
-from utils import Config, debug, json_stringify, load_config, normalize_directory, \
-    calculate_rate_delay, write, read
+from utils import *
 
 
 def tweepy_login(conf: Config) -> tweepy.API:
@@ -292,22 +288,6 @@ def download_users_execute(api: API, n: float, base_dir: str,
         time.sleep(rate_delay)
 
 
-def convert_to_generic(username: str, tweet: Tweet) -> Posting:
-    """
-    Convert a twitter's tweet to a generic posting
-
-    :param username: Username (for optimization, because including a user object in every tweet
-    slows computation significantly.)
-    :param tweet: Tweet data
-    :return: Generic posting
-    """
-    return Posting('twitter', username,
-                   text=tweet.full_text,
-                   popularity=tweet.favorite_count + tweet.retweet_count,
-                   repost=hasattr(tweet, 'retweeted_status'),
-                   date=tweet.created_at)
-
-
 if __name__ == '__main__':
     # python_ta.check_all(config={
     #     'max-line-length': 100,
diff --git a/src/raw_collect/twitter_individual.py b/src/raw_collect/twitter_individual.py
index a5f3617..d18a2a6 100644
--- a/src/raw_collect/twitter_individual.py
+++ b/src/raw_collect/twitter_individual.py
@@ -1,4 +1,7 @@
-from raw_collect.twitter import tweepy_login, download_user_tweets
+from tabulate import tabulate
+
+from process.twitter_process import *
+from raw_collect.twitter import *
 from utils import *
 
 
@@ -6,5 +9,11 @@ if __name__ == '__main__':
     conf = load_config()
     api = tweepy_login(conf)
 
-    download_user_tweets(api, 'sauricat')
+    users = load_users_popularity()
 
+    # Just curious, who are the 20 most popular individuals on twitter?
+    print(tabulate(((u.username, u.popularity) for u in users[:20]), headers=['Name', 'Followers']))
+
+    # Start download
+    for u in users:
+        download_all_tweets(api, u.username)