From d9a78baa3d2c5aff63952399670043b98c7b4163 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Mon, 22 Nov 2021 15:22:09 -0500 Subject: [PATCH] [O] Default to indent none to save space --- src/main.py | 6 ++++++ src/process/twitter_process.py | 4 ++-- src/raw_collect/twitter.py | 2 +- src/utils.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main.py b/src/main.py index d230db9..09fbb82 100644 --- a/src/main.py +++ b/src/main.py @@ -41,3 +41,9 @@ if __name__ == '__main__': # Data processing - Step P2 # (After step C2) Process the downloaded tweets, determine whether they are covid-related process_tweets() + + # Who posted the most covid tweets? (covid vs non-covid ratio) + # - Graph histogram of this ratio + # Who has the most covid tweet popularity (popularity of covid vs non-covid tweets ratio) + # - Graph histogram of this ratio + diff --git a/src/process/twitter_process.py b/src/process/twitter_process.py index 1c6988a..52da308 100644 --- a/src/process/twitter_process.py +++ b/src/process/twitter_process.py @@ -49,7 +49,7 @@ def process_users_popularity(user_dir: str = './data/twitter/user/') -> None: users.sort(key=lambda x: x.popularity, reverse=True) # Save data - write(f'{user_dir}/processed/popularity.json', json_stringify(users, indent=None)) + write(f'{user_dir}/processed/popularity.json', json_stringify(users)) def load_users_popularity(user_dir: str = './data/twitter/user/') -> list[UserPopularity]: @@ -110,7 +110,7 @@ def process_tweets(tweets_dir: str = './data/twitter/user-tweets/') -> None: for t in tweets] # Save data - write(f'{tweets_dir}/processed/{filename}', json_stringify(p, indent=None)) + write(f'{tweets_dir}/processed/{filename}', json_stringify(p)) debug(f'Processed: {filename}') diff --git a/src/raw_collect/twitter.py b/src/raw_collect/twitter.py index 31e0d3d..715893f 100644 --- a/src/raw_collect/twitter.py +++ b/src/raw_collect/twitter.py @@ -283,7 +283,7 @@ def download_users_execute(api: API, n: float, base_dir: str, # Update meta info so that downloading can be continued meta = {'downloaded': downloaded, 'done_set': done_set, 'current_set': current_set, 'next_set': next_set, 'n': n} - write(f'{base_dir}/meta/meta.json', json_stringify(meta, indent=None)) + write(f'{base_dir}/meta/meta.json', json_stringify(meta)) debug(f'Finished saving friends of {screen_name}') debug(f'============= Total {len(downloaded)} saved =============') diff --git a/src/utils.py b/src/utils.py index 567ca3d..6bc277f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -140,7 +140,7 @@ class EnhancedJSONEncoder(json.JSONEncoder): return super().default(o) -def json_stringify(obj, indent: Union[int, None] = 1) -> str: +def json_stringify(obj, indent: Union[int, None] = None) -> str: """ Serialize json string with support for dataclasses and datetime and sets and with custom configuration.