From b2edfc2003c3c4c579e50eef575df7d9dab257c5 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Sun, 21 Nov 2021 23:34:22 -0500 Subject: [PATCH] [O] Move rate delay to the end --- src/raw_collect/twitter.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/raw_collect/twitter.py b/src/raw_collect/twitter.py index f87a62a..9a23483 100644 --- a/src/raw_collect/twitter.py +++ b/src/raw_collect/twitter.py @@ -163,6 +163,9 @@ def download_users(api: API, start_point: str, n: float = math.inf, Path(f'{base_dir}/users').mkdir(parents=True, exist_ok=True) Path(f'{base_dir}/meta').mkdir(parents=True, exist_ok=True) + # Rate limit delay + rate_delay = 1 / rate_limit * 60 + 0.1 + # Set of all the downloaded users' screen names downloaded = set() @@ -177,9 +180,6 @@ def download_users(api: API, start_point: str, n: float = math.inf, # Loop until there are enough users while len(downloaded) < n: - # Rate limit - time.sleep(1 / rate_limit * 60 + 0.1) - # Take a screen name from the current list screen_name = current_set.pop() @@ -235,6 +235,9 @@ def download_users(api: API, start_point: str, n: float = math.inf, debug(f'Finished saving friends of {screen_name}') debug(f'============= Total {len(downloaded)} saved =============') + # Rate limit + time.sleep(rate_delay) + def convert_to_generic(username: str, tweet: Tweet) -> Posting: """