azalea/CSC110-Project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[F] Fix rate limit

Browse Source
This commit is contained in:
Hykilpikonna
2021-11-21 23:22:34 -05:00
parent ad3868406b
commit b4ca5f65af
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/raw_collect/twitter.py
+7 -3
View File
@@ -169,12 +169,16 @@ def download_users(api: API, start_point: str, n: float = math.inf,
# Ensure that the current request rate is under the specified rate limit. # Ensure that the current request rate is under the specified rate limit.
def ensure_rate_limit() -> None: def ensure_rate_limit() -> None:
requests_per_minute = num_requests / (time.time() - start_time) * 60 # Calculate current rate
seconds_elapsed = time.time() - start_time
requests_per_second = num_requests / seconds_elapsed
requests_per_minute = requests_per_second * 60
# We're over the rate limit. # We're over the rate limit.
if requests_per_minute > rate_limit: if requests_per_minute > rate_limit:
# Sleep and check again # Sleep and check again
debug(f'Rate-limit reached ({requests_per_minute} > {rate_limit} rpm), sleeping') debug(f'Rate-limit reached ({requests_per_minute} rpm > {rate_limit} rpm), sleeping')
time.sleep((rate_limit - requests_per_minute) * 60) time.sleep((requests_per_minute - rate_limit) * 60)
ensure_rate_limit() ensure_rate_limit()
# Set of all the downloaded users' screen names # Set of all the downloaded users' screen names