From 7e19250fe94af41dfc1104f24df2045384cb2feb Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Mon, 22 Nov 2021 00:36:33 -0500 Subject: [PATCH] [O] Handle too many requests exception --- src/raw_collect/twitter.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/raw_collect/twitter.py b/src/raw_collect/twitter.py index 4220257..9f2e9c8 100644 --- a/src/raw_collect/twitter.py +++ b/src/raw_collect/twitter.py @@ -10,7 +10,7 @@ from typing import Union import pytz import tweepy -from tweepy import API +from tweepy import API, TooManyRequests from raw_collect.utils import Config, debug, Posting, json_stringify, load_config @@ -223,7 +223,7 @@ def download_users_execute(api: API, n: float, base_dir: str, rate_limit: int, Path(f'{base_dir}/meta').mkdir(parents=True, exist_ok=True) # Rate limit delay - rate_delay = 1 / rate_limit * 60 + 0.1 + rate_delay = 1 / rate_limit * 60 + 1 print(f"Executing friends-chain download:") print(f"- n: {n}") @@ -239,8 +239,15 @@ def download_users_execute(api: API, n: float, base_dir: str, rate_limit: int, # Take a screen name from the current list screen_name = current_set.pop() - # Get a list of friends. - friends = api.get_friends(screen_name=screen_name, count=200) + try: + # Get a list of friends. + friends = api.get_friends(screen_name=screen_name, count=200) + except TooManyRequests: + # Rate limited, sleep and try again + debug('Caught TooManyRequests exception: Rate limited, sleep and try again.') + time.sleep(rate_delay) + current_set.add(screen_name) + continue # Save users for user in friends: