azalea/CSC110-Project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[O] Optimize network usage by trimming username

Browse Source
This commit is contained in:
Hykilpikonna
2021-11-05 00:19:52 -04:00
parent 7868b1ac07
commit 4ff72a47d7
1 changed files with 5 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
collect/twitter.py
+5 -6
View File
@@ -85,13 +85,13 @@ def download_user_tweets(api: API, screen_name: str) -> None:
start_date = pytz.UTC.localize(datetime(2020, 1, 1)) start_date = pytz.UTC.localize(datetime(2020, 1, 1))
# Get initial 200 tweets # Get initial 200 tweets
tweets = api.user_timeline(screen_name=screen_name, count=200, tweet_mode='extended') tweets = api.user_timeline(screen_name=screen_name, count=200, tweet_mode='extended', trim_user=True)
postings = [convert_to_generic(t) for t in tweets] postings = [convert_to_generic(t) for t in tweets]
# Get additional tweets # Get additional tweets
while True: while True:
debug(f'- Got {len(tweets)} tweets, getting additional tweets...') debug(f'- Got {len(tweets)} tweets, getting additional tweets...')
additional_tweets = api.user_timeline(screen_name=screen_name, count=200, tweet_mode='extended', additional_tweets = api.user_timeline(screen_name=screen_name, count=200, tweet_mode='extended', trim_user=True,
max_id=int(tweets[-1].id_str) - 1) max_id=int(tweets[-1].id_str) - 1)
if len(additional_tweets) == 0: if len(additional_tweets) == 0:
debug(f'- Got {len(tweets)} tweets, finished because no more tweets are available.') debug(f'- Got {len(tweets)} tweets, finished because no more tweets are available.')
@@ -102,7 +102,7 @@ def download_user_tweets(api: API, screen_name: str) -> None:
break break
tweets.extend(additional_tweets) tweets.extend(additional_tweets)
postings.extend([convert_to_generic(t) for t in additional_tweets]) postings.extend([convert_to_generic(screen_name, t) for t in additional_tweets])
# Make directory # Make directory
dir_raw = './data/twitter_users_raw/' dir_raw = './data/twitter_users_raw/'
@@ -117,15 +117,14 @@ def download_user_tweets(api: API, screen_name: str) -> None:
f.write(json_stringify(postings)) f.write(json_stringify(postings))
def convert_to_generic(tweet: Tweet) -> Posting: def convert_to_generic(username: str, tweet: Tweet) -> Posting:
""" """
Convert a twitter's tweet to a generic posting Convert a twitter's tweet to a generic posting
:param tweet: Tweet data :param tweet: Tweet data
:return: Generic posting :return: Generic posting
""" """
return Posting('twitter', return Posting('twitter', username,
username=tweet.user.screen_name,
text=tweet.full_text, text=tweet.full_text,
popularity=tweet.favorite_count + tweet.retweet_count, popularity=tweet.favorite_count + tweet.retweet_count,
repost=tweet.retweeted_status is not None, repost=tweet.retweeted_status is not None,