azalea/CSC110-Project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[O] Use read and write functions

Browse Source
This commit is contained in:
Hykilpikonna
2021-11-22 13:55:18 -05:00
parent 343e432df9
commit 4c83dd07a2
1 changed files with 11 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/raw_collect/twitter.py
+11 -21
View File
@@ -6,7 +6,6 @@ import math
import os import os
import random import random
import time import time
from pathlib import Path
from typing import Union, List from typing import Union, List
import tweepy import tweepy
@@ -14,7 +13,7 @@ from tweepy import API, TooManyRequests, User, Tweet
from process.twitter_process import Posting from process.twitter_process import Posting
from utils import Config, debug, json_stringify, load_config, normalize_directory, \ from utils import Config, debug, json_stringify, load_config, normalize_directory, \
calculate_rate_delay calculate_rate_delay, write, read
def tweepy_login(conf: Config) -> tweepy.API: def tweepy_login(conf: Config) -> tweepy.API:
@@ -72,7 +71,6 @@ def download_all_tweets(api: API, screen_name: str,
""" """
# Ensure directories exist # Ensure directories exist
base_dir = normalize_directory(base_dir) + '/user' base_dir = normalize_directory(base_dir) + '/user'
Path(base_dir).mkdir(parents=True, exist_ok=True)
file = f'{base_dir}/{screen_name}.json' file = f'{base_dir}/{screen_name}.json'
# Check if user already exists # Check if user already exists
@@ -99,18 +97,17 @@ def download_all_tweets(api: API, screen_name: str,
# No more tweets # No more tweets
if len(additional_tweets) == 0: if len(additional_tweets) == 0:
debug(f'- {screen_name}: {len(tweets)} tweets, no more tweets are available.') debug(f'- {screen_name}: {len(tweets)} tweets, no more tweets are available.\n')
break break
# Add tweets to the list # Add tweets to the list
tweets.extend(additional_tweets) tweets.extend(additional_tweets)
# Store in file # Store in file
with open(file, 'w', encoding='utf-8') as f: # Even though we are not supposed to use internal fields, there aren't any efficient way of
# Even though we are not supposed to use internal fields, there aren't any efficient way of # obtaining the json without the field. Using t.__dict__ will include the API object, which
# obtaining the json without the field. Using t.__dict__ will include the API object, which # is not serializable.
# is not serializable. write(file, json_stringify([t._json for t in tweets]))
f.write(json_stringify([t._json for t in tweets]))
def download_users_start(api: API, start_point: str, n: float = math.inf, def download_users_start(api: API, start_point: str, n: float = math.inf,
@@ -182,8 +179,7 @@ def download_users_resume_progress(api: API, base_dir: str = './data/twitter/use
:return: None :return: None
""" """
# Open file and read # Open file and read
with open(f'{base_dir}/meta/meta.json', 'r', encoding='utf-8') as f: meta = json.loads(read(f'{base_dir}/meta/meta.json'))
meta = json.load(f)
# Resume # Resume
download_users_execute(api, meta['n'], base_dir, download_users_execute(api, meta['n'], base_dir,
@@ -214,10 +210,6 @@ def download_users_execute(api: API, n: float, base_dir: str,
""" """
base_dir = normalize_directory(base_dir) base_dir = normalize_directory(base_dir)
# Ensure directory exists
Path(f'{base_dir}/users').mkdir(parents=True, exist_ok=True)
Path(f'{base_dir}/meta').mkdir(parents=True, exist_ok=True)
# Rate limit for this API endpoint is 1 request per minute, and rate delay defines how many # Rate limit for this API endpoint is 1 request per minute, and rate delay defines how many
# seconds to sleep for each request. # seconds to sleep for each request.
rate_delay = calculate_rate_delay(1) + 1 rate_delay = calculate_rate_delay(1) + 1
@@ -251,8 +243,7 @@ def download_users_execute(api: API, n: float, base_dir: str,
# This user was not saved, save the user. # This user was not saved, save the user.
if user not in downloaded: if user not in downloaded:
# Save user json # Save user json
with open(f'{base_dir}/users/{user.screen_name}.json', 'w', encoding='utf-8') as f: write(f'{base_dir}/users/{user.screen_name}.json', json_stringify(user._json))
f.write(json_stringify(user._json))
# Add to set # Add to set
downloaded.add(user.screen_name) downloaded.add(user.screen_name)
@@ -290,10 +281,9 @@ def download_users_execute(api: API, n: float, base_dir: str,
done_set.add(screen_name) done_set.add(screen_name)
# Update meta info so that downloading can be continued # Update meta info so that downloading can be continued
with open(f'{base_dir}/meta/meta.json', 'w', encoding='utf-8') as f: meta = {'downloaded': downloaded, 'done_set': done_set,
meta = {'downloaded': downloaded, 'done_set': done_set, 'current_set': current_set, 'next_set': next_set, 'n': n}
'current_set': current_set, 'next_set': next_set, 'n': n} write(f'{base_dir}/meta/meta.json', json_stringify(meta, indent=None))
f.write(json_stringify(meta, indent=None))
debug(f'Finished saving friends of {screen_name}') debug(f'Finished saving friends of {screen_name}')
debug(f'============= Total {len(downloaded)} saved =============') debug(f'============= Total {len(downloaded)} saved =============')