diff --git a/src/main.py b/src/main.py index feec6e8..d5e2704 100644 --- a/src/main.py +++ b/src/main.py @@ -1,7 +1,7 @@ from tabulate import tabulate from process.twitter_process import load_users_popularity, process_users_popularity -from raw_collect.twitter import tweepy_login +from raw_collect.twitter import tweepy_login, download_all_tweets from utils import load_config if __name__ == '__main__': @@ -33,7 +33,7 @@ if __name__ == '__main__': print(tabulate(((u.username, u.popularity) for u in users[:20]), headers=['Name', 'Followers'])) # Start download - + download_all_tweets(api, 'sauricat') ##################### # Data collection - Step C2 diff --git a/src/process/twitter_process.py b/src/process/twitter_process.py index 86c52a6..62c2b01 100644 --- a/src/process/twitter_process.py +++ b/src/process/twitter_process.py @@ -82,6 +82,6 @@ def load_users_popularity(user_dir: str = './data/twitter/user/') -> list[UserPo download_user_start. (Default: "./data/twitter/user/") :return: List of users' screen names and popularity, sorted descending by popularity. """ - user_dir = normalize_directory(user_dir) + '/users' + user_dir = normalize_directory(user_dir) with open(f'{user_dir}/processed_popularity.json', 'r', encoding='utf-8') as f: - return json.load(f) + return [UserPopularity(*u) for u in json.load(f)] diff --git a/src/raw_collect/twitter.py b/src/raw_collect/twitter.py index 8c3eab3..c26131a 100644 --- a/src/raw_collect/twitter.py +++ b/src/raw_collect/twitter.py @@ -5,12 +5,9 @@ import json import math import random import time -from dataclasses import dataclass -from datetime import datetime from pathlib import Path from typing import Union, List -import python_ta import tweepy from tweepy import API, TooManyRequests, User, Tweet @@ -19,34 +16,6 @@ from utils import Config, debug, json_stringify, load_config, normalize_director calculate_rate_delay -@dataclass -class Tweet: - created_at: datetime - id: int - id_str: str - full_text: str - truncated: bool - display_text_range: list[int] - entities: dict[str: list] - source: str - - in_reply_to_status_id: int - in_reply_to_status_id_str: str - in_reply_to_user_id: int - in_reply_to_user_id_str: str - in_reply_to_screen_name: str - - geo: str - coordinates: str - user: User - - is_quote_status: bool - - retweeted_status: Union[dict, None] - retweet_count: int - favorite_count: int - - def tweepy_login(conf: Config) -> tweepy.API: """ Login to tweepy @@ -127,8 +96,9 @@ def download_all_tweets(api: API, screen_name: str, # Store in file with open(f'{base_dir}/{screen_name}.json', 'w', encoding='utf-8') as f: # Even though we are not supposed to use internal fields, there aren't any efficient way of - # obtaining the json without the field. - f.write(json_stringify([t.__dict__ for t in tweets])) + # obtaining the json without the field. Using t.__dict__ will include the API object, which + # is not serializable. + f.write(json_stringify([t._json for t in tweets])) def download_users_start(api: API, start_point: str, n: float = math.inf,