[O] Process only if not processed
This commit is contained in:
@@ -85,6 +85,10 @@ def process_tweets(tweets_dir: str = './data/twitter/user-tweets/') -> None:
|
|||||||
for filename in os.listdir(f'{tweets_dir}/user'):
|
for filename in os.listdir(f'{tweets_dir}/user'):
|
||||||
# Only check json files and ignore macos dot files
|
# Only check json files and ignore macos dot files
|
||||||
if filename.endswith('.json') and not filename.startswith('.'):
|
if filename.endswith('.json') and not filename.startswith('.'):
|
||||||
|
# Check if already processed
|
||||||
|
if os.path.isfile(f'{tweets_dir}/processed/{filename}'):
|
||||||
|
continue
|
||||||
|
|
||||||
# Read
|
# Read
|
||||||
tweets = json.loads(read(f'{tweets_dir}/user/{filename}'))
|
tweets = json.loads(read(f'{tweets_dir}/user/{filename}'))
|
||||||
p = [Posting(is_covid_related(t['full_text']),
|
p = [Posting(is_covid_related(t['full_text']),
|
||||||
|
|||||||
Reference in New Issue
Block a user