diff --git a/CSC110-Project.iml b/CSC110-Project.iml index d3ce09f..02091c1 100644 --- a/CSC110-Project.iml +++ b/CSC110-Project.iml @@ -4,6 +4,7 @@ + diff --git a/requirements.txt b/requirements.txt index 2db2834..6d22f1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,9 @@ tweepy==4.4.0 # Pytz is for timezone manipulations pytz~=2021.1 +# 7zip packing utility for packing our processed data +py7zr==0.16.3 + ##################### # Data visualization # Print table data diff --git a/src/process/twitter_process.py b/src/process/twitter_process.py index cbcc27a..aff8ab3 100644 --- a/src/process/twitter_process.py +++ b/src/process/twitter_process.py @@ -3,6 +3,8 @@ import random from dataclasses import dataclass from typing import NamedTuple +from py7zr import py7zr, SevenZipFile + from utils import * @@ -246,3 +248,25 @@ def is_covid_related(text: str) -> bool: keywords += ['コロナ', '検疫'] return any(k in text.lower() for k in keywords) + + +def pack_data(data_dir: str = './data/') -> None: + """ + This function packs processed data and raw data separately. + + :param data_dir: Root directory of all data. + :return: None + """ + data_dir = normalize_directory(data_dir) + packed_dir = f'{data_dir}/packed' + Path(packed_dir).mkdir(parents=True, exist_ok=True) + + # Pack data for processed. + debug('Packing data...') + processed_dirs = ['/twitter/user/meta', '/twitter/user/processed', + '/twitter/user-tweets/processed'] + with SevenZipFile(f'{packed_dir}/processed.7z', 'w') as z: + z: SevenZipFile = z + for p in processed_dirs: + debug(f'- Packing {p}') + z.writeall(data_dir + p)