From c41931d3eb87b02ee881e26a48fa83d436dee99f Mon Sep 17 00:00:00 2001
From: Hykilpikonna <me@hydev.org>
Date: Wed, 24 Nov 2021 16:43:58 -0500
Subject: [PATCH] [+] Combine tweets for sample

---
 src/process/twitter_process.py | 33 ++++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/src/process/twitter_process.py b/src/process/twitter_process.py
index c707e59..998794b 100644
--- a/src/process/twitter_process.py
+++ b/src/process/twitter_process.py
@@ -5,6 +5,7 @@ import random
 from typing import NamedTuple
 from dataclasses import dataclass
 
+import dateutil.parser
 import requests
 from bs4 import BeautifulSoup
 from py7zr import SevenZipFile
@@ -206,8 +207,8 @@ class Posting(NamedTuple):
     popularity: int
     # Is it a repost
     repost: bool
-    # Date
-    date: datetime
+    # Date in ISO format
+    date: str
 
 
 def process_tweets() -> None:
@@ -235,7 +236,8 @@ def process_tweets() -> None:
             p = [Posting(is_covid_related(t['full_text']),
                          t['favorite_count'] + t['retweet_count'],
                          'retweeted_status' in t,
-                         datetime.strptime(t['created_at'], '%a %b %d %H:%M:%S +0000 %Y'))
+                         datetime.strptime(t['created_at'], '%a %b %d %H:%M:%S +0000 %Y')
+                         .isoformat())
                  for t in tweets]
 
             # Save data
@@ -280,6 +282,31 @@ def is_covid_related(text: str) -> bool:
     return any(k in text.lower() for k in keywords)
 
 
+def combine_tweets_for_sample(sample: list[str], name: str) -> None:
+    """
+    Combine tweets data for every user in a sample
+
+    Preconditions:
+      - name is a valid file name for your OS.
+
+    :param sample: Sample is a list of users' screen names
+    :param name: The name of the sample
+    :return: None
+    """
+    tweets: list[Posting] = []
+    for u in sample:
+        tweets += load_tweets(u)
+
+    # Sort by date, latest first
+    tweets.sort(key=lambda x: x.date, reverse=True)
+
+    # Ignore tweets that are earlier than the start of COVID
+    tweets = [t for t in tweets if t.date > '2020-01-01T01:01:01']
+
+    # Save
+    write(f'{TWEETS_DIR}/sample-combined/{name.replace(" ", "-")}.json', json_stringify(tweets))
+
+
 def pack_data() -> None:
     """
     This function packs processed data and raw data separately.