From 1cb1f58e1abe76e4246d08d7b58c2403762eb7d1 Mon Sep 17 00:00:00 2001
From: Hykilpikonna <me@hydev.org>
Date: Wed, 24 Nov 2021 17:00:44 -0500
Subject: [PATCH] [O] Filter news channels

---
 src/main.py                    |  6 ++++--
 src/process/twitter_process.py | 18 ++++++++++++++++++
 src/raw_collect/twitter.py     |  4 ++--
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/main.py b/src/main.py
index c222a89..9c18d42 100644
--- a/src/main.py
+++ b/src/main.py
@@ -58,8 +58,10 @@ if __name__ == '__main__':
     #####################
     # Data collection - Step C2.3
     # (After step P2) Download all tweets from the news channels we selected.
-    for u in load_user_sample().english_news:
-        download_all_tweets(api, u)
+    # for u in load_user_sample().english_news:
+    #     download_all_tweets(api, u)
+    # Filter out news channels that have been blocked by twitter or don't exist anymore
+    # filter_news_channels()
 
     #####################
     # Data processing - Step P3
diff --git a/src/process/twitter_process.py b/src/process/twitter_process.py
index 998794b..471ced7 100644
--- a/src/process/twitter_process.py
+++ b/src/process/twitter_process.py
@@ -184,6 +184,24 @@ def get_english_news_channels() -> list[str]:
     return list(news_channels)
 
 
+def filter_news_channels() -> None:
+    """
+    Filter out news channels that don't exist anymore or have been banned by Twitter.
+
+    Precondition:
+      - Run this after downloading all tweets from the news channels in Step 2.3 in main.
+
+    :return: None
+    """
+    sample = load_user_sample()
+    for u in list(sample.english_news):
+        u = u.lower()
+        if not (os.path.isfile(f'{TWEETS_DIR}/processed/{u}.json')
+                or os.path.isfile(f'{TWEETS_DIR}/users/{u}.json')):
+            sample.english_news.remove(u)
+    write(f'{USER_DIR}/processed/sample.json', json_stringify(sample))
+
+
 def load_user_sample() -> Sample:
     """
     Load the selected sample
diff --git a/src/raw_collect/twitter.py b/src/raw_collect/twitter.py
index 49a80be..ab7b98d 100644
--- a/src/raw_collect/twitter.py
+++ b/src/raw_collect/twitter.py
@@ -87,8 +87,8 @@ def download_all_tweets(api: API, screen_name: str,
         debug(f'- {screen_name}: Unauthorized. Probably a private account, ignoring.')
         return
     except NotFound:
-        debug(f'- {screen_name}: Not found. Probably a deleted account, writing []')
-        tweets = []
+        debug(f'- {screen_name}: Not found. Probably a deleted account, ignoring.')
+        return
 
     # This person has no tweets, done. (By the way, we discovered that @lorde has no tweets but has
     # 7 million followers... wow!)