[O] Filter news channels

src/main.py

@@ -58,8 +58,10 @@ if __name__ == '__main__':
     #####################
     # Data collection - Step C2.3
     # (After step P2) Download all tweets from the news channels we selected.
-    for u in load_user_sample().english_news:
+    # for u in load_user_sample().english_news:
-        download_all_tweets(api, u)
+    #     download_all_tweets(api, u)
+    # Filter out news channels that have been blocked by twitter or don't exist anymore
+    # filter_news_channels()
 
     #####################
     # Data processing - Step P3

src/process/twitter_process.py

@@ -184,6 +184,24 @@ def get_english_news_channels() -> list[str]:
     return list(news_channels)
 
 
+def filter_news_channels() -> None:
+    """
+    Filter out news channels that don't exist anymore or have been banned by Twitter.
+
+    Precondition:
+      - Run this after downloading all tweets from the news channels in Step 2.3 in main.
+
+    :return: None
+    """
+    sample = load_user_sample()
+    for u in list(sample.english_news):
+        u = u.lower()
+        if not (os.path.isfile(f'{TWEETS_DIR}/processed/{u}.json')
+                or os.path.isfile(f'{TWEETS_DIR}/users/{u}.json')):
+            sample.english_news.remove(u)
+    write(f'{USER_DIR}/processed/sample.json', json_stringify(sample))
+
+
 def load_user_sample() -> Sample:
     """
     Load the selected sample

src/raw_collect/twitter.py

@@ -87,8 +87,8 @@ def download_all_tweets(api: API, screen_name: str,
         debug(f'- {screen_name}: Unauthorized. Probably a private account, ignoring.')
         return
     except NotFound:
-        debug(f'- {screen_name}: Not found. Probably a deleted account, writing []')
+        debug(f'- {screen_name}: Not found. Probably a deleted account, ignoring.')
-        tweets = []
+        return
 
     # This person has no tweets, done. (By the way, we discovered that @lorde has no tweets but has
     # 7 million followers... wow!)