[F] Fix warnings

src/collect_twitter.py

@@ -68,7 +68,7 @@ def download_all_tweets(api: API, screen_name: str,
 
     :param api: Tweepy API object
     :param screen_name: Screen name of that individual
-    :param download_if_exists: Whether or not to download if it already exists (Default: False)
+    :param download_if_exists: Whether to download if it already exists (Default: False)
     :return: None
     """
     # Ensure directories exist
@@ -126,10 +126,10 @@ def download_all_tweets(api: API, screen_name: str,
 
 def download_users_start(api: API, start_point: str, n: float = math.inf) -> None:
     """
-    This function downloads n twitter users by using a friends-chain.
+    This function downloads n Twitter users by using a friends-chain.
 
-    Since there isn't an API or a database with all twitter users, we can't obtain a strict list
+    Since there isn't an API or a database with all Twitter users, we can't obtain a strict list
-    of all twitter users, nor can we obtain a list of strictly random or most popular twitter
+    of all Twitter users, nor can we obtain a list of strictly random or most popular Twitter
     users. Therefore, we use the method of follows chaining: we start from a specific individual,
     obtain their followers, and pick 6 random individuals from the friends list. Then, we repeat
     the process for the selected friends: we pick 6 random friends of the 6 random friends
@@ -149,7 +149,7 @@ def download_users_start(api: API, start_point: str, n: float = math.inf) -> Non
     https://developer.twitter.com/en/docs/twitter-api/v1/accounts-and-users/follow-search-get-users/api-reference/get-friends-list)
     This will limit the rate of requests to 15 requests in a 15-minute window, which is one request
     per minute. But it is actually the fastest method of downloading a wide range of users on
-    twitter because it can download a maximum of 200 users at a time while the API for downloading
+    Twitter because it can download a maximum of 200 users at a time while the API for downloading
     a single user is limited to only 900 queries per 15, which is only 60 users per minute.
 
     There is another API endpoint that might do the job, which is api.twitter.com/friends/ids (Doc:

src/main.py

@@ -21,7 +21,7 @@ if __name__ == '__main__':
     # manually stop it when there are enough users)
     # download_users_start(api, 'voxdotcom')
 
-    # This task will run for a very very long time to obtain a large dataset of twitter users. If
+    # This task will run for a very, very long time to obtain a large dataset of Twitter users. If
     # you want to stop the process, you can resume it later using the following line:
     # download_users_resume_progress(api)
 
@@ -32,7 +32,7 @@ if __name__ == '__main__':
 
     #####################
     # Data processing - Step P1
-    # (After step C1) Process the downloaded twitter users, extract screen name, popularity, and
+    # (After step C1) Process the downloaded Twitter users, extract screen name, popularity, and
     # number of tweets data.
     # process_users()
 
@@ -44,7 +44,7 @@ if __name__ == '__main__':
 
     #####################
     # Data collection - Step C2.1
-    # (After step P2) Load the downloaded twitter users by popularity, and start downloading all
+    # (After step P2) Load the downloaded Twitter users by popularity, and start downloading all
     # tweets from 500 of the most popular users. Takes around 2 hours.
     # for u in load_user_sample().most_popular:
     #     download_all_tweets(api, u.username)
@@ -60,7 +60,7 @@ if __name__ == '__main__':
     # (After step P2) Download all tweets from the news channels we selected.
     # for u in load_user_sample().english_news:
     #     download_all_tweets(api, u)
-    # Filter out news channels that have been blocked by twitter or don't exist anymore
+    # Filter out news channels that have been blocked by twitter or don't exist
     # filter_news_channels()
 
     #####################

src/processing.py

@@ -22,7 +22,7 @@ class ProcessedUser(NamedTuple):
     """
     User and popularity.
 
-    We use NamedTuple instead of dataclass because named tuples are easier to serialize in JSON and
+    We use NamedTuple instead of dataclass because named tuples are easier to serialize in JSON, and
     they require much less space in the stored json format because no key info is stored. For
     example, using dataclass, the json for one UserPopularity object will be:
     {"username": "a", "popularity": 1, "num_postings": 1}, while using NamedTuple, the json will be:
@@ -54,7 +54,7 @@ def process_users() -> None:
 
     # Loop through all the files
     for filename in os.listdir(f'{USER_DIR}/users'):
-        # Only check json files and ignore macos dot files
+        # Only check json files and ignore macOS dot files
         if filename.endswith('.json') and not filename.startswith('.'):
             # Read
             user = json.loads(read(f'{USER_DIR}/users/{filename}'))
@@ -190,7 +190,7 @@ def get_english_news_channels() -> list[str]:
     soup = BeautifulSoup(requests.get(url).text, 'html.parser')
     users = {h.text[1:] for h in soup.select('table tr td:nth-child(2) > a')}
 
-    # Combine two sets, ignoring case (since the ids in the 100 list are all lowercased)
+    # Combine two sets, ignoring case (since the ids in the 100 list are all lowercase)
     news_channels_lower = {n.lower() for n in news_channels}
     for u in users:
         if u not in news_channels_lower:
@@ -231,7 +231,7 @@ def load_user_sample() -> UserSample:
 
 class Posting(NamedTuple):
     """
-    Posting data stores the processed tweets data, and it contains info such as whether a tweet is
+    Posting data stores the processed tweets' data, and it contains info such as whether a tweet is
     covid-related
 
     Attributes:
@@ -251,19 +251,19 @@ class Posting(NamedTuple):
 
 def process_tweets() -> None:
     """
-    Process tweets, reduce the tweets data to only a few fields defined in the Posting class. These
+    Process tweets, reduce the tweets' data to only a few fields defined in the Posting class. These
-    include whether or not the tweet is covid-related, how popular is the tweet, if it is a repost,
+    include whether the tweet is covid-related, how popular is the tweet, if it is a repost, and its
-    and its date. The processed tweet does not contain its content.
+    date. The processed tweet does not contain its content.
 
     If a user's tweets is already processed, this function will skip over that user's data.
 
-    This function will save the processed tweets data to <tweets_dir>/processed/<username>.json
+    This function will save the processed tweets' data to <tweets_dir>/processed/<username>.json
 
     :return: None
     """
     # Loop through all the files
     for filename in os.listdir(f'{TWEETS_DIR}/user'):
-        # Only check json files and ignore macos dot files
+        # Only check json files and ignore macOS dot files
         if filename.endswith('.json') and not filename.startswith('.'):
             # Check if already processed
             if os.path.isfile(f'{TWEETS_DIR}/processed/{filename}'):
@@ -297,8 +297,8 @@ def load_tweets(username: str) -> list[Posting]:
 def is_covid_related(text: str) -> bool:
     """
     Is a tweet / article covid-related. Currently, this is done through keyword matching. Even
-    though we know that not all posts with covid-related words are covid-related posts, this is our
+    though we know that not all posts with covid-related words are covid-related posts, this is
-    current best method of classification.
+    currently our best method of classification.
 
     :param text: Text content
     :return: Whether the text is covid related

src/report.py

@@ -60,7 +60,7 @@ def generate_report() -> str:
 
         # Handle errors. (It prompts "too broad an exception clause" but I actually need to catch
         # every possible exception.)
-        except Exception as e:
+        except Exception:
             md[i] = f"<pre class=\"error\">" \
                     f"\nInvalid @include statement. \n{traceback.format_exc()}</pre>"
 
@@ -73,7 +73,7 @@ def generate_html() -> str:
 
     :return: HTML string
     """
-    # Generate markdown report and JSON encode it (which works as JS code! amazing
+    # Generate markdown report and JSON encode it (which works as JS code! amazing)
     md_json = json.dumps({'content': generate_report()})
     # Inject into HTML
     html = read(os.path.join(RES_DIR, 'report_page.html')) \
@@ -122,7 +122,7 @@ def serve_report() -> None:
     @app.route('/<path:path>')
     def res(path: str) -> Response:
         """
-        Resources endpoint. This maps report queries to the report directory
+        Resources endpoint. This function maps report queries to the report directory
 
         :param path: Path of the resource
         :return: File resource or 404

src/utils.py

@@ -181,7 +181,7 @@ def remove_outliers(points: list[float], z_threshold: float = 3.5) -> list[float
         - len(points) > 0
 
     :param points: Input points list
-    :param z_threshold: Z threshold for identifying whether or not a point is an outlier
+    :param z_threshold: Z threshold for identifying whether a point is an outlier
     :return: List with outliers removed
     """
     x = np.array(points)
@@ -296,7 +296,7 @@ def daterange(start_date: str, end_date: str) -> Generator[tuple[str, datetime],
         - end_date starts with the "YYYY-MM-DD" format
 
     :param start_date: Start date in "YYYY-MM-DD" format
-    :param end_date: End date in "YYYY-MM-DD" format
+    :param end_date: Ending date in "YYYY-MM-DD" format
     :return: Generator for looping through the dates one day at a time.
     """
     start = parse_date_only(start_date)
@@ -308,7 +308,7 @@ def daterange(start_date: str, end_date: str) -> Generator[tuple[str, datetime],
 def map_to_dates(y: dict[str, Union[int, float]], dates: list[str],
                  default: float = 0) -> list[float]:
     """
-    Takes y-axis data in the form of a mapping of date to values, and returns a list of all the
+    Takes y-axis data in the form of a mapping of dates to values, and returns a list of all the
     values mapped to the date in dates. If a date in dates isn't in y, then the default values is
     used instead.
 
@@ -325,7 +325,7 @@ def map_to_dates(y: dict[str, Union[int, float]], dates: list[str],
 
 def filter_days_avg(y: list[float], n: int) -> list[float]:
     """
-    Filter y by taking an average over a n-days window. If n = 0, then return y without processing.
+    Filter y by taking an average over an n-days window. If n = 0, then return y without processing.
 
     Preconditions:
         - n % 2 == 1
@@ -351,12 +351,12 @@ def filter_days_avg(y: list[float], n: int) -> list[float]:
 
     ret = []
     for i in range(len(y)):
-        l, r = i - radius, i + radius
+        left, right = i - radius, i + radius
-        l = max(0, l)  # avoid index out of bounds by "extending" first/last element
+        left = max(0, left)  # avoid index out of bounds by "extending" first/last element
-        r = min(r, len(y) - 1)
+        right = min(right, len(y) - 1)
-        current_sum += y[r]  # extend sliding window
+        current_sum += y[right]  # extend sliding window
         ret.append(current_sum / n)
-        current_sum -= y[l]  # remove old values
+        current_sum -= y[left]  # remove old values
     return ret
 
 
@@ -377,8 +377,9 @@ def divide_zeros(numerator: list[float], denominator: list[float]) -> list[float
             output[i] = 0
         else:
             output[i] = numerator[i] / denominator[i]
-    # This marks it as incorrect type but it's actually not incorrect type, just because numpy
+    # This marks it as incorrect type, but it's actually not incorrect type, just because numpy
     # doesn't specify its return types
+    # noinspection PyTypeChecker
     return output.tolist()
 
 

src/visualization.py

@@ -438,7 +438,6 @@ def graph_line_plot(x: list[datetime], y: Union[list[float], list[list[float]]],
         if freq:
             cases = get_covid_cases_us()
             c = map_to_dates(cases.cases, [d.isoformat()[:10] for d in x])
-            # c = scipy.signal.savgol_filter(c, 45, 2)
             c = filter_days_avg(c, 7)
             c = scipy.signal.lfilter([1.0 / n] * n, 1, c)