[+] Remove outliers

src/process/twitter_visualization.py

@@ -100,11 +100,16 @@ def view_covid_tweets_pop(users: list[ProcessedUser],
     print(tabulate([[u[0], f'{u[1]:.2f}'] for u in user_popularity[:20]],
                    ['Username', 'Popularity Ratio']))
 
+    # Remove outliers
+    print('As there are many outliers in the popularity ratio, they are removed in graphing.')
+    x_list = remove_outliers([f[1] for f in user_popularity])
+    print(x_list)
+
     # Graph histogram
     plt.title(f'COVID-related popularity ratios for {sample_name}')
     plt.xticks(rotation=90)
     plt.tight_layout()
-    plt.hist([f[1] for f in user_popularity], bins=100, color='#ffcccc')
+    plt.hist(x_list, bins=100, color='#ffcccc')
     plt.axvline([1], color='lightgray')
     plt.show()
 

src/utils.py

@@ -109,11 +109,11 @@ def remove_outliers(points: list[float], z_threshold: float = 3.5) -> list[float
     :param z_threshold: Z threshold for identifying whether or not a point is an outlier
     :return: List with outliers removed
     """
-    points = np.array(points)
+    x = np.array(points)
-    if len(points.shape) == 1:
+    if len(x.shape) == 1:
-        points = points[:, None]
+        x = x[:, None]
-    median = np.median(points, axis=0)
+    median = np.median(x, axis=0)
-    diff = np.sum((points - median)**2, axis=-1)
+    diff = np.sum((x - median) ** 2, axis=-1)
     diff = np.sqrt(diff)
     med_abs_deviation = np.median(diff)
 
@@ -121,7 +121,7 @@ def remove_outliers(points: list[float], z_threshold: float = 3.5) -> list[float
 
     is_outlier = modified_z_score > z_threshold
 
-    return [points[v] for v in range(len(points)) if not is_outlier[v]]
+    return [points[v] for v in range(len(x)) if not is_outlier[v]]
 
 
 class EnhancedJSONEncoder(json.JSONEncoder):