From 94005cc9d99f43fc2de0ce345c0915c733952607 Mon Sep 17 00:00:00 2001
From: Hykilpikonna <me@hydev.org>
Date: Wed, 24 Nov 2021 16:01:46 -0500
Subject: [PATCH] [+] Remove outliers

---
 src/process/twitter_visualization.py |  7 ++++++-
 src/utils.py                         | 12 ++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/process/twitter_visualization.py b/src/process/twitter_visualization.py
index 6cc5581..2c7a522 100644
--- a/src/process/twitter_visualization.py
+++ b/src/process/twitter_visualization.py
@@ -100,11 +100,16 @@ def view_covid_tweets_pop(users: list[ProcessedUser],
     print(tabulate([[u[0], f'{u[1]:.2f}'] for u in user_popularity[:20]],
                    ['Username', 'Popularity Ratio']))
 
+    # Remove outliers
+    print('As there are many outliers in the popularity ratio, they are removed in graphing.')
+    x_list = remove_outliers([f[1] for f in user_popularity])
+    print(x_list)
+
     # Graph histogram
     plt.title(f'COVID-related popularity ratios for {sample_name}')
     plt.xticks(rotation=90)
     plt.tight_layout()
-    plt.hist([f[1] for f in user_popularity], bins=100, color='#ffcccc')
+    plt.hist(x_list, bins=100, color='#ffcccc')
     plt.axvline([1], color='lightgray')
     plt.show()
 
diff --git a/src/utils.py b/src/utils.py
index 8b91c9f..1c50dec 100644
--- a/src/utils.py
+++ b/src/utils.py
@@ -109,11 +109,11 @@ def remove_outliers(points: list[float], z_threshold: float = 3.5) -> list[float
     :param z_threshold: Z threshold for identifying whether or not a point is an outlier
     :return: List with outliers removed
     """
-    points = np.array(points)
-    if len(points.shape) == 1:
-        points = points[:, None]
-    median = np.median(points, axis=0)
-    diff = np.sum((points - median)**2, axis=-1)
+    x = np.array(points)
+    if len(x.shape) == 1:
+        x = x[:, None]
+    median = np.median(x, axis=0)
+    diff = np.sum((x - median) ** 2, axis=-1)
     diff = np.sqrt(diff)
     med_abs_deviation = np.median(diff)
 
@@ -121,7 +121,7 @@ def remove_outliers(points: list[float], z_threshold: float = 3.5) -> list[float
 
     is_outlier = modified_z_score > z_threshold
 
-    return [points[v] for v in range(len(points)) if not is_outlier[v]]
+    return [points[v] for v in range(len(x)) if not is_outlier[v]]
 
 
 class EnhancedJSONEncoder(json.JSONEncoder):