[+] Add frequency stats

src/process/twitter_visualization.py

@@ -204,6 +204,7 @@ def report_histogram(x: list[float], path: str, title: str, clear_outliers: bool
     fig: plt.Figure
     ax: plt.Axes
     fig, ax = plt.subplots()
+    ax.margins(x=0, y=0)
 
     # Plot
     ax.set_title(title, color=border_color)
@@ -242,26 +243,25 @@ def report_histograms(sample: Sample) -> None:
     report_histogram(x, f'pop/{sample.name}-hist.png', title, True, axvline=[1])
 
 
-def report_pop_stats(samples: list[Sample]) -> None:
+def report_stats(samples: list[Sample]) -> None:
     """
-    Report popularity ratios' statistics
+    Report frequencies and popularity ratios' statistics
 
     :param samples: Samples
     :return: None
     """
     xs = [[d.data for d in s.popularity_ratios] for s in samples]
 
-    def tabulate_stats(stats: list[Stats]):
-        return [['Mean'] + [f'{s.mean:.2f}' for s in stats],
-                ['Median'] + [f'{s.median:.2f}' for s in stats],
-                ['StdDev'] + [f'{s.stddev:.2f}' for s in stats]]
-
     table = tabulate_stats([get_statistics(x) for x in xs])
     Reporter('pop/stats-with-outliers.md').table(table, [s.name for s in samples], True)
 
     table = tabulate_stats([get_statistics(remove_outliers(x)) for x in xs])
     Reporter('pop/stats.md').table(table, [s.name for s in samples], True)
 
+    xs = [[d.data for d in s.frequencies if d.data > 0.0005] for s in samples]
+    table = tabulate_stats([get_statistics(remove_outliers(x)) for x in xs])
+    Reporter('freq/stats.md').table(table, [s.name for s in samples], True)
+
 
 def view_covid_tweets_date(tweets: list[Posting]):
     # Graph histogram

src/utils.py

@@ -187,6 +187,18 @@ def get_statistics(points: list[float]) -> Stats:
     return Stats(statistics.mean(points), statistics.median(points), statistics.stdev(points))
 
 
+def tabulate_stats(stats: list[Stats]) -> list[list[str]]:
+    """
+    Create a table structure from statistics for tabulate
+
+    :param stats: Statistics
+    :return: Table for tabulate
+    """
+    return [['Mean'] + [f'{s.mean:.2f}' for s in stats],
+            ['Median'] + [f'{s.median:.2f}' for s in stats],
+            ['StdDev'] + [f'{s.stddev:.2f}' for s in stats]]
+
+
 def parse_date(iso: str) -> datetime:
     """
     Parse date faster. Running 1,000,000 trials, this parse_date function is 4.03 times faster than