[U] Update project structure

.gitignore

@@ -127,3 +127,4 @@ dmypy.json
 config.json5
 data/
 /report/
+/src/report

src/constants.py

@@ -5,6 +5,7 @@ DATA_DIR = '../data'
 TWEETS_DIR = f'{DATA_DIR}/twitter/user-tweets'
 USER_DIR = f'{DATA_DIR}/twitter/user'
 REPORT_DIR = './report'
+RES_DIR = './resources'
 
 # Debug mode, or developer mode. This affects two things:
 # 1. Whether debug messages are outputted

src/main.py

@@ -1,9 +1,6 @@
-from tabulate import tabulate
+from visualization import *
-
+from collect_twitter import *
-from process.twitter_process import *
+from report import serve_report
-from process.twitter_visualization import *
-from raw_collect.twitter import *
-from report.report import serve_report
 from utils import *
 
 

src/report.py

@@ -1,5 +1,6 @@
 import json
 import os.path
+import shutil
 import traceback
 import webbrowser
 from distutils.dir_util import copy_tree
@@ -7,12 +8,9 @@ from pathlib import Path
 
 from flask import Flask, send_from_directory, Response
 
-from constants import REPORT_DIR, DEBUG
+from constants import REPORT_DIR, DEBUG, RES_DIR
 from utils import read, write
 
-# Constants
-src_dir = Path(os.path.realpath(__file__)).parent
-
 
 def generate_report() -> str:
     """
@@ -21,7 +19,7 @@ def generate_report() -> str:
     :return: Markdown report
     """
     # Load markdown
-    md = read(str(src_dir.joinpath('report_document.md'))).replace('\r\n', '\n').split('\n')
+    md = read(os.path.join(RES_DIR, './report_document.md')).replace('\r\n', '\n').split('\n')
 
     # Process line by line
     for i in range(len(md)):
@@ -71,7 +69,7 @@ def generate_html() -> str:
     # Generate markdown report and JSON encode it (which works as JS code! amazing
     md_json = json.dumps({'content': generate_report()})
     # Inject into HTML
-    html = read(str(src_dir.joinpath('report_page.html'))) \
+    html = read(os.path.join(RES_DIR, 'report_page.html')) \
         .replace('`{{markdown}}`', md_json)
     return html
 
@@ -83,11 +81,11 @@ def write_html() -> None:
     :return: None
     """
     if os.path.isdir('./dist'):
-        os.remove('./dist')
+        shutil.rmtree('./dist')
-    Path('./dist/resources').mkdir(parents=True, exist_ok=True)
+    Path('./dist/html').mkdir(parents=True, exist_ok=True)
     write('./dist/index.html', generate_html())
 
-    copy_tree(str(src_dir.joinpath('resources/').absolute()), './dist/resources')
+    copy_tree(os.path.join(RES_DIR, 'html/'), './dist/html')
     copy_tree(REPORT_DIR, './dist')
 
 
@@ -124,7 +122,7 @@ def serve_report() -> None:
         """
         return send_from_directory(Path(REPORT_DIR).absolute(), path)
 
-    @app.route('/resources/<path:path>')
+    @app.route('/html/<path:path>')
     def js_res(path: str) -> Response:
         """
         JS Resource endpoint. This maps JS and CSS queries to the resources directory
@@ -132,7 +130,7 @@ def serve_report() -> None:
         :param path: Path of the resource
         :return: File resource or 404
         """
-        return send_from_directory(os.path.join(src_dir, 'resources'), path)
+        return send_from_directory(os.path.join(RES_DIR, 'html'), path)
 
     # Run app
     webbrowser.open("http://localhost:8080")

src/resources/report_document.md

@@ -80,7 +80,7 @@ $$ \text{pop_ratio}_i = \frac{ \sum_{u \in \text{Users}} \left(\frac{\sum\text{P
 After calculation, `freqs` and `pops` are plotted in line graphs against `dates`. Initially, we are seeing graphs with very high peaks such as the graph below. After some investigation, we found that these peaks are caused by not having enough tweets on each day to average out the random error of one single popular tweet. For example, in the graph below, we adjusted the program to print different users' popularity ratios when we found an average popularity ratio of greater than 20, which produced the output on the right. As it turns out, on 2020-07-11, the user @juniorbachchan posted that he and his father tested positive, and that single post is 163.84 times more popular than the average of all his posts. (The post is linked [here](https://twitter.com/juniorbachchan/status/1282018653215395840), it has 235k likes, 25k comments, and 32k retweets). Even though these data points are outliers, there isn't an effective way of removing them since we don't have enough tweets data from each user to calculate their range (for example, someone's COVID-related post might be the only one they've posted). So, we've decided to limit the viewing window to `y = [0, 2]` as shown in the graph on the right.
 
 <div class="image-row">
-    <div><img src="resources/peak-1.png" alt="graph"></div>
+    <div><img src="html/peak-1.png" alt="graph"></div>
     <div style="display: flex; flex-direction: column; justify-content: center"><pre>
 Date:  2020-07-11
 - JoeBiden 1.36
@@ -90,7 +90,7 @@ Date:  2020-07-11
 - gucci 0.13
 - StephenKing 0.61
     </pre></div>
-    <div><img src="resources/peak-2.png" alt="graph"></div>
+    <div><img src="html/peak-2.png" alt="graph"></div>
 </div>
 
 Then, we encountered the issue of noise. When we plot the graph without a filter, we found that the graph is actually very noisy. We decided to average the results over 7 days. Then, we also experimented with different filters from the `scipy` library and different parameter values, and chose to use an IIR filter with `n = 10`.

src/resources/report_page.html

@@ -3,16 +3,16 @@
 <head>
     <meta charset="UTF-8">
     <title>CSC110 Report</title>
-    <link rel="stylesheet" href="resources/style.css">
+    <link rel="stylesheet" href="html/style.css">
 </head>
 <body>
 <div id="content">
 
 </div>
-<script src="resources/marked.min.js"></script>
+<script src="html/marked.min.js"></script>
-<script src="resources/jquery.min.js"></script>
+<script src="html/jquery.min.js"></script>
-<script src="resources/polyfill.es6.min.js"></script>
+<script src="html/polyfill.es6.min.js"></script>
-<script src="resources/mathjax-tex-mml-chtml.js"></script>
+<script src="html/mathjax-tex-mml-chtml.js"></script>
 
 <script>
 

src/visualization.py

@@ -2,20 +2,17 @@
 This module uses matplotlib to visualize processed data as graphs. The results are stored in report directory.
 The graphs are created after processing the data, for example with filtering and removing outliers.
 """
-from datetime import timedelta
+import os.path
-from dataclasses import dataclass, field
 from typing import Optional
 
 import matplotlib.ticker
-import numpy as np
-import requests
 import scipy.signal
 from matplotlib import pyplot as plt, font_manager
 import matplotlib.dates as mdates
-from matplotlib import cm
 
-from process.twitter_process import *
+from constants import RES_DIR
-from raw_collect.others import get_covid_cases_us
+from processing import *
+from collect_others import get_covid_cases_us
 
 
 @dataclass()
@@ -287,7 +284,7 @@ def graph_load_font() -> None:
     """
     Load iosevka font for matplotlib
     """
-    font = Path(os.path.realpath(__file__)).absolute().parent.joinpath('iosevka-ss04-regular.ttf')
+    font = os.path.join(RES_DIR, 'iosevka-ss04-regular.ttf')
     fe = font_manager.FontEntry(font, 'iosevka')
     font_manager.fontManager.ttflist.insert(0, fe)
     plt.rcParams["font.family"] = "iosevka"