From 1da01de3aff4b1e9ef10827900bd832bac31f342 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Mon, 13 Dec 2021 00:24:20 -0500 Subject: [PATCH] [U] Update project structure --- .gitignore | 1 + .../others.py => collect_others.py} | 0 .../twitter.py => collect_twitter.py} | 0 src/constants.py | 1 + src/main.py | 9 +++----- .../twitter_process.py => processing.py} | 0 src/{report => }/report.py | 20 ++++++++---------- .../html}/jquery.min.js | 0 .../html}/marked.min.js | 0 .../html}/mathjax-tex-mml-chtml.js | 0 .../fonts/woff-v2/MathJax_AMS-Regular.woff | Bin .../woff-v2/MathJax_Calligraphic-Bold.woff | Bin .../woff-v2/MathJax_Calligraphic-Regular.woff | Bin .../fonts/woff-v2/MathJax_Fraktur-Bold.woff | Bin .../woff-v2/MathJax_Fraktur-Regular.woff | Bin .../fonts/woff-v2/MathJax_Main-Bold.woff | Bin .../fonts/woff-v2/MathJax_Main-Italic.woff | Bin .../fonts/woff-v2/MathJax_Main-Regular.woff | Bin .../woff-v2/MathJax_Math-BoldItalic.woff | Bin .../fonts/woff-v2/MathJax_Math-Italic.woff | Bin .../fonts/woff-v2/MathJax_Math-Regular.woff | Bin .../fonts/woff-v2/MathJax_SansSerif-Bold.woff | Bin .../woff-v2/MathJax_SansSerif-Italic.woff | Bin .../woff-v2/MathJax_SansSerif-Regular.woff | Bin .../fonts/woff-v2/MathJax_Script-Regular.woff | Bin .../fonts/woff-v2/MathJax_Size1-Regular.woff | Bin .../fonts/woff-v2/MathJax_Size2-Regular.woff | Bin .../fonts/woff-v2/MathJax_Size3-Regular.woff | Bin .../fonts/woff-v2/MathJax_Size4-Regular.woff | Bin .../woff-v2/MathJax_Typewriter-Regular.woff | Bin .../fonts/woff-v2/MathJax_Vector-Bold.woff | Bin .../fonts/woff-v2/MathJax_Vector-Regular.woff | Bin .../chtml/fonts/woff-v2/MathJax_Zero.woff | Bin .../output/chtml/fonts/woff-v2/source.txt | 0 .../resources => resources/html}/peak-1.png | Bin .../resources => resources/html}/peak-2.png | Bin .../html}/polyfill.es6.min.js | 0 .../resources => resources/html}/style.css | 0 .../iosevka-ss04-regular.ttf | Bin src/{report => resources}/report_document.md | 4 ++-- src/{report => resources}/report_page.html | 10 ++++----- ...tter_visualization.py => visualization.py} | 13 +++++------- 42 files changed, 26 insertions(+), 32 deletions(-) rename src/{raw_collect/others.py => collect_others.py} (100%) rename src/{raw_collect/twitter.py => collect_twitter.py} (100%) rename src/{process/twitter_process.py => processing.py} (100%) rename src/{report => }/report.py (86%) rename src/{report/resources => resources/html}/jquery.min.js (100%) rename src/{report/resources => resources/html}/marked.min.js (100%) rename src/{report/resources => resources/html}/mathjax-tex-mml-chtml.js (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_AMS-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Bold.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Fraktur-Bold.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Fraktur-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Main-Bold.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Main-Italic.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Main-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Math-BoldItalic.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Math-Italic.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Math-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_SansSerif-Bold.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_SansSerif-Italic.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_SansSerif-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Script-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Size1-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Size2-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Size3-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Size4-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Typewriter-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Vector-Bold.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Vector-Regular.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/MathJax_Zero.woff (100%) rename src/{report/resources => resources/html}/output/chtml/fonts/woff-v2/source.txt (100%) rename src/{report/resources => resources/html}/peak-1.png (100%) rename src/{report/resources => resources/html}/peak-2.png (100%) rename src/{report/resources => resources/html}/polyfill.es6.min.js (100%) rename src/{report/resources => resources/html}/style.css (100%) rename src/{process => resources}/iosevka-ss04-regular.ttf (100%) rename src/{report => resources}/report_document.md (99%) rename src/{report => resources}/report_page.html (83%) rename src/{process/twitter_visualization.py => visualization.py} (98%) diff --git a/.gitignore b/.gitignore index 8d55ce9..be8cda8 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,4 @@ dmypy.json config.json5 data/ /report/ +/src/report diff --git a/src/raw_collect/others.py b/src/collect_others.py similarity index 100% rename from src/raw_collect/others.py rename to src/collect_others.py diff --git a/src/raw_collect/twitter.py b/src/collect_twitter.py similarity index 100% rename from src/raw_collect/twitter.py rename to src/collect_twitter.py diff --git a/src/constants.py b/src/constants.py index 8b06a8a..757e82a 100644 --- a/src/constants.py +++ b/src/constants.py @@ -5,6 +5,7 @@ DATA_DIR = '../data' TWEETS_DIR = f'{DATA_DIR}/twitter/user-tweets' USER_DIR = f'{DATA_DIR}/twitter/user' REPORT_DIR = './report' +RES_DIR = './resources' # Debug mode, or developer mode. This affects two things: # 1. Whether debug messages are outputted diff --git a/src/main.py b/src/main.py index 4d17ba7..4bb6b77 100644 --- a/src/main.py +++ b/src/main.py @@ -1,9 +1,6 @@ -from tabulate import tabulate - -from process.twitter_process import * -from process.twitter_visualization import * -from raw_collect.twitter import * -from report.report import serve_report +from visualization import * +from collect_twitter import * +from report import serve_report from utils import * diff --git a/src/process/twitter_process.py b/src/processing.py similarity index 100% rename from src/process/twitter_process.py rename to src/processing.py diff --git a/src/report/report.py b/src/report.py similarity index 86% rename from src/report/report.py rename to src/report.py index e206fd5..072d279 100644 --- a/src/report/report.py +++ b/src/report.py @@ -1,5 +1,6 @@ import json import os.path +import shutil import traceback import webbrowser from distutils.dir_util import copy_tree @@ -7,12 +8,9 @@ from pathlib import Path from flask import Flask, send_from_directory, Response -from constants import REPORT_DIR, DEBUG +from constants import REPORT_DIR, DEBUG, RES_DIR from utils import read, write -# Constants -src_dir = Path(os.path.realpath(__file__)).parent - def generate_report() -> str: """ @@ -21,7 +19,7 @@ def generate_report() -> str: :return: Markdown report """ # Load markdown - md = read(str(src_dir.joinpath('report_document.md'))).replace('\r\n', '\n').split('\n') + md = read(os.path.join(RES_DIR, './report_document.md')).replace('\r\n', '\n').split('\n') # Process line by line for i in range(len(md)): @@ -71,7 +69,7 @@ def generate_html() -> str: # Generate markdown report and JSON encode it (which works as JS code! amazing md_json = json.dumps({'content': generate_report()}) # Inject into HTML - html = read(str(src_dir.joinpath('report_page.html'))) \ + html = read(os.path.join(RES_DIR, 'report_page.html')) \ .replace('`{{markdown}}`', md_json) return html @@ -83,11 +81,11 @@ def write_html() -> None: :return: None """ if os.path.isdir('./dist'): - os.remove('./dist') - Path('./dist/resources').mkdir(parents=True, exist_ok=True) + shutil.rmtree('./dist') + Path('./dist/html').mkdir(parents=True, exist_ok=True) write('./dist/index.html', generate_html()) - copy_tree(str(src_dir.joinpath('resources/').absolute()), './dist/resources') + copy_tree(os.path.join(RES_DIR, 'html/'), './dist/html') copy_tree(REPORT_DIR, './dist') @@ -124,7 +122,7 @@ def serve_report() -> None: """ return send_from_directory(Path(REPORT_DIR).absolute(), path) - @app.route('/resources/') + @app.route('/html/') def js_res(path: str) -> Response: """ JS Resource endpoint. This maps JS and CSS queries to the resources directory @@ -132,7 +130,7 @@ def serve_report() -> None: :param path: Path of the resource :return: File resource or 404 """ - return send_from_directory(os.path.join(src_dir, 'resources'), path) + return send_from_directory(os.path.join(RES_DIR, 'html'), path) # Run app webbrowser.open("http://localhost:8080") diff --git a/src/report/resources/jquery.min.js b/src/resources/html/jquery.min.js similarity index 100% rename from src/report/resources/jquery.min.js rename to src/resources/html/jquery.min.js diff --git a/src/report/resources/marked.min.js b/src/resources/html/marked.min.js similarity index 100% rename from src/report/resources/marked.min.js rename to src/resources/html/marked.min.js diff --git a/src/report/resources/mathjax-tex-mml-chtml.js b/src/resources/html/mathjax-tex-mml-chtml.js similarity index 100% rename from src/report/resources/mathjax-tex-mml-chtml.js rename to src/resources/html/mathjax-tex-mml-chtml.js diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_AMS-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_AMS-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_AMS-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_AMS-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Bold.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Bold.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Bold.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Bold.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Calligraphic-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Fraktur-Bold.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Fraktur-Bold.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Fraktur-Bold.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Fraktur-Bold.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Fraktur-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Fraktur-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Fraktur-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Fraktur-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Main-Bold.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Main-Bold.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Main-Bold.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Main-Bold.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Main-Italic.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Main-Italic.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Main-Italic.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Main-Italic.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Main-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Main-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Main-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Main-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Math-BoldItalic.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Math-BoldItalic.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Math-BoldItalic.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Math-BoldItalic.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Math-Italic.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Math-Italic.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Math-Italic.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Math-Italic.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Math-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Math-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Math-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Math-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_SansSerif-Bold.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_SansSerif-Bold.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_SansSerif-Bold.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_SansSerif-Bold.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_SansSerif-Italic.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_SansSerif-Italic.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_SansSerif-Italic.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_SansSerif-Italic.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_SansSerif-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_SansSerif-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_SansSerif-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_SansSerif-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Script-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Script-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Script-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Script-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Size1-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Size1-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Size1-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Size1-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Size2-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Size2-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Size2-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Size2-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Size3-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Size3-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Size3-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Size3-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Size4-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Size4-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Size4-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Size4-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Typewriter-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Typewriter-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Typewriter-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Typewriter-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Vector-Bold.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Vector-Bold.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Vector-Bold.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Vector-Bold.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Vector-Regular.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Vector-Regular.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Vector-Regular.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Vector-Regular.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/MathJax_Zero.woff b/src/resources/html/output/chtml/fonts/woff-v2/MathJax_Zero.woff similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/MathJax_Zero.woff rename to src/resources/html/output/chtml/fonts/woff-v2/MathJax_Zero.woff diff --git a/src/report/resources/output/chtml/fonts/woff-v2/source.txt b/src/resources/html/output/chtml/fonts/woff-v2/source.txt similarity index 100% rename from src/report/resources/output/chtml/fonts/woff-v2/source.txt rename to src/resources/html/output/chtml/fonts/woff-v2/source.txt diff --git a/src/report/resources/peak-1.png b/src/resources/html/peak-1.png similarity index 100% rename from src/report/resources/peak-1.png rename to src/resources/html/peak-1.png diff --git a/src/report/resources/peak-2.png b/src/resources/html/peak-2.png similarity index 100% rename from src/report/resources/peak-2.png rename to src/resources/html/peak-2.png diff --git a/src/report/resources/polyfill.es6.min.js b/src/resources/html/polyfill.es6.min.js similarity index 100% rename from src/report/resources/polyfill.es6.min.js rename to src/resources/html/polyfill.es6.min.js diff --git a/src/report/resources/style.css b/src/resources/html/style.css similarity index 100% rename from src/report/resources/style.css rename to src/resources/html/style.css diff --git a/src/process/iosevka-ss04-regular.ttf b/src/resources/iosevka-ss04-regular.ttf similarity index 100% rename from src/process/iosevka-ss04-regular.ttf rename to src/resources/iosevka-ss04-regular.ttf diff --git a/src/report/report_document.md b/src/resources/report_document.md similarity index 99% rename from src/report/report_document.md rename to src/resources/report_document.md index cacd935..4a5e761 100644 --- a/src/report/report_document.md +++ b/src/resources/report_document.md @@ -80,7 +80,7 @@ $$ \text{pop_ratio}_i = \frac{ \sum_{u \in \text{Users}} \left(\frac{\sum\text{P After calculation, `freqs` and `pops` are plotted in line graphs against `dates`. Initially, we are seeing graphs with very high peaks such as the graph below. After some investigation, we found that these peaks are caused by not having enough tweets on each day to average out the random error of one single popular tweet. For example, in the graph below, we adjusted the program to print different users' popularity ratios when we found an average popularity ratio of greater than 20, which produced the output on the right. As it turns out, on 2020-07-11, the user @juniorbachchan posted that he and his father tested positive, and that single post is 163.84 times more popular than the average of all his posts. (The post is linked [here](https://twitter.com/juniorbachchan/status/1282018653215395840), it has 235k likes, 25k comments, and 32k retweets). Even though these data points are outliers, there isn't an effective way of removing them since we don't have enough tweets data from each user to calculate their range (for example, someone's COVID-related post might be the only one they've posted). So, we've decided to limit the viewing window to `y = [0, 2]` as shown in the graph on the right.
-
graph
+
graph
 Date:  2020-07-11
 - JoeBiden 1.36
@@ -90,7 +90,7 @@ Date:  2020-07-11
 - gucci 0.13
 - StephenKing 0.61
-
graph
+
graph
Then, we encountered the issue of noise. When we plot the graph without a filter, we found that the graph is actually very noisy. We decided to average the results over 7 days. Then, we also experimented with different filters from the `scipy` library and different parameter values, and chose to use an IIR filter with `n = 10`. diff --git a/src/report/report_page.html b/src/resources/report_page.html similarity index 83% rename from src/report/report_page.html rename to src/resources/report_page.html index 79bf1b0..89b5f47 100644 --- a/src/report/report_page.html +++ b/src/resources/report_page.html @@ -3,16 +3,16 @@ CSC110 Report - +
- - - - + + + +