azalea/CSC110
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[+] Notes scraper

Browse Source
This commit is contained in:
Hykilpikonna
2021-12-07 22:21:31 -05:00
parent d29311c6a8
commit bf08d154e0
5 changed files with 449 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notes.7z.001
BIN
View File
Binary file not shown.
notes.7z.002
BIN
View File
Binary file not shown.
practice/prep11.py
Executable
+165
View File
@@ -0,0 +1,165 @@
"""CSC110 Fall 2021 Prep 11: Programming Exercises
Instructions (READ THIS FIRST!)
===============================
This Python module contains the data class definitions that serve as the foundation
for our computational model of a food delivery system, as we discussed in this week's
prep reading.
Please read through this file carefully, and follow all instructions described below.
We have marked each place you need to write code with the word "TODO".
As you complete your work in this file, delete each TODO comment.
Copyright and Usage Information
===============================
This file is provided solely for the personal and private use of students
taking CSC110 at the University of Toronto St. George campus. All forms of
distribution of this code, whether as given or with any changes, are
expressly prohibited. For more information on copyright for CSC110 materials,
please consult our Course Syllabus.
This file is Copyright (c) 2021 David Liu, Mario Badr, and Tom Fairgrieve.
"""
from __future__ import annotations
from dataclasses import dataclass
import datetime
from typing import Optional
@dataclass
class Restaurant:
"""A place that serves food.
Attributes:
- name: the name of the restaurant
- address: the address of the restaurant
- menu: the menu of the restaurant with the name of the dish mapping to
the price
- location: the location of the restaurant as (latitude, longitude)
Representation Invariants:
- self.name != ''
- self.address != ''
- all(self.menu[item] >= 0 for item in self.menu)
- -90 <= self.location[0] <= 90
- -180 <= self.location[1] <= 180
Sample Usage:
>>> mcdonalds = Restaurant(name='McDonalds', address='160 Spadina Ave',\
menu={'fries': 4.5}, location=(43.649, -79.397))
"""
name: str
address: str
menu: dict[str, float]
location: tuple[float, float]
@dataclass
class Customer:
"""A person who orders food.
Instance Attributes:
- name: Customer's name
- location: Customer's position in (latitutde, longitude) format.
Representation Invariants:
- self.name != ''
- -90 <= self.location[0] <= 90
- -180 <= self.location[1] <= 180
Sample Usage:
>>> david = Customer('David', (44.649, -79.115))
"""
name: str
location: tuple[float, float]
@dataclass
class Order:
"""A food order from a customer.
Attributes:
- customer: the customer who placed this order
- restaurant: the restaurant the order is placed for
- food_items: a mapping from names of food to the quantity being ordered
- start_time: the time the order was placed
- courier: the courier assigned to this order (initially None)
- end_time: the time the order was completed by the courier (initially None)
Representation Invariants:
- self.food_items != {}
- all(self.food_items[item] > 0 for item in self.food_items)
- all(f in self.restaurant.menu for f in self.food_items)
Sample Usage:
>>> david = Customer('David', (44.649, -79.115))
>>> mcdonalds = Restaurant(name='McDonalds', address='160 Spadina Ave',\
menu={'fries': 4.5}, location=(43.649, -79.397))
>>> order = Order(customer=david, restaurant=mcdonalds,\
food_items={'fries': 10},\
start_time=datetime.datetime(2020, 11, 5, 11, 30))
>>> order.courier is None # Illustrating default values
True
>>> order.end_time is None
True
"""
customer: Customer
restaurant: Restaurant
food_items: dict[str, int]
start_time: datetime.datetime
courier: Optional[Courier] = None
end_time: Optional[datetime.datetime] = None
@dataclass
class Courier:
"""A person who delivers food orders from restaurants to customers.
Deliberately left blank (you don't need to do anything here).
We'll discuss this data class in lecture!
"""
def total_cost(order: Order) -> float:
"""Return the total cost of the food items in this order.
Done: add one doctest example for this function that:
1. Creates a valid customer and restaurant, where the restaurant
has at least two different foods on the menu.
2. Creates an order for this customer and restaurant that orders
>= 2 different food items, and >= 3 quantity of each item.
3. Calculates the total cost of the order.
(Use math.isclose to avoid rounding error with floats.)
>>> import math
>>> david = Customer('David', (44.649, -79.115))
>>> mcdonalds = Restaurant(name='McDonalds', address='160 Spadina Ave',\
menu={'fries': 4.5, 'strawberries': 9999, 'ice cream': 1},\
location=(43.649, -79.397))
>>> order = Order(customer=david, restaurant=mcdonalds,\
food_items={'fries': 10, 'strawberries': 4},\
start_time=datetime.datetime(2020, 11, 5, 11, 30))
>>> math.isclose(total_cost(order), 40041.0)
True
"""
return sum([order.food_items[f] * order.restaurant.menu[f] for f in order.food_items])
if __name__ == '__main__':
import python_ta
python_ta.check_all(config={
'max-line-length': 100,
'extra-imports': ['python_ta.contracts', 'dataclasses', 'datetime'],
'disable': ['R1705', 'C0200'],
})
import python_ta.contracts
python_ta.contracts.DEBUG_CONTRACTS = False
python_ta.contracts.check_all_contracts()
import doctest
doctest.testmod()
scrape-addon.html
+155
View File
@@ -0,0 +1,155 @@
<div class="anchor-links">
<h1 id="anchor-links">Anchor Links</h1>
<h3>1. Working with Data</h3>
<a href="#anchor-01-01">1.1 The Different Types of Data</a>
<a href="#anchor-01-02">1.2 Introducing the Python Programming Language</a>
<a href="#anchor-01-03">1.3 Representing Data in Python</a>
<a href="#anchor-01-04">1.4 Storing Data in Variables</a>
<a href="#anchor-01-05">1.5 Building Up Data with Comprehensions</a>
<a href="#anchor-01-06">1.6 Application: Representing Colour</a>
<h3>2. Functions</h3>
<a href="#anchor-02-01">2.1 Python’s Built-In Functions</a>
<a href="#anchor-02-02">2.2 Defining Our Own Functions</a>
<a href="#anchor-02-03">2.3 Local Variables and Function Scope</a>
<a href="#anchor-02-04">2.4 Importing Modules</a>
<a href="#anchor-02-05">2.5 The Function Design Recipe</a>
<a href="#anchor-02-06">2.6 Testing Functions I: <code>doctest</code> and <code>pytest</code></a>
<a href="#anchor-02-07">2.7 Type Conversion Functions</a>
<a href="#anchor-02-08">2.8 Application: Representing Text</a>
<h3>3. Formal Logic</h3>
<a href="#anchor-03-01">3.1 Propositional Logic</a>
<a href="#anchor-03-02">3.2 Predicate Logic</a>
<a href="#anchor-03-03">3.3 Filtering Collections</a>
<a href="#anchor-03-04">3.4 Conditional Execution</a>
<a href="#anchor-03-05">3.5 Simplifying If Statements</a>
<a href="#anchor-03-06">3.6 <code>if __name__ == '__main__'</code></a>
<a href="#anchor-03-07">3.7 Function Specification</a>
<a href="#anchor-03-08">3.8 Richer Type Annotations</a>
<a href="#anchor-03-09">3.9 Working With Definitions</a>
<a href="#anchor-03-10">3.10 Testing Functions II: <code>hypothesis</code></a>
<a href="#anchor-03-11">3.11 Working with Multiple Quantifiers</a>
<h3>4. Working with Complex Data</h3>
<a href="#anchor-04-01">4.1 Tabular Data</a>
<a href="#anchor-04-02">4.2 Defining Our Own Data Types, Part 1</a>
<a href="#anchor-04-03">4.3 Defining Our Own Data Types, Part 2</a>
<a href="#anchor-04-04">4.4 Repeated Execution: Loops</a>
<a href="#anchor-04-05">4.5 For Loop Variations</a>
<a href="#anchor-04-06">4.6 Index-Based For loops</a>
<a href="#anchor-04-07">4.7 Nested For Loops</a>
<h3>5. Modifying Values and Variables</h3>
<a href="#anchor-05-01">5.1 Variable Reassignment and Object Mutation</a>
<a href="#anchor-05-02">5.2 Operations on Mutable Data Types</a>
<a href="#anchor-05-03">5.3 The Full Python Memory Model: Introduction</a>
<a href="#anchor-05-04">5.4 Aliasing and “Mutation at a Distance”</a>
<a href="#anchor-05-05">5.5 The Full Python Memory Model: Function Calls</a>
<a href="#anchor-05-06">5.6 Testing Functions III: Testing Mutation</a>
<h3>6. Formal Proofs</h3>
<a href="#anchor-06-01">6.1 An Introduction to Number Theory</a>
<a href="#anchor-06-02">6.2 Proofs with Number Theory</a>
<a href="#anchor-06-03">6.3 Proofs and Algorithms I: Primality Testing</a>
<a href="#anchor-06-04">6.4 Proof by Cases and Disproofs</a>
<a href="#anchor-06-05">6.5 Greatest Common Divisor</a>
<a href="#anchor-06-06">6.6 Proofs and Algorithms II: Computing the Greatest Common Divisor</a>
<a href="#anchor-06-07">6.7 Modular Arithmetic</a>
<h3>7. Cryptography</h3>
<a href="#anchor-07-01">7.1 Introduction to Cryptography</a>
<a href="#anchor-07-02">7.2 The One-Time Pad and Perfect Secrecy</a>
<a href="#anchor-07-03">7.3 Computing Shared Secret Keys</a>
<a href="#anchor-07-04">7.4 The RSA Cryptosystem</a>
<a href="#anchor-07-05">7.5 Implementing RSA in Python</a>
<a href="#anchor-07-06">7.6 Application: Securing Online Communications</a>
<h3>8. Analyzing Algorithm Running Time</h3>
<a href="#anchor-08-01">8.1 An Introduction to Running Time</a>
<a href="#anchor-08-02">8.2 Comparing Asymptotic Function Growth with Big-O</a>
<a href="#anchor-08-03">8.3 Big-O, Omega, Theta</a>
<a href="#anchor-08-04">8.4 Analyzing Algorithm Running Time</a>
<a href="#anchor-08-05">8.5 Analyzing Comprehensions and While Loops</a>
<a href="#anchor-08-06">8.6 Analyzing Built-In Data Type Operations</a>
<a href="#anchor-08-07">8.7 Worst-Case Running Time Analysis</a>
<a href="#anchor-08-08">8.8 Testing Functions IV: Efficiency</a>
<h3>9. Abstraction, Classes, and Software Design</h3>
<a href="#anchor-09-01">9.1 An Introduction to Abstraction</a>
<a href="#anchor-09-02">9.2 Defining Our Own Data Types, Part 3</a>
<a href="#anchor-09-03">9.3 Data Types, Abstract and Concrete</a>
<a href="#anchor-09-04">9.4 Stacks</a>
<a href="#anchor-09-05">9.5 Exceptions as a Part of the Public Interface</a>
<a href="#anchor-09-06">9.6 Queues</a>
<a href="#anchor-09-07">9.7 Priority Queues</a>
<a href="#anchor-09-08">9.8 Defining a Shared Public Interface with Inheritance</a>
<a href="#anchor-09-09">9.9 The <code>object</code> Superclass</a>
<h3>10. Building a Simulation</h3>
<a href="#anchor-10-01">10.1 The Problem Domain: Food Delivery Networks</a>
<a href="#anchor-10-02">10.2 Object-Oriented Modelling of Our Problem Domain</a>
<a href="#anchor-10-03">10.3 A “Manager” Class</a>
<a href="#anchor-10-04">10.4 Food Delivery Events</a>
<a href="#anchor-10-05">10.5 Creating a Discrete-Event Simulation</a>
<h3>A. Python Reference</h3>
<a href="#anchor-A--01">A.1 Python Built-In Function Reference</a>
<a href="#anchor-A--02">A.2 Python Built-In Data Types Reference</a>
<a href="#anchor-A--03">A.3 Python Special Method Reference</a>
<a href="#anchor-A--04">A.4 Python Exceptions Reference</a>
<a href="#anchor-A--05">A.5 Python Syntax Diagrams</a>
<h3>B. Python Libraries</h3>
<a href="#anchor-B--01">B.1 <code>doctest</code></a>
<a href="#anchor-B--02">B.2 <code>pytest</code></a>
<a href="#anchor-B--03">B.3 <code>python-ta</code></a>
<a href="#anchor-B--04">B.4 <code>typing</code></a>
<a href="#anchor-B--05">B.5 <code>pdb</code></a>
<h3>C. Math Reference</h3>
<a href="#anchor-C--01">C.1 Summations and Products</a>
<a href="#anchor-C--02">C.2 Inequalities</a>
<a id="anchors-button" href="#anchor-links">Links</a>
</div>
<style>
a:link
{
text-decoration: underline;
background: unset;
text-shadow: unset;
}
.anchor-links a
{
display: block;
font-size: 1.2em;
text-decoration: none !important;
}
#anchors-button
{
display: flex;
flex-flow: column;
justify-content: center;
align-items: center;
position: fixed;
bottom: 20px;
left: 20px;
text-decoration: none !important;
width: 80px;
height: 45px;
font-family: 'Roboto', sans-serif;
font-size: 11px;
text-transform: uppercase;
letter-spacing: 2px;
font-weight: 500;
color: #000;
background-color: #fff;
border: none;
border-radius: 45px;
box-shadow: 0 8px 15px rgba(0, 0, 0, 0.1);
transition: all 0.3s ease 0s;
cursor: pointer;
outline: none;
}
#anchors-button:hover {
background-color: #2EE59D;
box-shadow: 0px 15px 20px rgba(46, 229, 157, 0.4);
color: #fff;
transform: translateY(-7px);
}
</style>
<script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js" type="text/javascript"></script>
scrape.py
+129
View File
@@ -0,0 +1,129 @@
"""
Since the final test is open-notes, I've made this script to combine all course notes into one html
file, so I can ctrl+F without needing to find which chapter some definition is from.
"""
import binascii
import os
import re
import secrets
from pathlib import Path
import requests
def write(file: str, text: str) -> None:
"""
Write text to a file
:param file: File path
:param text: Text
:return: None
"""
if '/' in file:
Path(file).parent.mkdir(parents=True, exist_ok=True)
with open(file, 'w', encoding='utf-8') as f:
f.write(text)
def read(file: str) -> str:
"""
Read file content
:param file: File path (will be converted to lowercase)
:return: None
"""
with open(file, 'r', encoding='utf-8') as f:
return f.read()
def get(url: str) -> str:
req = requests.get(url)
req.encoding = req.apparent_encoding
return req.text
if __name__ == '__main__':
host = 'https://www.teach.cs.toronto.edu'
baseurl = f'{host}/~csc110y/fall/notes/'
basedir = 'notes'
r = get(baseurl)
# Replace references
g: set[str] = set(re.findall('(?<=href=").*?(?=")', r))
for href in g:
url = href
filename = href.replace('~', '-')
if not (url.startswith('//') or url.startswith('http')):
if url.startswith('/'):
url = host + url
else:
url = baseurl + url
else:
filename = filename.split("//")[1]
if filename.endswith('/'):
filename += 'index.html'
r = r.replace(href, filename)
path = os.path.join(basedir, filename)
if os.path.isfile(path):
continue
content = get(url)
write(path, content)
write(os.path.join(basedir, 'index.html'), r)
# Create full file
r = re.sub('<[/]*(section|p).*?>', '', r)
r = r.replace('<a href="www.teach.cs.toronto.edu/-csc110y/fall/notes/index.html">CSC110 Course Notes Home</a>',
'{INJECT_HERE}')
links: list[str] = re.findall('<a.*?>.*?</a>', r)
for link in links:
uid = secrets.token_hex(5)
href: str = re.findall('(?<=href=").*?(?=")', link)[0]
anchor = '-'.join([i[:2] for i in href.split('/')])
html = read(os.path.join(basedir, href))
html = re.sub('<(!DOCTYPE|html|/html|body|/body).*?>', '', html)
html = re.sub('<(head|div style="display:none"|footer)>.*?</(head|div|footer)>', '', html,
flags=re.DOTALL)
# Replace IDs
ids = set(re.findall('(?<=id=").*?(?=")', html))
for id in ids:
html = html.replace(f'"{id}"', f'"{id}-{uid}"')
html = html.replace(f'"#{id}"', f'"#{id}-{uid}"')
if '<section>' in html and '</section>' not in html:
html += '</section>'
# Download images
g: set[str] = set(re.findall('(?<=src=").*?(?=")', html))
for src in g:
if src.startswith('//') or src.startswith('http'):
continue
path = os.path.join(basedir, src)
# if os.path.isfile(path):
# continue
if src.startswith('/'):
url = host + src
else:
url = baseurl + f'{href.split("/")[-2]}/' + src
print(url)
img_data = requests.get(url).content
Path(path).parent.mkdir(parents=True, exist_ok=True)
with open(path, 'wb') as f:
f.write(img_data)
r = r.replace(link, f'<a id="anchor-{anchor}"></a>' + html)
# print(link.replace(href, f'#anchor-{anchor}'))
r = r.replace('{INJECT_HERE}', read('scrape-addon.html'))
write(os.path.join(basedir, 'full.html'), r)