From e0e4c6691c739b436e861c4d53b4a54c6dcd5d2b Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Wed, 2 Mar 2022 23:29:25 -0500 Subject: [PATCH 1/9] [O] Encapsulate --- src/PS5TrackingBot.py | 16 +++------------- src/StockTrackingBot.py | 15 +++------------ src/utils.py | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 25 deletions(-) create mode 100644 src/utils.py diff --git a/src/PS5TrackingBot.py b/src/PS5TrackingBot.py index 20a3161..9d014b4 100644 --- a/src/PS5TrackingBot.py +++ b/src/PS5TrackingBot.py @@ -1,14 +1,13 @@ import os -import re import time -import traceback import requests -from selenium.common.exceptions import StaleElementReferenceException from selenium.webdriver import Chrome from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By +from utils import parse_retry + # Config # Telegram chat ID that receives update messages (could be a channel in @channel_id format) # TG_RECEIVER = 1770239825 @@ -78,18 +77,9 @@ if __name__ == '__main__': # parse_page(browser) # browser.close() - def parse(tries: int = 0): - try: - parse_page(browser) - except StaleElementReferenceException: - if tries < 3: - parse(tries + 1) - except Exception as e: - traceback.print_exc() - # Refresh indefinitely while True: time.sleep(5) - parse() + parse_retry(parse_page, browser) browser.refresh() time.sleep(2) diff --git a/src/StockTrackingBot.py b/src/StockTrackingBot.py index a720b19..7dd4ad3 100644 --- a/src/StockTrackingBot.py +++ b/src/StockTrackingBot.py @@ -1,14 +1,14 @@ import os import re import time -import traceback import requests -from selenium.common.exceptions import StaleElementReferenceException from selenium.webdriver import Chrome from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By +from utils import parse_retry + # Config # Price increase ratio threshold (ignore everything higher than this ratio) INCR_MAX = 0.2 @@ -120,18 +120,9 @@ if __name__ == '__main__': # parse_page(browser) # browser.close() - def parse(tries: int = 0): - try: - parse_page(browser) - except StaleElementReferenceException: - if tries < 3: - parse(tries + 1) - except Exception as e: - traceback.print_exc() - # Refresh indefinitely while True: time.sleep(5) - parse() + parse_retry(parse_page, browser) browser.refresh() time.sleep(2) diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..0d45826 --- /dev/null +++ b/src/utils.py @@ -0,0 +1,15 @@ +import traceback +from typing import Callable + +from selenium.common.exceptions import StaleElementReferenceException +from selenium.webdriver.chrome.webdriver import WebDriver + + +def parse_retry(parser: Callable, browser: WebDriver, tries: int = 0): + try: + parser(browser) + except StaleElementReferenceException: + if tries < 3: + parse_retry(parser, browser, tries + 1) + except Exception as e: + traceback.print_exc() From b4422848af15a7a88b669b9d7e8641215b134107 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Wed, 2 Mar 2022 23:54:18 -0500 Subject: [PATCH 2/9] [-] Remove ignored --- src/PS5TrackingBot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/PS5TrackingBot.py b/src/PS5TrackingBot.py index 9d014b4..25897a7 100644 --- a/src/PS5TrackingBot.py +++ b/src/PS5TrackingBot.py @@ -14,11 +14,12 @@ from utils import parse_retry TG_RECEIVER = '@toronto_ps5_bestbuy' # Telegram bot token TG_TOKEN = os.environ['TG_TOKEN'] +# Alert receiver telegram chat ID +ALERT_RECEIVER = -1001655384423 # Constants CSS = By.CSS_SELECTOR AVAIL_TABLE: dict[str, bool] = {} -IGNORED = [] def send_message(msg: str): @@ -34,10 +35,6 @@ def parse_page(browser: Chrome): for item in browser.find_elements(By.CLASS_NAME, 'x-productListItem'): title = item.find_element(CSS, 'div[data-automation="productItemName"]').get_attribute('innerHTML') - # Ignored - if title in IGNORED: - continue - # Check availability avail = item.find_elements(CSS, 'div[data-automation="store-availability-messages"] span[data-automation="store-availability-checkmark"]') @@ -62,7 +59,10 @@ def parse_page(browser: Chrome): # Available and meets threshold criteria, notify user AVAIL_TABLE[title] = True send_message(f'PS5 Became Available!\n' - f'- [{title}]({link})') + f'- [{title}]({link}) ${price:.2f}') + + # Check alert + if __name__ == '__main__': From 4e93d80a687fa7131b9860e83d975e117732e637 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Thu, 3 Mar 2022 00:18:24 -0500 Subject: [PATCH 3/9] [O] Encapsulate telegram reporter --- src/PS5TrackingBot.py | 23 +++++++-------------- src/StockTrackingBot.py | 26 ++++++++++-------------- src/utils.py | 45 +++++++++++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 33 deletions(-) diff --git a/src/PS5TrackingBot.py b/src/PS5TrackingBot.py index 25897a7..c23a2f8 100644 --- a/src/PS5TrackingBot.py +++ b/src/PS5TrackingBot.py @@ -1,12 +1,11 @@ import os import time -import requests from selenium.webdriver import Chrome from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By -from utils import parse_retry +from utils import parse_retry, TelegramReporter, CSS # Config # Telegram chat ID that receives update messages (could be a channel in @channel_id format) @@ -18,16 +17,8 @@ TG_TOKEN = os.environ['TG_TOKEN'] ALERT_RECEIVER = -1001655384423 # Constants -CSS = By.CSS_SELECTOR AVAIL_TABLE: dict[str, bool] = {} - - -def send_message(msg: str): - r = requests.get(f'https://api.telegram.org/bot{TG_TOKEN}/sendMessage', - params={'chat_id': TG_RECEIVER, 'parse_mode': 'Markdown', 'text': msg}) - - if r.status_code != 200: - print('Request not OK:', r.status_code, r.text) +TG = TelegramReporter(TG_TOKEN, TG_RECEIVER, ALERT_RECEIVER) def parse_page(browser: Chrome): @@ -41,7 +32,7 @@ def parse_page(browser: Chrome): # Not available, check if it was previously available if len(avail) == 0: if title in AVAIL_TABLE: - send_message(f'Sold out: `{title}`') + TG.send(f'Sold out: `{title}`') del AVAIL_TABLE[title] continue @@ -58,11 +49,11 @@ def parse_page(browser: Chrome): # Available and meets threshold criteria, notify user AVAIL_TABLE[title] = True - send_message(f'PS5 Became Available!\n' - f'- [{title}]({link}) ${price:.2f}') + TG.send(f'PS5 Became Available!\n' + f'- [{title}]({link}) ${price:.2f}') # Check alert - + TG.alert() if __name__ == '__main__': @@ -72,7 +63,7 @@ if __name__ == '__main__': browser = Chrome(options=web_options) browser.get('https://www.bestbuy.ca/en-ca/category/ps5-consoles/17583383') - send_message('Bot started') + TG.send('Bot restarted') # parse_page(browser) # browser.close() diff --git a/src/StockTrackingBot.py b/src/StockTrackingBot.py index 7dd4ad3..fe2bc80 100644 --- a/src/StockTrackingBot.py +++ b/src/StockTrackingBot.py @@ -7,7 +7,7 @@ from selenium.webdriver import Chrome from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By -from utils import parse_retry +from utils import parse_retry, TelegramReporter, CSS # Config # Price increase ratio threshold (ignore everything higher than this ratio) @@ -17,9 +17,10 @@ INCR_MAX = 0.2 TG_RECEIVER = '@toronto_bestbuy_gpu' # Telegram bot token TG_TOKEN = os.environ['TG_TOKEN'] +# Alert receiver telegram chat ID +ALERT_RECEIVER = -1001655384423 # Constants -CSS = By.CSS_SELECTOR MODELS = [ ['3060 ti', 400, 132], ['3070 ti', 600, 167], @@ -34,6 +35,7 @@ USD_TO_CAD = 1.27 AVAIL_TABLE: dict[str, bool] = {} IGNORED = [] TITLE_SHORTEN = re.compile('(rtx|nvidia|geforce|edition|gddr[56]x*|video|card)', flags=re.IGNORECASE) +TG = TelegramReporter(TG_TOKEN, TG_RECEIVER, ALERT_RECEIVER) def shorten_title(title: str): @@ -43,14 +45,6 @@ def shorten_title(title: str): return short_title.strip() -def send_message(msg: str): - r = requests.get(f'https://api.telegram.org/bot{TG_TOKEN}/sendMessage', - params={'chat_id': TG_RECEIVER, 'parse_mode': 'Markdown', 'text': msg}) - - if r.status_code != 200: - print('Request not OK:', r.status_code, r.text) - - def parse_page(browser: Chrome): # Parse page for item in browser.find_elements(By.CLASS_NAME, 'x-productListItem'): @@ -67,7 +61,7 @@ def parse_page(browser: Chrome): # Not available, check if it was previously available if len(avail) == 0: if title in AVAIL_TABLE: - send_message(f'Sold out: `{title}`') + TG.send(f'Sold out: `{title}`') del AVAIL_TABLE[title] continue @@ -103,10 +97,10 @@ def parse_page(browser: Chrome): # Available and meets threshold criteria, notify user AVAIL_TABLE[title] = True - send_message(f'{model[0].upper()} Became Available!\n' - f'\n' - f'${price:.0f} | {price_incr * 100:.0f}% Incr | Value: {value:.0f}\n' - f'- [{title}]({link})') + TG.send(f'{model[0].upper()} Became Available!\n' + f'\n' + f'${price:.0f} | {price_incr * 100:.0f}% Incr | Value: {value:.0f}\n' + f'- [{title}]({link})') if __name__ == '__main__': @@ -120,6 +114,8 @@ if __name__ == '__main__': # parse_page(browser) # browser.close() + TG.send('Bot restarted') + # Refresh indefinitely while True: time.sleep(5) diff --git a/src/utils.py b/src/utils.py index 0d45826..5ec3762 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,8 +1,15 @@ -import traceback -from typing import Callable +from __future__ import annotations +import traceback +from typing import Callable, Optional + +import requests from selenium.common.exceptions import StaleElementReferenceException from selenium.webdriver.chrome.webdriver import WebDriver +from selenium.webdriver.common.by import By + + +CSS = By.CSS_SELECTOR def parse_retry(parser: Callable, browser: WebDriver, tries: int = 0): @@ -13,3 +20,37 @@ def parse_retry(parser: Callable, browser: WebDriver, tries: int = 0): parse_retry(parser, browser, tries + 1) except Exception as e: traceback.print_exc() + + +class TelegramReporter: + token: str + receiver: str | int + alert_receiver: Optional[str | int] + + def __init__(self, token: str, receiver: str, alert_receiver: Optional[str | int]): + self.token = token + self.receiver = receiver + self.alert_receiver = alert_receiver + + def send(self, msg: str, rec: Optional[str | int] = None) -> bool: + """ + Send a message + + :param msg: Message string + :param rec: Receiver + :return: Success or not + """ + if rec is None: + rec = self.receiver + + r = requests.get(f'https://api.telegram.org/bot{self.token}/sendMessage', + params={'chat_id': rec, 'parse_mode': 'Markdown', 'text': msg}) + + if r.status_code != 200: + print('Request not OK:', r.status_code, r.text) + + return r.status_code == 200 + + def alert(self) -> bool: + if self.alert_receiver: + return self.send('/alert', self.alert_receiver) From 943f4536fb953999e6308e7162de5d660c9f5b4c Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Thu, 3 Mar 2022 01:26:51 -0500 Subject: [PATCH 4/9] [O] Use telegram api --- src/utils.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/utils.py b/src/utils.py index 5ec3762..cc41cd5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,13 +1,14 @@ from __future__ import annotations +import os +import time import traceback from typing import Callable, Optional -import requests from selenium.common.exceptions import StaleElementReferenceException from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By - +from telegram import Bot, Message CSS = By.CSS_SELECTOR @@ -23,34 +24,28 @@ def parse_retry(parser: Callable, browser: WebDriver, tries: int = 0): class TelegramReporter: - token: str + bot: Bot receiver: str | int alert_receiver: Optional[str | int] - def __init__(self, token: str, receiver: str, alert_receiver: Optional[str | int]): - self.token = token + def __init__(self, token: str, receiver: str | int, alert_receiver: Optional[str | int]): + self.bot = Bot(token) self.receiver = receiver self.alert_receiver = alert_receiver - def send(self, msg: str, rec: Optional[str | int] = None) -> bool: + def send(self, msg: str, rec: Optional[str | int] = None) -> Message: """ Send a message :param msg: Message string :param rec: Receiver - :return: Success or not + :return: Edit code or None if failed """ if rec is None: rec = self.receiver - r = requests.get(f'https://api.telegram.org/bot{self.token}/sendMessage', - params={'chat_id': rec, 'parse_mode': 'Markdown', 'text': msg}) + return self.bot.send_message(chat_id=rec, parse_mode='Markdown', text=msg) - if r.status_code != 200: - print('Request not OK:', r.status_code, r.text) - - return r.status_code == 200 - - def alert(self) -> bool: + def alert(self) -> Message: if self.alert_receiver: return self.send('/alert', self.alert_receiver) From b1263c84af439456396a806519c7971ed3393836 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Thu, 3 Mar 2022 01:28:35 -0500 Subject: [PATCH 5/9] [+] Get availability --- src/bestbuy_utils.py | 51 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/bestbuy_utils.py diff --git a/src/bestbuy_utils.py b/src/bestbuy_utils.py new file mode 100644 index 0000000..a3c81c3 --- /dev/null +++ b/src/bestbuy_utils.py @@ -0,0 +1,51 @@ +import math +from dataclasses import dataclass +from html import unescape +from urllib.parse import unquote + +from selenium.webdriver import Chrome +from selenium.webdriver.chrome.webdriver import WebDriver +from selenium.webdriver.common.by import By + +from utils import CSS + + +@dataclass +class AvailableStore: + loc: str + avail: str + n: int + + +def get_availability(browser: WebDriver, url: str): + browser.get(url) + + # Get stores element + stores = browser.find_elements(CSS, '.x-nearby-stores > *') + + # Filter stores, not 'check other stores' button + stores = [s for s in stores if not s.get_attribute('data-automation')] + + # Filter stores that are available + stores = [s for s in stores if len(s.find_elements(CSS, '[data-automation="store-availability-checkmark"]'))] + + if not stores: + return [] + + # Get availability + result = [] + for store in stores: + loc = unescape(store.find_element(CSS, '[data-automation="pickup-store-list-item-store-name"]').get_attribute('innerHTML').strip()) + avail = store.find_elements(By.XPATH, '//*[@data-automation="pickup-store-list-item-reserve-button"]/following-sibling::span/following-sibling::span') + + if not avail: + avail = 'Available' + n = 100 + else: + avail = unescape(avail[0].get_attribute('innerHTML').strip()) + n = [int(s) for s in avail.split() if s.isdigit()][0] + + result.append(AvailableStore(loc, avail, n)) + + return result + From ad5707e7f62b5342686789910c939bab8bf94285 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Thu, 3 Mar 2022 02:17:15 -0500 Subject: [PATCH 6/9] [+] Availability v2 --- src/bestbuy_utils.py | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/bestbuy_utils.py b/src/bestbuy_utils.py index a3c81c3..6255cc4 100644 --- a/src/bestbuy_utils.py +++ b/src/bestbuy_utils.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from html import unescape from urllib.parse import unquote +import requests from selenium.webdriver import Chrome from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By @@ -10,14 +11,49 @@ from selenium.webdriver.common.by import By from utils import CSS -@dataclass +@dataclass() class AvailableStore: loc: str avail: str n: int -def get_availability(browser: WebDriver, url: str): +@dataclass() +class AvailableStoreV2(AvailableStore): + loc_key: str + fulfillment_key: str + + +def get_avail_v2(url: str, postal='M5H1N1') -> list[AvailableStore]: + sku = int(url.split('/')[-1]) + params = { + 'accept': 'application/vnd.bestbuy.standardproduct.v1+json', + 'accept-language': 'en-CA', + 'locations': '977|203|931|62|617|927|965|57|938|237|943|932|956|202|200|937|926|795|916|233|544|910|954|207|930|622|223|245|925|985|990|959|949|942', + 'postalCode': postal, + 'skus': sku + } + + headers = { + "accept": "*/*", + "accept-language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7", + "sec-ch-ua": "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"98\", \"Google Chrome\";v=\"98\"", + "sec-ch-ua-mobile": "?0", + "sec-ch-ua-platform": "\"macOS\"", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-origin", + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.109 Safari/537.36', + "referer": url, + } + + r = requests.get('https://www.bestbuy.ca/ecomm-api/availability/products', params=params, headers=headers) + o = r.json()['availabilities'][0]['pickup']['locations'] + return [AvailableStoreV2(loc['name'], str(loc['quantityOnHand']), loc['quantityOnHand'], loc['locationKey'], loc['fulfillmentKey']) + for loc in o if loc['quantityOnHand'] > 0] + + +def get_availability(browser: WebDriver, url: str) -> list[AvailableStore]: browser.get(url) # Get stores element @@ -36,7 +72,7 @@ def get_availability(browser: WebDriver, url: str): result = [] for store in stores: loc = unescape(store.find_element(CSS, '[data-automation="pickup-store-list-item-store-name"]').get_attribute('innerHTML').strip()) - avail = store.find_elements(By.XPATH, '//*[@data-automation="pickup-store-list-item-reserve-button"]/following-sibling::span/following-sibling::span') + avail = store.find_elements(By.XPATH, './/*[@data-automation="pickup-store-list-item-reserve-button"]/following-sibling::span/following-sibling::span') if not avail: avail = 'Available' From ac05e1acdb13aed30f4caea70d731a8b734207e3 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Thu, 3 Mar 2022 02:17:26 -0500 Subject: [PATCH 7/9] [+] Stringify avail --- src/bestbuy_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/bestbuy_utils.py b/src/bestbuy_utils.py index 6255cc4..924b8d9 100644 --- a/src/bestbuy_utils.py +++ b/src/bestbuy_utils.py @@ -85,3 +85,7 @@ def get_availability(browser: WebDriver, url: str) -> list[AvailableStore]: return result + +def avail_str(avail: list[AvailableStore]) -> str: + return '\n'.join([f'- {a.loc}: {"∞" if a.n == 100 else a.n}' for a in avail]) + From d2a0ce7da68acd64ea73855fea7622f020251784 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Thu, 3 Mar 2022 02:17:42 -0500 Subject: [PATCH 8/9] [+] GPU alert --- src/StockTrackingBot.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/StockTrackingBot.py b/src/StockTrackingBot.py index fe2bc80..65e9bd8 100644 --- a/src/StockTrackingBot.py +++ b/src/StockTrackingBot.py @@ -7,11 +7,15 @@ from selenium.webdriver import Chrome from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By +from bestbuy_utils import get_availability, avail_str, get_avail_v2 from utils import parse_retry, TelegramReporter, CSS # Config # Price increase ratio threshold (ignore everything higher than this ratio) INCR_MAX = 0.2 +ALERT_MODELS = ['3060 ti', '3070', '3070 ti', '3080'] +ALERT_MIN_AVAILABLE = 2 + # Telegram chat ID that receives update messages (could be a channel in @channel_id format) # TG_RECEIVER = 1770239825 TG_RECEIVER = '@toronto_bestbuy_gpu' @@ -46,6 +50,8 @@ def shorten_title(title: str): def parse_page(browser: Chrome): + become_available = [] + # Parse page for item in browser.find_elements(By.CLASS_NAME, 'x-productListItem'): title = item.find_element(CSS, 'div[data-automation="productItemName"]').get_attribute('innerHTML') @@ -97,10 +103,30 @@ def parse_page(browser: Chrome): # Available and meets threshold criteria, notify user AVAIL_TABLE[title] = True - TG.send(f'{model[0].upper()} Became Available!\n' - f'\n' - f'${price:.0f} | {price_incr * 100:.0f}% Incr | Value: {value:.0f}\n' - f'- [{title}]({link})') + msg = TG.send(f'{model[0].upper()} Became Available!\n' + f'\n' + f'${price:.0f} | {price_incr * 100:.0f}% Incr | Value: {value:.0f}\n' + f'- [{title}]({link})') + become_available.append((title, model, link, msg, price, price_incr, value)) + + # Notify user + for tup in become_available: + title, model, link, msg, price, price_incr, value = tup + + # Get availability + avail = get_avail_v2(link) + + # Edit message + msg.edit_text(f'{model[0].upper()} (${price:.0f} {price_incr * 100:.0f}% {value:.0f}) Became Available!\n' + f'\n' + f'[{title}]({link})\n' + f'{avail_str(avail[:5])}\n', + parse_mode='Markdown') + + # Check alert + if model[0] in ALERT_MODELS: + if sum(a.n for a in avail) >= ALERT_MIN_AVAILABLE: + TG.alert() if __name__ == '__main__': From 4c83af11a3425e328e9296df93a6f5827e787460 Mon Sep 17 00:00:00 2001 From: Hykilpikonna Date: Thu, 3 Mar 2022 02:17:51 -0500 Subject: [PATCH 9/9] [F] Fix typo --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index cc41cd5..7573fc8 100644 --- a/src/utils.py +++ b/src/utils.py @@ -48,4 +48,4 @@ class TelegramReporter: def alert(self) -> Message: if self.alert_receiver: - return self.send('/alert', self.alert_receiver) + return self.send('/alarm', self.alert_receiver)