[+] Crawl index data

index_crawler.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import datetime
+import json
+from pathlib import Path
+
+import requests
+
+
+ses = requests.Session()
+ses.headers = {'accept-language': 'zh-CN'}
+
+
+out_path = Path('index-data')
+out_path.mkdir(exist_ok=True)
+
+
+def write_entry(data):
+    id = data['guid']
+    time = datetime.datetime.fromtimestamp(data['delivery_time']).strftime('%Y-%m-%d %H-%M')
+
+    p = out_path / f'{time} {id}.json'
+    if p.is_file():
+        return False
+
+    p.write_text(json.dumps(data, ensure_ascii=False, indent=1))
+    return True
+
+
+def setup_proxy():
+    proxies = {
+        'http': 'socks5://localhost:9050',
+        'https': 'socks5://localhost:9050'
+    }
+
+    url = 'http://ifconfig.me/ip'
+
+    ip = requests.get(url).text.strip()
+    print(f'Raw ip: {ip}')
+
+    proxy_ip = requests.get(url, proxies=proxies).text.strip()
+    print(f'Proxy ip: {proxy_ip}')
+
+    assert ip != proxy_ip, 'Proxy did not start correctly.'
+
+
+if __name__ == '__main__':
+    setup_proxy()
+
+    prev_date_file = out_path / '0-prev-date.txt'
+
+    def send_req(prev_date: int | None):
+        add_param = {'logisticMinDeliveryTime': prev_date} if prev_date is not None else {}
+
+        r = requests.get('https://front.superbuy.com/logistic/get-index-pull-data',
+                         params={'onlyPackage': 1, **add_param}).json()
+
+        assert r['state'] == 0, 'Request failed.'
+
+        data = r['data']
+        successes = [write_entry(i) for i in data]
+
+        print(f'Out of {len(data)} entries, successfully wrote {sum(successes)} entries.')
+        prev_date_file.write_text(str(data[-1]['delivery_time'] - 1))
+
+    if not prev_date_file.is_file():
+        print('Getting first request...')
+        send_req(None)
+
+    while True:
+        prev_date = int(prev_date_file.read_text())
+        print(f'Getting entries before {prev_date}')
+        send_req(prev_date)