[+] Pickle zst

This commit is contained in:
2024-11-28 01:03:46 -05:00
parent cd3051a1b1
commit 27e3f92186
2 changed files with 14 additions and 2 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
__version__ = "1.0.25"
__version__ = "1.0.26"
import time
import logging
+13 -1
View File
@@ -1,3 +1,4 @@
import pickle
from pathlib import Path
import zstandard as zstd
@@ -5,7 +6,7 @@ import orjson
zstd_d = zstd.ZstdDecompressor()
zstd_c = zstd.ZstdCompressor(level=15, write_checksum=True, threads=-1)
zstd_c = zstd.ZstdCompressor(level=5, write_checksum=True, threads=-1)
def load_json_zst(file_path: str | Path) -> dict | list:
@@ -33,6 +34,17 @@ def write_json_zst(file_path: str | Path, data: dict | list, **kwargs):
Path(file_path).write_bytes(zstd_c.compress(orjson.dumps(data, **kwargs)))
def load_pickle_zst(file_path: str | Path) -> object:
with Path(file_path).open('rb') as f:
return pickle.loads(zstd_d.stream_reader(f).read())
def write_pickle_zst(file_path: str | Path, data: object):
Path(file_path).write_bytes(zstd_c.compress(pickle.dumps(data)))
if __name__ == '__main__':
write_pickle_zst('test.pickle.zst', {'a': 1, 'b': 2})
assert load_pickle_zst('test.pickle.zst') == {'a': 1, 'b': 2}
write_json_zst('test.json.zst', {'a': 1, 'b': 2})
assert load_json_zst('test.json.zst') == {'a': 1, 'b': 2}