From 0fe3e7c3c531176064b3ffb95b0d828b0170f9d3 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 16 Feb 2023 19:53:01 -0500 Subject: [PATCH 01/64] [F] Fix cargo warning --- backend/Cargo.lock | 2 +- backend/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c496c3b..1f357f9 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -3,5 +3,5 @@ version = 3 [[package]] -name = "MeowIndex" +name = "meow_index" version = "0.1.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index e29510f..c0c1e7f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "MeowIndex" +name = "meow_index" version = "0.1.0" edition = "2021" From 9687fc391b9854aba719249b3d83da7dfeb8c828 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 16 Feb 2023 19:58:07 -0500 Subject: [PATCH 02/64] [+] Add hyper dependency --- backend/Cargo.lock | 259 +++++++++++++++++++++++++++++++++++++++++++++ backend/Cargo.toml | 1 + 2 files changed, 260 insertions(+) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1f357f9..ce7da81 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2,6 +2,265 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures-channel" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" + +[[package]] +name = "futures-task" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" + +[[package]] +name = "futures-util" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "http" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "itoa" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + [[package]] name = "meow_index" version = "0.1.0" +dependencies = [ + "hyper", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "tokio" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +dependencies = [ + "autocfg", + "pin-project-lite", + "windows-sys", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c0c1e7f..7f138f8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -6,3 +6,4 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +hyper = "0.14.24" From a7cf931742267f5977b1f02dec9833c7e08e4d03 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 16 Feb 2023 20:35:34 -0500 Subject: [PATCH 03/64] [+] serde_json library --- backend/Cargo.lock | 337 ++++++++++++++++++++++++++++++++++++++++++++- backend/Cargo.toml | 5 +- 2 files changed, 340 insertions(+), 2 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ce7da81..e3e1c74 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -8,6 +8,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bytes" version = "1.4.0" @@ -41,6 +47,12 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +[[package]] +name = "futures-sink" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" + [[package]] name = "futures-task" version = "0.3.26" @@ -59,6 +71,40 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "h2" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "http" version = "0.2.8" @@ -103,24 +149,52 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2", "http", "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", + "socket2", "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "indexmap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +dependencies = [ + "autocfg", + "hashbrown", +] + [[package]] name = "itoa" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +[[package]] +name = "libc" +version = "0.2.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" + +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -130,11 +204,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + [[package]] name = "meow_index" version = "0.1.0" dependencies = [ "hyper", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi", + "libc", ] [[package]] @@ -143,6 +248,29 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.45.0", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -155,6 +283,121 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "proc-macro2" +version = "1.0.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ryu" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.152" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "socket2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "syn" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tokio" version = "1.25.0" @@ -162,8 +405,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "parking_lot", "pin-project-lite", - "windows-sys", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.42.0", +] + +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-util" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", ] [[package]] @@ -198,6 +475,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "unicode-ident" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" + [[package]] name = "want" version = "0.3.0" @@ -208,6 +491,34 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.42.0" @@ -223,6 +534,30 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.1" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7f138f8..febb579 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -6,4 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -hyper = "0.14.24" +hyper = { version = "0.14", features = ["full"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tokio = { version = "1", features = ["full"] } From b72f2ff7018f518fa9c9dbbc478d83dc83fdb97f Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 16 Feb 2023 20:35:48 -0500 Subject: [PATCH 04/64] [+] Simple file listing server --- backend/src/main.rs | 48 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index e7a11a9..1eb3b63 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1,3 +1,47 @@ -fn main() { - println!("Hello, world!"); +use std::convert::Infallible; +use std::{env, fs}; +use std::net::SocketAddr; +use std::os::unix::fs::MetadataExt; +use hyper::{Body, Request, Response, Server}; +use hyper::service::{make_service_fn, service_fn}; +use serde::{Deserialize, Serialize}; + +#[tokio::main] +async fn main() { + let cwd = env::current_dir().unwrap(); + let addr = SocketAddr::from(([127, 0, 0, 1], 3029)); + println!("Serving {} started on http://127.0.0.1:3029", cwd.display()); + + // A `Service` is needed for every connection, so this + // creates one from our `hello_world` function. + let make_svc = make_service_fn(|_conn| async { + // service_fn converts our function into a `Service` + Ok::<_, Infallible>(service_fn(hello_world)) + }); + + let server = Server::bind(&addr).serve(make_svc); + + // Run this server for... forever! + if let Err(e) = server.await { + eprintln!("server error: {}", e); + } +} + +#[derive(Serialize, Deserialize)] +struct ReturnPath { + name: String, + file_type: String, + mtime: i64 +} + +async fn hello_world(_req: Request) -> Result, Infallible> { + let paths: Vec = fs::read_dir("./").unwrap() + .map(|x| x.unwrap()) + .map(|x| ReturnPath { + name: x.file_name().to_str().unwrap().parse().unwrap(), + file_type: if x.path().is_file() { "file" } else { "directory" }.parse().unwrap(), + mtime: x.metadata().unwrap().mtime(), + }).collect(); + + Ok(Response::new(serde_json::to_string(&paths).unwrap().into())) } From 455aa9677e598edbd190492a873d91b2bbe35aa8 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Fri, 17 Feb 2023 10:09:26 -0500 Subject: [PATCH 05/64] [+] Macros, handle errors --- backend/src/macros.rs | 11 +++++++++++ backend/src/main.rs | 20 ++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 backend/src/macros.rs diff --git a/backend/src/macros.rs b/backend/src/macros.rs new file mode 100644 index 0000000..3adf71f --- /dev/null +++ b/backend/src/macros.rs @@ -0,0 +1,11 @@ +macro_rules! resp { + ($status:expr) => { + Response::builder().status($status) + }; +} + +macro_rules! ok { + () => { + resp!(StatusCode::OK) + }; +} diff --git a/backend/src/main.rs b/backend/src/main.rs index 1eb3b63..fced588 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1,8 +1,11 @@ +#[macro_use] +mod macros; + use std::convert::Infallible; use std::{env, fs}; use std::net::SocketAddr; use std::os::unix::fs::MetadataExt; -use hyper::{Body, Request, Response, Server}; +use hyper::{Body, http, Request, Response, Server, StatusCode}; use hyper::service::{make_service_fn, service_fn}; use serde::{Deserialize, Serialize}; @@ -34,8 +37,17 @@ struct ReturnPath { mtime: i64 } -async fn hello_world(_req: Request) -> Result, Infallible> { - let paths: Vec = fs::read_dir("./").unwrap() +async fn hello_world(_req: Request) -> http::Result> { + let path = _req.uri().path(); + println!("{path}"); + + // List files in directory + let read_dir = match fs::read_dir(".") { + Ok(file) => {file} + Err(e) => { return resp!(500).body(format!("Error {e}").into()) } + }; + + let paths: Vec = read_dir .map(|x| x.unwrap()) .map(|x| ReturnPath { name: x.file_name().to_str().unwrap().parse().unwrap(), @@ -43,5 +55,5 @@ async fn hello_world(_req: Request) -> Result, Infallible> mtime: x.metadata().unwrap().mtime(), }).collect(); - Ok(Response::new(serde_json::to_string(&paths).unwrap().into())) + ok!().body(serde_json::to_string(&paths).unwrap().into()) } From 303e65e2a2e956923089a8ebbfa10a6496a2bf64 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Fri, 17 Feb 2023 10:37:59 -0500 Subject: [PATCH 06/64] [O] Use trait instead of macros --- backend/src/macros.rs | 20 ++++++++++++-------- backend/src/main.rs | 14 ++++++++++---- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/backend/src/macros.rs b/backend/src/macros.rs index 3adf71f..e1f1bc4 100644 --- a/backend/src/macros.rs +++ b/backend/src/macros.rs @@ -1,11 +1,15 @@ -macro_rules! resp { - ($status:expr) => { - Response::builder().status($status) - }; +use hyper::{Body, http, Response, StatusCode}; + +pub trait StringExt { + fn resp(&self, status: u16) -> http::Result>; } -macro_rules! ok { - () => { - resp!(StatusCode::OK) - }; +impl StringExt for String { + fn resp(&self, status: u16) -> http::Result> { + Response::builder().status(StatusCode::from_u16(status).unwrap()).body(Body::from(self.to_owned())) + } } + +// fn main() { +// "a".resp() +// } diff --git a/backend/src/main.rs b/backend/src/main.rs index fced588..ada1f01 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,6 +8,8 @@ use std::os::unix::fs::MetadataExt; use hyper::{Body, http, Request, Response, Server, StatusCode}; use hyper::service::{make_service_fn, service_fn}; use serde::{Deserialize, Serialize}; +use path_clean::{clean}; +use macros::StringExt; #[tokio::main] async fn main() { @@ -38,13 +40,17 @@ struct ReturnPath { } async fn hello_world(_req: Request) -> http::Result> { - let path = _req.uri().path(); + let path = clean(_req.uri().path()); println!("{path}"); // List files in directory - let read_dir = match fs::read_dir(".") { + let read_dir = match fs::read_dir(path) { Ok(file) => {file} - Err(e) => { return resp!(500).body(format!("Error {e}").into()) } + Err(e) => { + let e_str = format!("Error {e}"); + if e.raw_os_error() == Some(2) { return e_str.resp(404) } + return e_str.resp(500) + } }; let paths: Vec = read_dir @@ -55,5 +61,5 @@ async fn hello_world(_req: Request) -> http::Result> { mtime: x.metadata().unwrap().mtime(), }).collect(); - ok!().body(serde_json::to_string(&paths).unwrap().into()) + serde_json::to_string(&paths).unwrap().resp(200) } From 5d89ffcfb9f0a3ab9790db3f3af08c41ea50943c Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Fri, 17 Feb 2023 10:38:17 -0500 Subject: [PATCH 07/64] [+] path-clean dependency --- backend/Cargo.lock | 7 +++++++ backend/Cargo.toml | 1 + 2 files changed, 8 insertions(+) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e3e1c74..be37228 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -215,6 +215,7 @@ name = "meow_index" version = "0.1.0" dependencies = [ "hyper", + "path-clean", "serde", "serde_json", "tokio", @@ -271,6 +272,12 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "path-clean" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd" + [[package]] name = "pin-project-lite" version = "0.2.9" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index febb579..16e5c69 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] hyper = { version = "0.14", features = ["full"] } +path-clean = "0.1.0" serde = { version = "1", features = ["derive"] } serde_json = "1" tokio = { version = "1", features = ["full"] } From 3f67edbc1ff22deca7094b7bb4b42ac353ce61fc Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Fri, 17 Feb 2023 10:40:28 -0500 Subject: [PATCH 08/64] [F] Fix path join --- backend/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index ada1f01..619d4fc 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -5,7 +5,7 @@ use std::convert::Infallible; use std::{env, fs}; use std::net::SocketAddr; use std::os::unix::fs::MetadataExt; -use hyper::{Body, http, Request, Response, Server, StatusCode}; +use hyper::{Body, http, Request, Response, Server}; use hyper::service::{make_service_fn, service_fn}; use serde::{Deserialize, Serialize}; use path_clean::{clean}; @@ -40,7 +40,7 @@ struct ReturnPath { } async fn hello_world(_req: Request) -> http::Result> { - let path = clean(_req.uri().path()); + let path = format!(".{}", clean(_req.uri().path())); println!("{path}"); // List files in directory From 47a1b71b5615603fec3d2af2a9d988851a30f984 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Fri, 17 Feb 2023 12:09:38 -0500 Subject: [PATCH 09/64] [O] More error handling --- backend/src/macros.rs | 16 +++++++++++++--- backend/src/main.rs | 22 +++++++++++++--------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/backend/src/macros.rs b/backend/src/macros.rs index e1f1bc4..eb329d9 100644 --- a/backend/src/macros.rs +++ b/backend/src/macros.rs @@ -1,3 +1,4 @@ +use std::path::PathBuf; use hyper::{Body, http, Response, StatusCode}; pub trait StringExt { @@ -10,6 +11,15 @@ impl StringExt for String { } } -// fn main() { -// "a".resp() -// } +pub trait PathExt { + fn file_type(&self) -> &str; +} + +impl PathExt for PathBuf { + fn file_type(&self) -> &str { + if self.is_file() { return "file" } + if self.is_dir() { return "directory" } + if self.is_symlink() { return "link" } + "unknown" + } +} diff --git a/backend/src/main.rs b/backend/src/main.rs index 619d4fc..49b72a4 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -10,6 +10,7 @@ use hyper::service::{make_service_fn, service_fn}; use serde::{Deserialize, Serialize}; use path_clean::{clean}; use macros::StringExt; +use crate::macros::PathExt; #[tokio::main] async fn main() { @@ -41,11 +42,11 @@ struct ReturnPath { async fn hello_world(_req: Request) -> http::Result> { let path = format!(".{}", clean(_req.uri().path())); - println!("{path}"); + println!("Raw path: {} | Sanitized path: {path}", _req.uri().path()); // List files in directory let read_dir = match fs::read_dir(path) { - Ok(file) => {file} + Ok(file) => { file } Err(e) => { let e_str = format!("Error {e}"); if e.raw_os_error() == Some(2) { return e_str.resp(404) } @@ -54,12 +55,15 @@ async fn hello_world(_req: Request) -> http::Result> { }; let paths: Vec = read_dir - .map(|x| x.unwrap()) - .map(|x| ReturnPath { - name: x.file_name().to_str().unwrap().parse().unwrap(), - file_type: if x.path().is_file() { "file" } else { "directory" }.parse().unwrap(), - mtime: x.metadata().unwrap().mtime(), - }).collect(); + .filter_map(|x| x.ok()) + .filter_map(|x| Some(ReturnPath { + name: x.file_name().to_str()?.to_string(), + file_type: x.path().file_type().to_string(), + mtime: x.metadata().ok()?.mtime(), + })).collect(); - serde_json::to_string(&paths).unwrap().resp(200) + match serde_json::to_string(&paths) { + Ok(json) => { json.resp(200) } + Err(e) => { e.to_string().resp(500) } + } } From 3eed35c71ed5ce8233f2b08dbd54234587868055 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Wed, 22 Feb 2023 17:12:23 -0500 Subject: [PATCH 10/64] [+] Add pathdiff, xdg-mime --- backend/Cargo.lock | 147 +++++++++++++++++++++++++++++++++++++++++++++ backend/Cargo.toml | 2 + 2 files changed, 149 insertions(+) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index be37228..33273a5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "autocfg" version = "1.1.0" @@ -26,6 +32,27 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "fnv" version = "1.0.7" @@ -71,6 +98,23 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "h2" version = "0.3.15" @@ -179,6 +223,19 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +[[package]] +name = "lexical-core" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if", + "ryu", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.139" @@ -216,11 +273,19 @@ version = "0.1.0" dependencies = [ "hyper", "path-clean", + "pathdiff", "serde", "serde_json", "tokio", + "xdg-mime", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "mio" version = "0.8.6" @@ -233,6 +298,17 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "nom" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +dependencies = [ + "lexical-core", + "memchr", + "version_check", +] + [[package]] name = "num_cpus" version = "1.15.0" @@ -278,6 +354,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -317,6 +399,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "ryu" version = "1.0.12" @@ -394,6 +487,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "syn" version = "1.0.107" @@ -405,6 +504,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio" version = "1.25.0" @@ -482,12 +601,27 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-ident" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "want" version = "0.3.0" @@ -606,3 +740,16 @@ name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" + +[[package]] +name = "xdg-mime" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87bf7b69bb50588d70a36e467be29d3df3e8c32580276d62eded9738c1a797aa" +dependencies = [ + "dirs-next", + "glob", + "mime", + "nom", + "unicase", +] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 16e5c69..43d1ad1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -8,6 +8,8 @@ edition = "2021" [dependencies] hyper = { version = "0.14", features = ["full"] } path-clean = "0.1.0" +pathdiff = "0.2.1" serde = { version = "1", features = ["derive"] } serde_json = "1" tokio = { version = "1", features = ["full"] } +xdg-mime = "0.3.3" From 1909b196588bb2f611fb5c7d87eff29b7fdd0417 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Wed, 22 Feb 2023 17:13:15 -0500 Subject: [PATCH 11/64] [+] Thumbnail generator --- backend/src/generator.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 backend/src/generator.rs diff --git a/backend/src/generator.rs b/backend/src/generator.rs new file mode 100644 index 0000000..4201606 --- /dev/null +++ b/backend/src/generator.rs @@ -0,0 +1,33 @@ +use std::fmt::Error; +use std::os::unix::fs::MetadataExt; +use pathdiff::diff_paths; +use std::path::{Path, PathBuf}; + +const DOT_PATH: &str = ".meow_index"; + +/// Generate thumbnail for a file if absent +pub fn generate_thumb(base: PathBuf, file: PathBuf) -> Result<(), String> +{ + let ext = file.extension().unwrap_or("".as_ref()).to_str().unwrap_or(""); + let relative = diff_paths(&file, &base).ok_or("Cannot unwrap path")?; + let thumb = base.join(DOT_PATH).join(relative) + .with_extension(format!("thumb.{}", ext)); + + if thumb.is_file() { + match (thumb.metadata(), file.metadata()) { + // Thumbnail already up to date + (Ok(tm), Ok(fm)) => if tm.mtime() >= fm.mtime() { return Ok(()) }, + _ => {} + } + } + + // Generate new thumbnail + println!("Generating thumbnail for {}\nto {}", file.display(), thumb.display()); + + Ok(()) +} + +fn generate(base: PathBuf, dir: PathBuf) +{ + // Check if already up-to-date +} From 9e8d9cb358563a56f8cbb5ba7badcd6851b3682f Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Wed, 22 Feb 2023 17:14:05 -0500 Subject: [PATCH 12/64] [O] Optimize imports --- backend/src/generator.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 4201606..dc2859b 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -1,7 +1,6 @@ -use std::fmt::Error; use std::os::unix::fs::MetadataExt; use pathdiff::diff_paths; -use std::path::{Path, PathBuf}; +use std::path::{PathBuf}; const DOT_PATH: &str = ".meow_index"; From bf050d78ef0497121f3209e41ccf9521793a7d77 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Wed, 22 Feb 2023 17:15:02 -0500 Subject: [PATCH 13/64] [F] Fix thumbnail extension --- backend/src/generator.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index dc2859b..c68ec0e 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -7,10 +7,9 @@ const DOT_PATH: &str = ".meow_index"; /// Generate thumbnail for a file if absent pub fn generate_thumb(base: PathBuf, file: PathBuf) -> Result<(), String> { - let ext = file.extension().unwrap_or("".as_ref()).to_str().unwrap_or(""); let relative = diff_paths(&file, &base).ok_or("Cannot unwrap path")?; let thumb = base.join(DOT_PATH).join(relative) - .with_extension(format!("thumb.{}", ext)); + .with_extension(format!("thumb.jpg")); if thumb.is_file() { match (thumb.metadata(), file.metadata()) { From fba8181a5bae490705cc44511df0a9a49213038b Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:49:14 -0500 Subject: [PATCH 14/64] [+] Add logger, anyhow dependencies --- backend/Cargo.lock | 182 ++++++++++++++++++++++++++++++++++++++++++++- backend/Cargo.toml | 3 + 2 files changed, 184 insertions(+), 1 deletion(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 33273a5..c57ddfc 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2,18 +2,77 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" +dependencies = [ + "backtrace", +] + [[package]] name = "arrayvec" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -26,6 +85,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + [[package]] name = "cfg-if" version = "1.0.0" @@ -53,6 +118,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "fnv" version = "1.0.7" @@ -109,6 +187,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" + [[package]] name = "glob" version = "0.3.1" @@ -140,6 +224,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hermit-abi" version = "0.2.6" @@ -183,6 +276,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + [[package]] name = "hyper" version = "0.14.24" @@ -271,9 +373,12 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" name = "meow_index" version = "0.1.0" dependencies = [ + "anyhow", "hyper", + "log", "path-clean", "pathdiff", + "pretty_env_logger", "serde", "serde_json", "tokio", @@ -286,6 +391,15 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.6" @@ -315,10 +429,19 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] +[[package]] +name = "object" +version = "0.30.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.17.1" @@ -372,6 +495,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pretty_env_logger" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" +dependencies = [ + "env_logger", + "log", +] + [[package]] name = "proc-macro2" version = "1.0.51" @@ -381,6 +514,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.23" @@ -410,6 +549,29 @@ dependencies = [ "thiserror", ] +[[package]] +name = "regex" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" + +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + [[package]] name = "ryu" version = "1.0.12" @@ -504,6 +666,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "termcolor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.38" @@ -654,6 +825,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 43d1ad1..6f4bfc0 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -6,9 +6,12 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = { version = "1.0.69", features = ["backtrace"] } hyper = { version = "0.14", features = ["full"] } +log = "0.4.17" path-clean = "0.1.0" pathdiff = "0.2.1" +pretty_env_logger = "0.4.0" serde = { version = "1", features = ["derive"] } serde_json = "1" tokio = { version = "1", features = ["full"] } From d4388a94e35f1a467e2f0afc9bfed101b18ba6ee Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:49:38 -0500 Subject: [PATCH 15/64] [+] Add test bin --- backend/Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6f4bfc0..acfe513 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -3,6 +3,10 @@ name = "meow_index" version = "0.1.0" edition = "2021" +[[bin]] +name = "test" +path = "src/test.rs" + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] From 732b289ca274b58f94525e34cf49430c718851c7 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:50:56 -0500 Subject: [PATCH 16/64] [+] Safe write util function --- backend/src/main.rs | 8 +++++++- backend/src/utils.rs | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 backend/src/utils.rs diff --git a/backend/src/main.rs b/backend/src/main.rs index 49b72a4..5db30d0 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1,5 +1,7 @@ #[macro_use] mod macros; +mod generator; +mod utils; use std::convert::Infallible; use std::{env, fs}; @@ -11,12 +13,16 @@ use serde::{Deserialize, Serialize}; use path_clean::{clean}; use macros::StringExt; use crate::macros::PathExt; +use generator::*; + +extern crate pretty_env_logger; +#[macro_use] extern crate log; #[tokio::main] async fn main() { let cwd = env::current_dir().unwrap(); let addr = SocketAddr::from(([127, 0, 0, 1], 3029)); - println!("Serving {} started on http://127.0.0.1:3029", cwd.display()); + info!("Serving {} started on http://127.0.0.1:3029", cwd.display()); // A `Service` is needed for every connection, so this // creates one from our `hello_world` function. diff --git a/backend/src/utils.rs b/backend/src/utils.rs new file mode 100644 index 0000000..9bfedbd --- /dev/null +++ b/backend/src/utils.rs @@ -0,0 +1,11 @@ +use std::{fs, io}; +use std::path::{PathBuf}; + +pub fn write_sf>(path: &PathBuf, contents: C) -> io::Result<()> { + // Create parent if it has parent + if let Some(p) = path.parent() { + fs::create_dir_all(p)? + } + + fs::write(path, contents) +} From 57d2b1946a9673e331a3545c94dd3ae67b42721a Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:51:20 -0500 Subject: [PATCH 17/64] [M] Move ReturnPath to Generator --- backend/src/generator.rs | 7 +++++++ backend/src/main.rs | 8 -------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index c68ec0e..4baeaeb 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -1,6 +1,7 @@ use std::os::unix::fs::MetadataExt; use pathdiff::diff_paths; use std::path::{PathBuf}; +use serde::{de, Deserialize, ser, Serialize}; const DOT_PATH: &str = ".meow_index"; @@ -16,6 +17,12 @@ pub fn generate_thumb(base: PathBuf, file: PathBuf) -> Result<(), String> // Thumbnail already up to date (Ok(tm), Ok(fm)) => if tm.mtime() >= fm.mtime() { return Ok(()) }, _ => {} +#[derive(Serialize, Deserialize)] +pub struct ReturnPath { + pub(crate) name: String, + pub(crate) file_type: String, + pub(crate) mtime: i64 +} } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 5db30d0..110de2c 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -9,7 +9,6 @@ use std::net::SocketAddr; use std::os::unix::fs::MetadataExt; use hyper::{Body, http, Request, Response, Server}; use hyper::service::{make_service_fn, service_fn}; -use serde::{Deserialize, Serialize}; use path_clean::{clean}; use macros::StringExt; use crate::macros::PathExt; @@ -39,13 +38,6 @@ async fn main() { } } -#[derive(Serialize, Deserialize)] -struct ReturnPath { - name: String, - file_type: String, - mtime: i64 -} - async fn hello_world(_req: Request) -> http::Result> { let path = format!(".{}", clean(_req.uri().path())); println!("Raw path: {} | Sanitized path: {path}", _req.uri().path()); From 5c408ad123f9ce1e730ac3181f15f0d50b60a4a3 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:52:26 -0500 Subject: [PATCH 18/64] [-] Remove generate functions --- backend/src/generator.rs | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 4baeaeb..3ca5ecc 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -5,34 +5,9 @@ use serde::{de, Deserialize, ser, Serialize}; const DOT_PATH: &str = ".meow_index"; -/// Generate thumbnail for a file if absent -pub fn generate_thumb(base: PathBuf, file: PathBuf) -> Result<(), String> -{ - let relative = diff_paths(&file, &base).ok_or("Cannot unwrap path")?; - let thumb = base.join(DOT_PATH).join(relative) - .with_extension(format!("thumb.jpg")); - - if thumb.is_file() { - match (thumb.metadata(), file.metadata()) { - // Thumbnail already up to date - (Ok(tm), Ok(fm)) => if tm.mtime() >= fm.mtime() { return Ok(()) }, - _ => {} #[derive(Serialize, Deserialize)] pub struct ReturnPath { pub(crate) name: String, pub(crate) file_type: String, pub(crate) mtime: i64 -} - } - } - - // Generate new thumbnail - println!("Generating thumbnail for {}\nto {}", file.display(), thumb.display()); - - Ok(()) -} - -fn generate(base: PathBuf, dir: PathBuf) -{ - // Check if already up-to-date } From 8d7ba396b2515dfa0c76a8c25f8122319fa25897 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:52:45 -0500 Subject: [PATCH 19/64] [+] make Generator class --- backend/src/generator.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 3ca5ecc..f891e40 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -1,6 +1,11 @@ use std::os::unix::fs::MetadataExt; use pathdiff::diff_paths; use std::path::{PathBuf}; +use std::fs::{File, Metadata}; +use std::fs::DirEntry; +use std::io::{BufReader}; +use xdg_mime::{SharedMimeInfo}; +use anyhow::{Result}; use serde::{de, Deserialize, ser, Serialize}; const DOT_PATH: &str = ".meow_index"; @@ -11,3 +16,9 @@ pub struct ReturnPath { pub(crate) file_type: String, pub(crate) mtime: i64 } + +pub struct Generator { + pub(crate) mime_db: SharedMimeInfo, + pub(crate) base: PathBuf, +} + From c724ce6712b9d00d484abcb74348755ee714a6ee Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:53:08 -0500 Subject: [PATCH 20/64] [+] Generator constructor --- backend/src/generator.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index f891e40..3178653 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -22,3 +22,13 @@ pub struct Generator { pub(crate) base: PathBuf, } +pub trait GeneratorTraits { + fn new(base: PathBuf) -> Generator; +} + +impl GeneratorTraits for Generator { + fn new(base: PathBuf) -> Generator { + Generator { mime_db: SharedMimeInfo::new(), base } + } +} + From 8e2072cbed597be4d1be48327c0ea78ec9719aa4 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:53:39 -0500 Subject: [PATCH 21/64] [+] Generator: Obtain .meow_index path from path --- backend/src/generator.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 3178653..63b9772 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -1,3 +1,7 @@ +use crate::macros::*; +use crate::utils::*; + +use std::fs; use std::os::unix::fs::MetadataExt; use pathdiff::diff_paths; use std::path::{PathBuf}; @@ -24,11 +28,17 @@ pub struct Generator { pub trait GeneratorTraits { fn new(base: PathBuf) -> Generator; + fn dot_path(&self, path: &PathBuf) -> PathBuf; } impl GeneratorTraits for Generator { fn new(base: PathBuf) -> Generator { Generator { mime_db: SharedMimeInfo::new(), base } } + + /// Get the same file location in DOT_PATH directory + fn dot_path(&self, path: &PathBuf) -> PathBuf { + self.base.join(DOT_PATH).join(diff_paths(&path, &self.base).unwrap()) + } } From 0f36380fd8a433a87474ef4fd37af4a180d4932f Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:53:50 -0500 Subject: [PATCH 22/64] [+] Generator: Cached results --- backend/src/generator.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 63b9772..f054102 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -29,6 +29,8 @@ pub struct Generator { pub trait GeneratorTraits { fn new(base: PathBuf) -> Generator; fn dot_path(&self, path: &PathBuf) -> PathBuf; + fn get_cached(&self, file: &PathBuf, token: &str, read: impl Fn(&PathBuf) -> Result, + gen: impl Fn(&PathBuf) -> Result<()>) -> Result; } impl GeneratorTraits for Generator { @@ -40,5 +42,19 @@ impl GeneratorTraits for Generator { fn dot_path(&self, path: &PathBuf) -> PathBuf { self.base.join(DOT_PATH).join(diff_paths(&path, &self.base).unwrap()) } + + /// Get the cached result + fn get_cached(&self, file: &PathBuf, token: &str, read: impl Fn(&PathBuf) -> Result, + gen: impl Fn(&PathBuf) -> Result<()>) -> Result { + let dot = self.dot_path(file).with_extension(token); + if ! dot.exists() || match (dot.metadata(), file.metadata()) { + (Ok(dm), Ok(fm)) => { fm.mtime() > dm.mtime() } + (_, _) => true + } { + debug!("Regenerating cached result {}", dot.display()); + gen(&dot)?; + } + Ok(read(&dot)?) + } } From 6e7a6b167b61e2206385c891e5f98fbb9103f044 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:53:59 -0500 Subject: [PATCH 23/64] [+] Generator: Cached json --- backend/src/generator.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index f054102..54a0197 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -31,6 +31,8 @@ pub trait GeneratorTraits { fn dot_path(&self, path: &PathBuf) -> PathBuf; fn get_cached(&self, file: &PathBuf, token: &str, read: impl Fn(&PathBuf) -> Result, gen: impl Fn(&PathBuf) -> Result<()>) -> Result; + fn get_cached_json(&self, file: &PathBuf, token: &str, gen: impl Fn() -> Result) -> Result + where T: de::DeserializeOwned + ?Sized + ser::Serialize; } impl GeneratorTraits for Generator { @@ -56,5 +58,20 @@ impl GeneratorTraits for Generator { } Ok(read(&dot)?) } + + /// Get the cached result + fn get_cached_json(&self, file: &PathBuf, token: &str, gen: impl Fn() -> Result) -> Result + where T: de::DeserializeOwned + ?Sized + ser::Serialize { + self.get_cached(&file, token, |f| { + let open = File::open(f)?; + let reader = BufReader::new(open); + let val: T = serde_json::from_reader(reader)?; + Ok(val) + }, |f| { + let res = gen()?; + fs::write(f, serde_json::to_string(&res)?)?; + Ok(()) + }) + } } From 7f4e06cb27f1463432b745ca0b7175005da25448 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:54:06 -0500 Subject: [PATCH 24/64] [+] Generator: Cached mime --- backend/src/generator.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 54a0197..c8cf6c2 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -33,6 +33,7 @@ pub trait GeneratorTraits { gen: impl Fn(&PathBuf) -> Result<()>) -> Result; fn get_cached_json(&self, file: &PathBuf, token: &str, gen: impl Fn() -> Result) -> Result where T: de::DeserializeOwned + ?Sized + ser::Serialize; + fn get_mime(&self, file: &PathBuf) -> Result; } impl GeneratorTraits for Generator { @@ -73,5 +74,16 @@ impl GeneratorTraits for Generator { Ok(()) }) } + + /// Get cached mime type + fn get_mime(&self, file: &PathBuf) -> Result { + self.get_cached(&file, "mime", |f| { + Ok(fs::read_to_string(f)?) + }, |f| { + let mut guesser = self.mime_db.guess_mime_type(); + write_sf(f, guesser.path(file).guess().mime_type().to_string())?; + Ok(()) + }) + } } From 5d83cf526021b39aa7e4cc96e0d7ee471cc6ca43 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 11:54:19 -0500 Subject: [PATCH 25/64] [T] Test mime --- backend/src/test.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 backend/src/test.rs diff --git a/backend/src/test.rs b/backend/src/test.rs new file mode 100644 index 0000000..d6e92fd --- /dev/null +++ b/backend/src/test.rs @@ -0,0 +1,20 @@ +mod generator; +mod macros; +mod utils; + +use std::path::PathBuf; +use generator::*; + +extern crate pretty_env_logger; +#[macro_use] extern crate log; + +fn main() { + pretty_env_logger::init(); + + let gen = Generator::new("/data".into()); + + let path: PathBuf = "/data/Anime/1977 Star Wars Collection/01 Star Wars Episode I The Phantom Menace - George Lucas 1999 Eng Subs 720p [H264-mp4].mp4".into(); + let mime = gen.get_mime(&path) + .expect("Panic"); + info!("mime {mime}") +} From 025d3922e779c022ad8bb359881d56e8aeb73013 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 12:23:38 -0500 Subject: [PATCH 26/64] [+] Use rust nightly --- backend/rust-toolchain | 1 + 1 file changed, 1 insertion(+) create mode 100644 backend/rust-toolchain diff --git a/backend/rust-toolchain b/backend/rust-toolchain new file mode 100644 index 0000000..bf867e0 --- /dev/null +++ b/backend/rust-toolchain @@ -0,0 +1 @@ +nightly From 63e9fd7e95176b07bb73f164e51c9b451eac807f Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 12:24:01 -0500 Subject: [PATCH 27/64] [-] Remove explicit trait --- backend/src/generator.rs | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index c8cf6c2..a19ad0e 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -1,3 +1,4 @@ + use crate::macros::*; use crate::utils::*; @@ -26,28 +27,18 @@ pub struct Generator { pub(crate) base: PathBuf, } -pub trait GeneratorTraits { - fn new(base: PathBuf) -> Generator; - fn dot_path(&self, path: &PathBuf) -> PathBuf; - fn get_cached(&self, file: &PathBuf, token: &str, read: impl Fn(&PathBuf) -> Result, - gen: impl Fn(&PathBuf) -> Result<()>) -> Result; - fn get_cached_json(&self, file: &PathBuf, token: &str, gen: impl Fn() -> Result) -> Result - where T: de::DeserializeOwned + ?Sized + ser::Serialize; - fn get_mime(&self, file: &PathBuf) -> Result; -} - -impl GeneratorTraits for Generator { - fn new(base: PathBuf) -> Generator { +impl Generator { + pub fn new(base: PathBuf) -> Generator { Generator { mime_db: SharedMimeInfo::new(), base } } /// Get the same file location in DOT_PATH directory - fn dot_path(&self, path: &PathBuf) -> PathBuf { + pub fn dot_path(&self, path: &PathBuf) -> PathBuf { self.base.join(DOT_PATH).join(diff_paths(&path, &self.base).unwrap()) } /// Get the cached result - fn get_cached(&self, file: &PathBuf, token: &str, read: impl Fn(&PathBuf) -> Result, + pub fn get_cached(&self, file: &PathBuf, token: &str, read: impl Fn(&PathBuf) -> Result, gen: impl Fn(&PathBuf) -> Result<()>) -> Result { let dot = self.dot_path(file).with_extension(token); if ! dot.exists() || match (dot.metadata(), file.metadata()) { @@ -61,7 +52,7 @@ impl GeneratorTraits for Generator { } /// Get the cached result - fn get_cached_json(&self, file: &PathBuf, token: &str, gen: impl Fn() -> Result) -> Result + pub fn get_cached_json(&self, file: &PathBuf, token: &str, gen: impl Fn() -> Result) -> Result where T: de::DeserializeOwned + ?Sized + ser::Serialize { self.get_cached(&file, token, |f| { let open = File::open(f)?; @@ -76,7 +67,7 @@ impl GeneratorTraits for Generator { } /// Get cached mime type - fn get_mime(&self, file: &PathBuf) -> Result { + pub fn get_mime(&self, file: &PathBuf) -> Result { self.get_cached(&file, "mime", |f| { Ok(fs::read_to_string(f)?) }, |f| { From 04233970bd77781d62742ee906b4a128e414839a Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:15:30 -0500 Subject: [PATCH 28/64] [+] Load thumbnailer --- backend/src/thumbnailer.rs | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 backend/src/thumbnailer.rs diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs new file mode 100644 index 0000000..baf49a7 --- /dev/null +++ b/backend/src/thumbnailer.rs @@ -0,0 +1,40 @@ +use std::collections::HashSet; +use std::fs; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use anyhow::{bail, Result}; + +#[derive(Debug)] +pub struct Thumbnailer { + try_exec: String, + exec: String, + mime_type: HashSet +} + +impl Thumbnailer { + /// Load an XDG thumbnailer (examples in /usr/share/thumbnailers) + pub fn load(p: &Path) -> Result { + let mut content = fs::read_to_string(p)?; + content = content.replace("\r\n", "\n"); + let lines = content.split("\n"); + + let mut t = Thumbnailer { + try_exec: "".to_string(), exec: "".to_string(), mime_type: HashSet::new() + }; + + lines.filter(|line| line.contains("=")) + .for_each(|line| { + let sp: Vec<&str> = line.splitn(2, "=").collect(); + let (key, val) = (sp[0].trim(), sp[1].trim().to_string()); + match key { + "TryExec" => t.try_exec = val, + "Exec" => t.exec = val, + "MimeType" => t.mime_type = HashSet::from_iter(val.split(";").map(str::to_string)), + _ => {}, + } + }); + + Ok(t) + } +} From efb13bdd0f8dd39210c04a54711848c645c7f866 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:15:41 -0500 Subject: [PATCH 29/64] [T] Test thumbnailer --- backend/src/main.rs | 1 + backend/src/test.rs | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 110de2c..d1c7581 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -2,6 +2,7 @@ mod macros; mod generator; mod utils; +mod thumbnailer; use std::convert::Infallible; use std::{env, fs}; diff --git a/backend/src/test.rs b/backend/src/test.rs index d6e92fd..8ccba23 100644 --- a/backend/src/test.rs +++ b/backend/src/test.rs @@ -1,9 +1,11 @@ mod generator; mod macros; mod utils; +mod thumbnailer; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use generator::*; +use crate::thumbnailer::Thumbnailer; extern crate pretty_env_logger; #[macro_use] extern crate log; @@ -16,5 +18,9 @@ fn main() { let path: PathBuf = "/data/Anime/1977 Star Wars Collection/01 Star Wars Episode I The Phantom Menace - George Lucas 1999 Eng Subs 720p [H264-mp4].mp4".into(); let mime = gen.get_mime(&path) .expect("Panic"); - info!("mime {mime}") + info!("mime {mime}"); + + let thumbnailer_path = "/usr/share/thumbnailers/totem.thumbnailer"; + let thumbnailer = Thumbnailer::load(Path::new(thumbnailer_path)).unwrap(); + info!("thumb {:?}", thumbnailer) } From 09723cac0504e092f64c82b06e407ff8a6de5ef4 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:16:06 -0500 Subject: [PATCH 30/64] [-] Remove unused imports --- backend/src/thumbnailer.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs index baf49a7..d9c7f08 100644 --- a/backend/src/thumbnailer.rs +++ b/backend/src/thumbnailer.rs @@ -1,9 +1,7 @@ use std::collections::HashSet; use std::fs; -use std::fs::File; -use std::io::{BufRead, BufReader}; use std::path::Path; -use anyhow::{bail, Result}; +use anyhow::{Result}; #[derive(Debug)] pub struct Thumbnailer { From 543f5e3aa8e80e253f778d8103a20b3e38399674 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:40:47 -0500 Subject: [PATCH 31/64] [+] shlex dependency --- backend/Cargo.lock | 7 +++++++ backend/Cargo.toml | 1 + 2 files changed, 8 insertions(+) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c57ddfc..b2d0615 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -381,6 +381,7 @@ dependencies = [ "pretty_env_logger", "serde", "serde_json", + "shlex", "tokio", "xdg-mime", ] @@ -615,6 +616,12 @@ dependencies = [ "serde", ] +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + [[package]] name = "signal-hook-registry" version = "1.4.1" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index acfe513..72db02c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -18,5 +18,6 @@ pathdiff = "0.2.1" pretty_env_logger = "0.4.0" serde = { version = "1", features = ["derive"] } serde_json = "1" +shlex = "1.1.0" tokio = { version = "1", features = ["full"] } xdg-mime = "0.3.3" From 24c4a0c8d486abb54ca2028f1732d77a59c5c0cd Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:41:04 -0500 Subject: [PATCH 32/64] [+] Thumbnailer check mime type --- backend/src/thumbnailer.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs index d9c7f08..cdbac55 100644 --- a/backend/src/thumbnailer.rs +++ b/backend/src/thumbnailer.rs @@ -35,4 +35,9 @@ impl Thumbnailer { Ok(t) } + + /// Check if this thumbnailer should run on a specific mime type + pub fn check(&self, mime: &str) -> bool { + self.mime_type.contains(mime) + } } From c1114d2bee826821b4c648ab0840397d7d1816d4 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:41:11 -0500 Subject: [PATCH 33/64] [+] Thumbnailer execute --- backend/src/thumbnailer.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs index cdbac55..2dd47da 100644 --- a/backend/src/thumbnailer.rs +++ b/backend/src/thumbnailer.rs @@ -1,7 +1,9 @@ use std::collections::HashSet; -use std::fs; +use std::{fs, process}; use std::path::Path; -use anyhow::{Result}; +use std::process::Command; +use anyhow::{bail, Result}; +use shlex::Shlex; #[derive(Debug)] pub struct Thumbnailer { @@ -40,4 +42,22 @@ impl Thumbnailer { pub fn check(&self, mime: &str) -> bool { self.mime_type.contains(mime) } + + /// Generate thumbnail + pub fn gen(&self, orig: &str, new: &str, pixels: i32) -> Result<()> { + let cmd = self.exec + .replace("%s", &*format!("'{pixels}'")) + .replace("%u", &*format!("'{orig}'")) + .replace("%o", &*format!("'{new}'")); + let args: Vec = Shlex::new(&*cmd).collect(); + let out = Command::new(args[0].to_owned()).args(&args[1..]).output()?; + if !out.status.success() { + error!("Command failed: {cmd}"); + error!("Command output: {:?}", out); + bail!(String::from_utf8(out.stderr)?); + } + debug!("Command output: {:?}", out); + + Ok(()) + } } From 127fbffb65564e9a1e26061f5f090ba6a431f7bf Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:41:18 -0500 Subject: [PATCH 34/64] [T] Test thumbnailer --- backend/src/test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/test.rs b/backend/src/test.rs index 8ccba23..680d95f 100644 --- a/backend/src/test.rs +++ b/backend/src/test.rs @@ -22,5 +22,7 @@ fn main() { let thumbnailer_path = "/usr/share/thumbnailers/totem.thumbnailer"; let thumbnailer = Thumbnailer::load(Path::new(thumbnailer_path)).unwrap(); - info!("thumb {:?}", thumbnailer) + info!("thumb {:?}", thumbnailer); + info!("check {:?}", thumbnailer.check("audio/x-mp3")); + thumbnailer.gen(path.to_str().unwrap(), "/tmp/test.png", 256).expect("Generation failed"); } From b92e835d1a06b9b6112cb97cf786f2621fa2cf42 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:47:19 -0500 Subject: [PATCH 35/64] [+] Load all thumbnailers --- backend/src/thumbnailer.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs index 2dd47da..7e34a79 100644 --- a/backend/src/thumbnailer.rs +++ b/backend/src/thumbnailer.rs @@ -38,6 +38,14 @@ impl Thumbnailer { Ok(t) } + /// Load all thumbanilers available in the system + pub fn load_all() -> Result> { + Ok(fs::read_dir("/usr/share/thumbnailers")? + .filter_map(|f| f.ok()) + .filter_map(|f| Thumbnailer::load(&*f.path()).ok()) + .collect()) + } + /// Check if this thumbnailer should run on a specific mime type pub fn check(&self, mime: &str) -> bool { self.mime_type.contains(mime) From ac4fe214d26ff68f0d0c87dc7129f395427a04b3 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:53:50 -0500 Subject: [PATCH 36/64] [+] Find thumbnailer with mime --- backend/src/thumbnailer.rs | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs index 7e34a79..2dae378 100644 --- a/backend/src/thumbnailer.rs +++ b/backend/src/thumbnailer.rs @@ -38,14 +38,6 @@ impl Thumbnailer { Ok(t) } - /// Load all thumbanilers available in the system - pub fn load_all() -> Result> { - Ok(fs::read_dir("/usr/share/thumbnailers")? - .filter_map(|f| f.ok()) - .filter_map(|f| Thumbnailer::load(&*f.path()).ok()) - .collect()) - } - /// Check if this thumbnailer should run on a specific mime type pub fn check(&self, mime: &str) -> bool { self.mime_type.contains(mime) @@ -69,3 +61,22 @@ impl Thumbnailer { Ok(()) } } + +pub struct Thumbnailers { + list: Vec +} + +impl Thumbnailers { + /// Load all thumbanilers available in the system + pub fn load_all() -> Result { + Ok(Thumbnailers { list: fs::read_dir("/usr/share/thumbnailers")? + .filter_map(|f| f.ok()) + .filter_map(|f| Thumbnailer::load(&*f.path()).ok()) + .collect() }) + } + + /// Find a thumbnailer for a mime type + pub fn find(&self, mime: &str) -> Option<&Thumbnailer> { + self.list.iter().find(|x| x.check(mime)) + } +} From b4a3991d0ee626611e1b6ca2a7ce4408e6c135ad Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:57:47 -0500 Subject: [PATCH 37/64] [T] Test thumbnailers --- backend/src/test.rs | 5 ++++- backend/src/thumbnailer.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/test.rs b/backend/src/test.rs index 680d95f..2f039b9 100644 --- a/backend/src/test.rs +++ b/backend/src/test.rs @@ -5,7 +5,7 @@ mod thumbnailer; use std::path::{Path, PathBuf}; use generator::*; -use crate::thumbnailer::Thumbnailer; +use crate::thumbnailer::{Thumbnailer, Thumbnailers}; extern crate pretty_env_logger; #[macro_use] extern crate log; @@ -25,4 +25,7 @@ fn main() { info!("thumb {:?}", thumbnailer); info!("check {:?}", thumbnailer.check("audio/x-mp3")); thumbnailer.gen(path.to_str().unwrap(), "/tmp/test.png", 256).expect("Generation failed"); + + let ts = Thumbnailers::load_all().unwrap(); + info!("Video thumbnailer: {:?}", ts.find("audio/x-mp3")) } diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs index 2dae378..2c31467 100644 --- a/backend/src/thumbnailer.rs +++ b/backend/src/thumbnailer.rs @@ -1,5 +1,5 @@ use std::collections::HashSet; -use std::{fs, process}; +use std::fs; use std::path::Path; use std::process::Command; use anyhow::{bail, Result}; From 83165256abf8b3833ea530fd9e0ec73719686c60 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 13:57:59 -0500 Subject: [PATCH 38/64] [+] Add thumbnailers to generatro --- backend/src/generator.rs | 64 ++++++++++++++++++++++++++++++++++++++-- backend/src/test.rs | 2 +- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index a19ad0e..62101d4 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -12,6 +12,7 @@ use std::io::{BufReader}; use xdg_mime::{SharedMimeInfo}; use anyhow::{Result}; use serde::{de, Deserialize, ser, Serialize}; +use crate::thumbnailer::{Thumbnailers}; const DOT_PATH: &str = ".meow_index"; @@ -24,12 +25,13 @@ pub struct ReturnPath { pub struct Generator { pub(crate) mime_db: SharedMimeInfo, + pub(crate) thumbnailers: Thumbnailers, pub(crate) base: PathBuf, } impl Generator { - pub fn new(base: PathBuf) -> Generator { - Generator { mime_db: SharedMimeInfo::new(), base } + pub fn new(base: PathBuf) -> Result { + Ok(Generator { mime_db: SharedMimeInfo::new(), thumbnailers: Thumbnailers::load_all()?, base }) } /// Get the same file location in DOT_PATH directory @@ -76,5 +78,63 @@ impl Generator { Ok(()) }) } + + /// Process a single file + pub fn get_thumb(&self, file: &PathBuf) -> Result<()> { + let dot = self.dot_path(&file); + let thumb = dot.with_extension("thumb.jpg"); + + if thumb.exists() { + match (thumb.metadata(), file.metadata()) { + // Thumbnail already up to date + (Ok(tm), Ok(fm)) => if tm.mtime() >= fm.mtime() { return Ok(()) }, + _ => {} + } + } + + // Generate new thumbnail + info!("Generating thumbnail for {}\nto {}", file.display(), thumb.display()); + + Ok(()) + } + + /// Process a directory + pub fn process_dir(&self, dir: &PathBuf) -> Result<()> { + // List files + let files: Vec<(DirEntry, Metadata)> = dir.to_owned().read_dir()? + .filter_map(|x| x.ok()) + .filter_map(|x| { + let meta = x.metadata(); + Some((x, meta.ok()?)) + }).collect(); + + // Recurse into directories + files.iter().for_each(|(f, m)| { + if f.path().is_dir() { + // Recurse into directory + let _ = self.process_dir(&f.path()); + } + }); + + // Check if directory listing needs update + let d_info = self.dot_path(&dir).with_extension(".json"); + let d_mtime = d_info.metadata()?.mtime(); + if files.iter().any(|(f, m)| m.mtime() > d_mtime) { + // Update directory listing + let paths: Vec = files.iter() + .filter_map(|(f, m)| Some(ReturnPath { + name: f.file_name().to_str()?.to_string(), + file_type: f.path().file_type().to_string(), + mtime: m.mtime(), + })).collect(); + + let json = serde_json::to_string(&paths)?; + // fs::write(d_info, json)?; + } + + + + Ok(()) + } } diff --git a/backend/src/test.rs b/backend/src/test.rs index 2f039b9..c2ad96e 100644 --- a/backend/src/test.rs +++ b/backend/src/test.rs @@ -13,7 +13,7 @@ extern crate pretty_env_logger; fn main() { pretty_env_logger::init(); - let gen = Generator::new("/data".into()); + let gen = Generator::new("/data".into()).unwrap(); let path: PathBuf = "/data/Anime/1977 Star Wars Collection/01 Star Wars Episode I The Phantom Menace - George Lucas 1999 Eng Subs 720p [H264-mp4].mp4".into(); let mime = gen.get_mime(&path) From 55cf1ec4a57a328d4096350dc7c7eedb39dd118e Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 14:09:06 -0500 Subject: [PATCH 39/64] [+] Implement get_thumb for generator --- backend/src/generator.rs | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 62101d4..bfa93b6 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -10,7 +10,7 @@ use std::fs::{File, Metadata}; use std::fs::DirEntry; use std::io::{BufReader}; use xdg_mime::{SharedMimeInfo}; -use anyhow::{Result}; +use anyhow::{Context, Result}; use serde::{de, Deserialize, ser, Serialize}; use crate::thumbnailer::{Thumbnailers}; @@ -80,22 +80,18 @@ impl Generator { } /// Process a single file - pub fn get_thumb(&self, file: &PathBuf) -> Result<()> { - let dot = self.dot_path(&file); - let thumb = dot.with_extension("thumb.jpg"); - - if thumb.exists() { - match (thumb.metadata(), file.metadata()) { - // Thumbnail already up to date - (Ok(tm), Ok(fm)) => if tm.mtime() >= fm.mtime() { return Ok(()) }, - _ => {} + pub fn get_thumb(&self, file: &PathBuf) -> Result> { + self.get_cached(file, "thumb-128.png", |thumb| { + Ok(fs::read(thumb)?) + }, |thumb| { + debug!("Generating thumbnail for {}\nto {}", file.display(), thumb.display()); + let mime = self.mime_db.guess_mime_type().path(file).guess().mime_type().to_string(); + if let Some(t) = self.thumbnailers.find(&*mime) { + t.gen(file.to_str().context("Orig file failed to convert to str")?, + thumb.to_str().context("New file failed to convert to str")?, 128)?; } - } - - // Generate new thumbnail - info!("Generating thumbnail for {}\nto {}", file.display(), thumb.display()); - - Ok(()) + Ok(()) + }) } /// Process a directory From 737e0826dd230dbcb514768bc363deb5d23d7f5a Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 14:09:30 -0500 Subject: [PATCH 40/64] [O] Use cached mime in get_thumb --- backend/src/generator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index bfa93b6..495564b 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -85,7 +85,7 @@ impl Generator { Ok(fs::read(thumb)?) }, |thumb| { debug!("Generating thumbnail for {}\nto {}", file.display(), thumb.display()); - let mime = self.mime_db.guess_mime_type().path(file).guess().mime_type().to_string(); + let mime = self.get_mime(file)?; if let Some(t) = self.thumbnailers.find(&*mime) { t.gen(file.to_str().context("Orig file failed to convert to str")?, thumb.to_str().context("New file failed to convert to str")?, 128)?; From 891647fbdb5ed1715bb3b51b97e42d30e101dd5c Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 14:17:37 -0500 Subject: [PATCH 41/64] [-] Remove listing generation --- backend/src/generator.rs | 27 --------------------------- backend/src/main.rs | 9 ++++++--- 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 495564b..4e10186 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -1,5 +1,3 @@ - -use crate::macros::*; use crate::utils::*; use std::fs; @@ -16,13 +14,6 @@ use crate::thumbnailer::{Thumbnailers}; const DOT_PATH: &str = ".meow_index"; -#[derive(Serialize, Deserialize)] -pub struct ReturnPath { - pub(crate) name: String, - pub(crate) file_type: String, - pub(crate) mtime: i64 -} - pub struct Generator { pub(crate) mime_db: SharedMimeInfo, pub(crate) thumbnailers: Thumbnailers, @@ -112,24 +103,6 @@ impl Generator { } }); - // Check if directory listing needs update - let d_info = self.dot_path(&dir).with_extension(".json"); - let d_mtime = d_info.metadata()?.mtime(); - if files.iter().any(|(f, m)| m.mtime() > d_mtime) { - // Update directory listing - let paths: Vec = files.iter() - .filter_map(|(f, m)| Some(ReturnPath { - name: f.file_name().to_str()?.to_string(), - file_type: f.path().file_type().to_string(), - mtime: m.mtime(), - })).collect(); - - let json = serde_json::to_string(&paths)?; - // fs::write(d_info, json)?; - } - - - Ok(()) } } diff --git a/backend/src/main.rs b/backend/src/main.rs index d1c7581..8e20f1b 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -4,16 +4,19 @@ mod generator; mod utils; mod thumbnailer; +use generator::*; +use macros::*; + use std::convert::Infallible; use std::{env, fs}; use std::net::SocketAddr; use std::os::unix::fs::MetadataExt; +use std::path::Path; use hyper::{Body, http, Request, Response, Server}; use hyper::service::{make_service_fn, service_fn}; use path_clean::{clean}; -use macros::StringExt; -use crate::macros::PathExt; -use generator::*; +use anyhow::{Result}; +use serde::{Deserialize, Serialize}; extern crate pretty_env_logger; #[macro_use] extern crate log; From 68dd0010759397bcb9924267f465c72c05b15fc8 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 14:18:07 -0500 Subject: [PATCH 42/64] [+] Add mime to json return --- backend/src/main.rs | 65 +++++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 8e20f1b..56baf49 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -42,30 +42,49 @@ async fn main() { } } -async fn hello_world(_req: Request) -> http::Result> { - let path = format!(".{}", clean(_req.uri().path())); - println!("Raw path: {} | Sanitized path: {path}", _req.uri().path()); +#[derive(Serialize, Deserialize)] +pub struct ReturnPath { + name: String, + file_type: String, + mtime: i64, + mime: Option +} - // List files in directory - let read_dir = match fs::read_dir(path) { - Ok(file) => { file } - Err(e) => { - let e_str = format!("Error {e}"); - if e.raw_os_error() == Some(2) { return e_str.resp(404) } - return e_str.resp(500) +struct MyApp { + generator: Generator +} + +impl MyApp { + fn new(base: &Path) -> Result { + Ok(MyApp { generator: Generator::new(base.into())? }) + } + + async fn hello_world(&self, _req: Request) -> http::Result> { + let path = format!(".{}", clean(_req.uri().path())); + println!("Raw path: {} | Sanitized path: {path}", _req.uri().path()); + + // List files in directory + let read_dir = match fs::read_dir(path) { + Ok(file) => { file } + Err(e) => { + let e_str = format!("Error {e}"); + if e.raw_os_error() == Some(2) { return e_str.resp(404) } + return e_str.resp(500) + } + }; + + let paths: Vec = read_dir + .filter_map(|x| x.ok()) + .filter_map(|x| Some(ReturnPath { + name: x.file_name().to_str()?.to_string(), + file_type: x.path().file_type().to_string(), + mtime: x.metadata().ok()?.mtime(), + mime: if x.path().is_file() { self.generator.get_mime(&x.path()).ok() } else { None } + })).collect(); + + match serde_json::to_string(&paths) { + Ok(json) => { json.resp(200) } + Err(e) => { e.to_string().resp(500) } } - }; - - let paths: Vec = read_dir - .filter_map(|x| x.ok()) - .filter_map(|x| Some(ReturnPath { - name: x.file_name().to_str()?.to_string(), - file_type: x.path().file_type().to_string(), - mtime: x.metadata().ok()?.mtime(), - })).collect(); - - match serde_json::to_string(&paths) { - Ok(json) => { json.resp(200) } - Err(e) => { e.to_string().resp(500) } } } From ef72ce84ab4c4aa4a4eb0519017bf957972bf577 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 14:56:12 -0500 Subject: [PATCH 43/64] [F] Fix relative paths --- backend/src/generator.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/src/generator.rs b/backend/src/generator.rs index 4e10186..7928063 100644 --- a/backend/src/generator.rs +++ b/backend/src/generator.rs @@ -9,7 +9,7 @@ use std::fs::DirEntry; use std::io::{BufReader}; use xdg_mime::{SharedMimeInfo}; use anyhow::{Context, Result}; -use serde::{de, Deserialize, ser, Serialize}; +use serde::{de, ser}; use crate::thumbnailer::{Thumbnailers}; const DOT_PATH: &str = ".meow_index"; @@ -22,12 +22,14 @@ pub struct Generator { impl Generator { pub fn new(base: PathBuf) -> Result { - Ok(Generator { mime_db: SharedMimeInfo::new(), thumbnailers: Thumbnailers::load_all()?, base }) + Ok(Generator { mime_db: SharedMimeInfo::new(), thumbnailers: Thumbnailers::load_all()?, base: fs::canonicalize(base)? }) } /// Get the same file location in DOT_PATH directory pub fn dot_path(&self, path: &PathBuf) -> PathBuf { - self.base.join(DOT_PATH).join(diff_paths(&path, &self.base).unwrap()) + debug!("Diffing {} to {}", path.display(), self.base.display()); + if path.is_relative() { self.base.join(DOT_PATH).join(path) } + else { self.base.join(DOT_PATH).join(diff_paths(&path, &self.base).unwrap()) } } /// Get the cached result @@ -96,7 +98,7 @@ impl Generator { }).collect(); // Recurse into directories - files.iter().for_each(|(f, m)| { + files.iter().for_each(|(f, _m)| { if f.path().is_dir() { // Recurse into directory let _ = self.process_dir(&f.path()); From bc975a1d290ec9802ec48ccd16a872244ba33c83 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:01:48 -0500 Subject: [PATCH 44/64] [F] Fix main execution --- backend/src/main.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 56baf49..8e33694 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -22,16 +22,19 @@ extern crate pretty_env_logger; #[macro_use] extern crate log; #[tokio::main] -async fn main() { +async fn main() -> Result<()> { + pretty_env_logger::init(); + let cwd = env::current_dir().unwrap(); let addr = SocketAddr::from(([127, 0, 0, 1], 3029)); info!("Serving {} started on http://127.0.0.1:3029", cwd.display()); + let app: &MyApp = Box::leak(Box::new(MyApp::new(&cwd).unwrap())) as &'static _; // A `Service` is needed for every connection, so this // creates one from our `hello_world` function. let make_svc = make_service_fn(|_conn| async { // service_fn converts our function into a `Service` - Ok::<_, Infallible>(service_fn(hello_world)) + Ok::<_, Infallible>(service_fn(|x| app.hello_world(x))) }); let server = Server::bind(&addr).serve(make_svc); @@ -40,6 +43,8 @@ async fn main() { if let Err(e) = server.await { eprintln!("server error: {}", e); } + + Ok(()) } #[derive(Serialize, Deserialize)] From d4c36be288d99b16bdbefb199ad9ecc7fc7888b1 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:10:38 -0500 Subject: [PATCH 45/64] [O] CORS allow --- backend/src/macros.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/macros.rs b/backend/src/macros.rs index eb329d9..9311965 100644 --- a/backend/src/macros.rs +++ b/backend/src/macros.rs @@ -1,5 +1,5 @@ use std::path::PathBuf; -use hyper::{Body, http, Response, StatusCode}; +use hyper::{Body, header, http, Response, StatusCode}; pub trait StringExt { fn resp(&self, status: u16) -> http::Result>; @@ -7,7 +7,8 @@ pub trait StringExt { impl StringExt for String { fn resp(&self, status: u16) -> http::Result> { - Response::builder().status(StatusCode::from_u16(status).unwrap()).body(Body::from(self.to_owned())) + Response::builder().header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .status(StatusCode::from_u16(status).unwrap()).body(Body::from(self.to_owned())) } } From f4cc0dd428995ed2e3ea9007f29262d800cf1dd8 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:10:45 -0500 Subject: [PATCH 46/64] [+] Add file size --- backend/src/main.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 8e33694..55f2493 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -52,7 +52,8 @@ pub struct ReturnPath { name: String, file_type: String, mtime: i64, - mime: Option + size: u64, + mime: Option, } struct MyApp { @@ -80,12 +81,15 @@ impl MyApp { let paths: Vec = read_dir .filter_map(|x| x.ok()) - .filter_map(|x| Some(ReturnPath { + .filter_map(|x| { + let m = x.metadata().ok()?; + Some(ReturnPath { name: x.file_name().to_str()?.to_string(), file_type: x.path().file_type().to_string(), - mtime: x.metadata().ok()?.mtime(), + mtime: m.mtime() * 1000, + size: m.len(), mime: if x.path().is_file() { self.generator.get_mime(&x.path()).ok() } else { None } - })).collect(); + })}).collect(); match serde_json::to_string(&paths) { Ok(json) => { json.resp(200) } From ae9c7a482c2bc8ce7ae318a9cf3acda5fb67115b Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:13:58 -0500 Subject: [PATCH 47/64] [O] UI: Display provided mime if present in data --- src/App.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/App.tsx b/src/App.tsx index 78ce560..c2da293 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -18,6 +18,7 @@ interface File { type: 'file' | 'directory' size: number mtime: string + mime?: string } // Placeholder for nginx to replace @@ -45,7 +46,7 @@ function getIcon(f: File) if (f.type == "directory") return urlJoin(deployPath, "mime/folder.svg") const sp = f.name.split(".") - const m = mime.getType(sp[sp.length - 1]) + const m = f.mime ?? mime.getType(sp[sp.length - 1]) if (m) return urlJoin(deployPath, `mime/${m.replace("/", "-")}.svg`) else return urlJoin(deployPath, 'mime/application-blank.svg') } From cfb8fb425cba14d957cef4a4f3ccf90ee05e5f5d Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:14:06 -0500 Subject: [PATCH 48/64] [O] Fix indent --- backend/src/main.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 55f2493..b0176d4 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -84,12 +84,13 @@ impl MyApp { .filter_map(|x| { let m = x.metadata().ok()?; Some(ReturnPath { - name: x.file_name().to_str()?.to_string(), - file_type: x.path().file_type().to_string(), - mtime: m.mtime() * 1000, - size: m.len(), - mime: if x.path().is_file() { self.generator.get_mime(&x.path()).ok() } else { None } - })}).collect(); + name: x.file_name().to_str()?.to_string(), + file_type: x.path().file_type().to_string(), + mtime: m.mtime() * 1000, + size: m.len(), + mime: if x.path().is_file() { self.generator.get_mime(&x.path()).ok() } else { None } + }) + }).collect(); match serde_json::to_string(&paths) { Ok(json) => { json.resp(200) } From 79292e89f20280d5f506754a1e989779aa3baa86 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:18:06 -0500 Subject: [PATCH 49/64] [O] UI: file_type compatibility --- src/App.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index c2da293..3147913 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -15,12 +15,15 @@ import InfiniteScroll from 'solid-infinite-scroll-fork'; interface File { name: string - type: 'file' | 'directory' + type?: 'file' | 'directory' + file_type?: 'file' | 'directory' | 'link' size: number mtime: string mime?: string } +const getType = (f: File) => f.type ?? f.file_type + // Placeholder for nginx to replace let deployPath = "{DEPLOY-PATH-PLACEHOLDER}" let host = "{HOST-PLACEHOLDER}" @@ -43,7 +46,7 @@ const fetchApi = async () => function getIcon(f: File) { - if (f.type == "directory") return urlJoin(deployPath, "mime/folder.svg") + if (getType(f) == "directory") return urlJoin(deployPath, "mime/folder.svg") const sp = f.name.split(".") const m = f.mime ?? mime.getType(sp[sp.length - 1]) @@ -53,7 +56,8 @@ function getIcon(f: File) function getHref(f: File) { - return f.type == "directory" ? urlJoin(fullPath, f.name) : urlJoin(host, filePath, f.name) + return getType(f) == "directory" ? urlJoin(fullPath, f.name) : urlJoin(host, filePath, f.name) + // return urlJoin(fullPath, f.name) } const alpNum = new Set("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") From df70bc2e2820920bef0bf6621f386fcd50138f69 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:23:03 -0500 Subject: [PATCH 50/64] [+] Add has_thumb return --- backend/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index b0176d4..e47ec4c 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -54,6 +54,7 @@ pub struct ReturnPath { mtime: i64, size: u64, mime: Option, + has_thumb: bool } struct MyApp { @@ -83,12 +84,14 @@ impl MyApp { .filter_map(|x| x.ok()) .filter_map(|x| { let m = x.metadata().ok()?; + let mime = if x.path().is_file() { self.generator.get_mime(&x.path()).ok() } else { None }; Some(ReturnPath { name: x.file_name().to_str()?.to_string(), file_type: x.path().file_type().to_string(), mtime: m.mtime() * 1000, size: m.len(), - mime: if x.path().is_file() { self.generator.get_mime(&x.path()).ok() } else { None } + mime: mime.to_owned(), + has_thumb: mime.is_some() && self.generator.thumbnailers.find(&*mime.unwrap()).is_some() }) }).collect(); From e612046e940c4ff67161bc8ece2f817ab1559a5c Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:52:08 -0500 Subject: [PATCH 51/64] [+] Add url, url-escape, duplicate dependencies --- backend/Cargo.lock | 118 +++++++++++++++++++++++++++++++++++++++++++++ backend/Cargo.toml | 3 ++ 2 files changed, 121 insertions(+) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b2d0615..fc85465 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -118,6 +118,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "duplicate" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a4be4cd710e92098de6ad258e6e7c24af11c29c5142f3c6f2a545652480ff8" +dependencies = [ + "heck", + "proc-macro-error", +] + [[package]] name = "env_logger" version = "0.7.1" @@ -137,6 +147,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures-channel" version = "0.3.26" @@ -224,6 +243,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -309,6 +334,16 @@ dependencies = [ "want", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "1.9.2" @@ -374,6 +409,7 @@ name = "meow_index" version = "0.1.0" dependencies = [ "anyhow", + "duplicate", "hyper", "log", "path-clean", @@ -383,6 +419,8 @@ dependencies = [ "serde_json", "shlex", "tokio", + "url", + "url-escape", "xdg-mime", ] @@ -484,6 +522,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -506,6 +550,30 @@ dependencies = [ "log", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.51" @@ -702,6 +770,21 @@ dependencies = [ "syn", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.25.0" @@ -788,12 +871,47 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicode-bidi" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" + [[package]] name = "unicode-ident" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "url-escape" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44e0ce4d1246d075ca5abec4b41d33e87a6054d08e2366b63205665e950db218" +dependencies = [ + "percent-encoding", +] + [[package]] name = "version_check" version = "0.9.4" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 72db02c..2c4c249 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -11,6 +11,7 @@ path = "src/test.rs" [dependencies] anyhow = { version = "1.0.69", features = ["backtrace"] } +duplicate = "0.4.1" hyper = { version = "0.14", features = ["full"] } log = "0.4.17" path-clean = "0.1.0" @@ -20,4 +21,6 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" shlex = "1.1.0" tokio = { version = "1", features = ["full"] } +url = "2.3.1" +url-escape = "0.1.1" xdg-mime = "0.3.3" From def0891fab0fd668dbf2f933c677237a3c56bb0d Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:52:24 -0500 Subject: [PATCH 52/64] [O] Implement resp for Vec also --- backend/src/macros.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/src/macros.rs b/backend/src/macros.rs index 9311965..a1ef477 100644 --- a/backend/src/macros.rs +++ b/backend/src/macros.rs @@ -1,11 +1,14 @@ +use std::collections::HashMap; use std::path::PathBuf; -use hyper::{Body, header, http, Response, StatusCode}; +use duplicate::duplicate_item; +use hyper::{Body, header, http, Request, Response, StatusCode}; -pub trait StringExt { +pub trait Resp { fn resp(&self, status: u16) -> http::Result>; } -impl StringExt for String { +#[duplicate_item(name; [String]; [Vec])] +impl Resp for name { fn resp(&self, status: u16) -> http::Result> { Response::builder().header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*") .status(StatusCode::from_u16(status).unwrap()).body(Body::from(self.to_owned())) From 9eac5bf863d66da1fc7d10f6403ac88c3937aa3d Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:52:38 -0500 Subject: [PATCH 53/64] [+] Request.params() --- backend/src/macros.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/src/macros.rs b/backend/src/macros.rs index a1ef477..05b2092 100644 --- a/backend/src/macros.rs +++ b/backend/src/macros.rs @@ -27,3 +27,15 @@ impl PathExt for PathBuf { "unknown" } } + +pub trait RequestExt { + fn params(&self) -> HashMap; +} + +impl RequestExt for Request { + fn params(&self) -> HashMap { + self.uri().query() + .map(|v| url::form_urlencoded::parse(v.as_bytes()).into_owned().collect()) + .unwrap_or_else(HashMap::new) + } +} From 7de31cc99e9c97c435bc7d57491385878cefe70a Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:52:50 -0500 Subject: [PATCH 54/64] [+] Generate thumbnail for file --- backend/src/main.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index e47ec4c..36f0865 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -11,7 +11,7 @@ use std::convert::Infallible; use std::{env, fs}; use std::net::SocketAddr; use std::os::unix::fs::MetadataExt; -use std::path::Path; +use std::path::{Path, PathBuf}; use hyper::{Body, http, Request, Response, Server}; use hyper::service::{make_service_fn, service_fn}; use path_clean::{clean}; @@ -66,9 +66,20 @@ impl MyApp { Ok(MyApp { generator: Generator::new(base.into())? }) } - async fn hello_world(&self, _req: Request) -> http::Result> { - let path = format!(".{}", clean(_req.uri().path())); - println!("Raw path: {} | Sanitized path: {path}", _req.uri().path()); + async fn hello_world(&self, req: Request) -> http::Result> { + let mut path: String = url_escape::decode(req.uri().path()).into_owned(); + path = format!(".{}", clean(&*path)); + println!("Raw path: {} | Sanitized path: {path}", req.uri().path()); + + let params = req.params(); + + // Reading thumbnail of a file + if params.contains_key("thumb") { + return match self.generator.get_thumb(&PathBuf::from(path.to_owned())) { + Ok(vec) => { vec.resp(200) } + Err(e) => { e.to_string().resp(500) } + } + } // List files in directory let read_dir = match fs::read_dir(path) { From df64020e074845e9299942a94091e68ecb538a68 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 15:55:15 -0500 Subject: [PATCH 55/64] [+] UI: Display image thumbnail --- src/App.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/App.tsx b/src/App.tsx index 3147913..aa90eac 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -20,6 +20,7 @@ interface File { size: number mtime: string mime?: string + has_thumb?: boolean } const getType = (f: File) => f.type ?? f.file_type @@ -47,6 +48,8 @@ const fetchApi = async () => function getIcon(f: File) { if (getType(f) == "directory") return urlJoin(deployPath, "mime/folder.svg") + + if (f.has_thumb) return urlJoin(host, filePath, f.name) + "?thumb=1" const sp = f.name.split(".") const m = f.mime ?? mime.getType(sp[sp.length - 1]) From bf42a51e8598f71932d58986be1a0fcef87b421a Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 16:01:47 -0500 Subject: [PATCH 56/64] [F] Fix path joining --- backend/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 36f0865..810cdfd 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -67,9 +67,9 @@ impl MyApp { } async fn hello_world(&self, req: Request) -> http::Result> { - let mut path: String = url_escape::decode(req.uri().path()).into_owned(); - path = format!(".{}", clean(&*path)); - println!("Raw path: {} | Sanitized path: {path}", req.uri().path()); + let rel: String = clean(&url_escape::decode(req.uri().path())); + let path = self.generator.base.join(&rel.strip_prefix("/").unwrap()); + println!("Raw path: {} | Sanitized path: {}", req.uri().path(), path.display()); let params = req.params(); From ce0f4e840415769019233ca0c04a00faf94230c4 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 16:14:53 -0500 Subject: [PATCH 57/64] [S] UI:Fix image positioning --- src/App.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/App.tsx b/src/App.tsx index aa90eac..3610b62 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -167,7 +167,7 @@ export default function App() { hasMore={scrollIndex() < api()?.length} next={scrollNext}>{(f, i) => - + {/* File name tooltip */} tippy(el, { From 6a216552d71e4d4069785965c555c1b054ec7085 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 16:15:05 -0500 Subject: [PATCH 58/64] [F] Check 404 --- backend/src/main.rs | 1 + backend/src/thumbnailer.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index 810cdfd..d153008 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -75,6 +75,7 @@ impl MyApp { // Reading thumbnail of a file if params.contains_key("thumb") { + if !path.is_file() { return "Error: File not found".to_string().resp(404) } return match self.generator.get_thumb(&PathBuf::from(path.to_owned())) { Ok(vec) => { vec.resp(200) } Err(e) => { e.to_string().resp(500) } diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs index 2c31467..b016df8 100644 --- a/backend/src/thumbnailer.rs +++ b/backend/src/thumbnailer.rs @@ -47,8 +47,8 @@ impl Thumbnailer { pub fn gen(&self, orig: &str, new: &str, pixels: i32) -> Result<()> { let cmd = self.exec .replace("%s", &*format!("'{pixels}'")) - .replace("%u", &*format!("'{orig}'")) - .replace("%o", &*format!("'{new}'")); + .replace("%u", &shlex::quote(orig)) + .replace("%o", &shlex::quote(new)); let args: Vec = Shlex::new(&*cmd).collect(); let out = Command::new(args[0].to_owned()).args(&args[1..]).output()?; if !out.status.success() { From 4acb36c0889e5e3fe0f2a93bba1fb1e98871b405 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 18:31:38 -0500 Subject: [PATCH 59/64] [+] Experimental thumbnailers --- backend/res/thumb/ffmpeg.thumbnailer | 4 ++++ backend/res/thumb/imagemagick.thumbnailer | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 backend/res/thumb/ffmpeg.thumbnailer create mode 100644 backend/res/thumb/imagemagick.thumbnailer diff --git a/backend/res/thumb/ffmpeg.thumbnailer b/backend/res/thumb/ffmpeg.thumbnailer new file mode 100644 index 0000000..5966d63 --- /dev/null +++ b/backend/res/thumb/ffmpeg.thumbnailer @@ -0,0 +1,4 @@ +[Thumbnailer Entry] +TryExec=ffmpeg +Exec=ffmpeg -i %i %o -fs %s +MimeType=application/mxf;application/ram;application/sdp;application/vnd.apple.mpegurl;application/vnd.ms-asf;application/vnd.ms-wpl;application/vnd.rn-realmedia;application/vnd.rn-realmedia-vbr;application/x-extension-m4a;application/x-extension-mp4;application/x-flash-video;application/x-matroska;application/x-netshow-channel;application/x-quicktimeplayer;application/x-shorten;image/vnd.rn-realpix;image/x-pict;misc/ultravox;text/x-google-video-pointer;video/3gp;video/3gpp;video/3gpp2;video/dv;video/divx;video/fli;video/flv;video/mp2t;video/mp4;video/mp4v-es;video/mpeg;video/mpeg-system;video/msvideo;video/ogg;video/quicktime;video/vivo;video/vnd.divx;video/vnd.mpegurl;video/vnd.rn-realvideo;video/vnd.vivo;video/webm;video/x-anim;video/x-avi;video/x-flc;video/x-fli;video/x-flic;video/x-flv;video/x-m4v;video/x-matroska;video/x-mjpeg;video/x-mpeg;video/x-mpeg2;video/x-ms-asf;video/x-ms-asf-plugin;video/x-ms-asx;video/x-msvideo;video/x-ms-wm;video/x-ms-wmv;video/x-ms-wmx;video/x-ms-wvx;video/x-nsv;video/x-ogm+ogg;video/x-theora;video/x-theora+ogg;video/x-totem-stream;audio/x-pn-realaudio;audio/3gpp;audio/3gpp2;audio/aac;audio/ac3;audio/AMR;audio/AMR-WB;audio/basic;audio/dv;audio/eac3;audio/flac;audio/m4a;audio/midi;audio/mp1;audio/mp2;audio/mp3;audio/mp4;audio/mpeg;audio/mpg;audio/ogg;audio/opus;audio/prs.sid;audio/scpls;audio/vnd.rn-realaudio;audio/wav;audio/webm;audio/x-aac;audio/x-aiff;audio/x-ape;audio/x-flac;audio/x-gsm;audio/x-it;audio/x-m4a;audio/x-m4b;audio/x-matroska;audio/x-mod;audio/x-mp1;audio/x-mp2;audio/x-mp3;audio/x-mpg;audio/x-mpeg;audio/x-ms-asf;audio/x-ms-asx;audio/x-ms-wax;audio/x-ms-wma;audio/x-musepack;audio/x-opus+ogg;audio/x-pn-aiff;audio/x-pn-au;audio/x-pn-wav;audio/x-pn-windows-acm;audio/x-realaudio;audio/x-real-audio;audio/x-s3m;audio/x-sbc;audio/x-shorten;audio/x-speex;audio/x-stm;audio/x-tta;audio/x-wav;audio/x-wavpack;audio/x-vorbis;audio/x-vorbis+ogg;audio/x-xm;application/x-flac; diff --git a/backend/res/thumb/imagemagick.thumbnailer b/backend/res/thumb/imagemagick.thumbnailer new file mode 100644 index 0000000..32a3402 --- /dev/null +++ b/backend/res/thumb/imagemagick.thumbnailer @@ -0,0 +1,4 @@ +[Thumbnailer Entry] +TryExec=convert +Exec=convert -resize %s -quality 50 %u %o +MimeType=application/x-navi-animation;image/x-pict;image/x-sun-raster;image/tiff;image/x-photo-cd;image/rle;image/x-fpx;image/x-olympus-orf;image/svg+xml-compressed;image/x-gimp-gih;image/cgm;image/vnd.zbrush.pcx;image/x-sony-arw;image/vnd.wap.wbmp;image/x-jng;image/vnd.dwg;image/x-dds;image/x-msod;image/x-eps;image/x-win-bitmap;image/x-canon-cr2;image/x-canon-cr3;image/vnd.microsoft.icon;image/x-cmu-raster;image/jpeg;image/jxl;image/x-minolta-mrw;image/x-bzeps;image/x-portable-graymap;image/x-ilbm;image/x-3ds;image/x-gzeps;image/x-xbitmap;image/bmp;image/x-xwindowdump;image/x-portable-pixmap;image/vnd.dxf;image/avif;image/x-xfig;image/x-panasonic-rw;image/svg+xml;image/x-nikon-nef;image/x-xcf;image/x-quicktime;image/x-tiff-multipage;image/x-pic;image/ktx;image/webp;image/x-sony-sr2;image/x-fuji-raf;image/jp2;image/x-portable-anymap;image/x-dcraw;image/x-lwo;image/x-jp2-codestream;image/x-lws;image/x-canon-crw;image/x-compressed-xcf;image/heif;image/x-kodak-dcr;image/x-icns;image/g3fax;image/x-exr;image/x-tga;image/x-gimp-pat;image/vnd.rn-realpix;image/wmf;image/x-sigma-x3f;image/dpx;image/vnd.ms-modi;image/astc;image/x-kodak-k25;image/x-hdr;image/x-xcursor;image/png;image/x-macpaint;image/x-niff;image/ktx2;image/x-panasonic-rw2;image/x-adobe-dng;image/vnd.djvu+multipage;image/x-sgi;image/openraster;image/x-sony-srf;image/x-nikon-nrw;image/x-portable-bitmap;image/x-kde-raw;image/x-kodak-kdc;image/jpm;image/x-xpixmap;image/x-skencil;image/x-rgb;image/emf;image/jpx;image/vnd.djvu;image/vnd.adobe.photoshop;image/x-gimp-gbr;image/gif;image/x-applix-graphics;image/ief;image/x-pentax-pef;image/x-dib; From 322773254952156c702af8a8af10ee60006fb24a Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 20:43:46 -0500 Subject: [PATCH 60/64] [+] Dockerfile --- backend/Dockerfile | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 backend/Dockerfile diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..ea4f858 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,33 @@ +#FROM rust:slim as builder +#WORKDIR /app +#COPY . . +#RUN cargo build --release + +FROM debian:sid-slim + +# Copy built files +WORKDIR /app +#COPY --from=builder /app/target/release/meow_index . +COPY ./target/release/meow_index . +COPY ./res/thumb/* /usr/share/thumbnailers/ + +RUN apt-get update \ + && apt-get install -y \ + # Video preview thumbnailer + # totem \ + # ffmpegthumbnailer \ + # Font preview thumbnailer + gnome-font-viewer \ + # Image thumbnailer + libgdk-pixbuf2.0-bin \ + # More image format supports + libavif-bin libavif-gdk-pixbuf heif-thumbnailer \ + # PDF thumbnailer + evince \ + # Office thumbnailer + libgsf-bin \ + # Video formatter + ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +CMD ["./meow_index"] From 8db3956911a040440d2b6a836a245680f95f0242 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 20:43:55 -0500 Subject: [PATCH 61/64] [+] dockerignore --- backend/.dockerignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 backend/.dockerignore diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..187b750 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,2 @@ +target +.idea/ From 72195c668ae6df2f2b85d1ccbbd02d28c5849e03 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 20:44:05 -0500 Subject: [PATCH 62/64] [+] Docker compose --- backend/docker-compose.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 backend/docker-compose.yml diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml new file mode 100644 index 0000000..7332f8a --- /dev/null +++ b/backend/docker-compose.yml @@ -0,0 +1,13 @@ +version: "3" + +services: + web: + build: . + environment: +# - RUST_LOG=info + - RUST_LOG=debug + - RUST_BACKTRACE=full + volumes: + - ./web:/data + ports: + - "3030:3029" From 56a1c3f81df8f1a4f53e8bcb728d62182d277071 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 20:44:19 -0500 Subject: [PATCH 63/64] [F] Fix host binding for docker --- backend/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index d153008..feae810 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -26,7 +26,7 @@ async fn main() -> Result<()> { pretty_env_logger::init(); let cwd = env::current_dir().unwrap(); - let addr = SocketAddr::from(([127, 0, 0, 1], 3029)); + let addr = SocketAddr::from(([0, 0, 0, 0], 3029)); info!("Serving {} started on http://127.0.0.1:3029", cwd.display()); let app: &MyApp = Box::leak(Box::new(MyApp::new(&cwd).unwrap())) as &'static _; From 72e019adad8c6372dc6022daa01811b4a8d32aa2 Mon Sep 17 00:00:00 2001 From: Azalea Gui Date: Thu, 23 Feb 2023 20:44:35 -0500 Subject: [PATCH 64/64] [F] Fix thumbnailer %i variable --- backend/src/thumbnailer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/thumbnailer.rs b/backend/src/thumbnailer.rs index b016df8..42b78f9 100644 --- a/backend/src/thumbnailer.rs +++ b/backend/src/thumbnailer.rs @@ -48,6 +48,7 @@ impl Thumbnailer { let cmd = self.exec .replace("%s", &*format!("'{pixels}'")) .replace("%u", &shlex::quote(orig)) + .replace("%i", &shlex::quote(orig)) .replace("%o", &shlex::quote(new)); let args: Vec = Shlex::new(&*cmd).collect(); let out = Command::new(args[0].to_owned()).args(&args[1..]).output()?;