[PR] Merge pull request #1 from hykilpikonna/staging

This commit is contained in:
Hykilpikonna
2023-02-23 20:45:57 -05:00
committed by GitHub
15 changed files with 1534 additions and 9 deletions
+2
View File
@@ -0,0 +1,2 @@
target
.idea/
+1054 -1
View File
File diff suppressed because it is too large Load Diff
+19 -1
View File
@@ -1,8 +1,26 @@
[package]
name = "MeowIndex"
name = "meow_index"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "test"
path = "src/test.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
anyhow = { version = "1.0.69", features = ["backtrace"] }
duplicate = "0.4.1"
hyper = { version = "0.14", features = ["full"] }
log = "0.4.17"
path-clean = "0.1.0"
pathdiff = "0.2.1"
pretty_env_logger = "0.4.0"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
shlex = "1.1.0"
tokio = { version = "1", features = ["full"] }
url = "2.3.1"
url-escape = "0.1.1"
xdg-mime = "0.3.3"
+33
View File
@@ -0,0 +1,33 @@
#FROM rust:slim as builder
#WORKDIR /app
#COPY . .
#RUN cargo build --release
FROM debian:sid-slim
# Copy built files
WORKDIR /app
#COPY --from=builder /app/target/release/meow_index .
COPY ./target/release/meow_index .
COPY ./res/thumb/* /usr/share/thumbnailers/
RUN apt-get update \
&& apt-get install -y \
# Video preview thumbnailer
# totem \
# ffmpegthumbnailer \
# Font preview thumbnailer
gnome-font-viewer \
# Image thumbnailer
libgdk-pixbuf2.0-bin \
# More image format supports
libavif-bin libavif-gdk-pixbuf heif-thumbnailer \
# PDF thumbnailer
evince \
# Office thumbnailer
libgsf-bin \
# Video formatter
ffmpeg \
&& rm -rf /var/lib/apt/lists/*
CMD ["./meow_index"]
+13
View File
@@ -0,0 +1,13 @@
version: "3"
services:
web:
build: .
environment:
# - RUST_LOG=info
- RUST_LOG=debug
- RUST_BACKTRACE=full
volumes:
- ./web:/data
ports:
- "3030:3029"
+4
View File
@@ -0,0 +1,4 @@
[Thumbnailer Entry]
TryExec=ffmpeg
Exec=ffmpeg -i %i %o -fs %s
MimeType=application/mxf;application/ram;application/sdp;application/vnd.apple.mpegurl;application/vnd.ms-asf;application/vnd.ms-wpl;application/vnd.rn-realmedia;application/vnd.rn-realmedia-vbr;application/x-extension-m4a;application/x-extension-mp4;application/x-flash-video;application/x-matroska;application/x-netshow-channel;application/x-quicktimeplayer;application/x-shorten;image/vnd.rn-realpix;image/x-pict;misc/ultravox;text/x-google-video-pointer;video/3gp;video/3gpp;video/3gpp2;video/dv;video/divx;video/fli;video/flv;video/mp2t;video/mp4;video/mp4v-es;video/mpeg;video/mpeg-system;video/msvideo;video/ogg;video/quicktime;video/vivo;video/vnd.divx;video/vnd.mpegurl;video/vnd.rn-realvideo;video/vnd.vivo;video/webm;video/x-anim;video/x-avi;video/x-flc;video/x-fli;video/x-flic;video/x-flv;video/x-m4v;video/x-matroska;video/x-mjpeg;video/x-mpeg;video/x-mpeg2;video/x-ms-asf;video/x-ms-asf-plugin;video/x-ms-asx;video/x-msvideo;video/x-ms-wm;video/x-ms-wmv;video/x-ms-wmx;video/x-ms-wvx;video/x-nsv;video/x-ogm+ogg;video/x-theora;video/x-theora+ogg;video/x-totem-stream;audio/x-pn-realaudio;audio/3gpp;audio/3gpp2;audio/aac;audio/ac3;audio/AMR;audio/AMR-WB;audio/basic;audio/dv;audio/eac3;audio/flac;audio/m4a;audio/midi;audio/mp1;audio/mp2;audio/mp3;audio/mp4;audio/mpeg;audio/mpg;audio/ogg;audio/opus;audio/prs.sid;audio/scpls;audio/vnd.rn-realaudio;audio/wav;audio/webm;audio/x-aac;audio/x-aiff;audio/x-ape;audio/x-flac;audio/x-gsm;audio/x-it;audio/x-m4a;audio/x-m4b;audio/x-matroska;audio/x-mod;audio/x-mp1;audio/x-mp2;audio/x-mp3;audio/x-mpg;audio/x-mpeg;audio/x-ms-asf;audio/x-ms-asx;audio/x-ms-wax;audio/x-ms-wma;audio/x-musepack;audio/x-opus+ogg;audio/x-pn-aiff;audio/x-pn-au;audio/x-pn-wav;audio/x-pn-windows-acm;audio/x-realaudio;audio/x-real-audio;audio/x-s3m;audio/x-sbc;audio/x-shorten;audio/x-speex;audio/x-stm;audio/x-tta;audio/x-wav;audio/x-wavpack;audio/x-vorbis;audio/x-vorbis+ogg;audio/x-xm;application/x-flac;
@@ -0,0 +1,4 @@
[Thumbnailer Entry]
TryExec=convert
Exec=convert -resize %s -quality 50 %u %o
MimeType=application/x-navi-animation;image/x-pict;image/x-sun-raster;image/tiff;image/x-photo-cd;image/rle;image/x-fpx;image/x-olympus-orf;image/svg+xml-compressed;image/x-gimp-gih;image/cgm;image/vnd.zbrush.pcx;image/x-sony-arw;image/vnd.wap.wbmp;image/x-jng;image/vnd.dwg;image/x-dds;image/x-msod;image/x-eps;image/x-win-bitmap;image/x-canon-cr2;image/x-canon-cr3;image/vnd.microsoft.icon;image/x-cmu-raster;image/jpeg;image/jxl;image/x-minolta-mrw;image/x-bzeps;image/x-portable-graymap;image/x-ilbm;image/x-3ds;image/x-gzeps;image/x-xbitmap;image/bmp;image/x-xwindowdump;image/x-portable-pixmap;image/vnd.dxf;image/avif;image/x-xfig;image/x-panasonic-rw;image/svg+xml;image/x-nikon-nef;image/x-xcf;image/x-quicktime;image/x-tiff-multipage;image/x-pic;image/ktx;image/webp;image/x-sony-sr2;image/x-fuji-raf;image/jp2;image/x-portable-anymap;image/x-dcraw;image/x-lwo;image/x-jp2-codestream;image/x-lws;image/x-canon-crw;image/x-compressed-xcf;image/heif;image/x-kodak-dcr;image/x-icns;image/g3fax;image/x-exr;image/x-tga;image/x-gimp-pat;image/vnd.rn-realpix;image/wmf;image/x-sigma-x3f;image/dpx;image/vnd.ms-modi;image/astc;image/x-kodak-k25;image/x-hdr;image/x-xcursor;image/png;image/x-macpaint;image/x-niff;image/ktx2;image/x-panasonic-rw2;image/x-adobe-dng;image/vnd.djvu+multipage;image/x-sgi;image/openraster;image/x-sony-srf;image/x-nikon-nrw;image/x-portable-bitmap;image/x-kde-raw;image/x-kodak-kdc;image/jpm;image/x-xpixmap;image/x-skencil;image/x-rgb;image/emf;image/jpx;image/vnd.djvu;image/vnd.adobe.photoshop;image/x-gimp-gbr;image/gif;image/x-applix-graphics;image/ief;image/x-pentax-pef;image/x-dib;
+1
View File
@@ -0,0 +1 @@
nightly
+111
View File
@@ -0,0 +1,111 @@
use crate::utils::*;
use std::fs;
use std::os::unix::fs::MetadataExt;
use pathdiff::diff_paths;
use std::path::{PathBuf};
use std::fs::{File, Metadata};
use std::fs::DirEntry;
use std::io::{BufReader};
use xdg_mime::{SharedMimeInfo};
use anyhow::{Context, Result};
use serde::{de, ser};
use crate::thumbnailer::{Thumbnailers};
const DOT_PATH: &str = ".meow_index";
pub struct Generator {
pub(crate) mime_db: SharedMimeInfo,
pub(crate) thumbnailers: Thumbnailers,
pub(crate) base: PathBuf,
}
impl Generator {
pub fn new(base: PathBuf) -> Result<Generator> {
Ok(Generator { mime_db: SharedMimeInfo::new(), thumbnailers: Thumbnailers::load_all()?, base: fs::canonicalize(base)? })
}
/// Get the same file location in DOT_PATH directory
pub fn dot_path(&self, path: &PathBuf) -> PathBuf {
debug!("Diffing {} to {}", path.display(), self.base.display());
if path.is_relative() { self.base.join(DOT_PATH).join(path) }
else { self.base.join(DOT_PATH).join(diff_paths(&path, &self.base).unwrap()) }
}
/// Get the cached result
pub fn get_cached<T>(&self, file: &PathBuf, token: &str, read: impl Fn(&PathBuf) -> Result<T>,
gen: impl Fn(&PathBuf) -> Result<()>) -> Result<T> {
let dot = self.dot_path(file).with_extension(token);
if ! dot.exists() || match (dot.metadata(), file.metadata()) {
(Ok(dm), Ok(fm)) => { fm.mtime() > dm.mtime() }
(_, _) => true
} {
debug!("Regenerating cached result {}", dot.display());
gen(&dot)?;
}
Ok(read(&dot)?)
}
/// Get the cached result
pub fn get_cached_json<T>(&self, file: &PathBuf, token: &str, gen: impl Fn() -> Result<T>) -> Result<T>
where T: de::DeserializeOwned + ?Sized + ser::Serialize {
self.get_cached(&file, token, |f| {
let open = File::open(f)?;
let reader = BufReader::new(open);
let val: T = serde_json::from_reader(reader)?;
Ok(val)
}, |f| {
let res = gen()?;
fs::write(f, serde_json::to_string(&res)?)?;
Ok(())
})
}
/// Get cached mime type
pub fn get_mime(&self, file: &PathBuf) -> Result<String> {
self.get_cached(&file, "mime", |f| {
Ok(fs::read_to_string(f)?)
}, |f| {
let mut guesser = self.mime_db.guess_mime_type();
write_sf(f, guesser.path(file).guess().mime_type().to_string())?;
Ok(())
})
}
/// Process a single file
pub fn get_thumb(&self, file: &PathBuf) -> Result<Vec<u8>> {
self.get_cached(file, "thumb-128.png", |thumb| {
Ok(fs::read(thumb)?)
}, |thumb| {
debug!("Generating thumbnail for {}\nto {}", file.display(), thumb.display());
let mime = self.get_mime(file)?;
if let Some(t) = self.thumbnailers.find(&*mime) {
t.gen(file.to_str().context("Orig file failed to convert to str")?,
thumb.to_str().context("New file failed to convert to str")?, 128)?;
}
Ok(())
})
}
/// Process a directory
pub fn process_dir(&self, dir: &PathBuf) -> Result<()> {
// List files
let files: Vec<(DirEntry, Metadata)> = dir.to_owned().read_dir()?
.filter_map(|x| x.ok())
.filter_map(|x| {
let meta = x.metadata();
Some((x, meta.ok()?))
}).collect();
// Recurse into directories
files.iter().for_each(|(f, _m)| {
if f.path().is_dir() {
// Recurse into directory
let _ = self.process_dir(&f.path());
}
});
Ok(())
}
}
+41
View File
@@ -0,0 +1,41 @@
use std::collections::HashMap;
use std::path::PathBuf;
use duplicate::duplicate_item;
use hyper::{Body, header, http, Request, Response, StatusCode};
pub trait Resp {
fn resp(&self, status: u16) -> http::Result<Response<Body>>;
}
#[duplicate_item(name; [String]; [Vec<u8>])]
impl Resp for name {
fn resp(&self, status: u16) -> http::Result<Response<Body>> {
Response::builder().header(header::ACCESS_CONTROL_ALLOW_ORIGIN, "*")
.status(StatusCode::from_u16(status).unwrap()).body(Body::from(self.to_owned()))
}
}
pub trait PathExt {
fn file_type(&self) -> &str;
}
impl PathExt for PathBuf {
fn file_type(&self) -> &str {
if self.is_file() { return "file" }
if self.is_dir() { return "directory" }
if self.is_symlink() { return "link" }
"unknown"
}
}
pub trait RequestExt {
fn params(&self) -> HashMap<String, String>;
}
impl <T> RequestExt for Request<T> {
fn params(&self) -> HashMap<String, String> {
self.uri().query()
.map(|v| url::form_urlencoded::parse(v.as_bytes()).into_owned().collect())
.unwrap_or_else(HashMap::new)
}
}
+114 -2
View File
@@ -1,3 +1,115 @@
fn main() {
println!("Hello, world!");
#[macro_use]
mod macros;
mod generator;
mod utils;
mod thumbnailer;
use generator::*;
use macros::*;
use std::convert::Infallible;
use std::{env, fs};
use std::net::SocketAddr;
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use hyper::{Body, http, Request, Response, Server};
use hyper::service::{make_service_fn, service_fn};
use path_clean::{clean};
use anyhow::{Result};
use serde::{Deserialize, Serialize};
extern crate pretty_env_logger;
#[macro_use] extern crate log;
#[tokio::main]
async fn main() -> Result<()> {
pretty_env_logger::init();
let cwd = env::current_dir().unwrap();
let addr = SocketAddr::from(([0, 0, 0, 0], 3029));
info!("Serving {} started on http://127.0.0.1:3029", cwd.display());
let app: &MyApp = Box::leak(Box::new(MyApp::new(&cwd).unwrap())) as &'static _;
// A `Service` is needed for every connection, so this
// creates one from our `hello_world` function.
let make_svc = make_service_fn(|_conn| async {
// service_fn converts our function into a `Service`
Ok::<_, Infallible>(service_fn(|x| app.hello_world(x)))
});
let server = Server::bind(&addr).serve(make_svc);
// Run this server for... forever!
if let Err(e) = server.await {
eprintln!("server error: {}", e);
}
Ok(())
}
#[derive(Serialize, Deserialize)]
pub struct ReturnPath {
name: String,
file_type: String,
mtime: i64,
size: u64,
mime: Option<String>,
has_thumb: bool
}
struct MyApp {
generator: Generator
}
impl MyApp {
fn new(base: &Path) -> Result<MyApp> {
Ok(MyApp { generator: Generator::new(base.into())? })
}
async fn hello_world(&self, req: Request<Body>) -> http::Result<Response<Body>> {
let rel: String = clean(&url_escape::decode(req.uri().path()));
let path = self.generator.base.join(&rel.strip_prefix("/").unwrap());
println!("Raw path: {} | Sanitized path: {}", req.uri().path(), path.display());
let params = req.params();
// Reading thumbnail of a file
if params.contains_key("thumb") {
if !path.is_file() { return "Error: File not found".to_string().resp(404) }
return match self.generator.get_thumb(&PathBuf::from(path.to_owned())) {
Ok(vec) => { vec.resp(200) }
Err(e) => { e.to_string().resp(500) }
}
}
// List files in directory
let read_dir = match fs::read_dir(path) {
Ok(file) => { file }
Err(e) => {
let e_str = format!("Error {e}");
if e.raw_os_error() == Some(2) { return e_str.resp(404) }
return e_str.resp(500)
}
};
let paths: Vec<ReturnPath> = read_dir
.filter_map(|x| x.ok())
.filter_map(|x| {
let m = x.metadata().ok()?;
let mime = if x.path().is_file() { self.generator.get_mime(&x.path()).ok() } else { None };
Some(ReturnPath {
name: x.file_name().to_str()?.to_string(),
file_type: x.path().file_type().to_string(),
mtime: m.mtime() * 1000,
size: m.len(),
mime: mime.to_owned(),
has_thumb: mime.is_some() && self.generator.thumbnailers.find(&*mime.unwrap()).is_some()
})
}).collect();
match serde_json::to_string(&paths) {
Ok(json) => { json.resp(200) }
Err(e) => { e.to_string().resp(500) }
}
}
}
+31
View File
@@ -0,0 +1,31 @@
mod generator;
mod macros;
mod utils;
mod thumbnailer;
use std::path::{Path, PathBuf};
use generator::*;
use crate::thumbnailer::{Thumbnailer, Thumbnailers};
extern crate pretty_env_logger;
#[macro_use] extern crate log;
fn main() {
pretty_env_logger::init();
let gen = Generator::new("/data".into()).unwrap();
let path: PathBuf = "/data/Anime/1977 Star Wars Collection/01 Star Wars Episode I The Phantom Menace - George Lucas 1999 Eng Subs 720p [H264-mp4].mp4".into();
let mime = gen.get_mime(&path)
.expect("Panic");
info!("mime {mime}");
let thumbnailer_path = "/usr/share/thumbnailers/totem.thumbnailer";
let thumbnailer = Thumbnailer::load(Path::new(thumbnailer_path)).unwrap();
info!("thumb {:?}", thumbnailer);
info!("check {:?}", thumbnailer.check("audio/x-mp3"));
thumbnailer.gen(path.to_str().unwrap(), "/tmp/test.png", 256).expect("Generation failed");
let ts = Thumbnailers::load_all().unwrap();
info!("Video thumbnailer: {:?}", ts.find("audio/x-mp3"))
}
+83
View File
@@ -0,0 +1,83 @@
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::process::Command;
use anyhow::{bail, Result};
use shlex::Shlex;
#[derive(Debug)]
pub struct Thumbnailer {
try_exec: String,
exec: String,
mime_type: HashSet<String>
}
impl Thumbnailer {
/// Load an XDG thumbnailer (examples in /usr/share/thumbnailers)
pub fn load(p: &Path) -> Result<Thumbnailer> {
let mut content = fs::read_to_string(p)?;
content = content.replace("\r\n", "\n");
let lines = content.split("\n");
let mut t = Thumbnailer {
try_exec: "".to_string(), exec: "".to_string(), mime_type: HashSet::new()
};
lines.filter(|line| line.contains("="))
.for_each(|line| {
let sp: Vec<&str> = line.splitn(2, "=").collect();
let (key, val) = (sp[0].trim(), sp[1].trim().to_string());
match key {
"TryExec" => t.try_exec = val,
"Exec" => t.exec = val,
"MimeType" => t.mime_type = HashSet::from_iter(val.split(";").map(str::to_string)),
_ => {},
}
});
Ok(t)
}
/// Check if this thumbnailer should run on a specific mime type
pub fn check(&self, mime: &str) -> bool {
self.mime_type.contains(mime)
}
/// Generate thumbnail
pub fn gen(&self, orig: &str, new: &str, pixels: i32) -> Result<()> {
let cmd = self.exec
.replace("%s", &*format!("'{pixels}'"))
.replace("%u", &shlex::quote(orig))
.replace("%i", &shlex::quote(orig))
.replace("%o", &shlex::quote(new));
let args: Vec<String> = Shlex::new(&*cmd).collect();
let out = Command::new(args[0].to_owned()).args(&args[1..]).output()?;
if !out.status.success() {
error!("Command failed: {cmd}");
error!("Command output: {:?}", out);
bail!(String::from_utf8(out.stderr)?);
}
debug!("Command output: {:?}", out);
Ok(())
}
}
pub struct Thumbnailers {
list: Vec<Thumbnailer>
}
impl Thumbnailers {
/// Load all thumbanilers available in the system
pub fn load_all() -> Result<Thumbnailers> {
Ok(Thumbnailers { list: fs::read_dir("/usr/share/thumbnailers")?
.filter_map(|f| f.ok())
.filter_map(|f| Thumbnailer::load(&*f.path()).ok())
.collect() })
}
/// Find a thumbnailer for a mime type
pub fn find(&self, mime: &str) -> Option<&Thumbnailer> {
self.list.iter().find(|x| x.check(mime))
}
}
+11
View File
@@ -0,0 +1,11 @@
use std::{fs, io};
use std::path::{PathBuf};
pub fn write_sf<C: AsRef<[u8]>>(path: &PathBuf, contents: C) -> io::Result<()> {
// Create parent if it has parent
if let Some(p) = path.parent() {
fs::create_dir_all(p)?
}
fs::write(path, contents)
}
+13 -5
View File
@@ -15,11 +15,16 @@ import InfiniteScroll from 'solid-infinite-scroll-fork';
interface File {
name: string
type: 'file' | 'directory'
type?: 'file' | 'directory'
file_type?: 'file' | 'directory' | 'link'
size: number
mtime: string
mime?: string
has_thumb?: boolean
}
const getType = (f: File) => f.type ?? f.file_type
// Placeholder for nginx to replace
let deployPath = "{DEPLOY-PATH-PLACEHOLDER}"
let host = "{HOST-PLACEHOLDER}"
@@ -42,17 +47,20 @@ const fetchApi = async () =>
function getIcon(f: File)
{
if (f.type == "directory") return urlJoin(deployPath, "mime/folder.svg")
if (getType(f) == "directory") return urlJoin(deployPath, "mime/folder.svg")
if (f.has_thumb) return urlJoin(host, filePath, f.name) + "?thumb=1"
const sp = f.name.split(".")
const m = mime.getType(sp[sp.length - 1])
const m = f.mime ?? mime.getType(sp[sp.length - 1])
if (m) return urlJoin(deployPath, `mime/${m.replace("/", "-")}.svg`)
else return urlJoin(deployPath, 'mime/application-blank.svg')
}
function getHref(f: File)
{
return f.type == "directory" ? urlJoin(fullPath, f.name) : urlJoin(host, filePath, f.name)
return getType(f) == "directory" ? urlJoin(fullPath, f.name) : urlJoin(host, filePath, f.name)
// return urlJoin(fullPath, f.name)
}
const alpNum = new Set("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
@@ -159,7 +167,7 @@ export default function App() {
hasMore={scrollIndex() < api()?.length} next={scrollNext}>{(f, i) =>
<a class="w-full flex gap-4 transition-all duration-300 bg-dark-800 hover:bg-dark-300 hover:duration-0 rounded-xl p-2 items-center"
href={getHref(f)}>
<img class="w-10" src={getIcon(f)} alt=""></img>
<img class="w-10 max-h-10 object-contain" src={getIcon(f)} alt=""></img>
{/* File name tooltip */}
<span class="flex-1 font-bold truncate" ref={el => tippy(el, {