feat: generate thumbhash for assets

package.json

@@ -3,7 +3,8 @@
   "version": "1.0.0",
   "type": "module",
   "scripts": {
-    "start": "bun src/index.ts",
+    "start": "bun src/index.ts metadata",
+    "start:thumb": "bun src/index.ts thumb",
     "typecheck": "tsc",
     "lint": "eslint --cache --ext .",
     "lint:fix": "eslint --cache --ext . --fix"
@@ -25,12 +26,14 @@
   },
   "dependencies": {
     "@guiiai/logg": "^1.2.5",
+    "@napi-rs/canvas": "^0.1.82",
     "@types/js-yaml": "^4.0.9",
     "@types/node": "^24.10.0",
     "bun": "^1.3.2",
     "es-toolkit": "^1.41.0",
     "fast-xml-parser": "^5.3.1",
     "glob": "^11.0.3",
+    "thumbhash-node": "^0.1.3",
     "tinypool": "^2.0.0",
     "zod": "^4.1.12"
   }

pnpm-lock.yaml

@@ -11,6 +11,9 @@ importers:
       '@guiiai/logg':
         specifier: ^1.2.5
         version: 1.2.5
+      '@napi-rs/canvas':
+        specifier: ^0.1.82
+        version: 0.1.82
       '@types/js-yaml':
         specifier: ^4.0.9
         version: 4.0.9
@@ -29,6 +32,9 @@ importers:
       glob:
         specifier: ^11.0.3
         version: 11.0.3
+      thumbhash-node:
+        specifier: ^0.1.3
+        version: 0.1.3
       tinypool:
         specifier: ^2.0.0
         version: 2.0.0
@@ -150,6 +156,70 @@ packages:
     resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
     engines: {node: '>=12'}
 
+  '@napi-rs/canvas-android-arm64@0.1.82':
+    resolution: {integrity: sha512-bvZhN0iI54ouaQOrgJV96H2q7J3ZoufnHf4E1fUaERwW29Rz4rgicohnAg4venwBJZYjGl5Yl3CGmlAl1LZowQ==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [android]
+
+  '@napi-rs/canvas-darwin-arm64@0.1.82':
+    resolution: {integrity: sha512-InuBHKCyuFqhNwNr4gpqazo5Xp6ltKflqOLiROn4hqAS8u21xAHyYCJRgHwd+a5NKmutFTaRWeUIT/vxWbU/iw==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [darwin]
+
+  '@napi-rs/canvas-darwin-x64@0.1.82':
+    resolution: {integrity: sha512-aQGV5Ynn96onSXcuvYb2y7TRXD/t4CL2EGmnGqvLyeJX1JLSNisKQlWN/1bPDDXymZYSdUqbXehj5qzBlOx+RQ==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [darwin]
+
+  '@napi-rs/canvas-linux-arm-gnueabihf@0.1.82':
+    resolution: {integrity: sha512-YIUpmHWeHGGRhWitT1KJkgj/JPXPfc9ox8oUoyaGPxolLGPp5AxJkq8wIg8CdFGtutget968dtwmx71m8o3h5g==}
+    engines: {node: '>= 10'}
+    cpu: [arm]
+    os: [linux]
+
+  '@napi-rs/canvas-linux-arm64-gnu@0.1.82':
+    resolution: {integrity: sha512-AwLzwLBgmvk7kWeUgItOUor/QyG31xqtD26w1tLpf4yE0hiXTGp23yc669aawjB6FzgIkjh1NKaNS52B7/qEBQ==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [linux]
+
+  '@napi-rs/canvas-linux-arm64-musl@0.1.82':
+    resolution: {integrity: sha512-moZWuqepAwWBffdF4JDadt8TgBD02iMhG6I1FHZf8xO20AsIp9rB+p0B8Zma2h2vAF/YMjeFCDmW5un6+zZz9g==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [linux]
+
+  '@napi-rs/canvas-linux-riscv64-gnu@0.1.82':
+    resolution: {integrity: sha512-w9++2df2kG9eC9LWYIHIlMLuhIrKGQYfUxs97CwgxYjITeFakIRazI9LYWgVzEc98QZ9x9GQvlicFsrROV59MQ==}
+    engines: {node: '>= 10'}
+    cpu: [riscv64]
+    os: [linux]
+
+  '@napi-rs/canvas-linux-x64-gnu@0.1.82':
+    resolution: {integrity: sha512-lZulOPwrRi6hEg/17CaqdwWEUfOlIJuhXxincx1aVzsVOCmyHf+xFq4i6liJl1P+x2v6Iz2Z/H5zHvXJCC7Bwg==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [linux]
+
+  '@napi-rs/canvas-linux-x64-musl@0.1.82':
+    resolution: {integrity: sha512-Be9Wf5RTv1w6GXlTph55K3PH3vsAh1Ax4T1FQY1UYM0QfD0yrwGdnJ8/fhqw7dEgMjd59zIbjJQC8C3msbGn5g==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [linux]
+
+  '@napi-rs/canvas-win32-x64-msvc@0.1.82':
+    resolution: {integrity: sha512-LN/i8VrvxTDmEEK1c10z2cdOTkWT76LlTGtyZe5Kr1sqoSomKeExAjbilnu1+oee5lZUgS5yfZ2LNlVhCeARuw==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [win32]
+
+  '@napi-rs/canvas@0.1.82':
+    resolution: {integrity: sha512-FGjyUBoF0sl1EenSiE4UV2WYu76q6F9GSYedq5EiOCOyGYoQ/Owulcv6rd7v/tWOpljDDtefXXIaOCJrVKem4w==}
+    engines: {node: '>= 10'}
+
   '@napi-rs/wasm-runtime@0.2.12':
     resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==}
 
@@ -1307,6 +1377,88 @@ packages:
     resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==}
     engines: {node: '>= 0.4'}
 
+  thumbhash-node-android-arm-eabi@0.1.3:
+    resolution: {integrity: sha512-xiT17jBTetB+TWMhJDDAVBQPQUVvpdQm39Eg1JCq9guY6/Mzt+GgSUzAveGF1pAt7Jif/zc1Ipcm6sfOWBZwIQ==}
+    engines: {node: '>= 10'}
+    cpu: [arm]
+    os: [android]
+
+  thumbhash-node-android-arm64@0.1.3:
+    resolution: {integrity: sha512-rpeWM7Z3EGnhTEI+mXOEP8cxpkqXZSPhV83T1Joa/UsZgXraQ/247rUxPVPzpEC/fwHmInvmxg5Lic+6lBFiyg==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [android]
+
+  thumbhash-node-darwin-arm64@0.1.3:
+    resolution: {integrity: sha512-McVwFidx7VbmkYbjpD3VR6v1kyEt+nNPnbH1KUYOeDf2nvqjEDmVoVq7wp0uru6zms17Qel4wOkaaF4xbd7M8A==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [darwin]
+
+  thumbhash-node-darwin-x64@0.1.3:
+    resolution: {integrity: sha512-2N2oVSDr9Q6HZliuVEc48FpEdPqWxxv1DwcX0KwsE7SEduH3T7UOFvOoEhwe1ViLAhU/LM9KVbI4EpBsgCFndA==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [darwin]
+
+  thumbhash-node-freebsd-x64@0.1.3:
+    resolution: {integrity: sha512-8CXIYYMP/nMkVKFqt++YfVuatuBOQv/cpzhrOr0kilmn6IbUEIxK909eeZ8l27SxVubdfPnaORtV6aB3VbTsmQ==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [freebsd]
+
+  thumbhash-node-linux-arm-gnueabihf@0.1.3:
+    resolution: {integrity: sha512-0puL2Owq8HUhSK517tLhWlxOLwg2Frvgee47WYU+grIOJEWNhT5xRenM7jvtvn3ABoP/9TbDO5B81gmB1JcsWw==}
+    engines: {node: '>= 10'}
+    cpu: [arm]
+    os: [linux]
+
+  thumbhash-node-linux-arm64-gnu@0.1.3:
+    resolution: {integrity: sha512-lowvXaSaqwc/bghNLckMZQD+wjSZFaaK0mEJgv94QbbKMZkYKU7ELh28q1v/mSMsFLB3etsxP+10dbrmkUoXog==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [linux]
+
+  thumbhash-node-linux-arm64-musl@0.1.3:
+    resolution: {integrity: sha512-1k4GXQA/iXD5hPPahOW1rNXyYWnc3C/Srp1UtdrqsH8A/rXIe6bPqaCKNr30GAcIifx1PoZIpvvEqfO14hqr3Q==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [linux]
+
+  thumbhash-node-linux-x64-gnu@0.1.3:
+    resolution: {integrity: sha512-5gtw1mJIPGG/fpFvPCv7CmN3hkvdDCU+wHe8Mnhq1r8Z+Vu/Rizx4Uxzqqo2QBGxZLHDbR/l24R0+8Oo4rV77w==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [linux]
+
+  thumbhash-node-linux-x64-musl@0.1.3:
+    resolution: {integrity: sha512-2qI7KARyY6kRgSEAvAi6rxYRS9GKO2JPyc9/Q/ai29IdJSvAXgnnvGZDJ6xdv7wzRwE9sjQoY1iME7AvM1x6Bw==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [linux]
+
+  thumbhash-node-win32-arm64-msvc@0.1.3:
+    resolution: {integrity: sha512-Q1Gh3JLL8Y/2zenQH44WitoMCi9juHgX9HXoDvtHwEzthbFpehYozMTFjZh5trttKDUeW9qWb26YYSBVXYys/g==}
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [win32]
+
+  thumbhash-node-win32-ia32-msvc@0.1.3:
+    resolution: {integrity: sha512-Wppy5xD05j7fks0gd4khE9JAXJtmmsPwvVuRHO+ftGWfVaYG44mcIZgygt6WqNSyXnyqXO4X9wuCJtoOtO8Xwg==}
+    engines: {node: '>= 10'}
+    cpu: [ia32]
+    os: [win32]
+
+  thumbhash-node-win32-x64-msvc@0.1.3:
+    resolution: {integrity: sha512-yz6XkszxbVnICc6NWVHHExzjoUvrRkbCUeeETnVDSJJgo3XYw7Ak04Z6XPmyd700+dJp/Lgi+qumQbdAXqGZBA==}
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [win32]
+
+  thumbhash-node@0.1.3:
+    resolution: {integrity: sha512-jP/j1zoNbTcYxN9wMaKU4WZYg+UIdY/hsCS8sKsiOmfK3VVmxProa3WE58rZ2yHafKEnK8wLWCqOjhdIB66SWQ==}
+    engines: {node: '>= 10'}
+
   tinyglobby@0.2.15:
     resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==}
     engines: {node: '>=12.0.0'}
@@ -1506,6 +1658,49 @@ snapshots:
       wrap-ansi: 8.1.0
       wrap-ansi-cjs: wrap-ansi@7.0.0
 
+  '@napi-rs/canvas-android-arm64@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-darwin-arm64@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-darwin-x64@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-linux-arm-gnueabihf@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-linux-arm64-gnu@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-linux-arm64-musl@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-linux-riscv64-gnu@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-linux-x64-gnu@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-linux-x64-musl@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas-win32-x64-msvc@0.1.82':
+    optional: true
+
+  '@napi-rs/canvas@0.1.82':
+    optionalDependencies:
+      '@napi-rs/canvas-android-arm64': 0.1.82
+      '@napi-rs/canvas-darwin-arm64': 0.1.82
+      '@napi-rs/canvas-darwin-x64': 0.1.82
+      '@napi-rs/canvas-linux-arm-gnueabihf': 0.1.82
+      '@napi-rs/canvas-linux-arm64-gnu': 0.1.82
+      '@napi-rs/canvas-linux-arm64-musl': 0.1.82
+      '@napi-rs/canvas-linux-riscv64-gnu': 0.1.82
+      '@napi-rs/canvas-linux-x64-gnu': 0.1.82
+      '@napi-rs/canvas-linux-x64-musl': 0.1.82
+      '@napi-rs/canvas-win32-x64-msvc': 0.1.82
+
   '@napi-rs/wasm-runtime@0.2.12':
     dependencies:
       '@emnapi/core': 1.7.0
@@ -2795,6 +2990,61 @@ snapshots:
 
   supports-preserve-symlinks-flag@1.0.0: {}
 
+  thumbhash-node-android-arm-eabi@0.1.3:
+    optional: true
+
+  thumbhash-node-android-arm64@0.1.3:
+    optional: true
+
+  thumbhash-node-darwin-arm64@0.1.3:
+    optional: true
+
+  thumbhash-node-darwin-x64@0.1.3:
+    optional: true
+
+  thumbhash-node-freebsd-x64@0.1.3:
+    optional: true
+
+  thumbhash-node-linux-arm-gnueabihf@0.1.3:
+    optional: true
+
+  thumbhash-node-linux-arm64-gnu@0.1.3:
+    optional: true
+
+  thumbhash-node-linux-arm64-musl@0.1.3:
+    optional: true
+
+  thumbhash-node-linux-x64-gnu@0.1.3:
+    optional: true
+
+  thumbhash-node-linux-x64-musl@0.1.3:
+    optional: true
+
+  thumbhash-node-win32-arm64-msvc@0.1.3:
+    optional: true
+
+  thumbhash-node-win32-ia32-msvc@0.1.3:
+    optional: true
+
+  thumbhash-node-win32-x64-msvc@0.1.3:
+    optional: true
+
+  thumbhash-node@0.1.3:
+    optionalDependencies:
+      thumbhash-node-android-arm-eabi: 0.1.3
+      thumbhash-node-android-arm64: 0.1.3
+      thumbhash-node-darwin-arm64: 0.1.3
+      thumbhash-node-darwin-x64: 0.1.3
+      thumbhash-node-freebsd-x64: 0.1.3
+      thumbhash-node-linux-arm-gnueabihf: 0.1.3
+      thumbhash-node-linux-arm64-gnu: 0.1.3
+      thumbhash-node-linux-arm64-musl: 0.1.3
+      thumbhash-node-linux-x64-gnu: 0.1.3
+      thumbhash-node-linux-x64-musl: 0.1.3
+      thumbhash-node-win32-arm64-msvc: 0.1.3
+      thumbhash-node-win32-ia32-msvc: 0.1.3
+      thumbhash-node-win32-x64-msvc: 0.1.3
+
   tinyglobby@0.2.15:
     dependencies:
       fdir: 6.5.0(picomatch@4.0.3)

src/index.ts

@@ -7,7 +7,8 @@ import yaml from 'js-yaml';
 import { z } from 'zod';
 
 import { MaimaiRegion, MaimaiMajorVersionId } from './interfaces/index';
-import { run } from './master';
+import { runMetadata } from './metadata/master';
+import { runThumb } from './thumb/master';
 
 const dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -15,7 +16,8 @@ export const Config = z.object({
   inputs: z.record(
     z.enum(MaimaiRegion),
     z.record(z.preprocess(Number, z.enum(MaimaiMajorVersionId)), z.string()),
-  ),
+  ).optional(),
+  assetsDir: z.string().optional(),
   outputDir: z.string(),
 });
 export type Config = z.infer<typeof Config>;
@@ -23,4 +25,13 @@ export type Config = z.infer<typeof Config>;
 const loadConfigFile = (filename: string) => yaml.load(fs.readFileSync(path.resolve(dirname, '..', filename), 'utf-8')) as Record<string, unknown>;
 const config = Config.parse(merge(loadConfigFile('config.base.yaml'), loadConfigFile('config.yaml')));
 
-await run(config);
+const command = process.argv[2];
+if (command === 'metadata') {
+  if (!config.inputs) throw new Error('config.inputs is required');
+  await runMetadata(config.inputs, config.outputDir);
+} else if (command === 'thumb') {
+  if (!config.assetsDir) throw new Error('config.assetsDir is required');
+  await runThumb(config.assetsDir, config.outputDir);
+} else {
+  throw new Error(`Unknown command: ${command}`);
+}

src/interfaces/index.ts

@@ -4,3 +4,6 @@ export * from './title';
 
 export const maimaiMetadataKinds = ['music', 'title', 'frame', 'icon', 'partner', 'plate', 'chara', 'card', 'loginBonus'] as const;
 export type MaimaiMetadataKind = (typeof maimaiMetadataKinds)[number];
+
+export const maimaiThumbKinds = ['music', 'frame', 'icon', 'plate'] as const;
+export type MaimaiThumbKind = (typeof maimaiThumbKinds)[number];

src/interfaces/music.ts

@@ -45,6 +45,7 @@ export interface MaimaiMusicMetadataRegionalInfo {
 
 export type MaimaiMusicLevelChangeLog = Partial<Record<MaimaiMajorVersionId, number>>[];
 export interface MaimaiMusicMetadata extends MaimaiMusicMetadataBase {
+  jacketThumbHash: string;
   /**
    * The level change log per chart (BASIC, ADVANCED, EXPERT, MASTER, Re:MASTER).
    */

src/interfaces/title.ts

@@ -15,3 +15,6 @@ export type BasicMetadata<TExtra = {}> = MetadataMaybeRegionalized<BasicMetadata
 };
 
 export type MaimaiTitleMetadataExtra = { rareType: MaimaiTitleRareType };
+export type MaimaiFrameMetadataExtra = { thumbHash: string };
+export type MaimaiIconMetadataExtra = { thumbHash: string };
+export type MaimaiPlateMetadataExtra = { thumbHash: string };

src/metadata/master.ts

@@ -3,14 +3,13 @@ import path from 'node:path';
 
 import Tinypool from 'tinypool';
 
-import type { Config } from './index';
+import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion, MaimaiThumbKind } from '../interfaces';
-import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion } from './interfaces';
+import { maimaiMetadataKinds, maimaiThumbKinds } from '../interfaces';
-import { maimaiMetadataKinds } from './interfaces';
+import { createLogger } from '../logger';
-import { createLogger } from './logger';
 import { basicDataTypes } from './processors/basic';
 import { mergeMusic } from './processors/music';
-import { arrayToObject, getOrSet, objectEntries, objectMap } from './utils/base';
 import type { WorkerArguments } from './worker';
+import { arrayToObject, getOrSet, objectEntries, objectMap } from '../utils/base';
 
 const logger = createLogger('Master');
 
@@ -21,15 +20,23 @@ const pool = new Tinypool({
 });
 
 export type IntermediateDataMap<TIntermediateData> = Map<MaimaiRegion, Map<MaimaiMajorVersionId, TIntermediateData>>;
-export type MetadataMerger<TIntermediateData, TResult> = (intermediateDataMap: IntermediateDataMap<TIntermediateData>) => TResult;
+export type MetadataMerger<TIntermediateData, TResult> = (intermediateDataMap: IntermediateDataMap<TIntermediateData>, thumbCache: Record<MaimaiThumbKind, Record<number, string>>) => TResult;
 
-export const run = async (config: Config) => {
+export const runMetadata = async (inputs: Record<MaimaiRegion, Record<MaimaiMajorVersionId, string>>, outputDir: string) => {
-  const outputDir = path.resolve(config.outputDir);
   await fs.promises.mkdir(outputDir, { recursive: true });
 
+  const thumbCacheFilePath = path.resolve(outputDir, 'thumb.json');
+  let thumbCache: Record<MaimaiThumbKind, Record<number, string>>;
+  if (await fs.promises.stat(thumbCacheFilePath).catch(() => false)) {
+    thumbCache = JSON.parse(await fs.promises.readFile(thumbCacheFilePath, 'utf-8')) as Record<MaimaiThumbKind, Record<number, string>>;
+  } else {
+    logger.warn('Thumb cache not found, generating metadata with empty thumb hashes');
+    thumbCache = arrayToObject(maimaiThumbKinds, () => ({}));
+  }
+
   const index: Map<MaimaiMetadataKind, Map<MaimaiRegion, Map<MaimaiMajorVersionId, string>>> = new Map();
   const tasks: Promise<void>[] = [];
-  for (const [regionName, versionPathMap] of objectEntries(config.inputs)) {
+  for (const [regionName, versionPathMap] of objectEntries(inputs)) {
     const region = regionName as MaimaiRegion;
     for (const [versionName, streamingAssetsPath] of objectEntries(versionPathMap)) {
       const version = Number(versionName) as MaimaiMajorVersionId;
@@ -46,6 +53,7 @@ export const run = async (config: Config) => {
         if (fs.existsSync(outputFilePath)) continue;
 
         tasks.push(pool.run({
+          thumbCache,
           region,
           version,
           streamingAssetsPath,
@@ -75,7 +83,7 @@ export const run = async (config: Config) => {
   };
   const result = objectEntries(mergers).reduce((result, [kind, merger]) => {
     try {
-      result[kind] = merger(dataMap.get(kind)!);
+      result[kind] = merger(dataMap.get(kind)!, thumbCache);
       return result;
     } catch (error) {
       logger.error(`Failed to merge ${kind} metadata`);

src/metadata/processors/basic.ts

@@ -1,20 +1,24 @@
 import { omit } from 'es-toolkit';
 
-import type { BasicMetadata, BasicMetadataBase, BasicMetadataIntermediate, MaimaiTitleMetadataExtra } from '../interfaces';
+import type { BasicMetadata, BasicMetadataBase, BasicMetadataIntermediate, MaimaiThumbKind, MaimaiTitleMetadataExtra } from '../../interfaces';
-import { MaimaiTitleRareType } from '../interfaces';
+import { MaimaiTitleRareType } from '../../interfaces';
+import { createLogger } from '../../logger';
+import { forEachParallel, objectMap } from '../../utils/base';
+import type { RegionalizedMap, RegionalizedNetOpenDate } from '../../utils/data';
+import { maybeCompactRegionalizedMap, parseNetOpenDate } from '../../utils/data';
+import { forEachRegionAndVersion } from '../../utils/each';
+import { globFiles, parseXmls } from '../../utils/fs';
+import { zCoerceNumber, zCoerceString, zParseEnum } from '../../utils/zod';
 import type { MetadataMerger } from '../master';
-import { forEachParallel, objectMap } from '../utils/base';
-import type { RegionalizedMap, RegionalizedNetOpenDate } from '../utils/data';
-import { maybeCompactRegionalizedMap, parseNetOpenDate } from '../utils/data';
-import { forEachRegionAndVersion } from '../utils/each';
-import { globFiles, parseXmls } from '../utils/fs';
-import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod';
 import type { WorkerProcessor } from '../worker';
 
+const logger = createLogger('Basic');
+
 type IntermediateData<TExtra> = Record<number, BasicMetadataIntermediate<TExtra>>;
 
 const defineDataType = <TExtra = {}>(
   globXmls: (axxxDir: string) => ReturnType<typeof globFiles>,
+  thumbKind: MaimaiThumbKind | undefined,
   xmlRootElementName: string,
   parseExtraFields: (xmlData: any) => TExtra = () => ({}) as any,
 ): {
@@ -28,7 +32,12 @@ const defineDataType = <TExtra = {}>(
       const id = zCoerceNumber(xmlData.name.id);
       const name = zCoerceString(xmlData.name.str);
       const netOpenDate = parseNetOpenDate(xmlData.netOpenName.str);
-      result[id] = { name, netOpenDate, ...parseExtraFields!(xmlData) };
+      result[id] = { name, netOpenDate, ...parseExtraFields(xmlData) };
+      if (thumbKind) {
+        const thumbHash = ctx.thumbCache[thumbKind][id];
+        if (!thumbHash) logger.warn(`Thumb hash not found for ${thumbKind} ${id}`);
+        (result[id] as { thumbHash?: string }).thumbHash = thumbHash ?? '';
+      }
     }));
     return result;
   },
@@ -55,14 +64,14 @@ const defineDataType = <TExtra = {}>(
 });
 
 export const basicDataTypes = {
-  title: defineDataType<MaimaiTitleMetadataExtra>(axxxDir => globFiles(axxxDir, 'title', 'title', 'Title.xml'), 'TitleData', TitleData => ({
+  title: defineDataType<MaimaiTitleMetadataExtra>(axxxDir => globFiles(axxxDir, 'title', 'title', 'Title.xml'), undefined, 'TitleData', TitleData => ({
     rareType: zParseEnum(MaimaiTitleRareType, TitleData.rareType),
   })),
-  frame: defineDataType(axxxDir => globFiles(axxxDir, 'frame', 'frame', 'Frame.xml'), 'FrameData'),
+  frame: defineDataType(axxxDir => globFiles(axxxDir, 'frame', 'frame', 'Frame.xml'), 'frame', 'FrameData'),
-  icon: defineDataType(axxxDir => globFiles(axxxDir, 'icon', 'icon', 'Icon.xml'), 'IconData'),
+  icon: defineDataType(axxxDir => globFiles(axxxDir, 'icon', 'icon', 'Icon.xml'), 'icon', 'IconData'),
-  partner: defineDataType(axxxDir => globFiles(axxxDir, 'partner', 'partner', 'Partner.xml'), 'PartnerData'),
+  partner: defineDataType(axxxDir => globFiles(axxxDir, 'partner', 'partner', 'Partner.xml'), undefined, 'PartnerData'),
-  plate: defineDataType(axxxDir => globFiles(axxxDir, 'plate', 'plate', 'Plate.xml'), 'PlateData'),
+  plate: defineDataType(axxxDir => globFiles(axxxDir, 'plate', 'plate', 'Plate.xml'), 'plate', 'PlateData'),
-  chara: defineDataType(axxxDir => globFiles(axxxDir, 'chara', 'chara', 'Chara.xml'), 'CharaData'),
+  chara: defineDataType(axxxDir => globFiles(axxxDir, 'chara', 'chara', 'Chara.xml'), undefined, 'CharaData'),
-  card: defineDataType(axxxDir => globFiles(axxxDir, 'card', 'card', 'Card.xml'), 'CardData'),
+  card: defineDataType(axxxDir => globFiles(axxxDir, 'card', 'card', 'Card.xml'), undefined, 'CardData'),
-  loginBonus: defineDataType(axxxDir => globFiles(axxxDir, 'loginBonus', 'LoginBonus', 'LoginBonus.xml'), 'LoginBonusData'),
+  loginBonus: defineDataType(axxxDir => globFiles(axxxDir, 'loginBonus', 'LoginBonus', 'LoginBonus.xml'), undefined, 'LoginBonusData'),
 };

src/metadata/processors/music.ts

@@ -1,12 +1,12 @@
-import type { MaimaiChartMetadataIntermediate, MaimaiMusicMetadata, MaimaiMusicMetadataIntermediate } from '../interfaces';
+import type { MaimaiChartMetadataIntermediate, MaimaiMusicMetadata, MaimaiMusicMetadataIntermediate } from '../../interfaces';
-import { MaimaiRegion, MaimaiMajorVersionId, maimaiMajorVersionIds, MaimaiMusicAddDeleteLogEntry } from '../interfaces';
+import { MaimaiRegion, MaimaiMajorVersionId, maimaiMajorVersionIds, MaimaiMusicAddDeleteLogEntry } from '../../interfaces';
-import { createLogger } from '../logger';
+import { createLogger } from '../../logger';
+import { forEachParallel, objectEntries, objectKeys } from '../../utils/base';
+import { parseEventIdAsNetOpenDate, parseNetOpenDate } from '../../utils/data';
+import { forEachRegionAndVersion } from '../../utils/each';
+import { globFiles, parseXmls } from '../../utils/fs';
+import { zCoerceNumber, zCoerceString, zParseEnum } from '../../utils/zod';
 import type { MetadataMerger } from '../master';
-import { forEachParallel, objectEntries, objectKeys } from '../utils/base';
-import { parseEventIdAsNetOpenDate, parseNetOpenDate } from '../utils/data';
-import { forEachRegionAndVersion } from '../utils/each';
-import { globFiles, parseXmls } from '../utils/fs';
-import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod';
 import type { WorkerProcessor } from '../worker';
 
 const logger = createLogger('Music');
@@ -74,19 +74,22 @@ export const processMusic: WorkerProcessor<IntermediateData> = async ctx => {
   return musics;
 };
 
-export const mergeMusic: MetadataMerger<IntermediateData, Record<number, MaimaiMusicMetadata>> = dataMap => {
+export const mergeMusic: MetadataMerger<IntermediateData, Record<number, MaimaiMusicMetadata>> = (dataMap, thumbCache) => {
   const result: Record<number, MaimaiMusicMetadata> = {};
 
   // Merge all musics. JPN first. Nerwer version first.
   const lowsetSeenVersionId: Record<number, MaimaiMajorVersionId> = {};
   forEachRegionAndVersion(dataMap, 'jpnFirst', 'newFirst', (region, version, musics) => Object.entries(musics).forEach(([idStr, music]) => {
     const id = Number(idStr);
+    const thumbHash = thumbCache.music[id % 10000];
+    if (!thumbHash) logger.warn(`Thumb hash not found for music ${id}`);
     const resultMusic = result[id] ??= {
       name: music.name,
       artist: music.artist,
       genre: music.genre,
       bpm: music.bpm,
       charts: music.charts,
+      jacketThumbHash: thumbHash ?? '',
       levelChangeLog: [],
       regionalInfo: {},
     };

src/metadata/worker.ts

@@ -1,15 +1,16 @@
 import fs from 'node:fs';
 
-import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId } from './interfaces';
+import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId, MaimaiThumbKind } from '../interfaces';
-import { createLogger } from './logger';
+import { createLogger } from '../logger';
 import { basicDataTypes } from './processors/basic';
 import { processMusic } from './processors/music';
-import { objectMap } from './utils/base';
+import { objectMap } from '../utils/base';
-import { globAxxxDirs } from './utils/fs';
+import { globAxxxDirs } from '../utils/fs';
 
 const logger = createLogger('Worker');
 
 export interface WorkerArguments {
+  thumbCache: Record<MaimaiThumbKind, Record<number, string>>;
   region: MaimaiRegion;
   version: MaimaiMajorVersionId;
   streamingAssetsPath: string;

src/thumb/master.ts

@@ -0,0 +1,54 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { glob } from 'glob';
+import Tinypool from 'tinypool';
+
+import type { MaimaiThumbKind } from '../interfaces';
+import { maimaiThumbKinds } from '../interfaces';
+import { createLogger } from '../logger';
+import type { WorkerArguments } from './worker';
+import { objectEntries, arrayToObject } from '../utils/base';
+
+const logger = createLogger('Master');
+
+const pool = new Tinypool({
+  filename: new URL('./worker.ts', import.meta.url).href,
+  minThreads: 8,
+  maxThreads: 32,
+});
+
+const NUM_WORKERS = 32;
+const ASSET_SUFFIX = '.png';
+
+export const runThumb = async (assetsDir: string, outputDir: string) => {
+  await fs.promises.mkdir(path.resolve(outputDir, 'thumb'), { recursive: true });
+
+  const workerArgs: WorkerArguments[] = Array.from({ length: NUM_WORKERS }).map((_, i) => ({ tasks: [], outputFile: path.resolve(outputDir, 'thumb', `${i}.json`) }));
+  let i = 0;
+  for (const kind of maimaiThumbKinds) {
+    const files = await glob(`${assetsDir}/${kind}/*${ASSET_SUFFIX}`);
+    const fileIds = files.map(file => {
+      const id = parseInt(path.basename(file, ASSET_SUFFIX));
+      if (path.basename(file) !== `${id}${ASSET_SUFFIX}`) {
+        logger.warn(`Invalid file name: ${file}`);
+        return 0;
+      }
+      workerArgs[i++ % NUM_WORKERS]!.tasks.push({ kind, id, filePath: file });
+      return id;
+    }).filter(id => id !== 0);
+    logger.log(`Found ${fileIds.length} files for ${kind}`);
+  }
+
+  await Promise.all(workerArgs.map(args => pool.run(args)));
+
+  const result: Record<MaimaiThumbKind, Record<number, string>> = arrayToObject(maimaiThumbKinds, () => ({}));
+  for (const args of workerArgs) {
+    const data = JSON.parse(await fs.promises.readFile(args.outputFile, 'utf-8')) as Record<MaimaiThumbKind, Record<number, string>>;
+    for (const [kind, entries] of objectEntries(data)) {
+      Object.assign(result[kind], entries);
+    }
+  }
+  await fs.promises.writeFile(`${outputDir}/thumb.json`, JSON.stringify(result, null, 2));
+  logger.log(`Merged thumb hashes to ${outputDir}/thumb.json`);
+};

src/thumb/worker.ts

@@ -0,0 +1,39 @@
+import fs from 'node:fs';
+
+import { createCanvas, loadImage } from '@napi-rs/canvas';
+import { rgbaToThumbHash } from 'thumbhash-node';
+
+import type { MaimaiThumbKind } from '../interfaces';
+import { maimaiThumbKinds } from '../interfaces';
+import { createLogger } from '../logger';
+import { arrayToObject } from '../utils/base';
+
+const logger = createLogger('Worker');
+
+export type WorkerArguments = {
+  tasks: { kind: MaimaiThumbKind; id: number; filePath: string }[];
+  outputFile: string;
+};
+
+const MAX_SIZE = 100;
+
+export default async (args: WorkerArguments) => {
+  const result: Record<MaimaiThumbKind, Record<number, string>> = arrayToObject(maimaiThumbKinds, () => ({}));
+  await Promise.all(args.tasks.map(async ({ kind, id, filePath }) => {
+    const image = await loadImage(filePath);
+    const width = image.width;
+    const height = image.height;
+    const scale = Math.min(MAX_SIZE / width, MAX_SIZE / height);
+    const resizedWidth = Math.round(width * scale);
+    const resizedHeight = Math.round(height * scale);
+    const canvas = createCanvas(resizedWidth, resizedHeight);
+    const ctx = canvas.getContext('2d');
+    ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
+    const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+    const rgba = new Uint8Array(imageData.data.buffer);
+    const thumbhash = rgbaToThumbHash(resizedWidth, resizedHeight, rgba);
+    logger.log(`Generated thumbhash for ${filePath}`);
+    result[kind][id] = Buffer.from(thumbhash).toString('base64url');
+  }));
+  await fs.promises.writeFile(args.outputFile, JSON.stringify(result, null, 2));
+};