feat: add asset file hash

src/index.ts

@@ -18,6 +18,7 @@ export const Config = z.object({
     z.record(z.preprocess(Number, z.enum(MaimaiMajorVersionId)), z.string()),
   ).optional(),
   assetsDir: z.string().optional(),
+  hashSalt: z.string().optional(),
   outputDir: z.string(),
 });
 export type Config = z.infer<typeof Config>;
@@ -31,7 +32,7 @@ if (command === 'metadata') {
   await runMetadata(config.inputs, config.outputDir);
 } else if (command === 'thumb') {
   if (!config.assetsDir) throw new Error('config.assetsDir is required');
-  await runThumb(config.assetsDir, config.outputDir);
+  await runThumb(config.assetsDir, config.hashSalt ?? '', config.outputDir);
 } else {
   throw new Error(`Unknown command: ${command}`);
 }

src/interfaces/index.ts

@@ -7,3 +7,4 @@ export type MaimaiMetadataKind = (typeof maimaiMetadataKinds)[number];
 
 export const maimaiThumbKinds = ['music', 'frame', 'icon', 'plate'] as const;
 export type MaimaiThumbKind = (typeof maimaiThumbKinds)[number];
+export type ThumbCache = Record<MaimaiThumbKind, Record<number, { thumbhash: string; hash: string }>>;

src/interfaces/music.ts

@@ -50,7 +50,7 @@ export interface MaimaiMusicMetadataRegionalInfo {
 
 export type MaimaiMusicLevelChangeLog = Partial<Record<MaimaiMajorVersionId, number>>[];
 export interface MaimaiMusicMetadata extends MaimaiMusicMetadataBase {
-  jacketThumbHash: string;
+  jacket?: { thumbhash: string; hash: string };
   /**
    * The level change log per chart (BASIC, ADVANCED, EXPERT, MASTER, Re:MASTER).
    */

src/metadata/master.ts

@@ -3,7 +3,7 @@ import path from 'node:path';
 
 import Tinypool from 'tinypool';
 
-import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion, MaimaiThumbKind } from '../interfaces';
+import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion, ThumbCache } from '../interfaces';
 import { maimaiMetadataKinds, maimaiThumbKinds } from '../interfaces';
 import { createLogger } from '../logger';
 import { basicDataTypes } from './processors/basic';
@@ -20,15 +20,15 @@ const pool = new Tinypool({
 });
 
 export type IntermediateDataMap<TIntermediateData> = Map<MaimaiRegion, Map<MaimaiMajorVersionId, TIntermediateData>>;
-export type MetadataMerger<TIntermediateData, TResult> = (intermediateDataMap: IntermediateDataMap<TIntermediateData>, thumbCache: Record<MaimaiThumbKind, Record<number, string>>) => TResult;
+export type MetadataMerger<TIntermediateData, TResult> = (intermediateDataMap: IntermediateDataMap<TIntermediateData>, thumbCache: ThumbCache) => TResult;
 
 export const runMetadata = async (inputs: Record<MaimaiRegion, Record<MaimaiMajorVersionId, string>>, outputDir: string) => {
   await fs.promises.mkdir(outputDir, { recursive: true });
 
   const thumbCacheFilePath = path.resolve(outputDir, 'thumb.json');
-  let thumbCache: Record<MaimaiThumbKind, Record<number, string>>;
+  let thumbCache: ThumbCache;
   if (await fs.promises.stat(thumbCacheFilePath).catch(() => false)) {
-    thumbCache = JSON.parse(await fs.promises.readFile(thumbCacheFilePath, 'utf-8')) as Record<MaimaiThumbKind, Record<number, string>>;
+    thumbCache = JSON.parse(await fs.promises.readFile(thumbCacheFilePath, 'utf-8')) as ThumbCache;
   } else {
     logger.warn('Thumb cache not found, generating metadata with empty thumb hashes');
     thumbCache = arrayToObject(maimaiThumbKinds, () => ({}));

src/metadata/processors/basic.ts

@@ -34,9 +34,9 @@ const defineDataType = <TExtra = {}>(
       const netOpenDate = parseNetOpenDate(xmlData.netOpenName.str);
       result[id] = { name, netOpenDate, ...parseExtraFields(xmlData) };
       if (thumbKind) {
-        const thumbHash = ctx.thumbCache[thumbKind][id];
-        if (!thumbHash) logger.warn(`Thumb hash not found for ${thumbKind} ${id}`);
-        (result[id] as { thumbHash?: string }).thumbHash = thumbHash ?? '';
+        const assetImage = ctx.thumbCache[thumbKind][id];
+        if (!assetImage) logger.warn(`Asset image ${thumbKind} ${id} not found in thumb cache`);
+        (result[id] as { assetImage?: { thumbhash: string; hash: string } }).assetImage = assetImage;
       }
     }));
     return result;

src/metadata/processors/music.ts

@@ -129,15 +129,15 @@ export const mergeMusic: MetadataMerger<IntermediateData, Record<number, MaimaiM
   const lowsetSeenVersionId: Record<number, MaimaiMajorVersionId> = {};
   forEachRegionAndVersion(dataMap, 'jpnFirst', 'newFirst', (region, version, musics) => Object.entries(musics).forEach(([idStr, music]) => {
     const id = Number(idStr);
-    const thumbHash = thumbCache.music[id % 10000];
-    if (!thumbHash) logger.warn(`Thumb hash not found for music ${id}`);
+    const jacket = thumbCache.music[id % 10000];
+    if (!jacket) logger.warn(`Jacket not found for music ${id}`);
     const resultMusic = result[id] ??= {
       name: music.name,
       artist: music.artist,
       genre: music.genre,
       bpm: music.bpm,
       charts: music.charts,
-      jacketThumbHash: thumbHash ?? '',
+      jacket,
       levelChangeLog: [],
       regionalInfo: {},
     };

src/metadata/worker.ts

@@ -1,6 +1,6 @@
 import fs from 'node:fs';
 
-import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId, MaimaiThumbKind } from '../interfaces';
+import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId, ThumbCache } from '../interfaces';
 import { createLogger } from '../logger';
 import { basicDataTypes } from './processors/basic';
 import { processMusic } from './processors/music';
@@ -10,7 +10,7 @@ import { globAxxxDirs } from '../utils/fs';
 const logger = createLogger('Worker');
 
 export interface WorkerArguments {
-  thumbCache: Record<MaimaiThumbKind, Record<number, string>>;
+  thumbCache: ThumbCache;
   region: MaimaiRegion;
   version: MaimaiMajorVersionId;
   streamingAssetsPath: string;

src/thumb/master.ts

@@ -4,7 +4,7 @@ import path from 'node:path';
 import { glob } from 'glob';
 import Tinypool from 'tinypool';
 
-import type { MaimaiThumbKind } from '../interfaces';
+import type { ThumbCache } from '../interfaces';
 import { maimaiThumbKinds } from '../interfaces';
 import { createLogger } from '../logger';
 import type { WorkerArguments } from './worker';
@@ -21,10 +21,10 @@ const pool = new Tinypool({
 const NUM_WORKERS = 32;
 const ASSET_SUFFIX = '.png';
 
-export const runThumb = async (assetsDir: string, outputDir: string) => {
+export const runThumb = async (assetsDir: string, hashSalt: string, outputDir: string) => {
   await fs.promises.mkdir(path.resolve(outputDir, 'thumb'), { recursive: true });
 
-  const workerArgs: WorkerArguments[] = Array.from({ length: NUM_WORKERS }).map((_, i) => ({ tasks: [], outputFile: path.resolve(outputDir, 'thumb', `${i}.json`) }));
+  const workerArgs: WorkerArguments[] = Array.from({ length: NUM_WORKERS }).map((_, i) => ({ hashSalt, tasks: [], outputFile: path.resolve(outputDir, 'thumb', `${i}.json`) }));
   let i = 0;
   for (const kind of maimaiThumbKinds) {
     const files = await glob(`${assetsDir}/${kind}/*${ASSET_SUFFIX}`);
@@ -42,9 +42,9 @@ export const runThumb = async (assetsDir: string, outputDir: string) => {
 
   await Promise.all(workerArgs.map(args => pool.run(args)));
 
-  const result: Record<MaimaiThumbKind, Record<number, string>> = arrayToObject(maimaiThumbKinds, () => ({}));
+  const result: ThumbCache = arrayToObject(maimaiThumbKinds, () => ({}));
   for (const args of workerArgs) {
-    const data = JSON.parse(await fs.promises.readFile(args.outputFile, 'utf-8')) as Record<MaimaiThumbKind, Record<number, string>>;
+    const data = JSON.parse(await fs.promises.readFile(args.outputFile, 'utf-8')) as ThumbCache;
     for (const [kind, entries] of objectEntries(data)) {
       Object.assign(result[kind], entries);
     }

src/thumb/worker.ts

@@ -1,9 +1,10 @@
+import crypto from 'node:crypto';
 import fs from 'node:fs';
 
 import { createCanvas, loadImage } from '@napi-rs/canvas';
 import { rgbaToThumbHash } from 'thumbhash-node';
 
-import type { MaimaiThumbKind } from '../interfaces';
+import type { MaimaiThumbKind, ThumbCache } from '../interfaces';
 import { maimaiThumbKinds } from '../interfaces';
 import { createLogger } from '../logger';
 import { arrayToObject } from '../utils/base';
@@ -11,6 +12,7 @@ import { arrayToObject } from '../utils/base';
 const logger = createLogger('Worker');
 
 export type WorkerArguments = {
+  hashSalt: string;
   tasks: { kind: MaimaiThumbKind; id: number; filePath: string }[];
   outputFile: string;
 };
@@ -18,9 +20,11 @@ export type WorkerArguments = {
 const MAX_SIZE = 100;
 
 export default async (args: WorkerArguments) => {
-  const result: Record<MaimaiThumbKind, Record<number, string>> = arrayToObject(maimaiThumbKinds, () => ({}));
+  const result: ThumbCache = arrayToObject(maimaiThumbKinds, () => ({}));
   await Promise.all(args.tasks.map(async ({ kind, id, filePath }) => {
-    const image = await loadImage(filePath);
+    const buffer = await fs.promises.readFile(filePath);
+    const hash = crypto.createHash('sha256').update(args.hashSalt).update(buffer).digest().subarray(8, 24).toString('base64url');
+    const image = await loadImage(buffer);
     const width = image.width;
     const height = image.height;
     const scale = Math.min(MAX_SIZE / width, MAX_SIZE / height);
@@ -33,7 +37,7 @@ export default async (args: WorkerArguments) => {
     const rgba = new Uint8Array(imageData.data.buffer);
     const thumbhash = rgbaToThumbHash(resizedWidth, resizedHeight, rgba);
     logger.log(`Generated thumbhash for ${filePath}`);
-    result[kind][id] = Buffer.from(thumbhash).toString('base64url');
+    result[kind][id] = { thumbhash: Buffer.from(thumbhash).toString('base64url'), hash };
   }));
   await fs.promises.writeFile(args.outputFile, JSON.stringify(result, null, 2));
 };