From 727b57c81e10eebd7464d41391229b78b3bcd71e Mon Sep 17 00:00:00 2001 From: Menci Date: Wed, 19 Nov 2025 11:48:55 +0800 Subject: [PATCH] feat: add asset file hash --- src/index.ts | 3 ++- src/interfaces/index.ts | 1 + src/interfaces/music.ts | 2 +- src/metadata/master.ts | 8 ++++---- src/metadata/processors/basic.ts | 6 +++--- src/metadata/processors/music.ts | 6 +++--- src/metadata/worker.ts | 4 ++-- src/thumb/master.ts | 10 +++++----- src/thumb/worker.ts | 12 ++++++++---- 9 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/index.ts b/src/index.ts index 0c19523..0db2eb6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,6 +18,7 @@ export const Config = z.object({ z.record(z.preprocess(Number, z.enum(MaimaiMajorVersionId)), z.string()), ).optional(), assetsDir: z.string().optional(), + hashSalt: z.string().optional(), outputDir: z.string(), }); export type Config = z.infer; @@ -31,7 +32,7 @@ if (command === 'metadata') { await runMetadata(config.inputs, config.outputDir); } else if (command === 'thumb') { if (!config.assetsDir) throw new Error('config.assetsDir is required'); - await runThumb(config.assetsDir, config.outputDir); + await runThumb(config.assetsDir, config.hashSalt ?? '', config.outputDir); } else { throw new Error(`Unknown command: ${command}`); } diff --git a/src/interfaces/index.ts b/src/interfaces/index.ts index 2eba40a..66a2c2a 100644 --- a/src/interfaces/index.ts +++ b/src/interfaces/index.ts @@ -7,3 +7,4 @@ export type MaimaiMetadataKind = (typeof maimaiMetadataKinds)[number]; export const maimaiThumbKinds = ['music', 'frame', 'icon', 'plate'] as const; export type MaimaiThumbKind = (typeof maimaiThumbKinds)[number]; +export type ThumbCache = Record>; diff --git a/src/interfaces/music.ts b/src/interfaces/music.ts index e52d2d5..7d5be5a 100644 --- a/src/interfaces/music.ts +++ b/src/interfaces/music.ts @@ -50,7 +50,7 @@ export interface MaimaiMusicMetadataRegionalInfo { export type MaimaiMusicLevelChangeLog = Partial>[]; export interface MaimaiMusicMetadata extends MaimaiMusicMetadataBase { - jacketThumbHash: string; + jacket?: { thumbhash: string; hash: string }; /** * The level change log per chart (BASIC, ADVANCED, EXPERT, MASTER, Re:MASTER). */ diff --git a/src/metadata/master.ts b/src/metadata/master.ts index 360811d..91ccd17 100644 --- a/src/metadata/master.ts +++ b/src/metadata/master.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import Tinypool from 'tinypool'; -import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion, MaimaiThumbKind } from '../interfaces'; +import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion, ThumbCache } from '../interfaces'; import { maimaiMetadataKinds, maimaiThumbKinds } from '../interfaces'; import { createLogger } from '../logger'; import { basicDataTypes } from './processors/basic'; @@ -20,15 +20,15 @@ const pool = new Tinypool({ }); export type IntermediateDataMap = Map>; -export type MetadataMerger = (intermediateDataMap: IntermediateDataMap, thumbCache: Record>) => TResult; +export type MetadataMerger = (intermediateDataMap: IntermediateDataMap, thumbCache: ThumbCache) => TResult; export const runMetadata = async (inputs: Record>, outputDir: string) => { await fs.promises.mkdir(outputDir, { recursive: true }); const thumbCacheFilePath = path.resolve(outputDir, 'thumb.json'); - let thumbCache: Record>; + let thumbCache: ThumbCache; if (await fs.promises.stat(thumbCacheFilePath).catch(() => false)) { - thumbCache = JSON.parse(await fs.promises.readFile(thumbCacheFilePath, 'utf-8')) as Record>; + thumbCache = JSON.parse(await fs.promises.readFile(thumbCacheFilePath, 'utf-8')) as ThumbCache; } else { logger.warn('Thumb cache not found, generating metadata with empty thumb hashes'); thumbCache = arrayToObject(maimaiThumbKinds, () => ({})); diff --git a/src/metadata/processors/basic.ts b/src/metadata/processors/basic.ts index ad3b361..782b1cb 100644 --- a/src/metadata/processors/basic.ts +++ b/src/metadata/processors/basic.ts @@ -34,9 +34,9 @@ const defineDataType = ( const netOpenDate = parseNetOpenDate(xmlData.netOpenName.str); result[id] = { name, netOpenDate, ...parseExtraFields(xmlData) }; if (thumbKind) { - const thumbHash = ctx.thumbCache[thumbKind][id]; - if (!thumbHash) logger.warn(`Thumb hash not found for ${thumbKind} ${id}`); - (result[id] as { thumbHash?: string }).thumbHash = thumbHash ?? ''; + const assetImage = ctx.thumbCache[thumbKind][id]; + if (!assetImage) logger.warn(`Asset image ${thumbKind} ${id} not found in thumb cache`); + (result[id] as { assetImage?: { thumbhash: string; hash: string } }).assetImage = assetImage; } })); return result; diff --git a/src/metadata/processors/music.ts b/src/metadata/processors/music.ts index ba8e845..439b3ec 100644 --- a/src/metadata/processors/music.ts +++ b/src/metadata/processors/music.ts @@ -129,15 +129,15 @@ export const mergeMusic: MetadataMerger = {}; forEachRegionAndVersion(dataMap, 'jpnFirst', 'newFirst', (region, version, musics) => Object.entries(musics).forEach(([idStr, music]) => { const id = Number(idStr); - const thumbHash = thumbCache.music[id % 10000]; - if (!thumbHash) logger.warn(`Thumb hash not found for music ${id}`); + const jacket = thumbCache.music[id % 10000]; + if (!jacket) logger.warn(`Jacket not found for music ${id}`); const resultMusic = result[id] ??= { name: music.name, artist: music.artist, genre: music.genre, bpm: music.bpm, charts: music.charts, - jacketThumbHash: thumbHash ?? '', + jacket, levelChangeLog: [], regionalInfo: {}, }; diff --git a/src/metadata/worker.ts b/src/metadata/worker.ts index 24f1cd8..d5cbe22 100644 --- a/src/metadata/worker.ts +++ b/src/metadata/worker.ts @@ -1,6 +1,6 @@ import fs from 'node:fs'; -import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId, MaimaiThumbKind } from '../interfaces'; +import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId, ThumbCache } from '../interfaces'; import { createLogger } from '../logger'; import { basicDataTypes } from './processors/basic'; import { processMusic } from './processors/music'; @@ -10,7 +10,7 @@ import { globAxxxDirs } from '../utils/fs'; const logger = createLogger('Worker'); export interface WorkerArguments { - thumbCache: Record>; + thumbCache: ThumbCache; region: MaimaiRegion; version: MaimaiMajorVersionId; streamingAssetsPath: string; diff --git a/src/thumb/master.ts b/src/thumb/master.ts index 406d5fd..de621ec 100644 --- a/src/thumb/master.ts +++ b/src/thumb/master.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { glob } from 'glob'; import Tinypool from 'tinypool'; -import type { MaimaiThumbKind } from '../interfaces'; +import type { ThumbCache } from '../interfaces'; import { maimaiThumbKinds } from '../interfaces'; import { createLogger } from '../logger'; import type { WorkerArguments } from './worker'; @@ -21,10 +21,10 @@ const pool = new Tinypool({ const NUM_WORKERS = 32; const ASSET_SUFFIX = '.png'; -export const runThumb = async (assetsDir: string, outputDir: string) => { +export const runThumb = async (assetsDir: string, hashSalt: string, outputDir: string) => { await fs.promises.mkdir(path.resolve(outputDir, 'thumb'), { recursive: true }); - const workerArgs: WorkerArguments[] = Array.from({ length: NUM_WORKERS }).map((_, i) => ({ tasks: [], outputFile: path.resolve(outputDir, 'thumb', `${i}.json`) })); + const workerArgs: WorkerArguments[] = Array.from({ length: NUM_WORKERS }).map((_, i) => ({ hashSalt, tasks: [], outputFile: path.resolve(outputDir, 'thumb', `${i}.json`) })); let i = 0; for (const kind of maimaiThumbKinds) { const files = await glob(`${assetsDir}/${kind}/*${ASSET_SUFFIX}`); @@ -42,9 +42,9 @@ export const runThumb = async (assetsDir: string, outputDir: string) => { await Promise.all(workerArgs.map(args => pool.run(args))); - const result: Record> = arrayToObject(maimaiThumbKinds, () => ({})); + const result: ThumbCache = arrayToObject(maimaiThumbKinds, () => ({})); for (const args of workerArgs) { - const data = JSON.parse(await fs.promises.readFile(args.outputFile, 'utf-8')) as Record>; + const data = JSON.parse(await fs.promises.readFile(args.outputFile, 'utf-8')) as ThumbCache; for (const [kind, entries] of objectEntries(data)) { Object.assign(result[kind], entries); } diff --git a/src/thumb/worker.ts b/src/thumb/worker.ts index d30603d..213822c 100644 --- a/src/thumb/worker.ts +++ b/src/thumb/worker.ts @@ -1,9 +1,10 @@ +import crypto from 'node:crypto'; import fs from 'node:fs'; import { createCanvas, loadImage } from '@napi-rs/canvas'; import { rgbaToThumbHash } from 'thumbhash-node'; -import type { MaimaiThumbKind } from '../interfaces'; +import type { MaimaiThumbKind, ThumbCache } from '../interfaces'; import { maimaiThumbKinds } from '../interfaces'; import { createLogger } from '../logger'; import { arrayToObject } from '../utils/base'; @@ -11,6 +12,7 @@ import { arrayToObject } from '../utils/base'; const logger = createLogger('Worker'); export type WorkerArguments = { + hashSalt: string; tasks: { kind: MaimaiThumbKind; id: number; filePath: string }[]; outputFile: string; }; @@ -18,9 +20,11 @@ export type WorkerArguments = { const MAX_SIZE = 100; export default async (args: WorkerArguments) => { - const result: Record> = arrayToObject(maimaiThumbKinds, () => ({})); + const result: ThumbCache = arrayToObject(maimaiThumbKinds, () => ({})); await Promise.all(args.tasks.map(async ({ kind, id, filePath }) => { - const image = await loadImage(filePath); + const buffer = await fs.promises.readFile(filePath); + const hash = crypto.createHash('sha256').update(args.hashSalt).update(buffer).digest().subarray(8, 24).toString('base64url'); + const image = await loadImage(buffer); const width = image.width; const height = image.height; const scale = Math.min(MAX_SIZE / width, MAX_SIZE / height); @@ -33,7 +37,7 @@ export default async (args: WorkerArguments) => { const rgba = new Uint8Array(imageData.data.buffer); const thumbhash = rgbaToThumbHash(resizedWidth, resizedHeight, rgba); logger.log(`Generated thumbhash for ${filePath}`); - result[kind][id] = Buffer.from(thumbhash).toString('base64url'); + result[kind][id] = { thumbhash: Buffer.from(thumbhash).toString('base64url'), hash }; })); await fs.promises.writeFile(args.outputFile, JSON.stringify(result, null, 2)); };