MaigoLabs/Kairos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: generate thumbhash for assets

Browse Source
This commit is contained in:
Menci
2025-11-17 02:52:00 +08:00
parent 96cfc41f7a
commit a956db4650
13 changed files with 429 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index.ts
+14 -3
View File
@@ -7,7 +7,8 @@ import yaml from 'js-yaml';
import { z } from 'zod';
import { MaimaiRegion, MaimaiMajorVersionId } from './interfaces/index';
import { run } from './master';
import { runMetadata } from './metadata/master';
import { runThumb } from './thumb/master';
const dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -15,7 +16,8 @@ export const Config = z.object({
inputs: z.record(
z.enum(MaimaiRegion),
z.record(z.preprocess(Number, z.enum(MaimaiMajorVersionId)), z.string()),
),
).optional(),
assetsDir: z.string().optional(),
outputDir: z.string(),
});
export type Config = z.infer<typeof Config>;
@@ -23,4 +25,13 @@ export type Config = z.infer<typeof Config>;
const loadConfigFile = (filename: string) => yaml.load(fs.readFileSync(path.resolve(dirname, '..', filename), 'utf-8')) as Record<string, unknown>;
const config = Config.parse(merge(loadConfigFile('config.base.yaml'), loadConfigFile('config.yaml')));
await run(config);
const command = process.argv[2];
if (command === 'metadata') {
if (!config.inputs) throw new Error('config.inputs is required');
await runMetadata(config.inputs, config.outputDir);
} else if (command === 'thumb') {
if (!config.assetsDir) throw new Error('config.assetsDir is required');
await runThumb(config.assetsDir, config.outputDir);
} else {
throw new Error(`Unknown command: ${command}`);
}
src/interfaces/index.ts
+3
View File
@@ -4,3 +4,6 @@ export * from './title';
export const maimaiMetadataKinds = ['music', 'title', 'frame', 'icon', 'partner', 'plate', 'chara', 'card', 'loginBonus'] as const;
export type MaimaiMetadataKind = (typeof maimaiMetadataKinds)[number];
export const maimaiThumbKinds = ['music', 'frame', 'icon', 'plate'] as const;
export type MaimaiThumbKind = (typeof maimaiThumbKinds)[number];
src/interfaces/music.ts
+1
View File
@@ -45,6 +45,7 @@ export interface MaimaiMusicMetadataRegionalInfo {
export type MaimaiMusicLevelChangeLog = Partial<Record<MaimaiMajorVersionId, number>>[];
export interface MaimaiMusicMetadata extends MaimaiMusicMetadataBase {
jacketThumbHash: string;
/**
* The level change log per chart (BASIC, ADVANCED, EXPERT, MASTER, Re:MASTER).
*/
src/interfaces/title.ts
+3
View File
@@ -15,3 +15,6 @@ export type BasicMetadata<TExtra = {}> = MetadataMaybeRegionalized<BasicMetadata
};
export type MaimaiTitleMetadataExtra = { rareType: MaimaiTitleRareType };
export type MaimaiFrameMetadataExtra = { thumbHash: string };
export type MaimaiIconMetadataExtra = { thumbHash: string };
export type MaimaiPlateMetadataExtra = { thumbHash: string };
src/master.ts → src/metadata/master.ts
+18 -10
View File
@@ -3,14 +3,13 @@ import path from 'node:path';
import Tinypool from 'tinypool';
import type { Config } from './index';
import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion } from './interfaces';
import { maimaiMetadataKinds } from './interfaces';
import { createLogger } from './logger';
import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion, MaimaiThumbKind } from '../interfaces';
import { maimaiMetadataKinds, maimaiThumbKinds } from '../interfaces';
import { createLogger } from '../logger';
import { basicDataTypes } from './processors/basic';
import { mergeMusic } from './processors/music';
import { arrayToObject, getOrSet, objectEntries, objectMap } from './utils/base';
import type { WorkerArguments } from './worker';
import { arrayToObject, getOrSet, objectEntries, objectMap } from '../utils/base';
const logger = createLogger('Master');
@@ -21,15 +20,23 @@ const pool = new Tinypool({
});
export type IntermediateDataMap<TIntermediateData> = Map<MaimaiRegion, Map<MaimaiMajorVersionId, TIntermediateData>>;
export type MetadataMerger<TIntermediateData, TResult> = (intermediateDataMap: IntermediateDataMap<TIntermediateData>) => TResult;
export type MetadataMerger<TIntermediateData, TResult> = (intermediateDataMap: IntermediateDataMap<TIntermediateData>, thumbCache: Record<MaimaiThumbKind, Record<number, string>>) => TResult;
export const run = async (config: Config) => {
const outputDir = path.resolve(config.outputDir);
export const runMetadata = async (inputs: Record<MaimaiRegion, Record<MaimaiMajorVersionId, string>>, outputDir: string) => {
await fs.promises.mkdir(outputDir, { recursive: true });
const thumbCacheFilePath = path.resolve(outputDir, 'thumb.json');
let thumbCache: Record<MaimaiThumbKind, Record<number, string>>;
if (await fs.promises.stat(thumbCacheFilePath).catch(() => false)) {
thumbCache = JSON.parse(await fs.promises.readFile(thumbCacheFilePath, 'utf-8')) as Record<MaimaiThumbKind, Record<number, string>>;
} else {
logger.warn('Thumb cache not found, generating metadata with empty thumb hashes');
thumbCache = arrayToObject(maimaiThumbKinds, () => ({}));
}
const index: Map<MaimaiMetadataKind, Map<MaimaiRegion, Map<MaimaiMajorVersionId, string>>> = new Map();
const tasks: Promise<void>[] = [];
for (const [regionName, versionPathMap] of objectEntries(config.inputs)) {
for (const [regionName, versionPathMap] of objectEntries(inputs)) {
const region = regionName as MaimaiRegion;
for (const [versionName, streamingAssetsPath] of objectEntries(versionPathMap)) {
const version = Number(versionName) as MaimaiMajorVersionId;
@@ -46,6 +53,7 @@ export const run = async (config: Config) => {
if (fs.existsSync(outputFilePath)) continue;
tasks.push(pool.run({
thumbCache,
region,
version,
streamingAssetsPath,
@@ -75,7 +83,7 @@ export const run = async (config: Config) => {
};
const result = objectEntries(mergers).reduce((result, [kind, merger]) => {
try {
result[kind] = merger(dataMap.get(kind)!);
result[kind] = merger(dataMap.get(kind)!, thumbCache);
return result;
} catch (error) {
logger.error(`Failed to merge ${kind} metadata`);
src/processors/basic.ts → src/metadata/processors/basic.ts
+26 -17
View File
@@ -1,20 +1,24 @@
import { omit } from 'es-toolkit';
import type { BasicMetadata, BasicMetadataBase, BasicMetadataIntermediate, MaimaiTitleMetadataExtra } from '../interfaces';
import { MaimaiTitleRareType } from '../interfaces';
import type { BasicMetadata, BasicMetadataBase, BasicMetadataIntermediate, MaimaiThumbKind, MaimaiTitleMetadataExtra } from '../../interfaces';
import { MaimaiTitleRareType } from '../../interfaces';
import { createLogger } from '../../logger';
import { forEachParallel, objectMap } from '../../utils/base';
import type { RegionalizedMap, RegionalizedNetOpenDate } from '../../utils/data';
import { maybeCompactRegionalizedMap, parseNetOpenDate } from '../../utils/data';
import { forEachRegionAndVersion } from '../../utils/each';
import { globFiles, parseXmls } from '../../utils/fs';
import { zCoerceNumber, zCoerceString, zParseEnum } from '../../utils/zod';
import type { MetadataMerger } from '../master';
import { forEachParallel, objectMap } from '../utils/base';
import type { RegionalizedMap, RegionalizedNetOpenDate } from '../utils/data';
import { maybeCompactRegionalizedMap, parseNetOpenDate } from '../utils/data';
import { forEachRegionAndVersion } from '../utils/each';
import { globFiles, parseXmls } from '../utils/fs';
import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod';
import type { WorkerProcessor } from '../worker';
const logger = createLogger('Basic');
type IntermediateData<TExtra> = Record<number, BasicMetadataIntermediate<TExtra>>;
const defineDataType = <TExtra = {}>(
globXmls: (axxxDir: string) => ReturnType<typeof globFiles>,
thumbKind: MaimaiThumbKind | undefined,
xmlRootElementName: string,
parseExtraFields: (xmlData: any) => TExtra = () => ({}) as any,
): {
@@ -28,7 +32,12 @@ const defineDataType = <TExtra = {}>(
const id = zCoerceNumber(xmlData.name.id);
const name = zCoerceString(xmlData.name.str);
const netOpenDate = parseNetOpenDate(xmlData.netOpenName.str);
result[id] = { name, netOpenDate, ...parseExtraFields!(xmlData) };
result[id] = { name, netOpenDate, ...parseExtraFields(xmlData) };
if (thumbKind) {
const thumbHash = ctx.thumbCache[thumbKind][id];
if (!thumbHash) logger.warn(`Thumb hash not found for ${thumbKind} ${id}`);
(result[id] as { thumbHash?: string }).thumbHash = thumbHash ?? '';
}
}));
return result;
},
@@ -55,14 +64,14 @@ const defineDataType = <TExtra = {}>(
});
export const basicDataTypes = {
title: defineDataType<MaimaiTitleMetadataExtra>(axxxDir => globFiles(axxxDir, 'title', 'title', 'Title.xml'), 'TitleData', TitleData => ({
title: defineDataType<MaimaiTitleMetadataExtra>(axxxDir => globFiles(axxxDir, 'title', 'title', 'Title.xml'), undefined, 'TitleData', TitleData => ({
rareType: zParseEnum(MaimaiTitleRareType, TitleData.rareType),
})),
frame: defineDataType(axxxDir => globFiles(axxxDir, 'frame', 'frame', 'Frame.xml'), 'FrameData'),
icon: defineDataType(axxxDir => globFiles(axxxDir, 'icon', 'icon', 'Icon.xml'), 'IconData'),
partner: defineDataType(axxxDir => globFiles(axxxDir, 'partner', 'partner', 'Partner.xml'), 'PartnerData'),
plate: defineDataType(axxxDir => globFiles(axxxDir, 'plate', 'plate', 'Plate.xml'), 'PlateData'),
chara: defineDataType(axxxDir => globFiles(axxxDir, 'chara', 'chara', 'Chara.xml'), 'CharaData'),
card: defineDataType(axxxDir => globFiles(axxxDir, 'card', 'card', 'Card.xml'), 'CardData'),
loginBonus: defineDataType(axxxDir => globFiles(axxxDir, 'loginBonus', 'LoginBonus', 'LoginBonus.xml'), 'LoginBonusData'),
frame: defineDataType(axxxDir => globFiles(axxxDir, 'frame', 'frame', 'Frame.xml'), 'frame', 'FrameData'),
icon: defineDataType(axxxDir => globFiles(axxxDir, 'icon', 'icon', 'Icon.xml'), 'icon', 'IconData'),
partner: defineDataType(axxxDir => globFiles(axxxDir, 'partner', 'partner', 'Partner.xml'), undefined, 'PartnerData'),
plate: defineDataType(axxxDir => globFiles(axxxDir, 'plate', 'plate', 'Plate.xml'), 'plate', 'PlateData'),
chara: defineDataType(axxxDir => globFiles(axxxDir, 'chara', 'chara', 'Chara.xml'), undefined, 'CharaData'),
card: defineDataType(axxxDir => globFiles(axxxDir, 'card', 'card', 'Card.xml'), undefined, 'CardData'),
loginBonus: defineDataType(axxxDir => globFiles(axxxDir, 'loginBonus', 'LoginBonus', 'LoginBonus.xml'), undefined, 'LoginBonusData'),
};
src/processors/music.ts → src/metadata/processors/music.ts
+12 -9
View File
@@ -1,12 +1,12 @@
import type { MaimaiChartMetadataIntermediate, MaimaiMusicMetadata, MaimaiMusicMetadataIntermediate } from '../interfaces';
import { MaimaiRegion, MaimaiMajorVersionId, maimaiMajorVersionIds, MaimaiMusicAddDeleteLogEntry } from '../interfaces';
import { createLogger } from '../logger';
import type { MaimaiChartMetadataIntermediate, MaimaiMusicMetadata, MaimaiMusicMetadataIntermediate } from '../../interfaces';
import { MaimaiRegion, MaimaiMajorVersionId, maimaiMajorVersionIds, MaimaiMusicAddDeleteLogEntry } from '../../interfaces';
import { createLogger } from '../../logger';
import { forEachParallel, objectEntries, objectKeys } from '../../utils/base';
import { parseEventIdAsNetOpenDate, parseNetOpenDate } from '../../utils/data';
import { forEachRegionAndVersion } from '../../utils/each';
import { globFiles, parseXmls } from '../../utils/fs';
import { zCoerceNumber, zCoerceString, zParseEnum } from '../../utils/zod';
import type { MetadataMerger } from '../master';
import { forEachParallel, objectEntries, objectKeys } from '../utils/base';
import { parseEventIdAsNetOpenDate, parseNetOpenDate } from '../utils/data';
import { forEachRegionAndVersion } from '../utils/each';
import { globFiles, parseXmls } from '../utils/fs';
import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod';
import type { WorkerProcessor } from '../worker';
const logger = createLogger('Music');
@@ -74,19 +74,22 @@ export const processMusic: WorkerProcessor<IntermediateData> = async ctx => {
return musics;
};
export const mergeMusic: MetadataMerger<IntermediateData, Record<number, MaimaiMusicMetadata>> = dataMap => {
export const mergeMusic: MetadataMerger<IntermediateData, Record<number, MaimaiMusicMetadata>> = (dataMap, thumbCache) => {
const result: Record<number, MaimaiMusicMetadata> = {};
// Merge all musics. JPN first. Nerwer version first.
const lowsetSeenVersionId: Record<number, MaimaiMajorVersionId> = {};
forEachRegionAndVersion(dataMap, 'jpnFirst', 'newFirst', (region, version, musics) => Object.entries(musics).forEach(([idStr, music]) => {
const id = Number(idStr);
const thumbHash = thumbCache.music[id % 10000];
if (!thumbHash) logger.warn(`Thumb hash not found for music ${id}`);
const resultMusic = result[id] ??= {
name: music.name,
artist: music.artist,
genre: music.genre,
bpm: music.bpm,
charts: music.charts,
jacketThumbHash: thumbHash ?? '',
levelChangeLog: [],
regionalInfo: {},
};
src/worker.ts → src/metadata/worker.ts
+5 -4
View File
@@ -1,15 +1,16 @@
import fs from 'node:fs';
import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId } from './interfaces';
import { createLogger } from './logger';
import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId, MaimaiThumbKind } from '../interfaces';
import { createLogger } from '../logger';
import { basicDataTypes } from './processors/basic';
import { processMusic } from './processors/music';
import { objectMap } from './utils/base';
import { globAxxxDirs } from './utils/fs';
import { objectMap } from '../utils/base';
import { globAxxxDirs } from '../utils/fs';
const logger = createLogger('Worker');
export interface WorkerArguments {
thumbCache: Record<MaimaiThumbKind, Record<number, string>>;
region: MaimaiRegion;
version: MaimaiMajorVersionId;
streamingAssetsPath: string;
src/thumb/master.ts
+54
View File
@@ -0,0 +1,54 @@
import fs from 'node:fs';
import path from 'node:path';
import { glob } from 'glob';
import Tinypool from 'tinypool';
import type { MaimaiThumbKind } from '../interfaces';
import { maimaiThumbKinds } from '../interfaces';
import { createLogger } from '../logger';
import type { WorkerArguments } from './worker';
import { objectEntries, arrayToObject } from '../utils/base';
const logger = createLogger('Master');
const pool = new Tinypool({
filename: new URL('./worker.ts', import.meta.url).href,
minThreads: 8,
maxThreads: 32,
});
const NUM_WORKERS = 32;
const ASSET_SUFFIX = '.png';
export const runThumb = async (assetsDir: string, outputDir: string) => {
await fs.promises.mkdir(path.resolve(outputDir, 'thumb'), { recursive: true });
const workerArgs: WorkerArguments[] = Array.from({ length: NUM_WORKERS }).map((_, i) => ({ tasks: [], outputFile: path.resolve(outputDir, 'thumb', `${i}.json`) }));
let i = 0;
for (const kind of maimaiThumbKinds) {
const files = await glob(`${assetsDir}/${kind}/*${ASSET_SUFFIX}`);
const fileIds = files.map(file => {
const id = parseInt(path.basename(file, ASSET_SUFFIX));
if (path.basename(file) !== `${id}${ASSET_SUFFIX}`) {
logger.warn(`Invalid file name: ${file}`);
return 0;
}
workerArgs[i++ % NUM_WORKERS]!.tasks.push({ kind, id, filePath: file });
return id;
}).filter(id => id !== 0);
logger.log(`Found ${fileIds.length} files for ${kind}`);
}
await Promise.all(workerArgs.map(args => pool.run(args)));
const result: Record<MaimaiThumbKind, Record<number, string>> = arrayToObject(maimaiThumbKinds, () => ({}));
for (const args of workerArgs) {
const data = JSON.parse(await fs.promises.readFile(args.outputFile, 'utf-8')) as Record<MaimaiThumbKind, Record<number, string>>;
for (const [kind, entries] of objectEntries(data)) {
Object.assign(result[kind], entries);
}
}
await fs.promises.writeFile(`${outputDir}/thumb.json`, JSON.stringify(result, null, 2));
logger.log(`Merged thumb hashes to ${outputDir}/thumb.json`);
};
src/thumb/utils.ts
View File
src/thumb/worker.ts
+39
View File
@@ -0,0 +1,39 @@
import fs from 'node:fs';
import { createCanvas, loadImage } from '@napi-rs/canvas';
import { rgbaToThumbHash } from 'thumbhash-node';
import type { MaimaiThumbKind } from '../interfaces';
import { maimaiThumbKinds } from '../interfaces';
import { createLogger } from '../logger';
import { arrayToObject } from '../utils/base';
const logger = createLogger('Worker');
export type WorkerArguments = {
tasks: { kind: MaimaiThumbKind; id: number; filePath: string }[];
outputFile: string;
};
const MAX_SIZE = 100;
export default async (args: WorkerArguments) => {
const result: Record<MaimaiThumbKind, Record<number, string>> = arrayToObject(maimaiThumbKinds, () => ({}));
await Promise.all(args.tasks.map(async ({ kind, id, filePath }) => {
const image = await loadImage(filePath);
const width = image.width;
const height = image.height;
const scale = Math.min(MAX_SIZE / width, MAX_SIZE / height);
const resizedWidth = Math.round(width * scale);
const resizedHeight = Math.round(height * scale);
const canvas = createCanvas(resizedWidth, resizedHeight);
const ctx = canvas.getContext('2d');
ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
const rgba = new Uint8Array(imageData.data.buffer);
const thumbhash = rgbaToThumbHash(resizedWidth, resizedHeight, rgba);
logger.log(`Generated thumbhash for ${filePath}`);
result[kind][id] = Buffer.from(thumbhash).toString('base64url');
}));
await fs.promises.writeFile(args.outputFile, JSON.stringify(result, null, 2));
};