diff --git a/package.json b/package.json index 5aab101..cd4ad31 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "version": "1.0.0", "type": "module", "scripts": { - "start": "bun src/index.ts", + "start": "bun src/index.ts metadata", + "start:thumb": "bun src/index.ts thumb", "typecheck": "tsc", "lint": "eslint --cache --ext .", "lint:fix": "eslint --cache --ext . --fix" @@ -25,12 +26,14 @@ }, "dependencies": { "@guiiai/logg": "^1.2.5", + "@napi-rs/canvas": "^0.1.82", "@types/js-yaml": "^4.0.9", "@types/node": "^24.10.0", "bun": "^1.3.2", "es-toolkit": "^1.41.0", "fast-xml-parser": "^5.3.1", "glob": "^11.0.3", + "thumbhash-node": "^0.1.3", "tinypool": "^2.0.0", "zod": "^4.1.12" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4413c39..a6de9e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@guiiai/logg': specifier: ^1.2.5 version: 1.2.5 + '@napi-rs/canvas': + specifier: ^0.1.82 + version: 0.1.82 '@types/js-yaml': specifier: ^4.0.9 version: 4.0.9 @@ -29,6 +32,9 @@ importers: glob: specifier: ^11.0.3 version: 11.0.3 + thumbhash-node: + specifier: ^0.1.3 + version: 0.1.3 tinypool: specifier: ^2.0.0 version: 2.0.0 @@ -150,6 +156,70 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@napi-rs/canvas-android-arm64@0.1.82': + resolution: {integrity: sha512-bvZhN0iI54ouaQOrgJV96H2q7J3ZoufnHf4E1fUaERwW29Rz4rgicohnAg4venwBJZYjGl5Yl3CGmlAl1LZowQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@napi-rs/canvas-darwin-arm64@0.1.82': + resolution: {integrity: sha512-InuBHKCyuFqhNwNr4gpqazo5Xp6ltKflqOLiROn4hqAS8u21xAHyYCJRgHwd+a5NKmutFTaRWeUIT/vxWbU/iw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@napi-rs/canvas-darwin-x64@0.1.82': + resolution: {integrity: sha512-aQGV5Ynn96onSXcuvYb2y7TRXD/t4CL2EGmnGqvLyeJX1JLSNisKQlWN/1bPDDXymZYSdUqbXehj5qzBlOx+RQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.82': + resolution: {integrity: sha512-YIUpmHWeHGGRhWitT1KJkgj/JPXPfc9ox8oUoyaGPxolLGPp5AxJkq8wIg8CdFGtutget968dtwmx71m8o3h5g==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@napi-rs/canvas-linux-arm64-gnu@0.1.82': + resolution: {integrity: sha512-AwLzwLBgmvk7kWeUgItOUor/QyG31xqtD26w1tLpf4yE0hiXTGp23yc669aawjB6FzgIkjh1NKaNS52B7/qEBQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/canvas-linux-arm64-musl@0.1.82': + resolution: {integrity: sha512-moZWuqepAwWBffdF4JDadt8TgBD02iMhG6I1FHZf8xO20AsIp9rB+p0B8Zma2h2vAF/YMjeFCDmW5un6+zZz9g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@napi-rs/canvas-linux-riscv64-gnu@0.1.82': + resolution: {integrity: sha512-w9++2df2kG9eC9LWYIHIlMLuhIrKGQYfUxs97CwgxYjITeFakIRazI9LYWgVzEc98QZ9x9GQvlicFsrROV59MQ==} + engines: {node: '>= 10'} + cpu: [riscv64] + os: [linux] + + '@napi-rs/canvas-linux-x64-gnu@0.1.82': + resolution: {integrity: sha512-lZulOPwrRi6hEg/17CaqdwWEUfOlIJuhXxincx1aVzsVOCmyHf+xFq4i6liJl1P+x2v6Iz2Z/H5zHvXJCC7Bwg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/canvas-linux-x64-musl@0.1.82': + resolution: {integrity: sha512-Be9Wf5RTv1w6GXlTph55K3PH3vsAh1Ax4T1FQY1UYM0QfD0yrwGdnJ8/fhqw7dEgMjd59zIbjJQC8C3msbGn5g==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@napi-rs/canvas-win32-x64-msvc@0.1.82': + resolution: {integrity: sha512-LN/i8VrvxTDmEEK1c10z2cdOTkWT76LlTGtyZe5Kr1sqoSomKeExAjbilnu1+oee5lZUgS5yfZ2LNlVhCeARuw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@napi-rs/canvas@0.1.82': + resolution: {integrity: sha512-FGjyUBoF0sl1EenSiE4UV2WYu76q6F9GSYedq5EiOCOyGYoQ/Owulcv6rd7v/tWOpljDDtefXXIaOCJrVKem4w==} + engines: {node: '>= 10'} + '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} @@ -1307,6 +1377,88 @@ packages: resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} engines: {node: '>= 0.4'} + thumbhash-node-android-arm-eabi@0.1.3: + resolution: {integrity: sha512-xiT17jBTetB+TWMhJDDAVBQPQUVvpdQm39Eg1JCq9guY6/Mzt+GgSUzAveGF1pAt7Jif/zc1Ipcm6sfOWBZwIQ==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + thumbhash-node-android-arm64@0.1.3: + resolution: {integrity: sha512-rpeWM7Z3EGnhTEI+mXOEP8cxpkqXZSPhV83T1Joa/UsZgXraQ/247rUxPVPzpEC/fwHmInvmxg5Lic+6lBFiyg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + thumbhash-node-darwin-arm64@0.1.3: + resolution: {integrity: sha512-McVwFidx7VbmkYbjpD3VR6v1kyEt+nNPnbH1KUYOeDf2nvqjEDmVoVq7wp0uru6zms17Qel4wOkaaF4xbd7M8A==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + thumbhash-node-darwin-x64@0.1.3: + resolution: {integrity: sha512-2N2oVSDr9Q6HZliuVEc48FpEdPqWxxv1DwcX0KwsE7SEduH3T7UOFvOoEhwe1ViLAhU/LM9KVbI4EpBsgCFndA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + thumbhash-node-freebsd-x64@0.1.3: + resolution: {integrity: sha512-8CXIYYMP/nMkVKFqt++YfVuatuBOQv/cpzhrOr0kilmn6IbUEIxK909eeZ8l27SxVubdfPnaORtV6aB3VbTsmQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + thumbhash-node-linux-arm-gnueabihf@0.1.3: + resolution: {integrity: sha512-0puL2Owq8HUhSK517tLhWlxOLwg2Frvgee47WYU+grIOJEWNhT5xRenM7jvtvn3ABoP/9TbDO5B81gmB1JcsWw==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + thumbhash-node-linux-arm64-gnu@0.1.3: + resolution: {integrity: sha512-lowvXaSaqwc/bghNLckMZQD+wjSZFaaK0mEJgv94QbbKMZkYKU7ELh28q1v/mSMsFLB3etsxP+10dbrmkUoXog==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + thumbhash-node-linux-arm64-musl@0.1.3: + resolution: {integrity: sha512-1k4GXQA/iXD5hPPahOW1rNXyYWnc3C/Srp1UtdrqsH8A/rXIe6bPqaCKNr30GAcIifx1PoZIpvvEqfO14hqr3Q==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + thumbhash-node-linux-x64-gnu@0.1.3: + resolution: {integrity: sha512-5gtw1mJIPGG/fpFvPCv7CmN3hkvdDCU+wHe8Mnhq1r8Z+Vu/Rizx4Uxzqqo2QBGxZLHDbR/l24R0+8Oo4rV77w==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + thumbhash-node-linux-x64-musl@0.1.3: + resolution: {integrity: sha512-2qI7KARyY6kRgSEAvAi6rxYRS9GKO2JPyc9/Q/ai29IdJSvAXgnnvGZDJ6xdv7wzRwE9sjQoY1iME7AvM1x6Bw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + thumbhash-node-win32-arm64-msvc@0.1.3: + resolution: {integrity: sha512-Q1Gh3JLL8Y/2zenQH44WitoMCi9juHgX9HXoDvtHwEzthbFpehYozMTFjZh5trttKDUeW9qWb26YYSBVXYys/g==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + thumbhash-node-win32-ia32-msvc@0.1.3: + resolution: {integrity: sha512-Wppy5xD05j7fks0gd4khE9JAXJtmmsPwvVuRHO+ftGWfVaYG44mcIZgygt6WqNSyXnyqXO4X9wuCJtoOtO8Xwg==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + thumbhash-node-win32-x64-msvc@0.1.3: + resolution: {integrity: sha512-yz6XkszxbVnICc6NWVHHExzjoUvrRkbCUeeETnVDSJJgo3XYw7Ak04Z6XPmyd700+dJp/Lgi+qumQbdAXqGZBA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + thumbhash-node@0.1.3: + resolution: {integrity: sha512-jP/j1zoNbTcYxN9wMaKU4WZYg+UIdY/hsCS8sKsiOmfK3VVmxProa3WE58rZ2yHafKEnK8wLWCqOjhdIB66SWQ==} + engines: {node: '>= 10'} + tinyglobby@0.2.15: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} @@ -1506,6 +1658,49 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@napi-rs/canvas-android-arm64@0.1.82': + optional: true + + '@napi-rs/canvas-darwin-arm64@0.1.82': + optional: true + + '@napi-rs/canvas-darwin-x64@0.1.82': + optional: true + + '@napi-rs/canvas-linux-arm-gnueabihf@0.1.82': + optional: true + + '@napi-rs/canvas-linux-arm64-gnu@0.1.82': + optional: true + + '@napi-rs/canvas-linux-arm64-musl@0.1.82': + optional: true + + '@napi-rs/canvas-linux-riscv64-gnu@0.1.82': + optional: true + + '@napi-rs/canvas-linux-x64-gnu@0.1.82': + optional: true + + '@napi-rs/canvas-linux-x64-musl@0.1.82': + optional: true + + '@napi-rs/canvas-win32-x64-msvc@0.1.82': + optional: true + + '@napi-rs/canvas@0.1.82': + optionalDependencies: + '@napi-rs/canvas-android-arm64': 0.1.82 + '@napi-rs/canvas-darwin-arm64': 0.1.82 + '@napi-rs/canvas-darwin-x64': 0.1.82 + '@napi-rs/canvas-linux-arm-gnueabihf': 0.1.82 + '@napi-rs/canvas-linux-arm64-gnu': 0.1.82 + '@napi-rs/canvas-linux-arm64-musl': 0.1.82 + '@napi-rs/canvas-linux-riscv64-gnu': 0.1.82 + '@napi-rs/canvas-linux-x64-gnu': 0.1.82 + '@napi-rs/canvas-linux-x64-musl': 0.1.82 + '@napi-rs/canvas-win32-x64-msvc': 0.1.82 + '@napi-rs/wasm-runtime@0.2.12': dependencies: '@emnapi/core': 1.7.0 @@ -2795,6 +2990,61 @@ snapshots: supports-preserve-symlinks-flag@1.0.0: {} + thumbhash-node-android-arm-eabi@0.1.3: + optional: true + + thumbhash-node-android-arm64@0.1.3: + optional: true + + thumbhash-node-darwin-arm64@0.1.3: + optional: true + + thumbhash-node-darwin-x64@0.1.3: + optional: true + + thumbhash-node-freebsd-x64@0.1.3: + optional: true + + thumbhash-node-linux-arm-gnueabihf@0.1.3: + optional: true + + thumbhash-node-linux-arm64-gnu@0.1.3: + optional: true + + thumbhash-node-linux-arm64-musl@0.1.3: + optional: true + + thumbhash-node-linux-x64-gnu@0.1.3: + optional: true + + thumbhash-node-linux-x64-musl@0.1.3: + optional: true + + thumbhash-node-win32-arm64-msvc@0.1.3: + optional: true + + thumbhash-node-win32-ia32-msvc@0.1.3: + optional: true + + thumbhash-node-win32-x64-msvc@0.1.3: + optional: true + + thumbhash-node@0.1.3: + optionalDependencies: + thumbhash-node-android-arm-eabi: 0.1.3 + thumbhash-node-android-arm64: 0.1.3 + thumbhash-node-darwin-arm64: 0.1.3 + thumbhash-node-darwin-x64: 0.1.3 + thumbhash-node-freebsd-x64: 0.1.3 + thumbhash-node-linux-arm-gnueabihf: 0.1.3 + thumbhash-node-linux-arm64-gnu: 0.1.3 + thumbhash-node-linux-arm64-musl: 0.1.3 + thumbhash-node-linux-x64-gnu: 0.1.3 + thumbhash-node-linux-x64-musl: 0.1.3 + thumbhash-node-win32-arm64-msvc: 0.1.3 + thumbhash-node-win32-ia32-msvc: 0.1.3 + thumbhash-node-win32-x64-msvc: 0.1.3 + tinyglobby@0.2.15: dependencies: fdir: 6.5.0(picomatch@4.0.3) diff --git a/src/index.ts b/src/index.ts index 4b72512..0c19523 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,8 @@ import yaml from 'js-yaml'; import { z } from 'zod'; import { MaimaiRegion, MaimaiMajorVersionId } from './interfaces/index'; -import { run } from './master'; +import { runMetadata } from './metadata/master'; +import { runThumb } from './thumb/master'; const dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -15,7 +16,8 @@ export const Config = z.object({ inputs: z.record( z.enum(MaimaiRegion), z.record(z.preprocess(Number, z.enum(MaimaiMajorVersionId)), z.string()), - ), + ).optional(), + assetsDir: z.string().optional(), outputDir: z.string(), }); export type Config = z.infer; @@ -23,4 +25,13 @@ export type Config = z.infer; const loadConfigFile = (filename: string) => yaml.load(fs.readFileSync(path.resolve(dirname, '..', filename), 'utf-8')) as Record; const config = Config.parse(merge(loadConfigFile('config.base.yaml'), loadConfigFile('config.yaml'))); -await run(config); +const command = process.argv[2]; +if (command === 'metadata') { + if (!config.inputs) throw new Error('config.inputs is required'); + await runMetadata(config.inputs, config.outputDir); +} else if (command === 'thumb') { + if (!config.assetsDir) throw new Error('config.assetsDir is required'); + await runThumb(config.assetsDir, config.outputDir); +} else { + throw new Error(`Unknown command: ${command}`); +} diff --git a/src/interfaces/index.ts b/src/interfaces/index.ts index c182c25..2eba40a 100644 --- a/src/interfaces/index.ts +++ b/src/interfaces/index.ts @@ -4,3 +4,6 @@ export * from './title'; export const maimaiMetadataKinds = ['music', 'title', 'frame', 'icon', 'partner', 'plate', 'chara', 'card', 'loginBonus'] as const; export type MaimaiMetadataKind = (typeof maimaiMetadataKinds)[number]; + +export const maimaiThumbKinds = ['music', 'frame', 'icon', 'plate'] as const; +export type MaimaiThumbKind = (typeof maimaiThumbKinds)[number]; diff --git a/src/interfaces/music.ts b/src/interfaces/music.ts index 15e3ded..4171eb6 100644 --- a/src/interfaces/music.ts +++ b/src/interfaces/music.ts @@ -45,6 +45,7 @@ export interface MaimaiMusicMetadataRegionalInfo { export type MaimaiMusicLevelChangeLog = Partial>[]; export interface MaimaiMusicMetadata extends MaimaiMusicMetadataBase { + jacketThumbHash: string; /** * The level change log per chart (BASIC, ADVANCED, EXPERT, MASTER, Re:MASTER). */ diff --git a/src/interfaces/title.ts b/src/interfaces/title.ts index e91d19b..2b41a50 100644 --- a/src/interfaces/title.ts +++ b/src/interfaces/title.ts @@ -15,3 +15,6 @@ export type BasicMetadata = MetadataMaybeRegionalized = Map>; -export type MetadataMerger = (intermediateDataMap: IntermediateDataMap) => TResult; +export type MetadataMerger = (intermediateDataMap: IntermediateDataMap, thumbCache: Record>) => TResult; -export const run = async (config: Config) => { - const outputDir = path.resolve(config.outputDir); +export const runMetadata = async (inputs: Record>, outputDir: string) => { await fs.promises.mkdir(outputDir, { recursive: true }); + const thumbCacheFilePath = path.resolve(outputDir, 'thumb.json'); + let thumbCache: Record>; + if (await fs.promises.stat(thumbCacheFilePath).catch(() => false)) { + thumbCache = JSON.parse(await fs.promises.readFile(thumbCacheFilePath, 'utf-8')) as Record>; + } else { + logger.warn('Thumb cache not found, generating metadata with empty thumb hashes'); + thumbCache = arrayToObject(maimaiThumbKinds, () => ({})); + } + const index: Map>> = new Map(); const tasks: Promise[] = []; - for (const [regionName, versionPathMap] of objectEntries(config.inputs)) { + for (const [regionName, versionPathMap] of objectEntries(inputs)) { const region = regionName as MaimaiRegion; for (const [versionName, streamingAssetsPath] of objectEntries(versionPathMap)) { const version = Number(versionName) as MaimaiMajorVersionId; @@ -46,6 +53,7 @@ export const run = async (config: Config) => { if (fs.existsSync(outputFilePath)) continue; tasks.push(pool.run({ + thumbCache, region, version, streamingAssetsPath, @@ -75,7 +83,7 @@ export const run = async (config: Config) => { }; const result = objectEntries(mergers).reduce((result, [kind, merger]) => { try { - result[kind] = merger(dataMap.get(kind)!); + result[kind] = merger(dataMap.get(kind)!, thumbCache); return result; } catch (error) { logger.error(`Failed to merge ${kind} metadata`); diff --git a/src/processors/basic.ts b/src/metadata/processors/basic.ts similarity index 68% rename from src/processors/basic.ts rename to src/metadata/processors/basic.ts index df7e966..ad3b361 100644 --- a/src/processors/basic.ts +++ b/src/metadata/processors/basic.ts @@ -1,20 +1,24 @@ import { omit } from 'es-toolkit'; -import type { BasicMetadata, BasicMetadataBase, BasicMetadataIntermediate, MaimaiTitleMetadataExtra } from '../interfaces'; -import { MaimaiTitleRareType } from '../interfaces'; +import type { BasicMetadata, BasicMetadataBase, BasicMetadataIntermediate, MaimaiThumbKind, MaimaiTitleMetadataExtra } from '../../interfaces'; +import { MaimaiTitleRareType } from '../../interfaces'; +import { createLogger } from '../../logger'; +import { forEachParallel, objectMap } from '../../utils/base'; +import type { RegionalizedMap, RegionalizedNetOpenDate } from '../../utils/data'; +import { maybeCompactRegionalizedMap, parseNetOpenDate } from '../../utils/data'; +import { forEachRegionAndVersion } from '../../utils/each'; +import { globFiles, parseXmls } from '../../utils/fs'; +import { zCoerceNumber, zCoerceString, zParseEnum } from '../../utils/zod'; import type { MetadataMerger } from '../master'; -import { forEachParallel, objectMap } from '../utils/base'; -import type { RegionalizedMap, RegionalizedNetOpenDate } from '../utils/data'; -import { maybeCompactRegionalizedMap, parseNetOpenDate } from '../utils/data'; -import { forEachRegionAndVersion } from '../utils/each'; -import { globFiles, parseXmls } from '../utils/fs'; -import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod'; import type { WorkerProcessor } from '../worker'; +const logger = createLogger('Basic'); + type IntermediateData = Record>; const defineDataType = ( globXmls: (axxxDir: string) => ReturnType, + thumbKind: MaimaiThumbKind | undefined, xmlRootElementName: string, parseExtraFields: (xmlData: any) => TExtra = () => ({}) as any, ): { @@ -28,7 +32,12 @@ const defineDataType = ( const id = zCoerceNumber(xmlData.name.id); const name = zCoerceString(xmlData.name.str); const netOpenDate = parseNetOpenDate(xmlData.netOpenName.str); - result[id] = { name, netOpenDate, ...parseExtraFields!(xmlData) }; + result[id] = { name, netOpenDate, ...parseExtraFields(xmlData) }; + if (thumbKind) { + const thumbHash = ctx.thumbCache[thumbKind][id]; + if (!thumbHash) logger.warn(`Thumb hash not found for ${thumbKind} ${id}`); + (result[id] as { thumbHash?: string }).thumbHash = thumbHash ?? ''; + } })); return result; }, @@ -55,14 +64,14 @@ const defineDataType = ( }); export const basicDataTypes = { - title: defineDataType(axxxDir => globFiles(axxxDir, 'title', 'title', 'Title.xml'), 'TitleData', TitleData => ({ + title: defineDataType(axxxDir => globFiles(axxxDir, 'title', 'title', 'Title.xml'), undefined, 'TitleData', TitleData => ({ rareType: zParseEnum(MaimaiTitleRareType, TitleData.rareType), })), - frame: defineDataType(axxxDir => globFiles(axxxDir, 'frame', 'frame', 'Frame.xml'), 'FrameData'), - icon: defineDataType(axxxDir => globFiles(axxxDir, 'icon', 'icon', 'Icon.xml'), 'IconData'), - partner: defineDataType(axxxDir => globFiles(axxxDir, 'partner', 'partner', 'Partner.xml'), 'PartnerData'), - plate: defineDataType(axxxDir => globFiles(axxxDir, 'plate', 'plate', 'Plate.xml'), 'PlateData'), - chara: defineDataType(axxxDir => globFiles(axxxDir, 'chara', 'chara', 'Chara.xml'), 'CharaData'), - card: defineDataType(axxxDir => globFiles(axxxDir, 'card', 'card', 'Card.xml'), 'CardData'), - loginBonus: defineDataType(axxxDir => globFiles(axxxDir, 'loginBonus', 'LoginBonus', 'LoginBonus.xml'), 'LoginBonusData'), + frame: defineDataType(axxxDir => globFiles(axxxDir, 'frame', 'frame', 'Frame.xml'), 'frame', 'FrameData'), + icon: defineDataType(axxxDir => globFiles(axxxDir, 'icon', 'icon', 'Icon.xml'), 'icon', 'IconData'), + partner: defineDataType(axxxDir => globFiles(axxxDir, 'partner', 'partner', 'Partner.xml'), undefined, 'PartnerData'), + plate: defineDataType(axxxDir => globFiles(axxxDir, 'plate', 'plate', 'Plate.xml'), 'plate', 'PlateData'), + chara: defineDataType(axxxDir => globFiles(axxxDir, 'chara', 'chara', 'Chara.xml'), undefined, 'CharaData'), + card: defineDataType(axxxDir => globFiles(axxxDir, 'card', 'card', 'Card.xml'), undefined, 'CardData'), + loginBonus: defineDataType(axxxDir => globFiles(axxxDir, 'loginBonus', 'LoginBonus', 'LoginBonus.xml'), undefined, 'LoginBonusData'), }; diff --git a/src/processors/music.ts b/src/metadata/processors/music.ts similarity index 94% rename from src/processors/music.ts rename to src/metadata/processors/music.ts index 51aff86..a9be616 100644 --- a/src/processors/music.ts +++ b/src/metadata/processors/music.ts @@ -1,12 +1,12 @@ -import type { MaimaiChartMetadataIntermediate, MaimaiMusicMetadata, MaimaiMusicMetadataIntermediate } from '../interfaces'; -import { MaimaiRegion, MaimaiMajorVersionId, maimaiMajorVersionIds, MaimaiMusicAddDeleteLogEntry } from '../interfaces'; -import { createLogger } from '../logger'; +import type { MaimaiChartMetadataIntermediate, MaimaiMusicMetadata, MaimaiMusicMetadataIntermediate } from '../../interfaces'; +import { MaimaiRegion, MaimaiMajorVersionId, maimaiMajorVersionIds, MaimaiMusicAddDeleteLogEntry } from '../../interfaces'; +import { createLogger } from '../../logger'; +import { forEachParallel, objectEntries, objectKeys } from '../../utils/base'; +import { parseEventIdAsNetOpenDate, parseNetOpenDate } from '../../utils/data'; +import { forEachRegionAndVersion } from '../../utils/each'; +import { globFiles, parseXmls } from '../../utils/fs'; +import { zCoerceNumber, zCoerceString, zParseEnum } from '../../utils/zod'; import type { MetadataMerger } from '../master'; -import { forEachParallel, objectEntries, objectKeys } from '../utils/base'; -import { parseEventIdAsNetOpenDate, parseNetOpenDate } from '../utils/data'; -import { forEachRegionAndVersion } from '../utils/each'; -import { globFiles, parseXmls } from '../utils/fs'; -import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod'; import type { WorkerProcessor } from '../worker'; const logger = createLogger('Music'); @@ -74,19 +74,22 @@ export const processMusic: WorkerProcessor = async ctx => { return musics; }; -export const mergeMusic: MetadataMerger> = dataMap => { +export const mergeMusic: MetadataMerger> = (dataMap, thumbCache) => { const result: Record = {}; // Merge all musics. JPN first. Nerwer version first. const lowsetSeenVersionId: Record = {}; forEachRegionAndVersion(dataMap, 'jpnFirst', 'newFirst', (region, version, musics) => Object.entries(musics).forEach(([idStr, music]) => { const id = Number(idStr); + const thumbHash = thumbCache.music[id % 10000]; + if (!thumbHash) logger.warn(`Thumb hash not found for music ${id}`); const resultMusic = result[id] ??= { name: music.name, artist: music.artist, genre: music.genre, bpm: music.bpm, charts: music.charts, + jacketThumbHash: thumbHash ?? '', levelChangeLog: [], regionalInfo: {}, }; diff --git a/src/worker.ts b/src/metadata/worker.ts similarity index 88% rename from src/worker.ts rename to src/metadata/worker.ts index f4016c0..24f1cd8 100644 --- a/src/worker.ts +++ b/src/metadata/worker.ts @@ -1,15 +1,16 @@ import fs from 'node:fs'; -import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId } from './interfaces'; -import { createLogger } from './logger'; +import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId, MaimaiThumbKind } from '../interfaces'; +import { createLogger } from '../logger'; import { basicDataTypes } from './processors/basic'; import { processMusic } from './processors/music'; -import { objectMap } from './utils/base'; -import { globAxxxDirs } from './utils/fs'; +import { objectMap } from '../utils/base'; +import { globAxxxDirs } from '../utils/fs'; const logger = createLogger('Worker'); export interface WorkerArguments { + thumbCache: Record>; region: MaimaiRegion; version: MaimaiMajorVersionId; streamingAssetsPath: string; diff --git a/src/thumb/master.ts b/src/thumb/master.ts new file mode 100644 index 0000000..406d5fd --- /dev/null +++ b/src/thumb/master.ts @@ -0,0 +1,54 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { glob } from 'glob'; +import Tinypool from 'tinypool'; + +import type { MaimaiThumbKind } from '../interfaces'; +import { maimaiThumbKinds } from '../interfaces'; +import { createLogger } from '../logger'; +import type { WorkerArguments } from './worker'; +import { objectEntries, arrayToObject } from '../utils/base'; + +const logger = createLogger('Master'); + +const pool = new Tinypool({ + filename: new URL('./worker.ts', import.meta.url).href, + minThreads: 8, + maxThreads: 32, +}); + +const NUM_WORKERS = 32; +const ASSET_SUFFIX = '.png'; + +export const runThumb = async (assetsDir: string, outputDir: string) => { + await fs.promises.mkdir(path.resolve(outputDir, 'thumb'), { recursive: true }); + + const workerArgs: WorkerArguments[] = Array.from({ length: NUM_WORKERS }).map((_, i) => ({ tasks: [], outputFile: path.resolve(outputDir, 'thumb', `${i}.json`) })); + let i = 0; + for (const kind of maimaiThumbKinds) { + const files = await glob(`${assetsDir}/${kind}/*${ASSET_SUFFIX}`); + const fileIds = files.map(file => { + const id = parseInt(path.basename(file, ASSET_SUFFIX)); + if (path.basename(file) !== `${id}${ASSET_SUFFIX}`) { + logger.warn(`Invalid file name: ${file}`); + return 0; + } + workerArgs[i++ % NUM_WORKERS]!.tasks.push({ kind, id, filePath: file }); + return id; + }).filter(id => id !== 0); + logger.log(`Found ${fileIds.length} files for ${kind}`); + } + + await Promise.all(workerArgs.map(args => pool.run(args))); + + const result: Record> = arrayToObject(maimaiThumbKinds, () => ({})); + for (const args of workerArgs) { + const data = JSON.parse(await fs.promises.readFile(args.outputFile, 'utf-8')) as Record>; + for (const [kind, entries] of objectEntries(data)) { + Object.assign(result[kind], entries); + } + } + await fs.promises.writeFile(`${outputDir}/thumb.json`, JSON.stringify(result, null, 2)); + logger.log(`Merged thumb hashes to ${outputDir}/thumb.json`); +}; diff --git a/src/thumb/utils.ts b/src/thumb/utils.ts new file mode 100644 index 0000000..e69de29 diff --git a/src/thumb/worker.ts b/src/thumb/worker.ts new file mode 100644 index 0000000..d30603d --- /dev/null +++ b/src/thumb/worker.ts @@ -0,0 +1,39 @@ +import fs from 'node:fs'; + +import { createCanvas, loadImage } from '@napi-rs/canvas'; +import { rgbaToThumbHash } from 'thumbhash-node'; + +import type { MaimaiThumbKind } from '../interfaces'; +import { maimaiThumbKinds } from '../interfaces'; +import { createLogger } from '../logger'; +import { arrayToObject } from '../utils/base'; + +const logger = createLogger('Worker'); + +export type WorkerArguments = { + tasks: { kind: MaimaiThumbKind; id: number; filePath: string }[]; + outputFile: string; +}; + +const MAX_SIZE = 100; + +export default async (args: WorkerArguments) => { + const result: Record> = arrayToObject(maimaiThumbKinds, () => ({})); + await Promise.all(args.tasks.map(async ({ kind, id, filePath }) => { + const image = await loadImage(filePath); + const width = image.width; + const height = image.height; + const scale = Math.min(MAX_SIZE / width, MAX_SIZE / height); + const resizedWidth = Math.round(width * scale); + const resizedHeight = Math.round(height * scale); + const canvas = createCanvas(resizedWidth, resizedHeight); + const ctx = canvas.getContext('2d'); + ctx.drawImage(image, 0, 0, canvas.width, canvas.height); + const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); + const rgba = new Uint8Array(imageData.data.buffer); + const thumbhash = rgbaToThumbHash(resizedWidth, resizedHeight, rgba); + logger.log(`Generated thumbhash for ${filePath}`); + result[kind][id] = Buffer.from(thumbhash).toString('base64url'); + })); + await fs.promises.writeFile(args.outputFile, JSON.stringify(result, null, 2)); +};