feat: initial commit
This commit is contained in:
Menci
+143
@@ -0,0 +1,143 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
|
||||
|
||||
# Runtime data
|
||||
pids
|
||||
*.pid
|
||||
*.seed
|
||||
*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
.stylelintcache
|
||||
|
||||
# Optional REPL history
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variable files
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
.cache
|
||||
.parcel-cache
|
||||
|
||||
# Next.js build output
|
||||
.next
|
||||
out
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
.nuxt
|
||||
dist
|
||||
.output
|
||||
|
||||
# Gatsby files
|
||||
.cache/
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
.temp
|
||||
.cache
|
||||
|
||||
# Sveltekit cache directory
|
||||
.svelte-kit/
|
||||
|
||||
# vitepress build output
|
||||
**/.vitepress/dist
|
||||
|
||||
# vitepress cache directory
|
||||
**/.vitepress/cache
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
.dynamodb/
|
||||
|
||||
# Firebase cache directory
|
||||
.firebase/
|
||||
|
||||
# TernJS port file
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
.vscode-test
|
||||
|
||||
# yarn v3
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
!.yarn/patches
|
||||
!.yarn/plugins
|
||||
!.yarn/releases
|
||||
!.yarn/sdks
|
||||
!.yarn/versions
|
||||
|
||||
# Vite files
|
||||
vite.config.js.timestamp-*
|
||||
vite.config.ts.timestamp-*
|
||||
.vite/
|
||||
|
||||
/config.yaml
|
||||
@@ -0,0 +1,2 @@
|
||||
inputs: {}
|
||||
outputDir: output
|
||||
@@ -0,0 +1,3 @@
|
||||
inputs:
|
||||
JPN:
|
||||
22: D:\maimai\BUDDiES_PLUS
|
||||
@@ -0,0 +1,153 @@
|
||||
import tsParser from '@typescript-eslint/parser';
|
||||
import tsPlugin from '@typescript-eslint/eslint-plugin';
|
||||
import importPlugin from 'eslint-plugin-import';
|
||||
import stylisticPlugin from '@stylistic/eslint-plugin';
|
||||
|
||||
import type { Linter } from 'eslint';
|
||||
|
||||
const commonConfig: Linter.Config = {
|
||||
plugins: {
|
||||
import: importPlugin,
|
||||
'@typescript-eslint': tsPlugin as any,
|
||||
stylistic: stylisticPlugin,
|
||||
},
|
||||
rules: {
|
||||
'import/order': [
|
||||
'error',
|
||||
{
|
||||
groups: ['builtin', 'external', ['internal', 'parent', 'sibling', 'index']],
|
||||
pathGroups: [
|
||||
{
|
||||
pattern: '@proj-marina/**',
|
||||
group: 'internal',
|
||||
position: 'before',
|
||||
},
|
||||
{
|
||||
pattern: '@/**',
|
||||
group: 'internal',
|
||||
position: 'before',
|
||||
},
|
||||
],
|
||||
'newlines-between': 'always',
|
||||
distinctGroup: false,
|
||||
alphabetize: {
|
||||
order: 'asc',
|
||||
caseInsensitive: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
'import/no-duplicates': 'error',
|
||||
|
||||
'@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
|
||||
'prefer-const': 'error',
|
||||
'no-var': 'error',
|
||||
'no-debugger': 'error',
|
||||
'object-shorthand': 'error',
|
||||
'prefer-template': 'error',
|
||||
eqeqeq: ['error', 'always', { null: 'ignore' }],
|
||||
|
||||
'@typescript-eslint/prefer-optional-chain': 'error',
|
||||
'@typescript-eslint/prefer-nullish-coalescing': 'error',
|
||||
'@typescript-eslint/return-await': ['error', 'always'],
|
||||
'@typescript-eslint/no-floating-promises': 'error',
|
||||
'@typescript-eslint/await-thenable': 'error',
|
||||
'@typescript-eslint/no-misused-promises': ['error'],
|
||||
'@typescript-eslint/prefer-as-const': 'error',
|
||||
'@typescript-eslint/prefer-for-of': 'error',
|
||||
'@typescript-eslint/prefer-includes': 'error',
|
||||
'@typescript-eslint/prefer-string-starts-ends-with': 'error',
|
||||
'@typescript-eslint/consistent-type-imports': ['error', { disallowTypeAnnotations: false }],
|
||||
|
||||
'stylistic/indent': ['error', 2, {
|
||||
'offsetTernaryExpressions': true
|
||||
}],
|
||||
'stylistic/linebreak-style': ['error', 'unix'],
|
||||
'stylistic/semi': ['error', 'always'],
|
||||
'stylistic/quotes': ['error', 'single', {
|
||||
'avoidEscape': true,
|
||||
'allowTemplateLiterals': 'avoidEscape',
|
||||
}],
|
||||
'stylistic/comma-dangle': ['error', 'always-multiline'],
|
||||
'stylistic/arrow-parens': ['error', 'as-needed'],
|
||||
'stylistic/object-curly-spacing': ['error', 'always'],
|
||||
'stylistic/array-bracket-spacing': ['error', 'never'],
|
||||
'stylistic/space-before-function-paren': ['error', {
|
||||
'anonymous': 'always',
|
||||
'named': 'never',
|
||||
'asyncArrow': 'always',
|
||||
}],
|
||||
'stylistic/space-in-parens': ['error', 'never'],
|
||||
'stylistic/comma-spacing': ['error', { 'before': false, 'after': true }],
|
||||
'stylistic/key-spacing': ['error', { 'beforeColon': false, 'afterColon': true }],
|
||||
'stylistic/keyword-spacing': ['error'],
|
||||
'stylistic/space-before-blocks': ['error', 'always'],
|
||||
'stylistic/space-infix-ops': ['error'],
|
||||
'stylistic/no-trailing-spaces': ['error'],
|
||||
'stylistic/eol-last': ['error', 'always'],
|
||||
'stylistic/no-multiple-empty-lines': ['error', { 'max': 1, 'maxEOF': 0 }],
|
||||
'stylistic/brace-style': ['error', '1tbs', { 'allowSingleLine': true }],
|
||||
'stylistic/object-curly-newline': ['error', {
|
||||
'ObjectExpression': { 'multiline': true, 'consistent': true },
|
||||
'ObjectPattern': { 'multiline': true, 'consistent': true },
|
||||
'ImportDeclaration': { 'multiline': true, 'consistent': true },
|
||||
'ExportDeclaration': { 'multiline': true, 'consistent': true }
|
||||
}],
|
||||
'stylistic/array-bracket-newline': ['error', 'consistent'],
|
||||
'stylistic/function-paren-newline': ['error', 'consistent'],
|
||||
'stylistic/member-delimiter-style': ['error', {
|
||||
'multiline': {
|
||||
'delimiter': 'semi',
|
||||
'requireLast': true
|
||||
},
|
||||
'singleline': {
|
||||
'delimiter': 'semi',
|
||||
'requireLast': false
|
||||
}
|
||||
}],
|
||||
'stylistic/type-annotation-spacing': ['error'],
|
||||
'stylistic/jsx-quotes': ['error', 'prefer-double'],
|
||||
|
||||
},
|
||||
settings: {
|
||||
'import/internal-regex': '^@proj-marina/',
|
||||
'import/resolver': {
|
||||
typescript: {
|
||||
project: ['./tsconfig.json'],
|
||||
noWarnOnMultipleProjects: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const parserOptions: Linter.ParserOptions = {
|
||||
parser: tsParser,
|
||||
ecmaVersion: 'latest',
|
||||
sourceType: 'module',
|
||||
project: ['./tsconfig.json'],
|
||||
noWarnOnMultipleProjects: true,
|
||||
};
|
||||
|
||||
const config: Linter.Config[] = [
|
||||
{
|
||||
...commonConfig,
|
||||
files: ['**/*.{ts,tsx}'],
|
||||
languageOptions: {
|
||||
parser: tsParser,
|
||||
ecmaVersion: 'latest',
|
||||
sourceType: 'module',
|
||||
parserOptions,
|
||||
},
|
||||
},
|
||||
{
|
||||
ignores: [
|
||||
'**/node_modules/**',
|
||||
// Build output
|
||||
'**/dist/**',
|
||||
'**/build/**',
|
||||
'**/coverage/**',
|
||||
'eslint.config.ts',
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
export default config;
|
||||
@@ -0,0 +1,2 @@
|
||||
*
|
||||
!.gitignore
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"name": "@maigolabs/kairos",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "bun src/index.ts",
|
||||
"typecheck": "tsc",
|
||||
"lint": "eslint --cache --ext .",
|
||||
"lint:fix": "eslint --cache --ext . --fix"
|
||||
},
|
||||
"license": "UNLICENSED",
|
||||
"packageManager": "pnpm@10.20.0",
|
||||
"private": true,
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.39.1",
|
||||
"@stylistic/eslint-plugin": "^5.5.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.46.3",
|
||||
"@typescript-eslint/parser": "^8.46.3",
|
||||
"cross-env": "^10.1.0",
|
||||
"eslint": "^9.39.1",
|
||||
"eslint-import-resolver-typescript": "^4.4.4",
|
||||
"eslint-plugin-import": "^2.32.0",
|
||||
"jiti": "^2.6.1",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@guiiai/logg": "^1.2.5",
|
||||
"@types/js-yaml": "^4.0.9",
|
||||
"@types/node": "^24.10.0",
|
||||
"bun": "^1.3.2",
|
||||
"es-toolkit": "^1.41.0",
|
||||
"fast-xml-parser": "^5.3.1",
|
||||
"glob": "^11.0.3",
|
||||
"tinypool": "^2.0.0",
|
||||
"zod": "^4.1.12"
|
||||
}
|
||||
}
|
||||
Generated
+2960
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
onlyBuiltDependencies:
|
||||
- bun
|
||||
- unrs-resolver
|
||||
@@ -0,0 +1,26 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { merge } from 'es-toolkit';
|
||||
import yaml from 'js-yaml';
|
||||
import { z } from 'zod';
|
||||
|
||||
import { MaimaiRegion, MaimaiMajorVersionId } from './interfaces/index';
|
||||
import { run } from './master';
|
||||
|
||||
const dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export const Config = z.object({
|
||||
inputs: z.record(
|
||||
z.enum(MaimaiRegion),
|
||||
z.record(z.preprocess(Number, z.enum(MaimaiMajorVersionId)), z.string()),
|
||||
),
|
||||
outputDir: z.string(),
|
||||
});
|
||||
export type Config = z.infer<typeof Config>;
|
||||
|
||||
const loadConfigFile = (filename: string) => yaml.load(fs.readFileSync(path.resolve(dirname, '..', filename), 'utf-8')) as Record<string, unknown>;
|
||||
const config = Config.parse(merge(loadConfigFile('config.base.yaml'), loadConfigFile('config.yaml')));
|
||||
|
||||
await run(config);
|
||||
@@ -0,0 +1,71 @@
|
||||
export enum MaimaiRegion {
|
||||
JPN = 'JPN',
|
||||
EXP = 'EXP',
|
||||
CHN = 'CHN',
|
||||
}
|
||||
|
||||
export function maimaiRegionFromGameId(gameId: string) {
|
||||
switch (gameId) {
|
||||
case '\x53\x44\x45\x5a':
|
||||
return MaimaiRegion.JPN;
|
||||
case '\x53\x44\x47\x41':
|
||||
return MaimaiRegion.EXP;
|
||||
case '\x53\x44\x47\x42':
|
||||
return MaimaiRegion.CHN;
|
||||
default: // Fallback to JPN
|
||||
return MaimaiRegion.JPN;
|
||||
}
|
||||
}
|
||||
|
||||
export enum MaimaiMajorVersionId {
|
||||
maimai = 0,
|
||||
maimai_PLUS = 1,
|
||||
GreeN = 2,
|
||||
GreeN_PLUS = 3,
|
||||
ORANGE = 4,
|
||||
ORANGE_PLUS = 5,
|
||||
PiNK = 6,
|
||||
PiNK_PLUS = 7,
|
||||
MURASAKi = 8,
|
||||
MURASAKi_PLUS = 9,
|
||||
MiLK = 10,
|
||||
MiLK_PLUS = 11,
|
||||
FiNALE = 12,
|
||||
DX = 13,
|
||||
DX_PLUS = 14,
|
||||
Splash = 15,
|
||||
Splash_PLUS = 16,
|
||||
UNiVERSE = 17,
|
||||
UNiVERSE_PLUS = 18,
|
||||
FESTiVAL = 19,
|
||||
FESTiVAL_PLUS = 20,
|
||||
BUDDiES = 21,
|
||||
BUDDiES_PLUS = 22,
|
||||
PRiSM = 23,
|
||||
PRiSM_PLUS = 24,
|
||||
CiRCLE = 25,
|
||||
}
|
||||
|
||||
export const maimaiMajorVersionIds = Object.values(MaimaiMajorVersionId).filter(
|
||||
v => typeof v === 'number',
|
||||
) as MaimaiMajorVersionId[];
|
||||
|
||||
export function maimaiVersionIdFromVersionString(gameVersion: string) {
|
||||
if (!gameVersion) {
|
||||
return maimaiMajorVersionIds[maimaiMajorVersionIds.length - 1];
|
||||
}
|
||||
const v = Number(gameVersion.split('.')[1]);
|
||||
const i = Math.floor(v / 5);
|
||||
const result = MaimaiMajorVersionId.DX + i;
|
||||
return maimaiMajorVersionIds.includes(result)
|
||||
? result
|
||||
: /* Fallback to the latest version */ maimaiMajorVersionIds[maimaiMajorVersionIds.length - 1];
|
||||
}
|
||||
|
||||
export type MetadataUnversioned<T> = { unversioned: T };
|
||||
export type MetadataVersioned<T> = { versioned: Partial<Record<MaimaiMajorVersionId, T>> };
|
||||
export type MetadataMaybeVersioned<T> = MetadataUnversioned<T> | MetadataVersioned<T>;
|
||||
|
||||
export type MetadataUnregionalized<T> = { unregionalized: MetadataMaybeVersioned<T> };
|
||||
export type MetadataRegionalized<T> = { regionalized: Partial<Record<MaimaiRegion, MetadataMaybeVersioned<T>>> };
|
||||
export type MetadataMaybeRegionalized<T> = MetadataUnregionalized<T> | MetadataRegionalized<T>;
|
||||
@@ -0,0 +1,6 @@
|
||||
export * from './base';
|
||||
export * from './music';
|
||||
export * from './title';
|
||||
|
||||
export const maimaiMetadataKinds = ['music', 'title', 'frame', 'icon', 'partner', 'plate', 'chara', 'card', 'loginBonus'] as const;
|
||||
export type MaimaiMetadataKind = (typeof maimaiMetadataKinds)[number];
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { MaimaiMajorVersionId, MaimaiRegion } from './base';
|
||||
|
||||
export type MaimaiChartMetadata = {
|
||||
// `level` is not included here and should be inferred from the change log.
|
||||
designer: string;
|
||||
// TODO: how many of TAP/HOLD/SLIDE/BREAK notes?
|
||||
};
|
||||
export type MaimaiChartMetadataIntermediate = MaimaiChartMetadata & {
|
||||
level: number;
|
||||
};
|
||||
|
||||
export type MaimaiMusicMetadataBase = {
|
||||
name: string;
|
||||
artist: string;
|
||||
genre: string;
|
||||
bpm: number;
|
||||
charts: MaimaiChartMetadata[];
|
||||
};
|
||||
|
||||
export type MaimaiMusicMetadataIntermediate = MaimaiMusicMetadataBase & {
|
||||
chartsWithLevel: MaimaiChartMetadataIntermediate[];
|
||||
versionId: MaimaiMajorVersionId;
|
||||
deletedInPatch: boolean;
|
||||
netOpenDate: string | null;
|
||||
};
|
||||
|
||||
export enum MaimaiMusicAddDeleteLogEntry {
|
||||
Added = 1,
|
||||
AddedReMaster = 2,
|
||||
DeletedFromPackage = 3,
|
||||
DeletedInPatch = 4,
|
||||
}
|
||||
export type MaimaiMusicAddDeleteLog = Partial<Record<MaimaiMajorVersionId, MaimaiMusicAddDeleteLogEntry>>;
|
||||
export interface MaimaiMusicMetadataRegionalInfo {
|
||||
/**
|
||||
* For a music deleted from package, it don't have the level value in that version.
|
||||
* For a music deleted in patch, it has the level value in that version.
|
||||
*/
|
||||
addDeleteLog: MaimaiMusicAddDeleteLog;
|
||||
netOpenDate: string | null;
|
||||
}
|
||||
|
||||
export type MaimaiMusicLevelChangeLog = Partial<Record<MaimaiMajorVersionId, number>>[];
|
||||
export interface MaimaiMusicMetadata extends MaimaiMusicMetadataBase {
|
||||
/**
|
||||
* The level change log per chart (BASIC, ADVANCED, EXPERT, MASTER, Re:MASTER).
|
||||
*/
|
||||
levelChangeLog: MaimaiMusicLevelChangeLog; // This is observed to be identical for all regions.
|
||||
regionalInfo: Partial<Record<MaimaiRegion, MaimaiMusicMetadataRegionalInfo>>;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { MaimaiRegion, MetadataMaybeRegionalized } from './base';
|
||||
|
||||
export enum MaimaiTitleRareType {
|
||||
Normal = 'Normal',
|
||||
Bronze = 'Bronze',
|
||||
Silver = 'Silver',
|
||||
Gold = 'Gold',
|
||||
Rainbow = 'Rainbow',
|
||||
}
|
||||
|
||||
export type BasicMetadataBase<TExtra> = TExtra & { name: string };
|
||||
export type BasicMetadataIntermediate<TExtra> = BasicMetadataBase<TExtra> & { netOpenDate: string | null };
|
||||
export type BasicMetadata<TExtra = {}> = MetadataMaybeRegionalized<BasicMetadataBase<TExtra>> & {
|
||||
regionalNetOpenDate: Partial<Record<MaimaiRegion, string | null>>;
|
||||
};
|
||||
|
||||
export type MaimaiTitleMetadataExtra = { rareType: MaimaiTitleRareType };
|
||||
@@ -0,0 +1,5 @@
|
||||
import { initLogger, useGlobalLogger } from '@guiiai/logg';
|
||||
|
||||
initLogger();
|
||||
|
||||
export const createLogger = (name: string) => useGlobalLogger(name);
|
||||
@@ -0,0 +1,89 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import Tinypool from 'tinypool';
|
||||
|
||||
import type { Config } from './index';
|
||||
import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion } from './interfaces';
|
||||
import { maimaiMetadataKinds } from './interfaces';
|
||||
import { createLogger } from './logger';
|
||||
import { basicDataTypes } from './processors/basic';
|
||||
import { mergeMusic } from './processors/music';
|
||||
import { arrayToObject, getOrSet, objectEntries, objectMap } from './utils/base';
|
||||
import type { WorkerArguments } from './worker';
|
||||
|
||||
const logger = createLogger('Master');
|
||||
|
||||
const pool = new Tinypool({
|
||||
filename: new URL('./worker.ts', import.meta.url).href,
|
||||
minThreads: 8,
|
||||
maxThreads: 32,
|
||||
});
|
||||
|
||||
export type IntermediateDataMap<TIntermediateData> = Map<MaimaiRegion, Map<MaimaiMajorVersionId, TIntermediateData>>;
|
||||
export type MetadataMerger<TIntermediateData, TResult> = (intermediateDataMap: IntermediateDataMap<TIntermediateData>) => TResult;
|
||||
|
||||
export const run = async (config: Config) => {
|
||||
const outputDir = path.resolve(config.outputDir);
|
||||
await fs.promises.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const index: Map<MaimaiMetadataKind, Map<MaimaiRegion, Map<MaimaiMajorVersionId, string>>> = new Map();
|
||||
const tasks: Promise<void>[] = [];
|
||||
for (const [regionName, versionPathMap] of objectEntries(config.inputs)) {
|
||||
const region = regionName as MaimaiRegion;
|
||||
for (const [versionName, streamingAssetsPath] of objectEntries(versionPathMap)) {
|
||||
const version = Number(versionName) as MaimaiMajorVersionId;
|
||||
const versionOutputDir = path.resolve(outputDir, 'intermediate', `${region}-${version}`);
|
||||
await fs.promises.mkdir(versionOutputDir, { recursive: true });
|
||||
|
||||
const filePaths = arrayToObject(maimaiMetadataKinds, kind => path.resolve(versionOutputDir, `${kind}.json`));
|
||||
|
||||
for (const [metadataKind, outputFilePath] of objectEntries(filePaths)) {
|
||||
const regionIndex = getOrSet(index, metadataKind as MaimaiMetadataKind, new Map());
|
||||
const versionIndex = getOrSet(regionIndex, region, new Map());
|
||||
versionIndex.set(version, outputFilePath);
|
||||
|
||||
if (fs.existsSync(outputFilePath)) continue;
|
||||
|
||||
tasks.push(pool.run({
|
||||
region,
|
||||
version,
|
||||
streamingAssetsPath,
|
||||
metadataKind,
|
||||
outputFilePath,
|
||||
} satisfies WorkerArguments));
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(tasks);
|
||||
|
||||
const dataMap: Map<MaimaiMetadataKind, IntermediateDataMap<unknown>> = new Map();
|
||||
for (const [kind, regionMergeIndex] of index.entries()) {
|
||||
const regionMap = getOrSet(dataMap, kind, new Map());
|
||||
for (const [region, versionMergeIndex] of regionMergeIndex.entries()) {
|
||||
const versionMap = getOrSet(regionMap, region, new Map());
|
||||
for (const [version, outputFilePath] of versionMergeIndex.entries()) {
|
||||
const data = JSON.parse(await fs.promises.readFile(outputFilePath, 'utf-8'));
|
||||
versionMap.set(version, data);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mergers: Record<MaimaiMetadataKind, MetadataMerger<any, any>> = {
|
||||
...objectMap(basicDataTypes, dataType => dataType.merge),
|
||||
music: mergeMusic,
|
||||
};
|
||||
const result = objectEntries(mergers).reduce((result, [kind, merger]) => {
|
||||
try {
|
||||
result[kind] = merger(dataMap.get(kind)!);
|
||||
return result;
|
||||
} catch (error) {
|
||||
logger.error(`Failed to merge ${kind} metadata`);
|
||||
throw error;
|
||||
}
|
||||
}, {} as Record<MaimaiMetadataKind, unknown>);
|
||||
|
||||
const mergedFilePath = path.resolve(outputDir, 'merged.json');
|
||||
await fs.promises.writeFile(mergedFilePath, JSON.stringify(result, null, 2));
|
||||
logger.log(`Merged metadata to ${mergedFilePath}`);
|
||||
};
|
||||
@@ -0,0 +1,68 @@
|
||||
import { omit } from 'es-toolkit';
|
||||
|
||||
import type { BasicMetadata, BasicMetadataBase, BasicMetadataIntermediate, MaimaiTitleMetadataExtra } from '../interfaces';
|
||||
import { MaimaiTitleRareType } from '../interfaces';
|
||||
import type { MetadataMerger } from '../master';
|
||||
import { forEachParallel, objectMap } from '../utils/base';
|
||||
import type { RegionalizedMap, RegionalizedNetOpenDate } from '../utils/data';
|
||||
import { maybeCompactRegionalizedMap, parseNetOpenDate } from '../utils/data';
|
||||
import { forEachRegionAndVersion } from '../utils/each';
|
||||
import { globFiles, parseXmls } from '../utils/fs';
|
||||
import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod';
|
||||
import type { WorkerProcessor } from '../worker';
|
||||
|
||||
type IntermediateData<TExtra> = Record<number, BasicMetadataIntermediate<TExtra>>;
|
||||
|
||||
const defineDataType = <TExtra = {}>(
|
||||
globXmls: (axxxDir: string) => ReturnType<typeof globFiles>,
|
||||
xmlRootElementName: string,
|
||||
parseExtraFields: (xmlData: any) => TExtra = () => ({}) as any,
|
||||
): {
|
||||
process: WorkerProcessor<IntermediateData<TExtra>>;
|
||||
merge: MetadataMerger<IntermediateData<TExtra>, Record<number, BasicMetadata<TExtra>>>;
|
||||
} => ({
|
||||
process: async ctx => {
|
||||
const result: Record<number, BasicMetadataIntermediate<TExtra>> = {};
|
||||
await ctx.forEachAxxxDirOrdered(async axxxDir => await forEachParallel(parseXmls(globXmls(axxxDir)), async ({ xml }) => {
|
||||
const xmlData = xml[xmlRootElementName];
|
||||
const id = zCoerceNumber(xmlData.name.id);
|
||||
const name = zCoerceString(xmlData.name.str);
|
||||
const netOpenDate = parseNetOpenDate(xmlData.netOpenName.str);
|
||||
result[id] = { name, netOpenDate, ...parseExtraFields!(xmlData) };
|
||||
}));
|
||||
return result;
|
||||
},
|
||||
merge: dataMap => {
|
||||
// Merge
|
||||
const regionalNetOpenDate: Record<number, RegionalizedNetOpenDate> = {};
|
||||
const mergedMap: Record<number, RegionalizedMap<BasicMetadataBase<TExtra>>> = {};
|
||||
forEachRegionAndVersion(dataMap, 'jpnFirst', 'oldFirst', (region, version, entries) => Object.entries(entries).forEach(([idStr, entry]) => {
|
||||
const id = Number(idStr);
|
||||
const regionalizedMap = mergedMap[id] ??= {};
|
||||
const versionedMap = regionalizedMap[region] ??= {};
|
||||
versionedMap[version] = omit(entry, ['netOpenDate']) as BasicMetadataBase<TExtra>;
|
||||
|
||||
(regionalNetOpenDate[id] ??= {})[region] ??= null;
|
||||
if (entry.netOpenDate) (regionalNetOpenDate[id] ??= {})[region] = entry.netOpenDate; // Newer version overrides the older one.
|
||||
}));
|
||||
|
||||
// Compact
|
||||
return objectMap(mergedMap, (regionalizedMap, idStr) => ({
|
||||
...maybeCompactRegionalizedMap(regionalizedMap!),
|
||||
regionalNetOpenDate: regionalNetOpenDate[Number(idStr)]!,
|
||||
}));
|
||||
},
|
||||
});
|
||||
|
||||
export const basicDataTypes = {
|
||||
title: defineDataType<MaimaiTitleMetadataExtra>(axxxDir => globFiles(axxxDir, 'title', 'title', 'Title.xml'), 'TitleData', TitleData => ({
|
||||
rareType: zParseEnum(MaimaiTitleRareType, TitleData.rareType),
|
||||
})),
|
||||
frame: defineDataType(axxxDir => globFiles(axxxDir, 'frame', 'frame', 'Frame.xml'), 'FrameData'),
|
||||
icon: defineDataType(axxxDir => globFiles(axxxDir, 'icon', 'icon', 'Icon.xml'), 'IconData'),
|
||||
partner: defineDataType(axxxDir => globFiles(axxxDir, 'partner', 'partner', 'Partner.xml'), 'PartnerData'),
|
||||
plate: defineDataType(axxxDir => globFiles(axxxDir, 'plate', 'plate', 'Plate.xml'), 'PlateData'),
|
||||
chara: defineDataType(axxxDir => globFiles(axxxDir, 'chara', 'chara', 'Chara.xml'), 'CharaData'),
|
||||
card: defineDataType(axxxDir => globFiles(axxxDir, 'card', 'card', 'Card.xml'), 'CardData'),
|
||||
loginBonus: defineDataType(axxxDir => globFiles(axxxDir, 'loginBonus', 'LoginBonus', 'LoginBonus.xml'), 'LoginBonusData'),
|
||||
};
|
||||
@@ -0,0 +1,241 @@
|
||||
import type { MaimaiChartMetadataIntermediate, MaimaiMusicMetadata, MaimaiMusicMetadataIntermediate } from '../interfaces';
|
||||
import { MaimaiRegion, MaimaiMajorVersionId, maimaiMajorVersionIds, MaimaiMusicAddDeleteLogEntry } from '../interfaces';
|
||||
import { createLogger } from '../logger';
|
||||
import type { MetadataMerger } from '../master';
|
||||
import { forEachParallel, objectEntries, objectKeys } from '../utils/base';
|
||||
import { parseNetOpenDate } from '../utils/data';
|
||||
import { forEachRegionAndVersion } from '../utils/each';
|
||||
import { globFiles, parseXmls } from '../utils/fs';
|
||||
import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod';
|
||||
import type { WorkerProcessor } from '../worker';
|
||||
|
||||
const logger = createLogger('Music');
|
||||
|
||||
type IntermediateData = Record<number, MaimaiMusicMetadataIntermediate>;
|
||||
|
||||
export const processMusic: WorkerProcessor<IntermediateData> = async ctx => {
|
||||
const musics: Record<number, MaimaiMusicMetadataIntermediate> = {};
|
||||
await ctx.forEachAxxxDirOrdered(async axxxDir => await forEachParallel(parseXmls(globFiles(axxxDir, 'music', 'music', 'Music.xml')), async ({ fileName, xml: { MusicData } }) => {
|
||||
const id = zCoerceNumber(MusicData.name.id);
|
||||
const name = zCoerceString(MusicData.name.str);
|
||||
const artist = zCoerceString(MusicData.artistName.str);
|
||||
const genre = zCoerceString(MusicData.genreName.str);
|
||||
const bpm = zCoerceNumber(MusicData.bpm);
|
||||
const versionId = zParseEnum(MaimaiMajorVersionId, MusicData.AddVersion.id);
|
||||
const netOpenDate = parseNetOpenDate(MusicData.netOpenName.str);
|
||||
|
||||
const charts = (MusicData.notesData.Notes as any[]).map(note =>
|
||||
// DX and DX+ version didn't set the `isEnable` flag.
|
||||
(
|
||||
ctx.version === MaimaiMajorVersionId.DX || ctx.version === MaimaiMajorVersionId.DX_PLUS
|
||||
? !!note.level
|
||||
: note.isEnable
|
||||
)
|
||||
? {
|
||||
level: zCoerceNumber(note.level) + zCoerceNumber(note.levelDecimal) / 10,
|
||||
designer: zCoerceString(note.notesDesigner.str),
|
||||
} satisfies MaimaiChartMetadataIntermediate
|
||||
: undefined);
|
||||
|
||||
// Remove charts of nonexistent difficulties.
|
||||
while (charts.length > 0 && charts[charts.length - 1] === undefined) charts.pop();
|
||||
|
||||
// Normally musics are deleted only on major version updates.
|
||||
// However, sometimes they delete musics in patches due to political or copyright reasons.
|
||||
const isDeleted = MusicData.eventName.id === 0;
|
||||
if (isDeleted) {
|
||||
logger.warn(`Music ${fileName} (${name}) deleted by patch!`);
|
||||
const existingMusic = musics[id];
|
||||
if (!existingMusic) logger.error(`Music ${fileName} (${name}) is detected to be deleted by patch but not found!`);
|
||||
else existingMusic.deletedInPatch = true;
|
||||
return;
|
||||
}
|
||||
|
||||
musics[id] = {
|
||||
name,
|
||||
artist,
|
||||
genre,
|
||||
bpm,
|
||||
versionId,
|
||||
charts: charts.map(c => ({ designer: c!.designer })),
|
||||
chartsWithLevel: charts.map(c => c!),
|
||||
netOpenDate,
|
||||
deletedInPatch: false,
|
||||
} satisfies MaimaiMusicMetadataIntermediate;
|
||||
}));
|
||||
return musics;
|
||||
};
|
||||
|
||||
export const mergeMusic: MetadataMerger<IntermediateData, Record<number, MaimaiMusicMetadata>> = dataMap => {
|
||||
const result: Record<number, MaimaiMusicMetadata> = {};
|
||||
|
||||
// Merge all musics. JPN first. Nerwer version first.
|
||||
const lowsetSeenVersionId: Record<number, MaimaiMajorVersionId> = {};
|
||||
forEachRegionAndVersion(dataMap, 'jpnFirst', 'newFirst', (region, version, musics) => Object.entries(musics).forEach(([idStr, music]) => {
|
||||
const id = Number(idStr);
|
||||
const resultMusic = result[id] ??= {
|
||||
name: music.name,
|
||||
artist: music.artist,
|
||||
genre: music.genre,
|
||||
bpm: music.bpm,
|
||||
charts: music.charts,
|
||||
levelChangeLog: [],
|
||||
regionalInfo: {},
|
||||
};
|
||||
if (resultMusic.charts.length < music.charts.length) {
|
||||
// Re:MASTER added.
|
||||
resultMusic.charts.push(...music.charts.slice(resultMusic.charts.length));
|
||||
}
|
||||
const regionalInfo = resultMusic.regionalInfo[region] ??= {
|
||||
addDeleteLog: {},
|
||||
netOpenDate: null,
|
||||
};
|
||||
regionalInfo.netOpenDate = music.netOpenDate; // The newest version comes first.
|
||||
lowsetSeenVersionId[id] = Math.min(lowsetSeenVersionId[id] ?? version, version);
|
||||
}));
|
||||
|
||||
const knownMusicIds = new Set(objectKeys(result).map(Number));
|
||||
const perRegionLevelChangeLog: Record<
|
||||
number,
|
||||
Partial<Record<MaimaiRegion, Partial<Record<MaimaiMajorVersionId, number | null>>[]>>
|
||||
> = Object.fromEntries(objectKeys(result).map(id => [id, {}]));
|
||||
// Process the change log - with the release version order.
|
||||
forEachRegionAndVersion(dataMap, 'jpnFirst', 'oldFirst', (region, version, musics) => {
|
||||
// Track the music IDs known but not seen in the current version.
|
||||
const unseenMusicIds = new Set(knownMusicIds);
|
||||
// Set the level in change log for the version, for each music and chart.
|
||||
for (const [idStr, music] of objectEntries(musics)) {
|
||||
const id = Number(idStr);
|
||||
const changeLog = (perRegionLevelChangeLog[id]![region] ??= []);
|
||||
for (const [i, chart] of music.chartsWithLevel.entries()) {
|
||||
unseenMusicIds.delete(id);
|
||||
if (chart) (changeLog[i] ||= {})[version] = chart.level;
|
||||
}
|
||||
}
|
||||
// For each unseen music IDs, if we've seen it before in current region, mark it as "deleted from package".
|
||||
for (const musicId of unseenMusicIds) {
|
||||
const changeLog = perRegionLevelChangeLog[musicId]![region];
|
||||
if (changeLog) {
|
||||
let deleted = false;
|
||||
for (const chart of changeLog) {
|
||||
if (chart) {
|
||||
chart[version] = null; // Mark as "deleted from package".
|
||||
deleted = true;
|
||||
}
|
||||
}
|
||||
if (deleted) logger.verbose(`Music ${musicId} (${result[musicId]!.name}) is deleted in ${MaimaiMajorVersionId[version]} package`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Check -- one chart's level should keep the same in the same version across regions.
|
||||
for (const musicId of knownMusicIds) {
|
||||
for (let difficulty = 0; difficulty <= 4; difficulty++) {
|
||||
for (const version of maimaiMajorVersionIds) {
|
||||
const resultMusic = result[musicId]!;
|
||||
let known: { region: string; level: number } | undefined;
|
||||
for (const region of objectKeys(resultMusic.regionalInfo)) {
|
||||
if (region === MaimaiRegion.CHN) continue;
|
||||
const changeLog = perRegionLevelChangeLog[musicId]![region]![difficulty];
|
||||
const level = changeLog?.[version];
|
||||
if (level) {
|
||||
if (known) {
|
||||
if (known.level !== level) {
|
||||
logger.error(
|
||||
`Chart ${musicId}[${difficulty}] (${result[musicId]!.name}) has different same-version level (${known.region}: ${known.level}, ${region}: ${level}) in ${MaimaiMajorVersionId[version]}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
known = { region, level };
|
||||
}
|
||||
|
||||
// Check -- if a music is added in a non-JPN region before the JPN version. the level should be the same.
|
||||
if (region !== MaimaiRegion.JPN) {
|
||||
const jpnChangeLog = perRegionLevelChangeLog[musicId]![MaimaiRegion.JPN]?.[difficulty];
|
||||
if (jpnChangeLog) {
|
||||
const firstVersion = lowsetSeenVersionId[musicId]!;
|
||||
const firstLevel = perRegionLevelChangeLog[musicId]![MaimaiRegion.JPN]![difficulty]?.[firstVersion];
|
||||
if (firstLevel && version < firstVersion) {
|
||||
if (level !== firstLevel) {
|
||||
logger.error(
|
||||
`Chart ${musicId}[${difficulty}] (${result[musicId]!.name}) has different level (${region}: ${level}, JPN: ${firstLevel}) in ${MaimaiMajorVersionId[version]} (JPN added in ${MaimaiMajorVersionId[firstVersion]})`,
|
||||
);
|
||||
} else {
|
||||
logger.verbose(
|
||||
`Chart ${musicId}[${difficulty}] (${result[musicId]!.name}) has the same level (${level}) in ${MaimaiMajorVersionId[version]} (JPN added in ${MaimaiMajorVersionId[firstVersion]})`,
|
||||
);
|
||||
}
|
||||
// else: missing data.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fill the `levelChangeLog` and `addDeleteLog` in the result with `perRegionLevelChangeLog`.
|
||||
for (const [idStr, music] of objectEntries(result)) {
|
||||
const id = Number(idStr);
|
||||
const levelChangeLog = music.levelChangeLog;
|
||||
for (const region of objectKeys(music.regionalInfo)) {
|
||||
const regionLevelChangeLog = perRegionLevelChangeLog[id]![region]!;
|
||||
if (region !== MaimaiRegion.CHN) {
|
||||
for (const [difficulty, regionLevelChangeLogEntries] of regionLevelChangeLog.entries()) {
|
||||
levelChangeLog[difficulty] ||= {};
|
||||
for (const [version, level] of objectEntries(regionLevelChangeLogEntries)) {
|
||||
if (level != null) {
|
||||
levelChangeLog[difficulty]![version] = level;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const { addDeleteLog } = music.regionalInfo[region]!;
|
||||
let previousVersion: MaimaiMajorVersionId | undefined;
|
||||
for (const version of dataMap.get(region)!.keys()) {
|
||||
let addDeleteLogEntry: MaimaiMusicAddDeleteLogEntry | undefined;
|
||||
if (
|
||||
regionLevelChangeLog[0]![version] != null &&
|
||||
(previousVersion === undefined || regionLevelChangeLog[0]![previousVersion] == null)
|
||||
) {
|
||||
addDeleteLogEntry = MaimaiMusicAddDeleteLogEntry.Added;
|
||||
} else if (
|
||||
previousVersion !== undefined &&
|
||||
regionLevelChangeLog[4]?.[version] != null &&
|
||||
regionLevelChangeLog[4]?.[previousVersion] == null
|
||||
) {
|
||||
addDeleteLogEntry = MaimaiMusicAddDeleteLogEntry.AddedReMaster;
|
||||
} else if (previousVersion !== undefined && regionLevelChangeLog[0]![previousVersion] != null) {
|
||||
const input = dataMap.get(region)!.get(version)!;
|
||||
if (regionLevelChangeLog[0]![version] == null) {
|
||||
addDeleteLogEntry = MaimaiMusicAddDeleteLogEntry.DeletedFromPackage;
|
||||
} else if (input[id]!.deletedInPatch) {
|
||||
addDeleteLogEntry = MaimaiMusicAddDeleteLogEntry.DeletedInPatch;
|
||||
}
|
||||
}
|
||||
if (addDeleteLogEntry) {
|
||||
addDeleteLog[version] = addDeleteLogEntry;
|
||||
}
|
||||
previousVersion = version;
|
||||
}
|
||||
}
|
||||
|
||||
// Delete the unchanged version entries from level change log.
|
||||
for (const levelChangeLogEntries of levelChangeLog.values()) {
|
||||
const entries = objectEntries(levelChangeLogEntries).sort(([a], [b]) => Number(a) - Number(b));
|
||||
for (const [i, [version, level]] of entries.entries()) {
|
||||
// The first entry is always kept.
|
||||
if (i === 0) continue;
|
||||
// Check if the level is changed from the previous version.
|
||||
const previousLevel = entries[i - 1]![1];
|
||||
if (level === previousLevel) {
|
||||
delete levelChangeLogEntries[version];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Merge to one change log and track unavailablity (done). CHN 1.20 levels mismatch?
|
||||
return result;
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
export const objectEntries = <T extends object>(obj: T) => Object.entries(obj) as Array<[keyof T, T[keyof T]]>;
|
||||
|
||||
export const objectFromEntries = <T extends [string | number | symbol, unknown][]>(entries: T) => Object.fromEntries(entries) as Record<T[number][0], T[number][1]>;
|
||||
|
||||
export const objectKeys = <T extends object>(obj: T) => Object.keys(obj) as (keyof T)[];
|
||||
|
||||
export const objectMap = <T extends object, R>(obj: T, fn: (value: T[keyof T], key: keyof T) => R) =>
|
||||
Object.fromEntries(objectEntries(obj).map(([key, value]) => [key, fn(value, key)])) as Record<keyof T, R>;
|
||||
|
||||
export const arrayToObject = <T extends string | number | symbol, R>(array: readonly T[], fn: (value: T, index: number) => R) =>
|
||||
Object.fromEntries(array.map((value, index) => [value, fn(value, index)])) as Record<T, R>;
|
||||
|
||||
export const objectFilter = <T extends object>(obj: T, fn: (value: T[keyof T], key: keyof T) => boolean) =>
|
||||
Object.fromEntries(objectEntries(obj).filter(([key, value]) => fn(value, key))) as T;
|
||||
|
||||
export const getOrSet = <M extends Map<unknown, unknown>>(
|
||||
map: M,
|
||||
key: Parameters<M['has']>[0],
|
||||
value: Parameters<M['set']>[1],
|
||||
): Parameters<M['set']>[1] => {
|
||||
if (!map.has(key)) {
|
||||
map.set(key, value);
|
||||
}
|
||||
return map.get(key)!;
|
||||
};
|
||||
|
||||
export async function forEachParallel<T>(
|
||||
iterable: AsyncIterable<T> | Promise<Iterable<T>>,
|
||||
callback: (value: T) => Promise<void>,
|
||||
) {
|
||||
const promises: Promise<void>[] = [];
|
||||
for await (const value of await iterable) {
|
||||
promises.push(callback(value));
|
||||
}
|
||||
await Promise.all(promises);
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { isEqual } from 'es-toolkit';
|
||||
|
||||
import type { MaimaiMajorVersionId, MaimaiRegion, MetadataMaybeRegionalized, MetadataMaybeVersioned } from '../interfaces';
|
||||
import { objectEntries, objectFromEntries, objectMap } from './base';
|
||||
import { zCoerceString } from './zod';
|
||||
|
||||
export const parseNetOpenDate = (input: unknown) => {
|
||||
const str = zCoerceString(input);
|
||||
const match = str.match(/^Net(\d{2})(\d{2})(\d{2})$/);
|
||||
if (!match) return null;
|
||||
const [, year, month, day] = match;
|
||||
if (year === '99') return null;
|
||||
return `20${year}-${month}-${day}`;
|
||||
};
|
||||
|
||||
export type VersionedMap<T> = Partial<Record<MaimaiMajorVersionId, T>>;
|
||||
export const maybeCompactVersionedMap = <T>(versionedMap: VersionedMap<T>): MetadataMaybeVersioned<T> => {
|
||||
const versionedEntries = objectEntries(versionedMap)
|
||||
.toSorted(([verA], [verB]) => verA - verB) // version: low -> high
|
||||
.filter(([, data], index, array) => index === 0 || !isEqual(data, array[index - 1]?.[1])); // Skip entries that are equal to the previous version.
|
||||
return versionedEntries.length === 1
|
||||
? { unversioned: versionedEntries[0]![1]! }
|
||||
: { versioned: objectFromEntries(versionedEntries) };
|
||||
};
|
||||
|
||||
export type RegionalizedMap<T> = Partial<Record<MaimaiRegion, VersionedMap<T>>>;
|
||||
export const maybeCompactRegionalizedMap = <T>(regionalizedMap: RegionalizedMap<T>): MetadataMaybeRegionalized<T> => {
|
||||
const regionalizedEntries = objectMap(regionalizedMap, versionedMap => maybeCompactVersionedMap(versionedMap!));
|
||||
return regionalizedEntries.JPN != null && isEqual(regionalizedEntries.JPN, regionalizedEntries.EXP) && isEqual(regionalizedEntries.JPN, regionalizedEntries.CHN)
|
||||
? { unregionalized: regionalizedEntries.JPN! }
|
||||
: { regionalized: regionalizedEntries };
|
||||
};
|
||||
|
||||
export type RegionalizedNetOpenDate = Partial<Record<MaimaiRegion, string | null>>;
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { MaimaiMajorVersionId } from '../interfaces';
|
||||
import { MaimaiRegion } from '../interfaces';
|
||||
|
||||
export const forEachRegion = <T>(
|
||||
regionMap: Map<MaimaiRegion, T>,
|
||||
order: 'jpnFirst' | 'jpnLast',
|
||||
callback: (region: MaimaiRegion, data: T) => void,
|
||||
) =>
|
||||
[...regionMap.entries()]
|
||||
.sort(
|
||||
([a], [b]) =>
|
||||
(order === 'jpnFirst' ? 1 : -1) *
|
||||
(Object.values(MaimaiRegion).indexOf(a) - Object.values(MaimaiRegion).indexOf(b)),
|
||||
)
|
||||
.forEach(([region, data]) => callback(region, data));
|
||||
|
||||
export const forEachVersion = <T>(
|
||||
versionMap: Map<MaimaiMajorVersionId, T>,
|
||||
order: 'oldFirst' | 'newFirst',
|
||||
callback: (version: MaimaiMajorVersionId, data: T) => void,
|
||||
) =>
|
||||
[...versionMap.entries()]
|
||||
.sort(([a], [b]) => (order === 'oldFirst' ? a - b : b - a))
|
||||
.forEach(([version, data]) => callback(version, data));
|
||||
|
||||
export const forEachRegionAndVersion = <T>(
|
||||
regionMap: Map<MaimaiRegion, Map<MaimaiMajorVersionId, T>>,
|
||||
regionOrder: 'jpnFirst' | 'jpnLast',
|
||||
versionOrder: 'oldFirst' | 'newFirst',
|
||||
callback: (region: MaimaiRegion, version: MaimaiMajorVersionId, data: T) => void,
|
||||
) => forEachRegion(
|
||||
regionMap, regionOrder,
|
||||
(region, versionMap) => forEachVersion(
|
||||
versionMap, versionOrder,
|
||||
(version, data) =>
|
||||
callback(region, version, data),
|
||||
),
|
||||
);
|
||||
@@ -0,0 +1,19 @@
|
||||
import fs from 'node:fs';
|
||||
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import { glob } from 'glob';
|
||||
|
||||
export async function* parseXmls(stream: ReturnType<typeof glob.stream>) {
|
||||
for await (const file of stream) {
|
||||
const fileName = typeof file === 'string' ? file : file.fullpath();
|
||||
const xmlString = await fs.promises.readFile(fileName, 'utf-8');
|
||||
const xml = new XMLParser({ ignoreAttributes: false, allowBooleanAttributes: true }).parse(xmlString);
|
||||
yield { fileName, xml };
|
||||
}
|
||||
}
|
||||
|
||||
export const globAxxxDirs = async (streamingAssetsPath: string) =>
|
||||
(await glob(`${streamingAssetsPath}/*`)).sort((a, b) => a.localeCompare(b));
|
||||
|
||||
export const globFiles = (axxxPath: string, dirName: string, subDirPrefix: string, fileName: string) =>
|
||||
glob.stream(`${axxxPath}/${dirName}/${subDirPrefix}*/${fileName}`);
|
||||
@@ -0,0 +1,6 @@
|
||||
import type { util as zUtil } from 'zod';
|
||||
import { z } from 'zod';
|
||||
|
||||
export const zCoerceNumber = (data: unknown) => z.coerce.number().parse(data);
|
||||
export const zCoerceString = (data: unknown) => z.coerce.string().parse(data);
|
||||
export const zParseEnum = <T extends zUtil.EnumLike>(enumType: T, data: unknown) => z.enum(enumType).parse(data);
|
||||
@@ -0,0 +1,57 @@
|
||||
import fs from 'node:fs';
|
||||
|
||||
import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId } from './interfaces';
|
||||
import { createLogger } from './logger';
|
||||
import { basicDataTypes } from './processors/basic';
|
||||
import { processMusic } from './processors/music';
|
||||
import { objectMap } from './utils/base';
|
||||
import { globAxxxDirs } from './utils/fs';
|
||||
|
||||
const logger = createLogger('Worker');
|
||||
|
||||
export interface WorkerArguments {
|
||||
region: MaimaiRegion;
|
||||
version: MaimaiMajorVersionId;
|
||||
streamingAssetsPath: string;
|
||||
metadataKind: MaimaiMetadataKind;
|
||||
outputFilePath: string;
|
||||
};
|
||||
|
||||
export interface WorkerContext extends WorkerArguments {
|
||||
forEachAxxxDirOrdered: (callback: (axxxDir: string) => Promise<void>) => Promise<void>;
|
||||
}
|
||||
|
||||
export type WorkerProcessor<TIntermediateData> = (ctx: WorkerContext) => TIntermediateData | Promise<TIntermediateData>;
|
||||
|
||||
const run = async (args: WorkerArguments) => {
|
||||
const { region, version, streamingAssetsPath, metadataKind, outputFilePath } = args;
|
||||
logger.log(`Generating metadata for maimai (region = ${region}, version = ${version})`);
|
||||
|
||||
// Axxx dirs are ordered, so don't use forEachParallel.
|
||||
const ctx: WorkerContext = {
|
||||
...args,
|
||||
forEachAxxxDirOrdered: async (callback: (axxxDir: string) => Promise<void>) => {
|
||||
for (const axxxDir of await globAxxxDirs(streamingAssetsPath)) {
|
||||
logger.log(`Processing ${axxxDir}`);
|
||||
await callback(axxxDir);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
const metadataKindProcessors: Record<MaimaiMetadataKind, WorkerProcessor<unknown>> = {
|
||||
...objectMap(basicDataTypes, dataType => dataType.process),
|
||||
music: processMusic,
|
||||
};
|
||||
|
||||
logger.log(`Writing metadata to ${outputFilePath}`);
|
||||
await fs.promises.writeFile(outputFilePath, JSON.stringify(await metadataKindProcessors[metadataKind](ctx), null, 2));
|
||||
};
|
||||
|
||||
export default async (args: WorkerArguments) => {
|
||||
try {
|
||||
await run(args);
|
||||
} catch (error) {
|
||||
logger.error(`Error processing metadata: ${error}`);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"jsx": "preserve",
|
||||
"lib": ["DOM", "DOM.Iterable", "ESNext", "WebWorker"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"resolveJsonModule": true,
|
||||
"allowJs": true,
|
||||
"strict": true,
|
||||
"strictNullChecks": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"skipLibCheck": true,
|
||||
"rootDir": ".",
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
Reference in New Issue
Block a user