feat: initial commit

src/index.ts

@@ -0,0 +1,26 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { merge } from 'es-toolkit';
+import yaml from 'js-yaml';
+import { z } from 'zod';
+
+import { MaimaiRegion, MaimaiMajorVersionId } from './interfaces/index';
+import { run } from './master';
+
+const dirname = path.dirname(fileURLToPath(import.meta.url));
+
+export const Config = z.object({
+  inputs: z.record(
+    z.enum(MaimaiRegion),
+    z.record(z.preprocess(Number, z.enum(MaimaiMajorVersionId)), z.string()),
+  ),
+  outputDir: z.string(),
+});
+export type Config = z.infer<typeof Config>;
+
+const loadConfigFile = (filename: string) => yaml.load(fs.readFileSync(path.resolve(dirname, '..', filename), 'utf-8')) as Record<string, unknown>;
+const config = Config.parse(merge(loadConfigFile('config.base.yaml'), loadConfigFile('config.yaml')));
+
+await run(config);

src/interfaces/base.ts

@@ -0,0 +1,71 @@
+export enum MaimaiRegion {
+  JPN = 'JPN',
+  EXP = 'EXP',
+  CHN = 'CHN',
+}
+
+export function maimaiRegionFromGameId(gameId: string) {
+  switch (gameId) {
+  case '\x53\x44\x45\x5a':
+    return MaimaiRegion.JPN;
+  case '\x53\x44\x47\x41':
+    return MaimaiRegion.EXP;
+  case '\x53\x44\x47\x42':
+    return MaimaiRegion.CHN;
+  default: // Fallback to JPN
+    return MaimaiRegion.JPN;
+  }
+}
+
+export enum MaimaiMajorVersionId {
+  maimai = 0,
+  maimai_PLUS = 1,
+  GreeN = 2,
+  GreeN_PLUS = 3,
+  ORANGE = 4,
+  ORANGE_PLUS = 5,
+  PiNK = 6,
+  PiNK_PLUS = 7,
+  MURASAKi = 8,
+  MURASAKi_PLUS = 9,
+  MiLK = 10,
+  MiLK_PLUS = 11,
+  FiNALE = 12,
+  DX = 13,
+  DX_PLUS = 14,
+  Splash = 15,
+  Splash_PLUS = 16,
+  UNiVERSE = 17,
+  UNiVERSE_PLUS = 18,
+  FESTiVAL = 19,
+  FESTiVAL_PLUS = 20,
+  BUDDiES = 21,
+  BUDDiES_PLUS = 22,
+  PRiSM = 23,
+  PRiSM_PLUS = 24,
+  CiRCLE = 25,
+}
+
+export const maimaiMajorVersionIds = Object.values(MaimaiMajorVersionId).filter(
+  v => typeof v === 'number',
+) as MaimaiMajorVersionId[];
+
+export function maimaiVersionIdFromVersionString(gameVersion: string) {
+  if (!gameVersion) {
+    return maimaiMajorVersionIds[maimaiMajorVersionIds.length - 1];
+  }
+  const v = Number(gameVersion.split('.')[1]);
+  const i = Math.floor(v / 5);
+  const result = MaimaiMajorVersionId.DX + i;
+  return maimaiMajorVersionIds.includes(result)
+    ? result
+    : /* Fallback to the latest version */ maimaiMajorVersionIds[maimaiMajorVersionIds.length - 1];
+}
+
+export type MetadataUnversioned<T> = { unversioned: T };
+export type MetadataVersioned<T> = { versioned: Partial<Record<MaimaiMajorVersionId, T>> };
+export type MetadataMaybeVersioned<T> = MetadataUnversioned<T> | MetadataVersioned<T>;
+
+export type MetadataUnregionalized<T> = { unregionalized: MetadataMaybeVersioned<T> };
+export type MetadataRegionalized<T> = { regionalized: Partial<Record<MaimaiRegion, MetadataMaybeVersioned<T>>> };
+export type MetadataMaybeRegionalized<T> = MetadataUnregionalized<T> | MetadataRegionalized<T>;

src/interfaces/index.ts

@@ -0,0 +1,6 @@
+export * from './base';
+export * from './music';
+export * from './title';
+
+export const maimaiMetadataKinds = ['music', 'title', 'frame', 'icon', 'partner', 'plate', 'chara', 'card', 'loginBonus'] as const;
+export type MaimaiMetadataKind = (typeof maimaiMetadataKinds)[number];

src/interfaces/music.ts

@@ -0,0 +1,50 @@
+import type { MaimaiMajorVersionId, MaimaiRegion } from './base';
+
+export type MaimaiChartMetadata = {
+  // `level` is not included here and should be inferred from the change log.
+  designer: string;
+  // TODO: how many of TAP/HOLD/SLIDE/BREAK notes?
+};
+export type MaimaiChartMetadataIntermediate = MaimaiChartMetadata & {
+  level: number;
+};
+
+export type MaimaiMusicMetadataBase = {
+  name: string;
+  artist: string;
+  genre: string;
+  bpm: number;
+  charts: MaimaiChartMetadata[];
+};
+
+export type MaimaiMusicMetadataIntermediate = MaimaiMusicMetadataBase & {
+  chartsWithLevel: MaimaiChartMetadataIntermediate[];
+  versionId: MaimaiMajorVersionId;
+  deletedInPatch: boolean;
+  netOpenDate: string | null;
+};
+
+export enum MaimaiMusicAddDeleteLogEntry {
+  Added = 1,
+  AddedReMaster = 2,
+  DeletedFromPackage = 3,
+  DeletedInPatch = 4,
+}
+export type MaimaiMusicAddDeleteLog = Partial<Record<MaimaiMajorVersionId, MaimaiMusicAddDeleteLogEntry>>;
+export interface MaimaiMusicMetadataRegionalInfo {
+  /**
+   * For a music deleted from package, it don't have the level value in that version.
+   * For a music deleted in patch, it has the level value in that version.
+   */
+  addDeleteLog: MaimaiMusicAddDeleteLog;
+  netOpenDate: string | null;
+}
+
+export type MaimaiMusicLevelChangeLog = Partial<Record<MaimaiMajorVersionId, number>>[];
+export interface MaimaiMusicMetadata extends MaimaiMusicMetadataBase {
+  /**
+   * The level change log per chart (BASIC, ADVANCED, EXPERT, MASTER, Re:MASTER).
+   */
+  levelChangeLog: MaimaiMusicLevelChangeLog; // This is observed to be identical for all regions.
+  regionalInfo: Partial<Record<MaimaiRegion, MaimaiMusicMetadataRegionalInfo>>;
+}

src/interfaces/title.ts

@@ -0,0 +1,17 @@
+import type { MaimaiRegion, MetadataMaybeRegionalized } from './base';
+
+export enum MaimaiTitleRareType {
+  Normal = 'Normal',
+  Bronze = 'Bronze',
+  Silver = 'Silver',
+  Gold = 'Gold',
+  Rainbow = 'Rainbow',
+}
+
+export type BasicMetadataBase<TExtra> = TExtra & { name: string };
+export type BasicMetadataIntermediate<TExtra> = BasicMetadataBase<TExtra> & { netOpenDate: string | null };
+export type BasicMetadata<TExtra = {}> = MetadataMaybeRegionalized<BasicMetadataBase<TExtra>> & {
+  regionalNetOpenDate: Partial<Record<MaimaiRegion, string | null>>;
+};
+
+export type MaimaiTitleMetadataExtra = { rareType: MaimaiTitleRareType };

src/logger.ts

@@ -0,0 +1,5 @@
+import { initLogger, useGlobalLogger } from '@guiiai/logg';
+
+initLogger();
+
+export const createLogger = (name: string) => useGlobalLogger(name);

src/master.ts

@@ -0,0 +1,89 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+import Tinypool from 'tinypool';
+
+import type { Config } from './index';
+import type { MaimaiMajorVersionId, MaimaiMetadataKind, MaimaiRegion } from './interfaces';
+import { maimaiMetadataKinds } from './interfaces';
+import { createLogger } from './logger';
+import { basicDataTypes } from './processors/basic';
+import { mergeMusic } from './processors/music';
+import { arrayToObject, getOrSet, objectEntries, objectMap } from './utils/base';
+import type { WorkerArguments } from './worker';
+
+const logger = createLogger('Master');
+
+const pool = new Tinypool({
+  filename: new URL('./worker.ts', import.meta.url).href,
+  minThreads: 8,
+  maxThreads: 32,
+});
+
+export type IntermediateDataMap<TIntermediateData> = Map<MaimaiRegion, Map<MaimaiMajorVersionId, TIntermediateData>>;
+export type MetadataMerger<TIntermediateData, TResult> = (intermediateDataMap: IntermediateDataMap<TIntermediateData>) => TResult;
+
+export const run = async (config: Config) => {
+  const outputDir = path.resolve(config.outputDir);
+  await fs.promises.mkdir(outputDir, { recursive: true });
+
+  const index: Map<MaimaiMetadataKind, Map<MaimaiRegion, Map<MaimaiMajorVersionId, string>>> = new Map();
+  const tasks: Promise<void>[] = [];
+  for (const [regionName, versionPathMap] of objectEntries(config.inputs)) {
+    const region = regionName as MaimaiRegion;
+    for (const [versionName, streamingAssetsPath] of objectEntries(versionPathMap)) {
+      const version = Number(versionName) as MaimaiMajorVersionId;
+      const versionOutputDir = path.resolve(outputDir, 'intermediate', `${region}-${version}`);
+      await fs.promises.mkdir(versionOutputDir, { recursive: true });
+
+      const filePaths = arrayToObject(maimaiMetadataKinds, kind => path.resolve(versionOutputDir, `${kind}.json`));
+
+      for (const [metadataKind, outputFilePath] of objectEntries(filePaths)) {
+        const regionIndex = getOrSet(index, metadataKind as MaimaiMetadataKind, new Map());
+        const versionIndex = getOrSet(regionIndex, region, new Map());
+        versionIndex.set(version, outputFilePath);
+
+        if (fs.existsSync(outputFilePath)) continue;
+
+        tasks.push(pool.run({
+          region,
+          version,
+          streamingAssetsPath,
+          metadataKind,
+          outputFilePath,
+        } satisfies WorkerArguments));
+      }
+    }
+  }
+  await Promise.all(tasks);
+
+  const dataMap: Map<MaimaiMetadataKind, IntermediateDataMap<unknown>> = new Map();
+  for (const [kind, regionMergeIndex] of index.entries()) {
+    const regionMap = getOrSet(dataMap, kind, new Map());
+    for (const [region, versionMergeIndex] of regionMergeIndex.entries()) {
+      const versionMap = getOrSet(regionMap, region, new Map());
+      for (const [version, outputFilePath] of versionMergeIndex.entries()) {
+        const data = JSON.parse(await fs.promises.readFile(outputFilePath, 'utf-8'));
+        versionMap.set(version, data);
+      }
+    }
+  }
+
+  const mergers: Record<MaimaiMetadataKind, MetadataMerger<any, any>> = {
+    ...objectMap(basicDataTypes, dataType => dataType.merge),
+    music: mergeMusic,
+  };
+  const result = objectEntries(mergers).reduce((result, [kind, merger]) => {
+    try {
+      result[kind] = merger(dataMap.get(kind)!);
+      return result;
+    } catch (error) {
+      logger.error(`Failed to merge ${kind} metadata`);
+      throw error;
+    }
+  }, {} as Record<MaimaiMetadataKind, unknown>);
+
+  const mergedFilePath = path.resolve(outputDir, 'merged.json');
+  await fs.promises.writeFile(mergedFilePath, JSON.stringify(result, null, 2));
+  logger.log(`Merged metadata to ${mergedFilePath}`);
+};

src/processors/basic.ts

@@ -0,0 +1,68 @@
+import { omit } from 'es-toolkit';
+
+import type { BasicMetadata, BasicMetadataBase, BasicMetadataIntermediate, MaimaiTitleMetadataExtra } from '../interfaces';
+import { MaimaiTitleRareType } from '../interfaces';
+import type { MetadataMerger } from '../master';
+import { forEachParallel, objectMap } from '../utils/base';
+import type { RegionalizedMap, RegionalizedNetOpenDate } from '../utils/data';
+import { maybeCompactRegionalizedMap, parseNetOpenDate } from '../utils/data';
+import { forEachRegionAndVersion } from '../utils/each';
+import { globFiles, parseXmls } from '../utils/fs';
+import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod';
+import type { WorkerProcessor } from '../worker';
+
+type IntermediateData<TExtra> = Record<number, BasicMetadataIntermediate<TExtra>>;
+
+const defineDataType = <TExtra = {}>(
+  globXmls: (axxxDir: string) => ReturnType<typeof globFiles>,
+  xmlRootElementName: string,
+  parseExtraFields: (xmlData: any) => TExtra = () => ({}) as any,
+): {
+  process: WorkerProcessor<IntermediateData<TExtra>>;
+  merge: MetadataMerger<IntermediateData<TExtra>, Record<number, BasicMetadata<TExtra>>>;
+} => ({
+  process: async ctx => {
+    const result: Record<number, BasicMetadataIntermediate<TExtra>> = {};
+    await ctx.forEachAxxxDirOrdered(async axxxDir => await forEachParallel(parseXmls(globXmls(axxxDir)), async ({ xml }) => {
+      const xmlData = xml[xmlRootElementName];
+      const id = zCoerceNumber(xmlData.name.id);
+      const name = zCoerceString(xmlData.name.str);
+      const netOpenDate = parseNetOpenDate(xmlData.netOpenName.str);
+      result[id] = { name, netOpenDate, ...parseExtraFields!(xmlData) };
+    }));
+    return result;
+  },
+  merge: dataMap => {
+    // Merge
+    const regionalNetOpenDate: Record<number, RegionalizedNetOpenDate> = {};
+    const mergedMap: Record<number, RegionalizedMap<BasicMetadataBase<TExtra>>> = {};
+    forEachRegionAndVersion(dataMap, 'jpnFirst', 'oldFirst', (region, version, entries) => Object.entries(entries).forEach(([idStr, entry]) => {
+      const id = Number(idStr);
+      const regionalizedMap = mergedMap[id] ??= {};
+      const versionedMap = regionalizedMap[region] ??= {};
+      versionedMap[version] = omit(entry, ['netOpenDate']) as BasicMetadataBase<TExtra>;
+
+      (regionalNetOpenDate[id] ??= {})[region] ??= null;
+      if (entry.netOpenDate) (regionalNetOpenDate[id] ??= {})[region] = entry.netOpenDate; // Newer version overrides the older one.
+    }));
+
+    // Compact
+    return objectMap(mergedMap, (regionalizedMap, idStr) => ({
+      ...maybeCompactRegionalizedMap(regionalizedMap!),
+      regionalNetOpenDate: regionalNetOpenDate[Number(idStr)]!,
+    }));
+  },
+});
+
+export const basicDataTypes = {
+  title: defineDataType<MaimaiTitleMetadataExtra>(axxxDir => globFiles(axxxDir, 'title', 'title', 'Title.xml'), 'TitleData', TitleData => ({
+    rareType: zParseEnum(MaimaiTitleRareType, TitleData.rareType),
+  })),
+  frame: defineDataType(axxxDir => globFiles(axxxDir, 'frame', 'frame', 'Frame.xml'), 'FrameData'),
+  icon: defineDataType(axxxDir => globFiles(axxxDir, 'icon', 'icon', 'Icon.xml'), 'IconData'),
+  partner: defineDataType(axxxDir => globFiles(axxxDir, 'partner', 'partner', 'Partner.xml'), 'PartnerData'),
+  plate: defineDataType(axxxDir => globFiles(axxxDir, 'plate', 'plate', 'Plate.xml'), 'PlateData'),
+  chara: defineDataType(axxxDir => globFiles(axxxDir, 'chara', 'chara', 'Chara.xml'), 'CharaData'),
+  card: defineDataType(axxxDir => globFiles(axxxDir, 'card', 'card', 'Card.xml'), 'CardData'),
+  loginBonus: defineDataType(axxxDir => globFiles(axxxDir, 'loginBonus', 'LoginBonus', 'LoginBonus.xml'), 'LoginBonusData'),
+};

src/processors/music.ts

@@ -0,0 +1,241 @@
+import type { MaimaiChartMetadataIntermediate, MaimaiMusicMetadata, MaimaiMusicMetadataIntermediate } from '../interfaces';
+import { MaimaiRegion, MaimaiMajorVersionId, maimaiMajorVersionIds, MaimaiMusicAddDeleteLogEntry } from '../interfaces';
+import { createLogger } from '../logger';
+import type { MetadataMerger } from '../master';
+import { forEachParallel, objectEntries, objectKeys } from '../utils/base';
+import { parseNetOpenDate } from '../utils/data';
+import { forEachRegionAndVersion } from '../utils/each';
+import { globFiles, parseXmls } from '../utils/fs';
+import { zCoerceNumber, zCoerceString, zParseEnum } from '../utils/zod';
+import type { WorkerProcessor } from '../worker';
+
+const logger = createLogger('Music');
+
+type IntermediateData = Record<number, MaimaiMusicMetadataIntermediate>;
+
+export const processMusic: WorkerProcessor<IntermediateData> = async ctx => {
+  const musics: Record<number, MaimaiMusicMetadataIntermediate> = {};
+  await ctx.forEachAxxxDirOrdered(async axxxDir => await forEachParallel(parseXmls(globFiles(axxxDir, 'music', 'music', 'Music.xml')), async ({ fileName, xml: { MusicData } }) => {
+    const id = zCoerceNumber(MusicData.name.id);
+    const name = zCoerceString(MusicData.name.str);
+    const artist = zCoerceString(MusicData.artistName.str);
+    const genre = zCoerceString(MusicData.genreName.str);
+    const bpm = zCoerceNumber(MusicData.bpm);
+    const versionId = zParseEnum(MaimaiMajorVersionId, MusicData.AddVersion.id);
+    const netOpenDate = parseNetOpenDate(MusicData.netOpenName.str);
+
+    const charts = (MusicData.notesData.Notes as any[]).map(note =>
+      // DX and DX+ version didn't set the `isEnable` flag.
+      (
+        ctx.version === MaimaiMajorVersionId.DX || ctx.version === MaimaiMajorVersionId.DX_PLUS
+          ? !!note.level
+          : note.isEnable
+      )
+        ? {
+          level: zCoerceNumber(note.level) + zCoerceNumber(note.levelDecimal) / 10,
+          designer: zCoerceString(note.notesDesigner.str),
+        } satisfies MaimaiChartMetadataIntermediate
+        : undefined);
+
+    // Remove charts of nonexistent difficulties.
+    while (charts.length > 0 && charts[charts.length - 1] === undefined) charts.pop();
+
+    // Normally musics are deleted only on major version updates.
+    // However, sometimes they delete musics in patches due to political or copyright reasons.
+    const isDeleted = MusicData.eventName.id === 0;
+    if (isDeleted) {
+      logger.warn(`Music ${fileName} (${name}) deleted by patch!`);
+      const existingMusic = musics[id];
+      if (!existingMusic) logger.error(`Music ${fileName} (${name}) is detected to be deleted by patch but not found!`);
+      else existingMusic.deletedInPatch = true;
+      return;
+    }
+
+    musics[id] = {
+      name,
+      artist,
+      genre,
+      bpm,
+      versionId,
+      charts: charts.map(c => ({ designer: c!.designer })),
+      chartsWithLevel: charts.map(c => c!),
+      netOpenDate,
+      deletedInPatch: false,
+    } satisfies MaimaiMusicMetadataIntermediate;
+  }));
+  return musics;
+};
+
+export const mergeMusic: MetadataMerger<IntermediateData, Record<number, MaimaiMusicMetadata>> = dataMap => {
+  const result: Record<number, MaimaiMusicMetadata> = {};
+
+  // Merge all musics. JPN first. Nerwer version first.
+  const lowsetSeenVersionId: Record<number, MaimaiMajorVersionId> = {};
+  forEachRegionAndVersion(dataMap, 'jpnFirst', 'newFirst', (region, version, musics) => Object.entries(musics).forEach(([idStr, music]) => {
+    const id = Number(idStr);
+    const resultMusic = result[id] ??= {
+      name: music.name,
+      artist: music.artist,
+      genre: music.genre,
+      bpm: music.bpm,
+      charts: music.charts,
+      levelChangeLog: [],
+      regionalInfo: {},
+    };
+    if (resultMusic.charts.length < music.charts.length) {
+      // Re:MASTER added.
+      resultMusic.charts.push(...music.charts.slice(resultMusic.charts.length));
+    }
+    const regionalInfo = resultMusic.regionalInfo[region] ??= {
+      addDeleteLog: {},
+      netOpenDate: null,
+    };
+    regionalInfo.netOpenDate = music.netOpenDate; // The newest version comes first.
+    lowsetSeenVersionId[id] = Math.min(lowsetSeenVersionId[id] ?? version, version);
+  }));
+
+  const knownMusicIds = new Set(objectKeys(result).map(Number));
+  const perRegionLevelChangeLog: Record<
+    number,
+    Partial<Record<MaimaiRegion, Partial<Record<MaimaiMajorVersionId, number | null>>[]>>
+  > = Object.fromEntries(objectKeys(result).map(id => [id, {}]));
+  // Process the change log - with the release version order.
+  forEachRegionAndVersion(dataMap, 'jpnFirst', 'oldFirst', (region, version, musics) => {
+    // Track the music IDs known but not seen in the current version.
+    const unseenMusicIds = new Set(knownMusicIds);
+    // Set the level in change log for the version, for each music and chart.
+    for (const [idStr, music] of objectEntries(musics)) {
+      const id = Number(idStr);
+      const changeLog = (perRegionLevelChangeLog[id]![region] ??= []);
+      for (const [i, chart] of music.chartsWithLevel.entries()) {
+        unseenMusicIds.delete(id);
+        if (chart) (changeLog[i] ||= {})[version] = chart.level;
+      }
+    }
+    // For each unseen music IDs, if we've seen it before in current region, mark it as "deleted from package".
+    for (const musicId of unseenMusicIds) {
+      const changeLog = perRegionLevelChangeLog[musicId]![region];
+      if (changeLog) {
+        let deleted = false;
+        for (const chart of changeLog) {
+          if (chart) {
+            chart[version] = null; // Mark as "deleted from package".
+            deleted = true;
+          }
+        }
+        if (deleted) logger.verbose(`Music ${musicId} (${result[musicId]!.name}) is deleted in ${MaimaiMajorVersionId[version]} package`);
+      }
+    }
+  });
+
+  // Check -- one chart's level should keep the same in the same version across regions.
+  for (const musicId of knownMusicIds) {
+    for (let difficulty = 0; difficulty <= 4; difficulty++) {
+      for (const version of maimaiMajorVersionIds) {
+        const resultMusic = result[musicId]!;
+        let known: { region: string; level: number } | undefined;
+        for (const region of objectKeys(resultMusic.regionalInfo)) {
+          if (region === MaimaiRegion.CHN) continue;
+          const changeLog = perRegionLevelChangeLog[musicId]![region]![difficulty];
+          const level = changeLog?.[version];
+          if (level) {
+            if (known) {
+              if (known.level !== level) {
+                logger.error(
+                  `Chart ${musicId}[${difficulty}] (${result[musicId]!.name}) has different same-version level (${known.region}: ${known.level}, ${region}: ${level}) in ${MaimaiMajorVersionId[version]}`,
+                );
+              }
+            } else {
+              known = { region, level };
+            }
+
+            // Check -- if a music is added in a non-JPN region before the JPN version. the level should be the same.
+            if (region !== MaimaiRegion.JPN) {
+              const jpnChangeLog = perRegionLevelChangeLog[musicId]![MaimaiRegion.JPN]?.[difficulty];
+              if (jpnChangeLog) {
+                const firstVersion = lowsetSeenVersionId[musicId]!;
+                const firstLevel = perRegionLevelChangeLog[musicId]![MaimaiRegion.JPN]![difficulty]?.[firstVersion];
+                if (firstLevel && version < firstVersion) {
+                  if (level !== firstLevel) {
+                    logger.error(
+                      `Chart ${musicId}[${difficulty}] (${result[musicId]!.name}) has different level (${region}: ${level}, JPN: ${firstLevel}) in ${MaimaiMajorVersionId[version]} (JPN added in ${MaimaiMajorVersionId[firstVersion]})`,
+                    );
+                  } else {
+                    logger.verbose(
+                      `Chart ${musicId}[${difficulty}] (${result[musicId]!.name}) has the same level (${level}) in ${MaimaiMajorVersionId[version]} (JPN added in ${MaimaiMajorVersionId[firstVersion]})`,
+                    );
+                  }
+                  // else: missing data.
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // Fill the `levelChangeLog` and `addDeleteLog` in the result with `perRegionLevelChangeLog`.
+  for (const [idStr, music] of objectEntries(result)) {
+    const id = Number(idStr);
+    const levelChangeLog = music.levelChangeLog;
+    for (const region of objectKeys(music.regionalInfo)) {
+      const regionLevelChangeLog = perRegionLevelChangeLog[id]![region]!;
+      if (region !== MaimaiRegion.CHN) {
+        for (const [difficulty, regionLevelChangeLogEntries] of regionLevelChangeLog.entries()) {
+          levelChangeLog[difficulty] ||= {};
+          for (const [version, level] of objectEntries(regionLevelChangeLogEntries)) {
+            if (level != null) {
+              levelChangeLog[difficulty]![version] = level;
+            }
+          }
+        }
+      }
+      const { addDeleteLog } = music.regionalInfo[region]!;
+      let previousVersion: MaimaiMajorVersionId | undefined;
+      for (const version of dataMap.get(region)!.keys()) {
+        let addDeleteLogEntry: MaimaiMusicAddDeleteLogEntry | undefined;
+        if (
+          regionLevelChangeLog[0]![version] != null &&
+          (previousVersion === undefined || regionLevelChangeLog[0]![previousVersion] == null)
+        ) {
+          addDeleteLogEntry = MaimaiMusicAddDeleteLogEntry.Added;
+        } else if (
+          previousVersion !== undefined &&
+          regionLevelChangeLog[4]?.[version] != null &&
+          regionLevelChangeLog[4]?.[previousVersion] == null
+        ) {
+          addDeleteLogEntry = MaimaiMusicAddDeleteLogEntry.AddedReMaster;
+        } else if (previousVersion !== undefined && regionLevelChangeLog[0]![previousVersion] != null) {
+          const input = dataMap.get(region)!.get(version)!;
+          if (regionLevelChangeLog[0]![version] == null) {
+            addDeleteLogEntry = MaimaiMusicAddDeleteLogEntry.DeletedFromPackage;
+          } else if (input[id]!.deletedInPatch) {
+            addDeleteLogEntry = MaimaiMusicAddDeleteLogEntry.DeletedInPatch;
+          }
+        }
+        if (addDeleteLogEntry) {
+          addDeleteLog[version] = addDeleteLogEntry;
+        }
+        previousVersion = version;
+      }
+    }
+
+    // Delete the unchanged version entries from level change log.
+    for (const levelChangeLogEntries of levelChangeLog.values()) {
+      const entries = objectEntries(levelChangeLogEntries).sort(([a], [b]) => Number(a) - Number(b));
+      for (const [i, [version, level]] of entries.entries()) {
+        // The first entry is always kept.
+        if (i === 0) continue;
+        // Check if the level is changed from the previous version.
+        const previousLevel = entries[i - 1]![1];
+        if (level === previousLevel) {
+          delete levelChangeLogEntries[version];
+        }
+      }
+    }
+  }
+
+  // TODO: Merge to one change log and track unavailablity (done). CHN 1.20 levels mismatch?
+  return result;
+};

src/utils/base.ts

@@ -0,0 +1,36 @@
+export const objectEntries = <T extends object>(obj: T) => Object.entries(obj) as Array<[keyof T, T[keyof T]]>;
+
+export const objectFromEntries = <T extends [string | number | symbol, unknown][]>(entries: T) => Object.fromEntries(entries) as Record<T[number][0], T[number][1]>;
+
+export const objectKeys = <T extends object>(obj: T) => Object.keys(obj) as (keyof T)[];
+
+export const objectMap = <T extends object, R>(obj: T, fn: (value: T[keyof T], key: keyof T) => R) =>
+  Object.fromEntries(objectEntries(obj).map(([key, value]) => [key, fn(value, key)])) as Record<keyof T, R>;
+
+export const arrayToObject = <T extends string | number | symbol, R>(array: readonly T[], fn: (value: T, index: number) => R) =>
+  Object.fromEntries(array.map((value, index) => [value, fn(value, index)])) as Record<T, R>;
+
+export const objectFilter = <T extends object>(obj: T, fn: (value: T[keyof T], key: keyof T) => boolean) =>
+  Object.fromEntries(objectEntries(obj).filter(([key, value]) => fn(value, key))) as T;
+
+export const getOrSet = <M extends Map<unknown, unknown>>(
+  map: M,
+  key: Parameters<M['has']>[0],
+  value: Parameters<M['set']>[1],
+): Parameters<M['set']>[1] => {
+  if (!map.has(key)) {
+    map.set(key, value);
+  }
+  return map.get(key)!;
+};
+
+export async function forEachParallel<T>(
+  iterable: AsyncIterable<T> | Promise<Iterable<T>>,
+  callback: (value: T) => Promise<void>,
+) {
+  const promises: Promise<void>[] = [];
+  for await (const value of await iterable) {
+    promises.push(callback(value));
+  }
+  await Promise.all(promises);
+}

src/utils/data.ts

@@ -0,0 +1,34 @@
+import { isEqual } from 'es-toolkit';
+
+import type { MaimaiMajorVersionId, MaimaiRegion, MetadataMaybeRegionalized, MetadataMaybeVersioned } from '../interfaces';
+import { objectEntries, objectFromEntries, objectMap } from './base';
+import { zCoerceString } from './zod';
+
+export const parseNetOpenDate = (input: unknown) => {
+  const str = zCoerceString(input);
+  const match = str.match(/^Net(\d{2})(\d{2})(\d{2})$/);
+  if (!match) return null;
+  const [, year, month, day] = match;
+  if (year === '99') return null;
+  return `20${year}-${month}-${day}`;
+};
+
+export type VersionedMap<T> = Partial<Record<MaimaiMajorVersionId, T>>;
+export const maybeCompactVersionedMap = <T>(versionedMap: VersionedMap<T>): MetadataMaybeVersioned<T> => {
+  const versionedEntries = objectEntries(versionedMap)
+    .toSorted(([verA], [verB]) => verA - verB) // version: low -> high
+    .filter(([, data], index, array) => index === 0 || !isEqual(data, array[index - 1]?.[1])); // Skip entries that are equal to the previous version.
+  return versionedEntries.length === 1
+    ? { unversioned: versionedEntries[0]![1]! }
+    : { versioned: objectFromEntries(versionedEntries) };
+};
+
+export type RegionalizedMap<T> = Partial<Record<MaimaiRegion, VersionedMap<T>>>;
+export const maybeCompactRegionalizedMap = <T>(regionalizedMap: RegionalizedMap<T>): MetadataMaybeRegionalized<T> => {
+  const regionalizedEntries = objectMap(regionalizedMap, versionedMap => maybeCompactVersionedMap(versionedMap!));
+  return regionalizedEntries.JPN != null && isEqual(regionalizedEntries.JPN, regionalizedEntries.EXP) && isEqual(regionalizedEntries.JPN, regionalizedEntries.CHN)
+    ? { unregionalized: regionalizedEntries.JPN! }
+    : { regionalized: regionalizedEntries };
+};
+
+export type RegionalizedNetOpenDate = Partial<Record<MaimaiRegion, string | null>>;

src/utils/each.ts

@@ -0,0 +1,38 @@
+import type { MaimaiMajorVersionId } from '../interfaces';
+import { MaimaiRegion } from '../interfaces';
+
+export const forEachRegion = <T>(
+  regionMap: Map<MaimaiRegion, T>,
+  order: 'jpnFirst' | 'jpnLast',
+  callback: (region: MaimaiRegion, data: T) => void,
+) =>
+  [...regionMap.entries()]
+    .sort(
+      ([a], [b]) =>
+        (order === 'jpnFirst' ? 1 : -1) *
+        (Object.values(MaimaiRegion).indexOf(a) - Object.values(MaimaiRegion).indexOf(b)),
+    )
+    .forEach(([region, data]) => callback(region, data));
+
+export const forEachVersion = <T>(
+  versionMap: Map<MaimaiMajorVersionId, T>,
+  order: 'oldFirst' | 'newFirst',
+  callback: (version: MaimaiMajorVersionId, data: T) => void,
+) =>
+  [...versionMap.entries()]
+    .sort(([a], [b]) => (order === 'oldFirst' ? a - b : b - a))
+    .forEach(([version, data]) => callback(version, data));
+
+export const forEachRegionAndVersion = <T>(
+  regionMap: Map<MaimaiRegion, Map<MaimaiMajorVersionId, T>>,
+  regionOrder: 'jpnFirst' | 'jpnLast',
+  versionOrder: 'oldFirst' | 'newFirst',
+  callback: (region: MaimaiRegion, version: MaimaiMajorVersionId, data: T) => void,
+) => forEachRegion(
+  regionMap, regionOrder,
+  (region, versionMap) => forEachVersion(
+    versionMap, versionOrder,
+    (version, data) =>
+      callback(region, version, data),
+  ),
+);

src/utils/fs.ts

@@ -0,0 +1,19 @@
+import fs from 'node:fs';
+
+import { XMLParser } from 'fast-xml-parser';
+import { glob } from 'glob';
+
+export async function* parseXmls(stream: ReturnType<typeof glob.stream>) {
+  for await (const file of stream) {
+    const fileName = typeof file === 'string' ? file : file.fullpath();
+    const xmlString = await fs.promises.readFile(fileName, 'utf-8');
+    const xml = new XMLParser({ ignoreAttributes: false, allowBooleanAttributes: true }).parse(xmlString);
+    yield { fileName, xml };
+  }
+}
+
+export const globAxxxDirs = async (streamingAssetsPath: string) =>
+  (await glob(`${streamingAssetsPath}/*`)).sort((a, b) => a.localeCompare(b));
+
+export const globFiles = (axxxPath: string, dirName: string, subDirPrefix: string, fileName: string) =>
+  glob.stream(`${axxxPath}/${dirName}/${subDirPrefix}*/${fileName}`);

src/utils/zod.ts

@@ -0,0 +1,6 @@
+import type { util as zUtil } from 'zod';
+import { z } from 'zod';
+
+export const zCoerceNumber = (data: unknown) => z.coerce.number().parse(data);
+export const zCoerceString = (data: unknown) => z.coerce.string().parse(data);
+export const zParseEnum = <T extends zUtil.EnumLike>(enumType: T, data: unknown) => z.enum(enumType).parse(data);

src/worker.ts

@@ -0,0 +1,57 @@
+import fs from 'node:fs';
+
+import type { MaimaiMetadataKind, MaimaiRegion, MaimaiMajorVersionId } from './interfaces';
+import { createLogger } from './logger';
+import { basicDataTypes } from './processors/basic';
+import { processMusic } from './processors/music';
+import { objectMap } from './utils/base';
+import { globAxxxDirs } from './utils/fs';
+
+const logger = createLogger('Worker');
+
+export interface WorkerArguments {
+  region: MaimaiRegion;
+  version: MaimaiMajorVersionId;
+  streamingAssetsPath: string;
+  metadataKind: MaimaiMetadataKind;
+  outputFilePath: string;
+};
+
+export interface WorkerContext extends WorkerArguments {
+  forEachAxxxDirOrdered: (callback: (axxxDir: string) => Promise<void>) => Promise<void>;
+}
+
+export type WorkerProcessor<TIntermediateData> = (ctx: WorkerContext) => TIntermediateData | Promise<TIntermediateData>;
+
+const run = async (args: WorkerArguments) => {
+  const { region, version, streamingAssetsPath, metadataKind, outputFilePath } = args;
+  logger.log(`Generating metadata for maimai (region = ${region}, version = ${version})`);
+
+  // Axxx dirs are ordered, so don't use forEachParallel.
+  const ctx: WorkerContext = {
+    ...args,
+    forEachAxxxDirOrdered: async (callback: (axxxDir: string) => Promise<void>) => {
+      for (const axxxDir of await globAxxxDirs(streamingAssetsPath)) {
+        logger.log(`Processing ${axxxDir}`);
+        await callback(axxxDir);
+      }
+    },
+  };
+
+  const metadataKindProcessors: Record<MaimaiMetadataKind, WorkerProcessor<unknown>> = {
+    ...objectMap(basicDataTypes, dataType => dataType.process),
+    music: processMusic,
+  };
+
+  logger.log(`Writing metadata to ${outputFilePath}`);
+  await fs.promises.writeFile(outputFilePath, JSON.stringify(await metadataKindProcessors[metadataKind](ctx), null, 2));
+};
+
+export default async (args: WorkerArguments) => {
+  try {
+    await run(args);
+  } catch (error) {
+    logger.error(`Error processing metadata: ${error}`);
+    throw error;
+  }
+};