[+] Music mode

2025-11-22 10:56:36 +08:00
parent c543eeae62
commit e15c7eb641
6 changed files with 145 additions and 20 deletions
@@ -13,4 +13,7 @@ Practice Japanese Karaoke lyrics reading and typing at the same time with KashiD
 5. 纯前端存储数据

 TODO: 404 page
-TODO: Update an existing playlist
+TODO: Update an existing playlist
+
+TODO: Allow users to correct lyric pronunciations through correction feedback
+TODO: Correct lyrics timing inconsistencies (i.e. 网易云的歌词因为是业余用户上传的，时间戳不一定准确。但是 waveform 里面可以分析出每句歌词的具体开始结束时间，也许可以自动修正)
@@ -6,12 +6,14 @@
    icon,
    textIcon,
    disabled = false,
+    sub,
    onclick,
    children,
  }: {
    icon?: string;
    textIcon?: string;
    disabled?: boolean;
+    sub?: string;
    onclick: () => void;
    children: Snippet;
  } = $props();
@@ -26,7 +28,12 @@
      {textIcon}
    </span>
  {/if}
-  {@render children()}
+  <div class="vbox items-start">
+    {@render children()}
+    {#if sub}
+      <div class="m3-font-body-small opacity-60">{sub}</div>
+    {/if}
+  </div>
 </button>

 <style lang="scss">
@@ -51,7 +51,8 @@
 <ProgressList percentage={progressPercentage} items={progressItems} />

 {#if loadStatus === "done"}
-  <div class="vbox p-16px">
-    <Button big icon="i-material-symbols:play-arrow-rounded" onclick={() => goto(`/song/${data.song.id}/play`)}>开始</Button>
+  <div class="hbox gap-4 p-16px">
+    <Button big icon="i-material-symbols:keyboard-rounded" onclick={() => goto(`/song/${data.song.id}/play`)}>打字模式</Button>
+    <Button big icon="i-material-symbols:music-note-rounded" onclick={() => goto(`/song/${data.song.id}/play?music=true`)}>音乐模式</Button>
  </div>
 {/if}
@@ -2,7 +2,7 @@ import type { PageServerLoad } from './$types'
 import { getLyricsProcessed, getSongRaw, getSongUrl, checkLyricsProcessed } from "$lib/server/songs.ts"
 import { redirect } from '@sveltejs/kit'

-export const load: PageServerLoad = async ({ params }) => {
+export const load: PageServerLoad = async ({ params, url }) => {
  const songId = +params.id
  const song = await getSongRaw(songId)
  const hasLrc = await checkLyricsProcessed(songId)
@@ -10,6 +10,6 @@ export const load: PageServerLoad = async ({ params }) => {
  if (!hasLrc) throw redirect(302, `/song/${songId}`)

  const lrc = await getLyricsProcessed(songId)!
-  // const audioUrl = await getSongUrl(songId)
-  return { song, lrc, audioUrl: undefined }
+  const audioUrl = url.searchParams.get('music') === 'true' ? await getSongUrl(songId) : undefined
+  return { song, lrc, audioUrl }
 }
@@ -12,6 +12,7 @@
  import { animateCaret } from "./animation.ts"
  import { goto } from '$app/navigation'
  import { artistAndAlbum } from "../../../../shared/tools.ts"
+  import { MusicControl } from "./MusicControl.ts"

  let { data }: PageProps = $props()

@@ -34,12 +35,17 @@
  const preprocessKana = (kana: string, state?: string) => (settings.showRomaji || (settings.showRomajiOnError && state === 'wrong')) ? `<ruby>${_preprocessKana(kana)}<rt>${toRomaji(kana)}</rt></ruby>` : _preprocessKana(kana)

  // Process each line into segments with swi (start word index) and kanji/kana
-  let processedLrc: ProcLrcLine[] = $derived(dedupLines(data.lrc, settings.hideRepeated).map(line => processLrcLine(line.lyric)))
+  const isHideRepeated = $derived(settings.hideRepeated && !data.audioUrl)
+  let deduplicatedLyrics = $derived(dedupLines(data.lrc, isHideRepeated))
+  let processedLrc: ProcLrcLine[] = $derived(deduplicatedLyrics.map(line => processLrcLine(line.lyric)))
  // State tracking for each kana character: UNSEEN, RIGHT, WRONG
-  let states = $state(dedupLines(data.lrc, settings.hideRepeated).map(line => processLrcLine(line.lyric)).map(line => new Array(line.totalLen).fill('unseen')))
+  let states = $state(processedLrc.map(line => new Array(line.totalLen).fill('unseen')))
  
+  let musicControl: MusicControl | undefined
+  $effect(() => { li; musicControl?.updateLine(li) })
+
+  // Reset when processedLrc changes (settings changed)
  $effect(() => {
-    // Reset when processedLrc changes (settings changed)
    states = processedLrc.map(line => new Array(line.totalLen).fill('unseen'))
    li = 0
    wi = 0
@@ -73,10 +79,17 @@
    const onBlur = () => setTimeout(() => hiddenInput?.focus(), 10)
    hiddenInput.addEventListener('blur', onBlur)

+    if (data.audioUrl) {
+      musicControl = new MusicControl(data.audioUrl)
+      musicControl.setLyrics(deduplicatedLyrics)
+      musicControl.start()
+    }
+
    const interval = setInterval(() => { if (startTime) now = Date.now() }, 1000)
    return () => {
      clearInterval(interval)
      hiddenInput?.removeEventListener('blur', onBlur)
+      musicControl?.dispose()
    }
  })

@@ -89,6 +102,11 @@
    inp = toHiragana(inp.replaceAll(' ', ''), { IMEMode: true })
    const imeUsed = input !== inp

+    // Prevent IME stuck (If the user enters "yto" -> "yと", the "y" should be ignored)
+    if (imeUsed && inp && !isKana(inp[0]) && inp.split('').some(c => isKana(c)) ) {
+      while (inp && !isKana(inp[0])) inp = inp.slice(1)
+    }
+
    function findLoc() {
      let cLine = processedLrc[li]
      let cSeg = cLine.parts.find(seg => wi >= seg.swi && wi < seg.swi + seg.kana.length)!
@@ -128,13 +146,7 @@
    }

    // Check next expected character, if it's neither kana nor kanji, skip it
-    while (findLoc().let(({ exp, cLine }) => !isKana(exp) && !isKanji(exp) && incr(cLine)))
-
-    // Prevent IME stuck
-    if (imeUsed && inp && !isKana(inp[0]) && inp.split('').some(c => isKana(c)) ) {
-      console.log("Clearing input to prevent IME stuck")
-      inp = ""
-    }
+    while (findLoc().let(({ exp, cLine }) => !isKana(exp) && !isKanji(exp) && incr(cLine))) {}
  }
  $effect(() => inputChanged(inp, false))

@@ -177,7 +189,8 @@
  }
 </script>

-<svelte:window onresize={() => caret && animateCaret(caret)} />
+<!-- Window events -->
+<svelte:window onresize={() => caret && animateCaret(caret)} onclick={() => musicControl?.ready()} onkeydown={() => musicControl?.ready()}/>


 <AppBar title={data.song.name} sub={artistAndAlbum(data.song)}>
@@ -185,9 +198,12 @@
  <MenuItem textIcon="カ" onclick={() => settings.allKata = !settings.allKata}>{settings.allKata ? "恢复平假名" : "全部转换为片假名"}</MenuItem>
  <MenuItem icon="i-material-symbols:language-japanese-kana-rounded" onclick={() => settings.showRomaji = !settings.showRomaji}>{settings.showRomaji ? "隐藏罗马音" : "显示罗马音"}</MenuItem>
  <MenuItem icon="i-material-symbols:error-circle-rounded" onclick={() => settings.showRomajiOnError = !settings.showRomajiOnError}>{settings.showRomajiOnError ? "不在错误时显示罗马音" : "错误时显示罗马音"}</MenuItem>
-  <MenuItem icon="i-material-symbols:compress-rounded" onclick={() => settings.hideRepeated = !settings.hideRepeated}>{settings.hideRepeated ? "显示重复行" : "隐藏重复行"}</MenuItem>
+  <MenuItem icon="i-material-symbols:compress-rounded" 
+    disabled={!!data.audioUrl}
+    sub={data.audioUrl ? "音乐模式下不可用" : ""}
+    onclick={() => settings.hideRepeated = !settings.hideRepeated}>{isHideRepeated ? "显示重复行" : "隐藏重复行"}</MenuItem>
  {#if loc}
-    <MenuItem icon={loc.playMode === 'random' ? "i-material-symbols:shuffle-rounded" : "i-material-symbols:repeat-rounded"} onclick={() =>
+  <MenuItem icon={loc.playMode === 'random' ? "i-material-symbols:shuffle-rounded" : "i-material-symbols:repeat-rounded"} onclick={() =>
    loc.playMode = loc.playMode === 'random' ? 'sequential' : 'random'}>{loc.playMode === 'random' ? "当前：随机播放" : "当前：顺序播放"}</MenuItem>
  {/if}
 </AppBar>
@@ -0,0 +1,98 @@
+import * as Tone from 'tone'
+import type { LyricLine } from '../../../../shared/types'
+
+export class MusicControl {
+  player: Tone.Player
+  lyrics: LyricLine[] = []
+  currentLineIndex: number = 0
+  checkInterval: any
+  isLoaded = false
+
+  audioUrl: string
+
+  constructor(audioUrl: string) {
+    this.audioUrl = audioUrl
+    this.player = new Tone.Player(audioUrl).toDestination()
+  }
+
+  log(msg: string) {
+    console.log(`%c[MusicControl] ${msg}`, 'color: red')
+  }
+
+  setLyrics(lyrics: LyricLine[]) {
+    this.lyrics = lyrics
+  }
+
+  updateLine(index: number) {
+    this.log(`updateLine: ${index}`)
+    this.currentLineIndex = index
+    if (Tone.getTransport().state === 'paused') {
+      Tone.getTransport().start()
+    }
+  }
+
+  async ready() {
+    if (Tone.getContext().state !== 'running') {
+      await Tone.start()
+      this.log(`Tone started! Context state: ${Tone.getContext().state}`)
+    }
+  }
+
+  async start() {
+    this.log('start() called')
+    await this.ready().catch(e => this.log(`Tone.start() failed: ${e}`))
+
+    if (!this.player.loaded) {
+      this.log('Loading audio...')
+      await this.player.load(this.audioUrl)
+      this.log('Audio loaded')
+    }
+
+    // Sync player to transport and schedule start at 0
+    // We do this regardless of transport state to ensure it's scheduled
+    this.player.sync().start(0)
+
+    if (Tone.getTransport().state !== 'started') {
+      this.log('Starting Transport')
+      Tone.getTransport().start()
+    }
+    this.startCheckLoop()
+  }
+
+  startCheckLoop() {
+    if (this.checkInterval) return
+    this.checkInterval = setInterval(() => this.check(), 50)
+  }
+
+  check() {
+    if (Tone.getTransport().state !== 'started') return
+    const ct = Tone.getTransport().seconds
+    const ni = this.currentLineIndex + 1
+    if (ni >= this.lyrics.length) return
+    const nt = this.parseTime(this.lyrics[ni].time)
+    if (ct >= nt) {
+      this.log(`Current time: ${ct.toFixed(2)}, Next line time: ${nt.toFixed(2)}, Pausing!`)
+      Tone.getTransport().pause()
+    }
+  }
+
+  parseTime(timeStr: string): number {
+    // mm:ss.xx or mm:ss.xxx
+    // Example: 00:12.34
+    const match = timeStr.match(/(\d+):(\d+(\.\d+)?)/)
+    if (match) {
+      const min = parseFloat(match[1])
+      const sec = parseFloat(match[2])
+      return min * 60 + sec
+    }
+    this.log(`Invalid time string: ${timeStr}`)
+    return 0
+  }
+
+  dispose() {
+    if (this.checkInterval) clearInterval(this.checkInterval)
+    this.player.dispose()
+    Tone.getTransport().stop()
+    Tone.getTransport().cancel()
+  }
+}