[O] Rewrite song and playlist logic

src/components/listitem/SongInfo.svelte

@@ -1,8 +1,9 @@
 <script lang="ts">
-  import type { NeteaseSongBrief } from "../../shared/types.ts";
+  import { artistAndAlbum } from "../../shared/tools.ts";
+  import type { NeteaseSong } from "../../shared/types.ts";
   import ImageListItem from "./ImageListItem.svelte";
 
-  let { info }: { info: NeteaseSongBrief } = $props();
+  let { info }: { info: NeteaseSong } = $props();
 </script>
 
-<ImageListItem photoUrl={info.albumPic} title={info.name} text={`${info.artists.map(a => a.name).join(", ")} - ${info.album}`} />
+<ImageListItem photoUrl={info.al.picUrl} title={info.name} text={artistAndAlbum(info)} />

src/lib/server/neteaseImport.ts

@@ -1,99 +0,0 @@
-import { getSongsFromPlaylist, getLyricsRaw } from './songs'
-import { db } from './db'
-import type { NeteaseSongBrief, UserDocument } from '../../shared/types'
-
-export interface ImportSession {
-    id: string
-    playlistId: number
-    userId: any
-    songs: {
-        song: NeteaseSongBrief
-        status: 'importing' | 'success' | 'failed-not-japanese' | 'failed-unknown'
-    }[]
-    done: boolean
-}
-
-const sessions = new Map<string, ImportSession>()
-
-export const getSession = (id: string) => sessions.get(id)
-
-/**
- * Start an import session
- * @param link Netease playlist link
- * @param userId User ID to associate the imported playlist with
- * @returns Import session
- */
-export async function startImport(link: string, user: UserDocument): Promise<ImportSession> {
-    const { meta, songs } = await getSongsFromPlaylist(link)
-    const importId = crypto.randomUUID()
-    
-    const session: ImportSession = {
-        id: importId,
-        playlistId: meta.id,
-        userId: user._id,
-        songs: songs.map(s => ({ song: s, status: 'importing' })),
-        done: false
-    }
-
-    // If there is another session importing the same playlist, return it
-    if (sessions.has(importId)) {
-        console.log(`Import session ${importId} already exists`)
-        return sessions.get(importId)!
-    }
-    
-    sessions.set(importId, session)
-    
-    // Start background processing
-    processImport(session).catch(err => console.error('Import failed', err))
-    
-    return session
-}
-
-/**
- * Process an import session
- * @param session Session to store data for retriving progress
- */
-async function processImport(session: ImportSession) {
-    const validSongs: NeteaseSongBrief[] = []
-    console.log(`Starting import: Playlist ${session.playlistId}`)
-
-    for (let i = 0; i < session.songs.length; i++) {
-        const item = session.songs[i]
-        try {
-            console.log(`Processing song ${item.song.id}`)
-            const lyrics = await getLyricsRaw(item.song.id)
-            if (lyrics.lang === 'jpn') {
-                item.status = 'success'
-                console.log(`Song ${item.song.id} is valid`)
-                validSongs.push(item.song)
-            } else {
-                item.status = 'failed-not-japanese'
-                console.log(`Song ${item.song.id} is not Japanese (is ${lyrics.lang})`)
-            }
-        } catch (e) {
-            console.error(`Failed to process song ${item.song.id}`, e)
-            item.status = 'failed-unknown'
-        }
-    }
-
-    session.done = true
-
-    // Save to database
-    if (validSongs.length > 0) {
-        await db.collection('playlists_imported').replaceOne(
-            { _id: session.playlistId as any },
-            { 
-                _id: session.playlistId, 
-                songs: validSongs,
-                importedAt: new Date()
-            },
-            { upsert: true }
-        )
-        
-        // Add to user's favorites
-        await db.collection('users').updateOne(
-            { _id: session.userId },
-            { $addToSet: { "data.myPlaylists": session.playlistId } }
-        )
-    }
-}

src/lib/server/songs.ts

@@ -1,6 +1,6 @@
 import * as ne from '@neteasecloudmusicapienhanced/api'
 import { aiParseLyrics } from './tools/lyrics'
-import type { NeteaseSongBrief, UserDocument } from '../../shared/types'
+import type { NeteaseSong, UserDocument } from '../../shared/types'
 import { db } from './db'
 import { franc } from 'franc'
 import { error } from '@sveltejs/kit'
@@ -36,61 +36,36 @@ function parsePlaylistRef(ref: string): number {
     throw new Error('Invalid playlist reference')
 }
 
-/**
+const getPlaylistRaw = cached('playlists_raw',
- * Get raw playlist data from cache or netease API.
- */
-export const getPlaylistRaw = cached('playlists',
     async (id: number) => {
         const pl = ((await ne.playlist_detail({ id })).body as any).playlist
 
         // Save each song
-        for (const track of pl.tracks) {
+        for (const track of pl.tracks)
-            await db.collection('songs').replaceOne({ _id: track.id }, { _id: track.id, data: track }, { upsert: true })
+            await db.collection('songs_raw').replaceOne({ _id: track.id }, { _id: track.id, data: track }, { upsert: true })
-        }
+
         return pl
     })
 
-// TODO: A better recommendation system
+function normalizeTimestamps(text: string): string {
-export const listRecPlaylists = async () => await db.collection('playlists').find()
+    // Replace all [dd:dd:dd] wit [dd:dd.dd]
-  .map(it => it.data).toArray()
+    return text.replace(/\[(\d+):(\d+):(\d+)\]/g, '[$1:$2.$3]')
-
-export const listMyPlaylists = async (user: UserDocument) => (await user.data.myPlaylists?.let(pl => db.collection('playlists').find({
-  _id: { $in: pl as any as ObjectId[] }
-}).map(it => it.data).toArray())) ?? []
-
-export const getSongMeta = cached('songs',
-    async (songId: number) => {
-        const detail = await ne.song_detail({ ids: songId.toString() })
-        return detail.body.songs[0]
-    })
-
-export const parseBrief = (songData: any): NeteaseSongBrief => ({
-    id: songData.id,
-    name: songData.name,
-    album: songData.al.name,
-    albumId: songData.al.id,
-    albumPic: songData.al.picUrl,
-    artists: songData.ar.map((ar: any) => ({ id: ar.id, name: ar.name }))
-})
-
-/**
- * Get a list of songs from a playlist reference.
- */
-export async function getSongsFromPlaylist(ref: string): Promise<{meta: any, songs: NeteaseSongBrief[]}> {
-    const playlistId = parsePlaylistRef(ref)
-    const plData = await getPlaylistRaw(playlistId)
-    return {meta: plData, songs: plData.tracks.map(parseBrief)}
 }
 
 interface NeteaseLyricsResponse { lrc: { lyric: string }, lang: string }
-
+const getLyricsRaw = cached('lyrics_raw',
-export const getLyricsRaw = cached('lyrics_raw',
     async (songId: number) => {
         const raw = (await ne.lyric({ id: songId })).body as any as NeteaseLyricsResponse
-        const lang = franc(raw.lrc.lyric)
+        const lang = franc(raw.lrc.lyric.replace(/\[.*?\]/g, '').replace(/\s+/g, ' ').trim())
+        raw.lrc.lyric = normalizeTimestamps(raw.lrc.lyric)
         return { ...raw, lang }
-    }
+    })
-)
+
+export const getSongRaw = cached('songs_raw',
+    async (songId: number) => {
+        const detail = await ne.song_detail({ ids: songId.toString() })
+        return detail.body.songs[0] as NeteaseSong
+    })
 
 export const getLyricsProcessed = cached('lyrics_processed',
     async (songId: number) => {
@@ -100,10 +75,110 @@ export const getLyricsProcessed = cached('lyrics_processed',
         return aiParseLyrics(raw.lrc.lyric)
     })
 
-await getSongsFromPlaylist("13555799996")
+
-await getSongsFromPlaylist("https://music.163.com/playlist?id=14348145982")
+export interface ImportSession {
-await getSongsFromPlaylist("https://music.163.com/playlist?id=14392963638")
+    id: string
-await getSongsFromPlaylist("https://music.163.com/playlist?id=580208139")
+    playlistId: number
-await getSongsFromPlaylist("https://music.163.com/playlist?id=17404030548")
+    userId?: any
+    songs: {
+        song: NeteaseSong
+        status: 'importing' | 'success' | 'failed-not-japanese' | 'failed-unknown'
+    }[]
+    done: boolean
+}
+
+const sessions = new Map<string, ImportSession>()
+
+export const getSession = (id: string) => sessions.get(id)
+
+/**
+ * Start an import session
+ * @param link Netease playlist link
+ * @param userId User ID to associate the imported playlist with
+ * @returns Import session
+ */
+export async function startImport(link: string, userId?: number): Promise<ImportSession> {
+    const meta = await getPlaylistRaw(parsePlaylistRef(link))
+    const importId = crypto.randomUUID()
+    
+    const session: ImportSession = {
+        id: importId,
+        playlistId: meta.id,
+        userId,
+        songs: meta.tracks.map((s: any) => ({ song: s, status: 'importing' })),
+        done: false
+    }
+
+    // If there is another session importing the same playlist, return it
+    if (sessions.has(importId)) {
+        console.log(`Import session ${importId} already exists`)
+        return sessions.get(importId)!
+    }
+    
+    sessions.set(importId, session)
+    
+    // Start background processing
+    processImport(session, meta).catch(err => console.error('Import failed', err))
+    
+    return session
+}
+
+/**
+ * Process an import session
+ * @param session Session to store data for retriving progress
+ * @param data Playlist metadata
+ */
+async function processImport(session: ImportSession, data: any) {
+    console.log(`Starting import: Playlist ${session.playlistId}`)
+    data.tracks = (await Promise.all(session.songs.map(async item => {
+        try {
+            const lyrics = await getLyricsRaw(item.song.id)
+            console.log(`Song ${item.song.id} lang ${lyrics.lang}`)
+            if (lyrics.lang === 'jpn') {
+                item.status = 'success'
+                return item.song
+            } else item.status = 'failed-not-japanese'
+        } catch (e) {
+            console.error(`Failed to process song ${item.song.id}`, e)
+            item.status = 'failed-unknown'
+        }
+    }))).filter(it => it !== undefined)
+
+    session.done = true
+
+    // Save to database
+    await db.collection('playlists').replaceOne(
+        { _id: session.playlistId as any },
+        { _id: session.playlistId, data, importedAt: new Date() },
+        { upsert: true }
+    )
+    
+    // Add to user's favorites
+    session.userId?.let(async (uid: any) => await db.collection('users').updateOne(
+        { _id: uid },
+        { $addToSet: { "data.myPlaylists": session.playlistId } }
+    ))
+}
+
+
+
+
+// TODO: A better recommendation system
+export const listRecPlaylists = async () => await db.collection('playlists').find().limit(10).map(it => it.data).toArray()
+export const listMyPlaylists = async (user: UserDocument) => (await user.data.myPlaylists?.let(pl => db.collection('playlists').find({
+  _id: { $in: pl as any as ObjectId[] }
+}).map(it => it.data).toArray())) ?? []
+
+export const getPlaylist = async (playlistId: number | string) => {
+    const plData = await db.collection('playlists').findOne({ _id: +playlistId as any })
+    if (!plData) throw error(404, 'Playlist not found')
+    return plData.data
+}
+
+await startImport("13555799996")
+await startImport("https://music.163.com/playlist?id=14348145982")
+await startImport("https://music.163.com/playlist?id=14392963638")
+await startImport("https://music.163.com/playlist?id=580208139")
+await startImport("https://music.163.com/playlist?id=17404030548")
 
 // TODO: Filter out non-Japanese songs

src/lib/server/tools/lyrics.ts

@@ -187,7 +187,8 @@ export async function aiParseLyricsRaw(raw: string): Promise<LyricLine[]> {
   
   // Clean up potential markdown blocks
   const responseText = text.replace(/```/g, '').trim()
-  console.log('AI response text:\n', responseText)
+  console.log('AI request:\n', raw)
+  console.log('AI response:\n', responseText)
   console.log(`Finish reason: ${response.choices[0].finish_reason}`)
   // If response does not contain any timestamp, something is wrong
   if (!/\[\d+:\d+\.\d+\]/.test(responseText)) {

src/routes/+page.server.ts

@@ -1,9 +1,9 @@
 // import { log } from 'console';
-import { getSongMeta, listMyPlaylists, listRecPlaylists, parseBrief } from '../lib/server/songs';
+import { listMyPlaylists, listRecPlaylists } from '../lib/server/songs';
 import type { PageServerLoad } from './$types';
 
 export const load: PageServerLoad = async ({ params, parent }) => {
-  let last = parseBrief(await getSongMeta(25723366))
+  let last = undefined
   const { user } = await parent()
   let myPlaylists = await listMyPlaylists(user)
   let recPlaylists = await listRecPlaylists()

src/routes/api/import/netease/progress/+server.ts

@@ -1,5 +1,5 @@
 import { error, json } from '@sveltejs/kit';
-import { getSession } from '$lib/server/neteaseImport';
+import { getSession } from '$lib/server/songs';
 import type { RequestHandler } from './$types';
 
 export const POST: RequestHandler = async ({ request }) => {

src/routes/api/import/netease/start/+server.ts

@@ -1,7 +1,7 @@
 import { error, json } from '@sveltejs/kit';
-import { startImport } from '$lib/server/neteaseImport';
 import { login } from '$lib/server/user';
 import type { RequestHandler } from './$types';
+import { startImport } from '$lib/server/songs';
 
 export const POST: RequestHandler = async ({ request, cookies }) => {
     const { link } = await request.json();
@@ -11,7 +11,7 @@ export const POST: RequestHandler = async ({ request, cookies }) => {
     if (!user) throw error(401, 'Unauthorized');
 
     try {
-        return json(await startImport(link, user))
+        return json(await startImport(link, user._id))
     } catch (e) {
         console.error(e)
         throw error(500, 'Failed to start import')

src/routes/import/netease/+page.svelte

@@ -2,14 +2,14 @@
   import { LinearProgress, TextFieldOutlined } from "m3-svelte"
   import AppBar from "../../../components/appbar/AppBar.svelte"
   import Button from "../../../components/Button.svelte"
-  import type { NeteaseSongBrief } from "../../../shared/types"
+  import type { NeteaseSong } from "../../../shared/types"
   import { API } from "../../../lib/client"
     import ErrorDialog from "../../../components/status/ErrorDialog.svelte";
 
   let link = $state('')
 
   interface SongImportStatus {
-    song: NeteaseSongBrief
+    song: NeteaseSong
     status: 'importing' | 'success' | 'failed-not-japanese' | 'failed-unknown'
   }
 
@@ -90,7 +90,7 @@
           <span class="{statusToIcon(song.status)} text-xl"></span>
           <div class="vbox">
             <span class="m3-font-title-medium">{song.song.name}</span>
-            <span class="m3-font-body-small mfg-on-surface-variant">{song.song.artists.map(a => a.name).join(', ')}</span>
+            <span class="m3-font-body-small mfg-on-surface-variant">{song.song.ar.map(a => a.name).join(', ')}</span>
           </div>
         </div>
       {/each}

src/routes/playlist/[id]/+page.server.ts

@@ -1,6 +1,6 @@
 import type { PageServerLoad } from './$types';
-import { getSongsFromPlaylist, listRecPlaylists } from "$lib/server/songs.ts";
+import { getPlaylist } from "$lib/server/songs.ts";
 
 export const load: PageServerLoad = async ({ params }) => ({
-  playlist: await getSongsFromPlaylist(params.id)
+  playlist: await getPlaylist(params.id)
 })

src/routes/playlist/[id]/+page.svelte

@@ -7,8 +7,8 @@
 
   let { data }: PageProps = $props()
 
-  let meta = $derived(data.playlist.meta)
+  let meta = $derived(data.playlist)
-  let songs = $derived(data.playlist.songs)
+  let songs = $derived(data.playlist.tracks)
   let isFavorite = $derived(data.user.data.myPlaylists?.includes(meta.id) ?? false)
 
   async function toggleFavorite() {
@@ -44,11 +44,11 @@
   </div>
 </div>
 
-<div class="vbox gap-12px">
+<div class="hbox gap-12px items-end! h-48px p-content">
-  <div class="hbox gap-12px items-end! h-48px p-content">
+  <div class="m3-font-headline-small">歌曲列表</div>
-    <div class="m3-font-headline-small">歌曲列表</div>
+  <div class="m3-font-label-small pb-3px">{songs.length} 首歌曲</div>
-    <div class="m3-font-label-small pb-3px">{songs.length} 首歌曲</div>
+</div>
-  </div>
+<div class="vbox gap-12px mt-12px min-h-0 overflow-y-auto">
   {#each songs as song, index}
     <a href="/song/{song.id}" class="p-content">
       <SongInfo info={song} />

src/routes/results/[id]/+page.server.ts

@@ -1,17 +1,17 @@
 import { error } from '@sveltejs/kit'
 import { getResult } from '$lib/server/result'
 import type { PageServerLoad } from './$types'
-import { getLyricsProcessed, getSongMeta, parseBrief } from "$lib/server/songs.ts";
+import { getLyricsProcessed, getSongRaw } from "$lib/server/songs.ts";
 
 export const load: PageServerLoad = async ({ params }) => {
   const result = await getResult(params.id)
   if (!result) throw error(404, 'Result not found')
 
-  const song = await getSongMeta(result.songId)
+  const song = await getSongRaw(result.songId)
 
   return { 
     result: structuredClone(result),
     lrc: await getLyricsProcessed(result.songId),
-    song, brief: parseBrief(song)
+    song
   }
 }

src/routes/results/[id]/+page.svelte

@@ -97,7 +97,7 @@
   });
 </script>
 
-<AppBar title={data.brief.name} sub={artistAndAlbum(data.brief)} />
+<AppBar title={data.song.name} sub={artistAndAlbum(data.song)} />
 
 <div class="vbox gap-16px p-content flex-1 overflow-y-auto">
   <div class="hbox gap-12px items-end! h-48px">

src/routes/song/[id]/+page.server.ts

@@ -1,10 +1,9 @@
 import type { PageServerLoad } from './$types'
-import { getLyricsProcessed, getSongMeta, listRecPlaylists, parseBrief } from "$lib/server/songs.ts";
+import { getLyricsProcessed, getSongRaw, listRecPlaylists } from "$lib/server/songs.ts";
 
 export const load: PageServerLoad = async ({ params }) => {
   const songId = +params.id
-  const raw = await getSongMeta(songId)
+  const song = await getSongRaw(songId)
-  const brief = parseBrief(raw)
   const lrc = await getLyricsProcessed(songId)
-  return { raw, brief, lrc }
+  return { song, lrc }
 }

src/routes/song/[id]/+page.svelte

@@ -146,16 +146,16 @@
   // Result is stored on the server and is fetched from a separate results page
   async function submitResult() {
     const res = await API.saveResult({
-      songId: data.raw.id,
+      songId: data.song.id,
       endTime: Date.now(),
-      realTimeFactor: (Date.now() - startTime) / (data.raw.dt / 1000),
+      realTimeFactor: (Date.now() - startTime) / (data.song.dt / 1000),
       totalTyped, totalRight, startTime, statsHistory
     })
     goto(`/results/${res.id}`)
   }
 </script>
 
-<AppBar title={data.brief.name} sub={artistAndAlbum(data.brief)}>
+<AppBar title={data.song.name} sub={artistAndAlbum(data.song)}>
   <MenuItem textIcon="あ" onclick={() => settings.isFuri = !settings.isFuri}>{settings.isFuri ? "隐藏" : "显示"}假名标注</MenuItem>
   <MenuItem textIcon="カ" onclick={() => settings.allKata = !settings.allKata}>{settings.allKata ? "恢复平假名" : "全部转换为片假名"}</MenuItem>
   <MenuItem icon="i-material-symbols:language-japanese-kana-rounded" onclick={() => settings.showRomaji = !settings.showRomaji}>{settings.showRomaji ? "隐藏罗马音" : "显示罗马音"}</MenuItem>

src/shared/tools.ts

@@ -1,3 +1,3 @@
-import type { NeteaseSongBrief } from "./types";
+import type { NeteaseSong } from "./types";
 
-export const artistAndAlbum = (song: NeteaseSongBrief) => `${song.artists.map(it => it.name).join(', ')} - ${song.album}`
+export const artistAndAlbum = (song: NeteaseSong) => `${song.ar.map(it => it.name).join(', ')} - ${song.al.name}`

src/shared/types.ts

@@ -10,13 +10,26 @@ export interface Song {
     lyrics: LyricLine[]
 }
 
-export interface NeteaseSongBrief { 
+export interface NeteaseSong { 
     id: number
     name: string
-    album: string
+    al: {
-    albumId: number
+        id: number
-    albumPic: string
+        name: string
-    artists: { id: number, name: string }[]
+        picUrl: string
+    }
+    ar: { id: number, name: string }[]
+    dt: number
+}
+
+export interface NeteasePlaylist {
+    id: number
+    name: string
+    coverImgUrl: string
+    creator: {
+        nickname: string
+    }
+    tracks: NeteaseSong[]
 }
 
 export interface ProcessedLyricLine {