feat: initial commit
This commit is contained in:
Menci
@@ -0,0 +1,12 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>MaigoLabs :: needLe</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
<script type="module" src="/src/main.tsx"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "@maigolabs/needle-demo",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"typecheck": "tsc",
|
||||
"dev": "vite --port 5172",
|
||||
"build": "tsc -b && vite build"
|
||||
},
|
||||
"license": "AGPL-3.0",
|
||||
"packageManager": "pnpm@10.20.0",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@maigolabs/needle": "workspace:*",
|
||||
"react": "^19.2.0",
|
||||
"react-dom": "^19.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@iconify-json/svg-spinners": "^1.2.4",
|
||||
"@types/node": "^24.10.1",
|
||||
"@types/react": "^19.2.5",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@vitejs/plugin-react": "^5.1.1",
|
||||
"unocss": "^66.5.12",
|
||||
"vite": "^7.2.4",
|
||||
"vite-plugin-top-level-await": "^1.6.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
/fonts
|
||||
@@ -0,0 +1,168 @@
|
||||
import { TokenType } from '@maigolabs/needle/common';
|
||||
import {
|
||||
searchInvertedIndex,
|
||||
highlightSearchResult,
|
||||
type SearchResult,
|
||||
} from '@maigolabs/needle/searcher';
|
||||
import { useState, type FunctionComponent } from 'react';
|
||||
|
||||
type Tab = 'search' | 'tokenize';
|
||||
|
||||
type AppData = typeof import('./data');
|
||||
export const Layout: FunctionComponent<{ dataPromise: Promise<AppData> }> = ({ dataPromise }) => {
|
||||
const [appData, setAppData] = useState<AppData | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
void dataPromise.then(props => setAppData(props)).catch(error => setError((error instanceof Error ? error.stack : undefined) ?? String(error)));
|
||||
return (
|
||||
<div className="min-h-screen bg-[#f9f2e0] text-[#8b7355] font-mono selection:bg-[#d4c4b0]/70">
|
||||
<div className="max-w-200 mx-auto px-4 pt-8 pb-6">
|
||||
<header className="mb-8">
|
||||
<h1 className="pb-3 text-2xl text-[#a08060]">MaigoLabs :: needLe</h1>
|
||||
<div className="pb-4 text-sm">
|
||||
<p>Fuzzy search engine for small text pieces, with Chinese/Japanese pronunciation support</p>
|
||||
<p>(Available in TypeScript and C#)</p>
|
||||
</div>
|
||||
<div className="flex gap-4 text-sm">
|
||||
<a href="https://github.com/MaigoLabs/needLe" target="_blank" rel="noopener" className="text-[#b8a890] hover:text-[#8b7355]">[GitHub]</a>
|
||||
<a href="https://www.npmjs.com/package/@maigolabs/needle" target="_blank" rel="noopener" className="text-[#b8a890] hover:text-[#8b7355]">[NPM]</a>
|
||||
<a href="https://www.nuget.org/packages/MaigoLabs.NeedLe" target="_blank" rel="noopener" className="text-[#b8a890] hover:text-[#8b7355]">[NuGet]</a>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{
|
||||
appData
|
||||
? <App appData={appData} />
|
||||
: error
|
||||
? <div className="text-sm bg-[#efe5d0] px-4 py-3 rounded-lg whitespace-pre-wrap">{error}</div>
|
||||
: <div>
|
||||
<div className="flex flex-row items-center gap-2"><div className="i-svg-spinners:ring-resize" /> Loading...</div>
|
||||
<div className="mt-6 text-sm bg-[#efe5d0] px-4 py-3 rounded-lg">
|
||||
<div className="font-bold mb-2">Tips:</div>
|
||||
<div>This demo loads Kuromoji/OpenCC/pinyin-pro for tokenization and index building.</div>
|
||||
<div>However, searching on a prebuilt index doesn't require loading any external library/dictionary.</div>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
interface AppProps {
|
||||
appData: AppData;
|
||||
}
|
||||
|
||||
export const App: FunctionComponent<AppProps> = ({ appData: { kuromoji, createTokenizer, invertedIndex } }) => {
|
||||
const [input, setInput] = useState('');
|
||||
const [tab, setTab] = useState<Tab>('search');
|
||||
|
||||
const searchResults = tab === 'search' && input.trim()
|
||||
? searchInvertedIndex(invertedIndex, input).slice(0, 50)
|
||||
: [];
|
||||
|
||||
const tokenizeResults = tab === 'tokenize' && input.trim()
|
||||
? (() => {
|
||||
const tokenizer = createTokenizer({ kuromoji });
|
||||
const tokens = tokenizer.tokenize(input);
|
||||
const tokenDefs = tokenizer.tokens;
|
||||
const codePoints = [...input];
|
||||
return tokens.map(t => {
|
||||
const def = [...tokenDefs.values()].find(d => d.id === t.id)!;
|
||||
const original = codePoints.slice(t.start, t.end).join('');
|
||||
return { ...t, type: def.type, text: def.text, original };
|
||||
});
|
||||
})()
|
||||
: [];
|
||||
|
||||
return (
|
||||
<>
|
||||
<input
|
||||
type="text"
|
||||
value={input}
|
||||
onChange={e => setInput(e.target.value)}
|
||||
placeholder={`Type something to ${tab}...`}
|
||||
className="w-full bg-[#efe5d0] text-[#6b5a48] px-3 py-2 mb-2 outline-none placeholder-[#b8a890] rounded-lg"
|
||||
/>
|
||||
|
||||
<div className="flex gap-4 mb-6 text-sm">
|
||||
<button
|
||||
onClick={() => setTab('search')}
|
||||
className={`bg-transparent border-none cursor-pointer ${tab === 'search' ? 'text-[#6b5a48]' : 'text-[#c0b0a0]'}`}
|
||||
>
|
||||
Search
|
||||
</button>
|
||||
<button
|
||||
onClick={() => setTab('tokenize')}
|
||||
className={`bg-transparent border-none cursor-pointer ${tab === 'tokenize' ? 'text-[#6b5a48]' : 'text-[#c0b0a0]'}`}
|
||||
>
|
||||
Tokenize
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
{tab === 'search' && searchResults.map((result, i) => (
|
||||
<SearchResultItem key={i} result={result} input={input} />
|
||||
))}
|
||||
|
||||
{tab === 'tokenize' && tokenizeResults.length > 0 && (
|
||||
<div className="grid grid-cols-[repeat(auto-fill,minmax(280px,1fr))] gap-1">
|
||||
{tokenizeResults.map((token, i) => (
|
||||
<div key={i} className="bg-[#efe5d0] px-3 py-2 text-sm truncate rounded-lg">
|
||||
<span className="text-[#a08060]">{TokenType[token.type]}: </span>
|
||||
<span className="text-[#6b5a48]">{JSON.stringify(token.text)}</span>
|
||||
<span className="text-[#c0b0a0]">{' <- '}</span>
|
||||
<span className="text-[#8b7355]">{JSON.stringify(token.original)}</span>
|
||||
<span className="text-[#c8bba8]">{` [${token.start}, ${token.end}]`}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{input.trim() && tab === 'search' && searchResults.length === 0 && (
|
||||
<div className="text-[#b8a890] text-sm">No results.</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const SearchResultItem: FunctionComponent<{ result: SearchResult; input: string }> = ({ result, input }) => {
|
||||
const highlighted = highlightSearchResult(result);
|
||||
const inputCodePoints = [...input];
|
||||
|
||||
const stats = [
|
||||
`${result.rangeCount} range(s)`,
|
||||
`${Math.round(result.matchRatio * 100)}%`,
|
||||
result.prefixMatchCount > 0 ? `${result.prefixMatchCount} prefix` : null,
|
||||
].filter(Boolean).join(', ');
|
||||
|
||||
return (
|
||||
<div className="bg-[#efe5d0] px-3 py-2 text-sm rounded-lg">
|
||||
<div className="flex gap-2">
|
||||
<div className="flex-1 truncate">
|
||||
{highlighted.map((part, i) =>
|
||||
typeof part === 'string'
|
||||
? <span key={i} className="text-[#b8a890]">{part}</span>
|
||||
: <span key={i} className="text-[#5a4a38]">{part.highlight}</span>)}
|
||||
</div>
|
||||
<div className="text-[#c8bba8] shrink-0">{stats}</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-[repeat(auto-fill,minmax(200px,1fr))] gap-x-2 mt-1">
|
||||
{result.tokens.map((token, i) => {
|
||||
const inputText = inputCodePoints.slice(token.inputOffset.start, token.inputOffset.end).join('');
|
||||
const docText = result.documentCodePoints.slice(token.documentOffset.start, token.documentOffset.end).join('');
|
||||
return (
|
||||
<div key={i} className="text-[11px] truncate">
|
||||
<span className="text-[#b8a890]">{TokenType[token.definition.type]}: </span>
|
||||
<span className="text-[#8b7355]">{JSON.stringify(inputText)}</span>
|
||||
<span className="text-[#c8bba8]">{' -> '}</span>
|
||||
<span className="text-[#6b5a48]">{JSON.stringify(docText)}</span>
|
||||
{token.isTokenPrefixMatching && <span className="text-[#b8a890]">{' (prefix)'}</span>}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,26 @@
|
||||
import { buildInvertedIndex } from '@maigolabs/needle/indexer';
|
||||
import { loadInvertedIndex } from '@maigolabs/needle/searcher';
|
||||
import { TokenizerBuilder } from '@patdx/kuromoji';
|
||||
|
||||
// Indexer loads OpenCC and pinyin-pro which is large, put them in data.ts for dynamic importing.
|
||||
export { createTokenizer } from '@maigolabs/needle/indexer';
|
||||
|
||||
const musicNames: string[] = [...new Set(
|
||||
Object.values(
|
||||
await (await fetch('https://sekai-world.github.io/sekai-master-db-diff/musics.json')).json(),
|
||||
).map(music => (music as { title: string }).title),
|
||||
)];
|
||||
|
||||
export const kuromoji = await new TokenizerBuilder({
|
||||
loader: {
|
||||
loadArrayBuffer: async (url: string) => {
|
||||
url = `https://cdn.jsdelivr.net/npm/@aiktb/kuromoji@1.0.2/dict/${url.replace('.gz', '')}`;
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`Failed to fetch ${url}`);
|
||||
return await res.arrayBuffer();
|
||||
},
|
||||
},
|
||||
}).build();
|
||||
|
||||
export const compressed = buildInvertedIndex(musicNames, { kuromoji });
|
||||
export const invertedIndex = loadInvertedIndex(compressed);
|
||||
@@ -0,0 +1,12 @@
|
||||
import { StrictMode } from 'react';
|
||||
import { createRoot } from 'react-dom/client';
|
||||
|
||||
import { Layout } from './App';
|
||||
import 'virtual:uno.css';
|
||||
import '@unocss/reset/tailwind.css';
|
||||
|
||||
createRoot(document.getElementById('root')!).render(
|
||||
<StrictMode>
|
||||
<Layout dataPromise={import('./data')} />
|
||||
</StrictMode>,
|
||||
);
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"jsx": "preserve",
|
||||
"lib": ["DOM", "DOM.Iterable", "ESNext", "WebWorker"],
|
||||
"types": ["vite/client"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"resolveJsonModule": true,
|
||||
"allowJs": true,
|
||||
"strict": true,
|
||||
"strictNullChecks": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"skipLibCheck": true,
|
||||
"rootDir": ".",
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*.ts", "src/**/*.tsx"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { createLocalFontProcessor } from '@unocss/preset-web-fonts/local';
|
||||
import { defineConfig, presetWind3, presetTypography, presetWebFonts, transformerVariantGroup, transformerDirectives, presetIcons } from 'unocss';
|
||||
|
||||
export default defineConfig({
|
||||
presets: [
|
||||
presetWind3(),
|
||||
presetTypography(),
|
||||
presetIcons({
|
||||
scale: 1.2,
|
||||
warn: true,
|
||||
}),
|
||||
presetWebFonts({
|
||||
fonts: {
|
||||
mono: {
|
||||
name: 'Maple Mono',
|
||||
provider: 'fontsource',
|
||||
},
|
||||
},
|
||||
processors: createLocalFontProcessor({
|
||||
cacheDir: 'node_modules/.cache/unocss/fonts',
|
||||
fontAssetsDir: 'public/assets/fonts/cache',
|
||||
fontServeBaseUrl: '/assets/fonts/cache',
|
||||
}),
|
||||
}),
|
||||
],
|
||||
transformers: [
|
||||
transformerDirectives(),
|
||||
transformerVariantGroup(),
|
||||
],
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
import { defineConfig } from 'vite'
|
||||
import UnoCSS from 'unocss/vite'
|
||||
import react from '@vitejs/plugin-react'
|
||||
import topLevelAwait from 'vite-plugin-top-level-await'
|
||||
|
||||
// https://vite.dev/config/
|
||||
export default defineConfig({
|
||||
plugins: [react(), UnoCSS(), topLevelAwait()],
|
||||
build: {
|
||||
assetsInlineLimit: 0,
|
||||
minify: true
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "@maigolabs/needle-playground-bot",
|
||||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "tsx src/index.ts",
|
||||
"typecheck": "tsc"
|
||||
},
|
||||
"license": "AGPL-3.0",
|
||||
"packageManager": "pnpm@10.20.0",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@maigolabs/needle": "workspace:*",
|
||||
"telegraf": "^4.16.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.10.4"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import url from 'node:url';
|
||||
|
||||
import { TokenType } from '@maigolabs/needle/common';
|
||||
import { buildInvertedIndex, createTokenizer } from '@maigolabs/needle/indexer';
|
||||
import { loadInvertedIndex, inspectSearchResult, searchInvertedIndex } from '@maigolabs/needle/searcher';
|
||||
import { TokenizerBuilder } from '@patdx/kuromoji';
|
||||
import NodeDictionaryLoader from '@patdx/kuromoji/node';
|
||||
import { Telegraf } from 'telegraf';
|
||||
|
||||
const botToken = process.env.TELEGRAM_BOT_TOKEN!;
|
||||
const targetChatId = parseInt(process.env.TARGET_CHAT_ID!);
|
||||
if (!botToken || isNaN(targetChatId)) throw new Error('Missing environment variables TELEGRAM_BOT_TOKEN or TARGET_CHAT_ID');
|
||||
|
||||
const bot = new Telegraf(botToken);
|
||||
|
||||
const escapeHtml = (s: string) => s.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>');
|
||||
|
||||
const commands = await (async () => {
|
||||
const kuromojiDictPath = path.resolve(url.fileURLToPath(import.meta.resolve('@patdx/kuromoji')), '..', '..', 'dict');
|
||||
const kuromoji = await new TokenizerBuilder({ loader: new NodeDictionaryLoader({ dic_path: kuromojiDictPath }) }).build();
|
||||
|
||||
const documents = (await fs.promises.readFile('../../example.txt', 'utf-8')).split('\n').filter(line => line.length > 0);
|
||||
const startBuildInvertedIndex = performance.now();
|
||||
const compressed = buildInvertedIndex(documents, { kuromoji });
|
||||
const endBuildInvertedIndex = performance.now();
|
||||
console.log(`Built inverted index in ${endBuildInvertedIndex - startBuildInvertedIndex}ms`);
|
||||
|
||||
const startLoadInvertedIndex = performance.now();
|
||||
const invertedIndex = loadInvertedIndex(compressed);
|
||||
const endLoadInvertedIndex = performance.now();
|
||||
console.log(`Loaded inverted index in ${endLoadInvertedIndex - startLoadInvertedIndex}ms`);
|
||||
|
||||
const codify = (text: string) => `<code>${escapeHtml(text)}</code>`;
|
||||
return {
|
||||
needle: (text: string) => {
|
||||
const startSearch = performance.now();
|
||||
const results = searchInvertedIndex(invertedIndex, text);
|
||||
const endSearch = performance.now();
|
||||
const searchDuration = (endSearch - startSearch).toFixed(3);
|
||||
const showingResults = results.slice(0, 5);
|
||||
return results.length === 0 ? codify(`No results found after ${searchDuration}ms`) : [
|
||||
codify(`Search completed in ${searchDuration}ms, showing ${showingResults.length}/${results.length} results:\n`),
|
||||
...showingResults.map(result => inspectSearchResult(result, true)),
|
||||
].join('\n').trimEnd();
|
||||
},
|
||||
tokenize: (text: string) => {
|
||||
const startTokenize = performance.now();
|
||||
const tokenizer = createTokenizer({ kuromoji });
|
||||
const tokens = tokenizer.tokenize(text);
|
||||
const tokenDefinitions = [...tokenizer.tokens.values()];
|
||||
const endTokenize = performance.now();
|
||||
const tokenizeDuration = (endTokenize - startTokenize).toFixed(3);
|
||||
return codify(tokens.length === 0 ? `No tokens emitted after ${tokenizeDuration}ms` : [
|
||||
`Tokenization completed in ${tokenizeDuration}ms, emitted ${tokens.length} tokens:`,
|
||||
...tokens
|
||||
.map(token => [tokenDefinitions[token.id]!, token, [...text].slice(token.start, token.end).join('')] as const)
|
||||
.map(([token, { start, end }, originalPhrase]) => ` ${TokenType[token.type]}: ${JSON.stringify(token.text)} <- ${JSON.stringify(originalPhrase)} [${start}, ${end}]`),
|
||||
].join('\n'));
|
||||
},
|
||||
};
|
||||
})();
|
||||
|
||||
bot.on('message', async ctx => {
|
||||
const text = 'text' in ctx.message ? ctx.message.text : undefined;
|
||||
console.log(`${ctx.chat.id ?? 'N/A'}:${ctx.from!.id} ${JSON.stringify(text)}`);
|
||||
if (ctx.chat.id === targetChatId) {
|
||||
if (text?.startsWith('/needle ')) {
|
||||
await ctx.reply(commands.needle(text.slice('/needle '.length)), { parse_mode: 'HTML' });
|
||||
} else if (text?.startsWith('/tokenize ')) {
|
||||
await ctx.reply(commands.tokenize(text.slice('/tokenize '.length)), { parse_mode: 'HTML' });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await bot.launch();
|
||||
void bot.telegram.getMe().then(me => console.log(`Bot logged in as ${me.first_name} (@${me.username})`));
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"jsx": "preserve",
|
||||
"lib": ["DOM", "DOM.Iterable", "ESNext", "WebWorker"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"resolveJsonModule": true,
|
||||
"allowJs": true,
|
||||
"strict": true,
|
||||
"strictNullChecks": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"skipLibCheck": true,
|
||||
"rootDir": ".",
|
||||
"outDir": "dist"
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
Reference in New Issue
Block a user