feat: initial commit
This commit is contained in:
Menci
@@ -0,0 +1,371 @@
|
||||
|
||||
# Created by https://www.toptal.com/developers/gitignore/api/git,visualstudio
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=git,visualstudio
|
||||
|
||||
### Git ###
|
||||
# Created by git for backups. To disable backups in Git:
|
||||
# $ git config --global mergetool.keepBackup false
|
||||
*.orig
|
||||
|
||||
# Created by git when using merge tools for conflicts
|
||||
*.BACKUP.*
|
||||
*.BASE.*
|
||||
*.LOCAL.*
|
||||
*.REMOTE.*
|
||||
*_BACKUP_*.txt
|
||||
*_BASE_*.txt
|
||||
*_LOCAL_*.txt
|
||||
*_REMOTE_*.txt
|
||||
|
||||
### VisualStudio ###
|
||||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.rsuser
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Mono auto generated files
|
||||
mono_crash.*
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
[Aa][Rr][Mm]/
|
||||
[Aa][Rr][Mm]64/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
[Ll]ogs/
|
||||
|
||||
# Visual Studio 2015/2017 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# Visual Studio 2017 auto generated files
|
||||
Generated\ Files/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUnit
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
nunit-*.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# Benchmark Results
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
|
||||
# StyleCop
|
||||
StyleCopReport.xml
|
||||
|
||||
# Files built by Visual Studio
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_h.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.iobj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.ipdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*_wpftmp.csproj
|
||||
*.log
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# Visual Studio Trace Files
|
||||
*.e2e
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# AxoCover is a Code Coverage Tool
|
||||
.axoCover/*
|
||||
!.axoCover/settings.json
|
||||
|
||||
# Coverlet is a free, cross platform Code Coverage Tool
|
||||
coverage*[.json, .xml, .info]
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# NuGet Symbol Packages
|
||||
*.snupkg
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/[Pp]ackages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
*.appx
|
||||
*.appxbundle
|
||||
*.appxupload
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!?*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Including strong name files can present a security risk
|
||||
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||
#*.snk
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
ServiceFabricBackup/
|
||||
*.rptproj.bak
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
*.rptproj.rsuser
|
||||
*- [Bb]ackup.rdl
|
||||
*- [Bb]ackup ([0-9]).rdl
|
||||
*- [Bb]ackup ([0-9][0-9]).rdl
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# CodeRush personal settings
|
||||
.cr/personal
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Tabs Studio
|
||||
*.tss
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
|
||||
# OpenCover UI analysis results
|
||||
OpenCover/
|
||||
|
||||
# Azure Stream Analytics local run output
|
||||
ASALocalRun/
|
||||
|
||||
# MSBuild Binary and Structured Log
|
||||
*.binlog
|
||||
|
||||
# NVidia Nsight GPU debugger configuration file
|
||||
*.nvuser
|
||||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
|
||||
# Local History for Visual Studio
|
||||
.localhistory/
|
||||
|
||||
# BeatPulse healthcheck temp database
|
||||
healthchecksdb
|
||||
|
||||
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
||||
MigrationBackup/
|
||||
|
||||
# Ionide (cross platform F# VS Code tools) working folder
|
||||
.ionide/
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/git,visualstudio
|
||||
@@ -0,0 +1,31 @@
|
||||
<Project>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<PreserveCompilationContext>true</PreserveCompilationContext>
|
||||
<LangVersion>14</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<NoWarn>CA1822</NoWarn>
|
||||
<ProjectName>MaigoLabs.NeedLe</ProjectName>
|
||||
<VSTestLogger>console%3Bverbosity=detailed</VSTestLogger>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>false</IsPackable>
|
||||
<Version>1.0.0</Version>
|
||||
<Authors>Menci</Authors>
|
||||
<Description>Fuzzy search engine for small text pieces, with Chinese/Japanese pronunciation support</Description>
|
||||
<PackageLicenseExpression>AGPL-3.0-only</PackageLicenseExpression>
|
||||
<RepositoryUrl>https://github.com/MaigoLabs/needLe</RepositoryUrl>
|
||||
<RepositoryType>git</RepositoryType>
|
||||
<PackageProjectUrl>https://github.com/MaigoLabs/needLe</PackageProjectUrl>
|
||||
<PackageTags>search;fuzzy;cjk;chinese;japanese;pinyin;romaji</PackageTags>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Include="..\README.md" Pack="true" PackagePath="\" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,19 @@
|
||||
<Project>
|
||||
<PropertyGroup>
|
||||
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
|
||||
<PackageVersion Include="coverlet.collector" Version="6.0.4" />
|
||||
<PackageVersion Include="xunit" Version="2.9.3" />
|
||||
<PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageVersion Include="DotNetCampus.LatestCSharpFeatures" Version="13.0.1" />
|
||||
<PackageVersion Include="hyjiacan.pinyin4net" Version="4.1.1" />
|
||||
<PackageVersion Include="MeCab.DotNet" Version="1.2.0" />
|
||||
<PackageVersion Include="MyNihongo.KanaConverter" Version="1.0.5" />
|
||||
<PackageVersion Include="OpenccNetLib" Version="1.4.0" />
|
||||
<PackageVersion Include="Telegram.Bot" Version="22.5.0" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -0,0 +1,45 @@
|
||||
namespace MaigoLabs.NeedLe.Common;
|
||||
|
||||
// This is for global normalization for any input and documents.
|
||||
public static class CommonNormalization
|
||||
{
|
||||
public static int NormalizeCodePoint(int codePoint)
|
||||
{
|
||||
// Fullwidth ASCII -> Halfwidth ASCII
|
||||
if (codePoint >= 0xFF01 && codePoint <= 0xFF5E) return ToLowerCaseAscii(codePoint - 0xFEE0);
|
||||
// Fullwidth space -> Halfwidth space
|
||||
else if (codePoint == /* ' ' */ 0x3000) return ' ';
|
||||
// Halfwidth kana (U+FF66 - U+FF9D) -> Fullwidth kana
|
||||
else if (codePoint >= 0xFF66 && codePoint <= 0xFF9D) return HALF_TO_FULL_KANA.TryGetValue(codePoint, out var value) ? value : codePoint;
|
||||
else if (codePoint == /* '｡' */ 0xFF61) return '。';
|
||||
else if (codePoint == /* '｢' */ 0xFF62) return '「';
|
||||
else if (codePoint == /* '｣' */ 0xFF63) return '」';
|
||||
else if (codePoint == /* '､' */ 0xFF64) return '、';
|
||||
else if (codePoint == /* '･' */ 0xFF65) return '・';
|
||||
else if (codePoint == /* 'ﾞ' */ 0xFF9E || codePoint == /* '゛' */ 0x309B) return 0x3099; // -> COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK
|
||||
else if (codePoint == /* 'ﾟ' */ 0xFF9F || codePoint == /* '゜' */ 0x309C) return 0x309A; // -> COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
|
||||
else return ToLowerCaseAscii(codePoint);
|
||||
}
|
||||
|
||||
private static readonly Dictionary<int, int> HALF_TO_FULL_KANA = new Dictionary<int, int> {
|
||||
['ｦ'] = 'ヲ', ['ｧ'] = 'ァ', ['ｨ'] = 'ィ', ['ｩ'] = 'ゥ', ['ｪ'] = 'ェ', ['ｫ'] = 'ォ',
|
||||
['ｬ'] = 'ャ', ['ｭ'] = 'ュ', ['ｮ'] = 'ョ', ['ｯ'] = 'ッ',
|
||||
['ｰ'] = 'ー',
|
||||
['ｱ'] = 'ア', ['ｲ'] = 'イ', ['ｳ'] = 'ウ', ['ｴ'] = 'エ', ['ｵ'] = 'オ',
|
||||
['ｶ'] = 'カ', ['ｷ'] = 'キ', ['ｸ'] = 'ク', ['ｹ'] = 'ケ', ['ｺ'] = 'コ',
|
||||
['ｻ'] = 'サ', ['ｼ'] = 'シ', ['ｽ'] = 'ス', ['ｾ'] = 'セ', ['ｿ'] = 'ソ',
|
||||
['ﾀ'] = 'タ', ['ﾁ'] = 'チ', ['ﾂ'] = 'ツ', ['ﾃ'] = 'テ', ['ﾄ'] = 'ト',
|
||||
['ﾅ'] = 'ナ', ['ﾆ'] = 'ニ', ['ﾇ'] = 'ヌ', ['ﾈ'] = 'ネ', ['ﾉ'] = 'ノ',
|
||||
['ﾊ'] = 'ハ', ['ﾋ'] = 'ヒ', ['ﾌ'] = 'フ', ['ﾍ'] = 'ヘ', ['ﾎ'] = 'ホ',
|
||||
['ﾏ'] = 'マ', ['ﾐ'] = 'ミ', ['ﾑ'] = 'ム', ['ﾒ'] = 'メ', ['ﾓ'] = 'モ',
|
||||
['ﾔ'] = 'ヤ', ['ﾕ'] = 'ユ', ['ﾖ'] = 'ヨ',
|
||||
['ﾗ'] = 'ラ', ['ﾘ'] = 'リ', ['ﾙ'] = 'ル', ['ﾚ'] = 'レ', ['ﾛ'] = 'ロ',
|
||||
['ﾜ'] = 'ワ', ['ﾝ'] = 'ン',
|
||||
};
|
||||
|
||||
public static int ToLowerCaseAscii(int codePoint) => codePoint >= 0x41 && codePoint <= 0x5A ? codePoint + 0x20 : codePoint;
|
||||
|
||||
public static bool IsHiraganaRange(int codePoint) => (codePoint >= 0x3041 && codePoint <= 0x3096) || (codePoint >= 0x309D && codePoint <= 0x309E);
|
||||
public static int ToKatakana(int codePoint) => IsHiraganaRange(codePoint) ? codePoint + 0x60 : codePoint;
|
||||
public static string ToKatakana(string text) => string.Concat(text.Select(c => (char)ToKatakana(c)));
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
namespace MaigoLabs.NeedLe.Common;
|
||||
|
||||
public static class CommonUtils
|
||||
{
|
||||
public static bool IsWhitespace(int codePoint) =>
|
||||
codePoint == 0x0009 /* \t */ ||
|
||||
codePoint == 0x000A /* \n */ ||
|
||||
codePoint == 0x000B /* Vertical Tab */ ||
|
||||
codePoint == 0x000C /* \f */ ||
|
||||
codePoint == 0x000D /* \r */ ||
|
||||
codePoint == 0x0020 /* Space */ ||
|
||||
codePoint == 0x0085 /* Next Line (NEL) */ ||
|
||||
codePoint == 0x00A0 /* No-Break Space */ ||
|
||||
codePoint == 0x1680 /* Ogham Space Mark */ ||
|
||||
codePoint >= 0x2000 && codePoint <= 0x200A ||
|
||||
codePoint == 0x2028 /* Line Separator */ ||
|
||||
codePoint == 0x2029 /* Paragraph Separator */ ||
|
||||
codePoint == 0x202F /* Narrow No-Break Space */ ||
|
||||
codePoint == 0x205F /* Medium Mathematical Space */ ||
|
||||
codePoint == 0x3000 /* Ideographic Space */;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
using System.Text;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Common.Extensions;
|
||||
|
||||
public static class UnicodeExtensions
|
||||
{
|
||||
public static IEnumerable<int> ToCodePoints(this string s)
|
||||
{
|
||||
for (int i = 0; i < s.Length; i++)
|
||||
{
|
||||
int codePoint = char.ConvertToUtf32(s, i);
|
||||
if (codePoint > 0xffff) i++;
|
||||
yield return codePoint;
|
||||
}
|
||||
}
|
||||
|
||||
public static StringBuilder ToUtf32StringBuilder(this IEnumerable<int> codePoints)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
foreach (var codePoint in codePoints) sb.Append(char.ConvertFromUtf32(codePoint));
|
||||
return sb;
|
||||
}
|
||||
|
||||
public static string ToUtf32String(this IEnumerable<int> codePoints) => ToUtf32StringBuilder(codePoints).ToString();
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<OutputType>Library</OutputType>
|
||||
<RootNamespace>$(ProjectName).Common</RootNamespace>
|
||||
<AssemblyName>$(RootNamespace)</AssemblyName>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>$(RootNamespace)</PackageId>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="DotNetCampus.LatestCSharpFeatures" PrivateAssets="all" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace MaigoLabs.NeedLe.Common;
|
||||
|
||||
public class TrieNode
|
||||
{
|
||||
public required TrieNode? Parent { get; set; }
|
||||
public required Dictionary<int, TrieNode> Children { get; set; } // Unicode code point -> child node
|
||||
public required List<int> TokenIds { get; set; }
|
||||
public required List<int> SubTreeTokenIds { get; set; } // Empty on root.
|
||||
}
|
||||
|
||||
public static class TrieNodeExtensions
|
||||
{
|
||||
public static TrieNode? TraverseStep(this TrieNode? node, int codePoint, bool isIgnorable = false) =>
|
||||
(node?.Children.TryGetValue(codePoint, out var child) ?? false)
|
||||
? child
|
||||
: isIgnorable ? node : null;
|
||||
|
||||
public static TrieNode? Traverse(this TrieNode? node, int[] codePoints, bool isIgnorable = false)
|
||||
{
|
||||
if (node == null) return null;
|
||||
foreach (var codePoint in codePoints)
|
||||
{
|
||||
node = node?.TraverseStep(codePoint, isIgnorable);
|
||||
if (node == null) return null;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
public static List<int> GetTokenIds(this TrieNode? node, bool includeSubTree = false) =>
|
||||
(includeSubTree ? node?.SubTreeTokenIds : node?.TokenIds) ?? [];
|
||||
|
||||
public static bool IsTokenExactMatch(this TrieNode? node, int tokenId) => node?.TokenIds.Contains(tokenId) ?? false;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace MaigoLabs.NeedLe.Common.Types;
|
||||
|
||||
#pragma warning disable IDE1006 // Naming rule violation
|
||||
|
||||
// For compatibility with TypeScript, we use camelCase property names here.
|
||||
|
||||
public class CompressedInvertedIndex
|
||||
{
|
||||
public required string[] documents { get; set; }
|
||||
public required int[] tokenTypes { get; set; } // Use int values here instead of TokenType enum to avoid JSON serialization issues.
|
||||
public required List<int[]>[] tokenReferences { get; set; } // tokenId -> [documentId, start1, end1, start2, end2, ...]
|
||||
public required CompressedInvertedIndexTries tries { get; set; }
|
||||
}
|
||||
|
||||
public class CompressedInvertedIndexTries
|
||||
{
|
||||
public required int[] romaji { get; set; }
|
||||
public required int[] kana { get; set; }
|
||||
public required int[] other { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace MaigoLabs.NeedLe.Common.Types;
|
||||
|
||||
public class OffsetSpan
|
||||
{
|
||||
public required int Start { get; init; }
|
||||
public required int End { get; init; }
|
||||
|
||||
public int Length => End - Start;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace MaigoLabs.NeedLe.Common.Types;
|
||||
|
||||
public class TokenDefinition
|
||||
{
|
||||
public required int Id { get; set; }
|
||||
public required TokenType Type { get; set; }
|
||||
public required string Text { get; set; }
|
||||
public required int CodePointLength { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace MaigoLabs.NeedLe.Common.Types;
|
||||
|
||||
public enum TokenType
|
||||
{
|
||||
Raw,
|
||||
Kana,
|
||||
Romaji,
|
||||
Han,
|
||||
Pinyin,
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using OpenccNetLib;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer.Han;
|
||||
|
||||
public class HanVariantProvider
|
||||
{
|
||||
private readonly Dictionary<int, int[]> EXCHANGE_MAP;
|
||||
public HanVariantProvider(DictWithMaxLength[]? dicts = null)
|
||||
{
|
||||
dicts ??=
|
||||
[
|
||||
DictionaryLib.Provider.hk_variants,
|
||||
DictionaryLib.Provider.hk_variants_rev,
|
||||
DictionaryLib.Provider.jp_variants,
|
||||
DictionaryLib.Provider.jp_variants_rev,
|
||||
DictionaryLib.Provider.st_characters,
|
||||
DictionaryLib.Provider.ts_characters,
|
||||
DictionaryLib.Provider.tw_variants,
|
||||
DictionaryLib.Provider.tw_variants_rev,
|
||||
];
|
||||
EXCHANGE_MAP = BuildHanExchangeMap(dicts);
|
||||
}
|
||||
|
||||
private Dictionary<int, int[]> BuildHanExchangeMap(DictWithMaxLength[] dicts)
|
||||
{
|
||||
var unionFindSet = new UnionFindSet();
|
||||
foreach (var dict in dicts) foreach (var item in dict.Dict)
|
||||
{
|
||||
var from = item.Key.ToCodePoints().ToArray();
|
||||
var to = item.Value.ToCodePoints().ToArray();
|
||||
if (from.Length != 1 || to.Length != 1) continue;
|
||||
unionFindSet.Union(from[0], to[0]);
|
||||
}
|
||||
var variants = new Dictionary<int, List<int>>();
|
||||
foreach (var x in unionFindSet.Keys)
|
||||
{
|
||||
var parent = unionFindSet.Find(x);
|
||||
if (!variants.TryGetValue(parent, out var list)) variants[parent] = list = [];
|
||||
if (x != parent) variants[x] = list;
|
||||
list.Add(x);
|
||||
}
|
||||
return variants.ToDictionary(item => item.Key, item => item.Value.OrderBy(x => x).ToArray());
|
||||
}
|
||||
|
||||
// https://github.com/google/re2/blob/e7aec5985072c1dbe735add802653ef4b36c231a/re2/unicode_groups.cc#L5590-L5615
|
||||
private static readonly (int Min, int Max)[] RE2_SCRIPT_HAN_RENAGES =
|
||||
[
|
||||
// Han_range16
|
||||
(11904, 11929),
|
||||
(11931, 12019),
|
||||
(12032, 12245),
|
||||
(12293, 12293),
|
||||
(12295, 12295),
|
||||
(12321, 12329),
|
||||
(12344, 12347),
|
||||
(13312, 19903),
|
||||
(19968, 40959),
|
||||
(63744, 64109),
|
||||
(64112, 64217),
|
||||
// Han_range32
|
||||
(94178, 94179),
|
||||
(94192, 94193),
|
||||
(131072, 173791),
|
||||
(173824, 177977),
|
||||
(177984, 178205),
|
||||
(178208, 183969),
|
||||
(183984, 191456),
|
||||
(191472, 192093),
|
||||
(194560, 195101),
|
||||
(196608, 201546),
|
||||
(201552, 205743),
|
||||
];
|
||||
|
||||
public static bool IsHanCharacter(int codePoint) => RE2_SCRIPT_HAN_RENAGES.Any(range => codePoint >= range.Min && codePoint <= range.Max);
|
||||
|
||||
public int[] GetHanVariants(int codePoint) => EXCHANGE_MAP.TryGetValue(codePoint, out var variants)
|
||||
? variants
|
||||
: IsHanCharacter(codePoint) ? [codePoint] : [];
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
using hyjiacan.py4n;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer.Han;
|
||||
|
||||
public static class PinyinHelper
|
||||
{
|
||||
private static readonly string[] PINYIN_INITIALS = ["b", "p", "m", "f", "d", "t", "n", "l", "g", "k", "h", "j", "q", "x", "zh", "ch", "sh", "r", "z", "c", "s", "y", "w"];
|
||||
private static readonly Dictionary<string, string> PINYIN_FINALS_FUZZY_MAP = new() { ["ang"] = "an", ["eng"] = "en", ["ing"] = "in" };
|
||||
|
||||
public static IEnumerable<string> GetPinyinCandidates(int codePoint) => codePoint < char.MinValue || codePoint > char.MaxValue || !PinyinUtil.IsHanzi((char)codePoint) ? [] :
|
||||
Pinyin4Net.GetPinyin((char)codePoint, PinyinFormat.LOWERCASE | PinyinFormat.WITHOUT_TONE).Where(pinyin => pinyin.Length > 0).SelectMany(pinyin =>
|
||||
{
|
||||
var initial = PINYIN_INITIALS.FirstOrDefault(initial => pinyin.StartsWith(initial));
|
||||
var initialAlphabet = initial != null ? initial[..1] : pinyin[..1];
|
||||
var fuzzySuffix = pinyin.Length < 3 ? null : pinyin[^3..];
|
||||
var fuzzyPinyin = fuzzySuffix != null && PINYIN_FINALS_FUZZY_MAP.TryGetValue(fuzzySuffix, out var fuzzySuffixTarget) ? pinyin[..^3] + fuzzySuffixTarget : null;
|
||||
return new string?[] { pinyin, initial, initialAlphabet, fuzzyPinyin }.OfType<string>();
|
||||
}).Distinct();
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
namespace MaigoLabs.NeedLe.Indexer.Han;
|
||||
|
||||
public class UnionFindSet
|
||||
{
|
||||
private Dictionary<int, int> Parent { get; set; } = [];
|
||||
private Dictionary<int, int> Rank { get; set; } = [];
|
||||
|
||||
public IEnumerable<int> Keys => Parent.Keys;
|
||||
|
||||
public int Find(int x)
|
||||
{
|
||||
if (!Parent.TryGetValue(x, out var parent)) return Parent[x] = x;
|
||||
else if (x == parent) return x;
|
||||
else return Parent[x] = Find(parent);
|
||||
}
|
||||
|
||||
public void Union(int x, int y)
|
||||
{
|
||||
x = Find(x);
|
||||
y = Find(y);
|
||||
if (x == y) return;
|
||||
int rankX = GetRank(x), rankY = GetRank(y);
|
||||
if (rankX < rankY) Parent[x] = y;
|
||||
else if (rankX > rankY) Parent[y] = x;
|
||||
else
|
||||
{
|
||||
Parent[y] = x;
|
||||
Rank[x] = rankX + 1;
|
||||
}
|
||||
}
|
||||
|
||||
private int GetRank(int x) => !Rank.TryGetValue(x, out var rank) ? 0 : rank;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MaigoLabs.NeedLe.Common.Types;
|
||||
using MaigoLabs.NeedLe.Indexer.Japanese;
|
||||
using MaigoLabs.NeedLe.Indexer.Trie;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer;
|
||||
|
||||
public static class InvertedIndexBuilder
|
||||
{
|
||||
private static TrieNode BuildTypedTrie(IEnumerable<TokenDefinition> tokenDefinitions, Func<TokenType, bool> typePredicate) =>
|
||||
TrieBuilder.BuildTrie(tokenDefinitions
|
||||
.Where(token => typePredicate(token.Type))
|
||||
.Select(token => (token.Id, CodePoints: token.Text.ToCodePoints())));
|
||||
|
||||
public static CompressedInvertedIndex BuildInvertedIndex(string[] documents, TokenizerOptions? tokenizerOptions = null)
|
||||
{
|
||||
var tokenizer = new Tokenizer(tokenizerOptions);
|
||||
var documentTokens = documents.Select(tokenizer.Tokenize).ToArray();
|
||||
|
||||
var tokenDefinitions = tokenizer.Tokens.Values;
|
||||
var romajiRoot = BuildTypedTrie(tokenDefinitions, type => type == TokenType.Romaji);
|
||||
var kanaRoot = BuildTypedTrie(tokenDefinitions, type => type == TokenType.Kana);
|
||||
var otherRoot = BuildTypedTrie(tokenDefinitions, type => type != TokenType.Romaji && type != TokenType.Kana);
|
||||
TrieBuilder.GraftTriePaths(romajiRoot, JapaneseNormalization.NORMALIZE_RULES_ROMAJI_CODEPOINTS);
|
||||
TrieBuilder.GraftTriePaths(kanaRoot, JapaneseNormalization.NORMALIZE_RULES_KANA_DAKUTEN_CODEPOINTS);
|
||||
|
||||
var invertedIndex = new CompressedInvertedIndex
|
||||
{
|
||||
documents = documents,
|
||||
tokenTypes = [.. tokenDefinitions.Select(token => (int)token.Type)],
|
||||
tokenReferences = [.. tokenDefinitions.Select(_ => new List<int[]>())],
|
||||
tries = new CompressedInvertedIndexTries
|
||||
{
|
||||
romaji = TrieSerializer.Serialize(romajiRoot),
|
||||
kana = TrieSerializer.Serialize(kanaRoot),
|
||||
other = TrieSerializer.Serialize(otherRoot),
|
||||
},
|
||||
};
|
||||
for (var documentId = 0; documentId < documents.Length; documentId++)
|
||||
{
|
||||
var tokens = documentTokens[documentId];
|
||||
var tokenOccurrences = new Dictionary<int, List<int>>();
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
if (!tokenOccurrences.TryGetValue(token.Id, out var occurrences)) tokenOccurrences[token.Id] = occurrences = [];
|
||||
occurrences.Add(token.Start);
|
||||
occurrences.Add(token.End);
|
||||
}
|
||||
foreach (var (tokenId, occurrences) in tokenOccurrences)
|
||||
{
|
||||
invertedIndex.tokenReferences[tokenId].Add([documentId, .. occurrences]);
|
||||
}
|
||||
}
|
||||
return invertedIndex;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer.Japanese;
|
||||
|
||||
public static class JapaneseNormalization
|
||||
{
|
||||
public delegate string Normalizer(string text);
|
||||
|
||||
public static Normalizer CreateNormalizer(Dictionary<string, string> rules) => text =>
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var beforeCurrentIteration = text;
|
||||
foreach (var (from, to) in rules) text = text.Replace(from, to);
|
||||
if (text == beforeCurrentIteration) break;
|
||||
}
|
||||
return text;
|
||||
};
|
||||
|
||||
public static IEnumerable<(int[] From, int[] To)> ToCodePointPairs(Dictionary<string, string> rules) =>
|
||||
rules.Select(rule => (From: rule.Key.ToCodePoints().ToArray(), To: rule.Value.ToCodePoints().ToArray()));
|
||||
|
||||
public static readonly Dictionary<string, string> NORMALIZE_RULES_ROMAJI = new()
|
||||
{
|
||||
// Remove all long vowels (sa-ba- -> saba)
|
||||
["-"] = "",
|
||||
// Collapse consecutive vowels
|
||||
["aa"] = "a",
|
||||
["ii"] = "i",
|
||||
["uu"] = "u",
|
||||
["ee"] = "e",
|
||||
["oo"] = "o",
|
||||
["ou"] = "o",
|
||||
// mb/mp/mm -> nb/np/nm (shimbun -> shinbun)
|
||||
["mb"] = "nb",
|
||||
["mp"] = "np",
|
||||
["mm"] = "nm",
|
||||
// Others
|
||||
["sha"] = "sya",
|
||||
["tsu"] = "tu",
|
||||
["chi"] = "ti",
|
||||
["shi"] = "si",
|
||||
["ji"] = "zi",
|
||||
};
|
||||
public static readonly IEnumerable<(int[] From, int[] To)> NORMALIZE_RULES_ROMAJI_CODEPOINTS = ToCodePointPairs(NORMALIZE_RULES_ROMAJI);
|
||||
public static readonly Normalizer NormalizeRomaji = CreateNormalizer(NORMALIZE_RULES_ROMAJI);
|
||||
|
||||
public static readonly Dictionary<string, string> NORMALIZE_RULES_KANA_DAKUTEN = new()
|
||||
{
|
||||
["う\u3099"] = "ゔ",
|
||||
["か\u3099"] = "が", ["き\u3099"] = "ぎ", ["く\u3099"] = "ぐ", ["け\u3099"] = "げ", ["こ\u3099"] = "ご",
|
||||
["さ\u3099"] = "ざ", ["し\u3099"] = "じ", ["す\u3099"] = "ず", ["せ\u3099"] = "ぜ", ["そ\u3099"] = "ぞ",
|
||||
["た\u3099"] = "だ", ["ち\u3099"] = "ぢ", ["つ\u3099"] = "づ", ["て\u3099"] = "で", ["と\u3099"] = "ど",
|
||||
["は\u3099"] = "ば", ["ひ\u3099"] = "び", ["ふ\u3099"] = "ぶ", ["へ\u3099"] = "べ", ["ほ\u3099"] = "ぼ",
|
||||
["は\u309A"] = "ぱ", ["ひ\u309A"] = "ぴ", ["ふ\u309A"] = "ぷ", ["へ\u309A"] = "ぺ", ["ほ\u309A"] = "ぽ",
|
||||
["ゝ\u3099"] = "ゞ",
|
||||
|
||||
["ウ\u3099"] = "ヴ",
|
||||
["カ\u3099"] = "ガ", ["キ\u3099"] = "ギ", ["ク\u3099"] = "グ", ["ケ\u3099"] = "ゲ", ["コ\u3099"] = "ゴ",
|
||||
["サ\u3099"] = "ザ", ["シ\u3099"] = "ジ", ["ス\u3099"] = "ズ", ["セ\u3099"] = "ゼ", ["ソ\u3099"] = "ゾ",
|
||||
["タ\u3099"] = "ダ", ["チ\u3099"] = "ヂ", ["ツ\u3099"] = "ヅ", ["テ\u3099"] = "デ", ["ト\u3099"] = "ド",
|
||||
["ハ\u3099"] = "バ", ["ヒ\u3099"] = "ビ", ["フ\u3099"] = "ブ", ["ヘ\u3099"] = "ベ", ["ホ\u3099"] = "ボ",
|
||||
["ハ\u309A"] = "パ", ["ヒ\u309A"] = "ピ", ["フ\u309A"] = "プ", ["ヘ\u309A"] = "ペ", ["ホ\u309A"] = "ポ",
|
||||
["ワ\u3099"] = "ヷ", ["ヰ\u3099"] = "ヸ", ["ヱ\u3099"] = "ヹ", ["ヲ\u3099"] = "ヺ",
|
||||
["ヽ\u3099"] = "ヾ",
|
||||
};
|
||||
public static readonly IEnumerable<(int[] From, int[] To)> NORMALIZE_RULES_KANA_DAKUTEN_CODEPOINTS = ToCodePointPairs(NORMALIZE_RULES_KANA_DAKUTEN);
|
||||
public static readonly Normalizer NormalizeKanaDakuten = CreateNormalizer(NORMALIZE_RULES_KANA_DAKUTEN);
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
using MaigoLabs.NeedLe.Indexer.Han;
|
||||
using MyNihongo.KanaConverter;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer.Japanese;
|
||||
|
||||
public static class JapaneseUtils
|
||||
{
|
||||
public static bool IsMaybeJapanese(int codePoint) =>
|
||||
HanVariantProvider.IsHanCharacter(codePoint) ||
|
||||
IsKana(codePoint) ||
|
||||
IsJapaneseSoundMark(codePoint) ||
|
||||
codePoint == 0x3005 || codePoint == 0x3006 || codePoint == 0x30FC;
|
||||
|
||||
// See also Common/Normalization.cs
|
||||
public static bool IsJapaneseSoundMark(int codePoint) => codePoint == 0x3099 || codePoint == 0x309A;
|
||||
public static string StripJapaneseSoundMarks(string text) => string.Concat(text.Where(codePoint => !IsJapaneseSoundMark(codePoint)));
|
||||
|
||||
public static bool IsKana(int codePoint) => (codePoint >= 0x3041 && codePoint <= 0x309F) || (codePoint >= 0x30A0 && codePoint <= 0x30FF);
|
||||
|
||||
private static readonly int[] KANAS_CANNOT_BE_FIRST =
|
||||
[
|
||||
'ァ', 'ィ', 'ゥ', 'ェ', 'ォ',
|
||||
'ぁ', 'ぃ', 'ぅ', 'ぇ', 'ぉ',
|
||||
'ャ', 'ュ', 'ョ',
|
||||
'ゃ', 'ゅ', 'ょ',
|
||||
'ヮ', 'ゎ',
|
||||
'ㇰ', 'ㇱ', 'ㇲ', 'ㇳ', 'ㇴ', 'ㇵ', 'ㇶ', 'ㇷ', 'ㇸ', 'ㇹ', 'ㇺ', 'ㇻ', 'ㇼ', 'ㇽ', 'ㇾ', 'ㇿ',
|
||||
'ー',
|
||||
];
|
||||
|
||||
private static readonly int[] KANAS_CANNOT_BE_LAST =
|
||||
[
|
||||
'ッ', 'っ'
|
||||
];
|
||||
|
||||
public static string ToRomajiStrictly(string kanaText)
|
||||
{
|
||||
if (kanaText.Length == 0) return "";
|
||||
if (KANAS_CANNOT_BE_FIRST.Contains(kanaText[0])) return "";
|
||||
if (KANAS_CANNOT_BE_LAST.Contains(kanaText[^1])) return "";
|
||||
string romaji;
|
||||
try { romaji = kanaText.ToRomaji(); }
|
||||
catch { return ""; }
|
||||
if (!romaji.All(c => c is >= 'a' and <= 'z')) return "";
|
||||
return romaji;
|
||||
}
|
||||
|
||||
public static bool IsValidJapanesePhrase(ReadOnlySpan<int> codePoints, int start, int length) =>
|
||||
// Skip splittings that cause sound marks to occur in the first position of a phrase
|
||||
!IsJapaneseSoundMark(codePoints[start]) && (start + length == codePoints.Length || !IsJapaneseSoundMark(codePoints[start + length]));
|
||||
public static bool IsValidJapanesePhrase(ReadOnlyMemory<int> codePoints, int start, int length) => IsValidJapanesePhrase(codePoints.Span, start, length);
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
using System.Runtime.InteropServices;
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MeCab;
|
||||
using MeCab.Core;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer.Japanese;
|
||||
|
||||
public class Transcription
|
||||
{
|
||||
public required int Start { get; set; }
|
||||
public required int Length { get; set; }
|
||||
public required string[] Transcriptions { get; set; }
|
||||
}
|
||||
|
||||
public delegate IEnumerable<Transcription> TranscriptionEnumerator(ReadOnlyMemory<int> codePoints);
|
||||
public delegate bool IsValidPhraseDelegate(ReadOnlyMemory<int> codePoints, int start, int length);
|
||||
public delegate HashSet<string> GetAllTranscriptionsDelegate(string phrase);
|
||||
|
||||
public class TranscriptionProvider
|
||||
{
|
||||
public MeCabDictionary[] Dictionaries { get; set; }
|
||||
|
||||
public TranscriptionProvider(MeCabDictionary[]? dictionaries = null)
|
||||
{
|
||||
if (dictionaries == null)
|
||||
{
|
||||
var param = new MeCabParam();
|
||||
param.LoadDicRC();
|
||||
var dictionary = new MeCabDictionary();
|
||||
dictionary.Open(Path.Combine(param.DicDir, "sys.dic"));
|
||||
dictionaries = [dictionary];
|
||||
}
|
||||
Dictionaries = dictionaries;
|
||||
}
|
||||
|
||||
public static TranscriptionEnumerator CreateTranscriptionEnumerator(IsValidPhraseDelegate isValidPhrase, GetAllTranscriptionsDelegate getAllTranscriptions) => codePoints =>
|
||||
{
|
||||
var resultMap = new Dictionary<(int Start, int Length), Transcription>();
|
||||
for (int phraseLength = 1; phraseLength <= codePoints.Length; phraseLength++) for (int start = 0; start + phraseLength <= codePoints.Length; start++)
|
||||
{
|
||||
if (!isValidPhrase(codePoints, start, phraseLength)) continue;
|
||||
var phrase = MemoryMarshal.ToEnumerable(codePoints.Slice(start, phraseLength)).ToUtf32String();
|
||||
var atomicTranscriptions = getAllTranscriptions(phrase).Where(transcription => transcription != null).Where(candidateTranscription =>
|
||||
{
|
||||
if (candidateTranscription.Length == 0) return false;
|
||||
// Ensure the transcription is atomic (not a combination of multiple shorter transcriptions, separated by any midpoints)
|
||||
var visitedStates = new HashSet<(int PhrasePosition, int TranscriptionPosition)>();
|
||||
var queue = new Queue<(int PhrasePosition, int TranscriptionPosition)>();
|
||||
queue.Enqueue((0, 0));
|
||||
while (queue.Count > 0)
|
||||
{
|
||||
var (phrasePosition, transcriptionPosition) = queue.Dequeue();
|
||||
for (int prefixLength = 1; prefixLength <= phraseLength - phrasePosition; prefixLength++)
|
||||
{
|
||||
if (!resultMap.TryGetValue((start + phrasePosition, prefixLength), out var prefixResult)) continue;
|
||||
foreach (var transcription in prefixResult.Transcriptions) if (string.Compare(candidateTranscription, transcriptionPosition, transcription, 0, transcription.Length) == 0)
|
||||
{
|
||||
var nextState = (PhrasePosition: phrasePosition + prefixLength, TranscriptionPosition: transcriptionPosition + transcription.Length);
|
||||
if (nextState.PhrasePosition == phraseLength && nextState.TranscriptionPosition == candidateTranscription.Length) return false; // Found a valid combination
|
||||
if (visitedStates.Contains(nextState)) continue;
|
||||
visitedStates.Add(nextState);
|
||||
queue.Enqueue(nextState);
|
||||
}
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}).ToArray();
|
||||
if (atomicTranscriptions.Length > 0) resultMap[(start, phraseLength)] = new() { Start = start, Length = phraseLength, Transcriptions = atomicTranscriptions };
|
||||
}
|
||||
return resultMap.Values;
|
||||
};
|
||||
|
||||
public HashSet<string> GetAllKanaReadings(string phrase)
|
||||
{
|
||||
var result = new HashSet<string>();
|
||||
var isKana = phrase.All(ch => JapaneseUtils.IsKana(ch));
|
||||
if (isKana) result.Add(CommonNormalization.ToKatakana(phrase));
|
||||
if (isKana && phrase.Length == 1) return result;
|
||||
|
||||
foreach (var dictionary in Dictionaries)
|
||||
{
|
||||
var searchResult = dictionary.ExactMatchSearch(phrase);
|
||||
if (searchResult.Value == -1) continue;
|
||||
var tokens = dictionary.GetToken(searchResult);
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
var feature = dictionary.GetFeature(token.Feature);
|
||||
var parts = feature.Split(',');
|
||||
if (parts.Length > 7) result.Add(CommonNormalization.ToKatakana(parts[7]));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public HashSet<string> GetAllKanaReadingsWithNormalization(string phrase) =>
|
||||
GetAllKanaReadings(JapaneseUtils.StripJapaneseSoundMarks(JapaneseNormalization.NormalizeKanaDakuten(phrase)));
|
||||
|
||||
public TranscriptionEnumerator EnumerateKanaTranscriptions => CreateTranscriptionEnumerator(
|
||||
JapaneseUtils.IsValidJapanesePhrase,
|
||||
GetAllKanaReadingsWithNormalization);
|
||||
public TranscriptionEnumerator EnumerateRomajiTranscriptions => CreateTranscriptionEnumerator(
|
||||
JapaneseUtils.IsValidJapanesePhrase,
|
||||
phrase => [.. GetAllKanaReadingsWithNormalization(phrase).Select(kana => JapaneseNormalization.NormalizeRomaji(JapaneseUtils.ToRomajiStrictly(kana)))]);
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<OutputType>Library</OutputType>
|
||||
<RootNamespace>$(ProjectName).Indexer</RootNamespace>
|
||||
<AssemblyName>$(RootNamespace)</AssemblyName>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>$(RootNamespace)</PackageId>
|
||||
<!-- Don't include MeCab dictionaries in this package; let MeCab.DotNet provide them to end users -->
|
||||
<MeCabUseDefaultDictionary>False</MeCabUseDefaultDictionary>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe.Common\MaigoLabs.NeedLe.Common.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="DotNetCampus.LatestCSharpFeatures" PrivateAssets="all" />
|
||||
<PackageReference Include="hyjiacan.pinyin4net" />
|
||||
<PackageReference Include="MeCab.DotNet" PrivateAssets="analyzers" />
|
||||
<PackageReference Include="MyNihongo.KanaConverter" />
|
||||
<PackageReference Include="OpenccNetLib" PrivateAssets="analyzers" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,104 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MaigoLabs.NeedLe.Common.Types;
|
||||
using MaigoLabs.NeedLe.Indexer.Han;
|
||||
using MaigoLabs.NeedLe.Indexer.Japanese;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer;
|
||||
|
||||
public class TokenizerOptions
|
||||
{
|
||||
public HanVariantProvider? HanVariantProvider { get; set; }
|
||||
public TranscriptionProvider? TranscriptionProvider { get; set; }
|
||||
}
|
||||
|
||||
public class Tokenizer(TokenizerOptions? options = null)
|
||||
{
|
||||
public HanVariantProvider HanVariantProvider { get; set; } = options?.HanVariantProvider ?? new HanVariantProvider();
|
||||
public TranscriptionProvider TranscriptionProvider { get; set; } = options?.TranscriptionProvider ?? new TranscriptionProvider();
|
||||
|
||||
public class Token
|
||||
{
|
||||
public required int Id { get; set; }
|
||||
public required int Start { get; set; }
|
||||
public required int End { get; set; }
|
||||
}
|
||||
|
||||
public Dictionary<(TokenType Type, string Text), TokenDefinition> Tokens { get; } = [];
|
||||
private TokenDefinition EnsureToken(TokenType type, string text)
|
||||
{
|
||||
var key = (type, text);
|
||||
if (Tokens.TryGetValue(key, out var tokenDefinition)) return tokenDefinition;
|
||||
tokenDefinition = new TokenDefinition { Id = Tokens.Count, Type = type, Text = text, CodePointLength = text.ToCodePoints().Count() };
|
||||
Tokens.Add(key, tokenDefinition);
|
||||
return tokenDefinition;
|
||||
}
|
||||
|
||||
public List<Token> Tokenize(string text)
|
||||
{
|
||||
var codePoints = text.ToCodePoints().Select(CommonNormalization.NormalizeCodePoint).ToArray();
|
||||
var results = new List<Token>();
|
||||
Action<TokenType /* tokenType */, string /* text */> Emitter(int start, int end) =>
|
||||
(tokenType, codePoints) => results.Add(new Token { Id = EnsureToken(tokenType, codePoints).Id, Start = start, End = end });
|
||||
|
||||
void EmitMaybeJapanese(ReadOnlyMemory<int> codePoints, int offset)
|
||||
{
|
||||
foreach (var combination in TranscriptionProvider.EnumerateKanaTranscriptions(codePoints))
|
||||
{
|
||||
var emit = Emitter(offset + combination.Start, offset + combination.Start + combination.Length);
|
||||
foreach (var transcription in combination.Transcriptions) emit(TokenType.Kana, transcription);
|
||||
}
|
||||
foreach (var combination in TranscriptionProvider.EnumerateRomajiTranscriptions(codePoints))
|
||||
{
|
||||
var emit = Emitter(offset + combination.Start, offset + combination.Start + combination.Length);
|
||||
foreach (var transcription in combination.Transcriptions) emit(TokenType.Romaji, transcription);
|
||||
}
|
||||
for (int i = 0; i < codePoints.Length; i++)
|
||||
{
|
||||
// Single character may have not only kana readings, but also Chinese pronunciations or Simplified/Traditional/Japanese variants.
|
||||
var hanAlternates = HanVariantProvider.GetHanVariants(codePoints.Span[i]); // All possible variant characters (Simplified/Traditional/Japanese)
|
||||
var pinyinAlternates = hanAlternates.SelectMany(PinyinHelper.GetPinyinCandidates).Distinct();
|
||||
var emit = Emitter(offset + i, offset + i + 1);
|
||||
foreach (var han in hanAlternates) emit(TokenType.Han, char.ConvertFromUtf32(han));
|
||||
foreach (var pinyin in pinyinAlternates) emit(TokenType.Pinyin, pinyin);
|
||||
}
|
||||
}
|
||||
|
||||
var consequentCharsets = new (Func<int, bool> Is, Action<ReadOnlyMemory<int>, int> Emit)[]
|
||||
{
|
||||
(Is: JapaneseUtils.IsMaybeJapanese, Emit: EmitMaybeJapanese),
|
||||
};
|
||||
|
||||
void EmitRaw(int codePoint, int offset) => Emitter(offset, offset + 1)(TokenType.Raw, char.ConvertFromUtf32(codePoint));
|
||||
|
||||
for (int start = 0; start < codePoints.Length; )
|
||||
{
|
||||
var codePoint = codePoints[start];
|
||||
var emitted = false;
|
||||
foreach (var (Is, Emit) in consequentCharsets)
|
||||
{
|
||||
var length = 0;
|
||||
while (start + length < codePoints.Length && Is(codePoints[start + length])) length++;
|
||||
if (length > 0)
|
||||
{
|
||||
Emit(new Memory<int>(codePoints, start, length), start);
|
||||
start += length;
|
||||
emitted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (emitted) continue;
|
||||
|
||||
// Skip whitespaces
|
||||
if (CommonUtils.IsWhitespace(codePoint))
|
||||
{
|
||||
start++;
|
||||
continue;
|
||||
}
|
||||
|
||||
EmitRaw(codePoint, start);
|
||||
start++;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer.Trie;
|
||||
|
||||
public static class TrieBuilder
|
||||
{
|
||||
private static TrieNode NewNode(TrieNode? parent) => new() { Parent = parent, Children = [], TokenIds = [], SubTreeTokenIds = [] };
|
||||
|
||||
public static TrieNode BuildTrie(IEnumerable<(int Id, IEnumerable<int> CodePoints)> tokens)
|
||||
{
|
||||
var root = NewNode(null);
|
||||
foreach (var (id, codePoints) in tokens)
|
||||
{
|
||||
var node = root;
|
||||
foreach (var codePoint in codePoints)
|
||||
{
|
||||
node.Children.TryGetValue(codePoint, out var childNode);
|
||||
if (childNode == null) node.Children[codePoint] = childNode = NewNode(node);
|
||||
node = childNode;
|
||||
node.SubTreeTokenIds.Add(id);
|
||||
}
|
||||
node.TokenIds.Add(id);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
public static void GraftTriePaths(TrieNode root, IEnumerable<(int[] From, int[] To)> rules)
|
||||
{
|
||||
foreach (var (inputPhrase, graftTo) in rules) if (graftTo.Length > inputPhrase.Length) throw new ArgumentException($"Graft rule {inputPhrase} -> {graftTo} maps to longer string and may cause infinite loop");
|
||||
var visitedNodes = new HashSet<TrieNode>();
|
||||
void GraftFromNode(TrieNode node, bool recursiveChildren)
|
||||
{
|
||||
if (!visitedNodes.Add(node)) return;
|
||||
if (recursiveChildren) foreach (var child in node.Children.Values) GraftFromNode(child, true);
|
||||
while (true)
|
||||
{
|
||||
var nodesWithNewGraftedChildren = new Dictionary<TrieNode, /* depth from initial node */ int>();
|
||||
foreach (var (inputPhrase, graftTo) in rules)
|
||||
{
|
||||
var targetNode = node.Traverse(graftTo);
|
||||
if (targetNode == null) continue;
|
||||
var graftedPath = new TrieNode[inputPhrase.Length - 1];
|
||||
var isGrafted = false;
|
||||
var currentNode = node;
|
||||
for (var i = 0; i < inputPhrase.Length; i++)
|
||||
{
|
||||
var codePoint = inputPhrase[i];
|
||||
currentNode.Children.TryGetValue(codePoint, out var childNode);
|
||||
if (i == inputPhrase.Length - 1)
|
||||
{
|
||||
if (childNode != null)
|
||||
{
|
||||
if (childNode != targetNode) throw new ArgumentException($"Grafted path {inputPhrase} conflicts with existing path");
|
||||
// Already grafted
|
||||
}
|
||||
else
|
||||
{
|
||||
currentNode.Children[codePoint] = childNode = targetNode;
|
||||
isGrafted = true;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (childNode == null)
|
||||
{
|
||||
childNode = NewNode(currentNode);
|
||||
childNode.SubTreeTokenIds = targetNode.SubTreeTokenIds;
|
||||
currentNode.Children[codePoint] = childNode;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Part of another grafted path?
|
||||
childNode.SubTreeTokenIds = new HashSet<int>(childNode.SubTreeTokenIds.Concat(targetNode.SubTreeTokenIds)).ToList();
|
||||
}
|
||||
graftedPath[i] = currentNode = childNode;
|
||||
}
|
||||
}
|
||||
if (isGrafted) for (var i = 0; i < graftedPath.Length; i++) nodesWithNewGraftedChildren[graftedPath[i]!] = i + 1;
|
||||
}
|
||||
if (nodesWithNewGraftedChildren.Count > 0)
|
||||
{
|
||||
// Re-check graft rules on the newly grafted path
|
||||
// 1. No need to recursive other children (not on this path) since their children are not affected
|
||||
// 2. No need to consider ancestors of this node since they're handled later (we run in DFS order)
|
||||
var sortedNodes = nodesWithNewGraftedChildren.OrderByDescending(x => x.Value);
|
||||
foreach (var (changedNode, _) in sortedNodes) GraftFromNode(changedNode, false);
|
||||
}
|
||||
else break; // No new grafts applied
|
||||
}
|
||||
}
|
||||
GraftFromNode(root, true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Indexer.Trie;
|
||||
|
||||
public static class TrieSerializer
|
||||
{
|
||||
private class NodeEntry
|
||||
{
|
||||
public int Id { get; set; }
|
||||
public bool Visited { get; set; }
|
||||
public int[]? Data { get; set; }
|
||||
}
|
||||
|
||||
public static int[] Serialize(TrieNode root)
|
||||
{
|
||||
var nodeEntries = new Dictionary<TrieNode, NodeEntry>();
|
||||
var currentId = 0;
|
||||
NodeEntry GetNodeEntry(TrieNode node) => nodeEntries.TryGetValue(node, out var nodeEntry) ? nodeEntry :
|
||||
nodeEntries[node] = new NodeEntry { Id = ++currentId, Visited = false, Data = null };
|
||||
int SerializeNode(TrieNode node)
|
||||
{
|
||||
var entry = GetNodeEntry(node);
|
||||
if (entry.Visited) return entry.Id;
|
||||
entry.Visited = true;
|
||||
var children = node.Children.Select(child => (CodePoint: child.Key, ChildId: SerializeNode(child.Value))).ToArray();
|
||||
entry.Data =
|
||||
[
|
||||
node.Parent != null ? GetNodeEntry(node.Parent).Id : 0,
|
||||
.. children.Select(child => child.CodePoint),
|
||||
.. children.Select(child => child.ChildId),
|
||||
// End of children list (<= 0 are not valid code points nor node IDs)
|
||||
.. node.TokenIds.Count > 0
|
||||
? node.TokenIds.Select(tokenId => -(tokenId + 1)) // Use the negative value of (tokenId + 1)
|
||||
: [0], // End of children list, no token IDs (token IDs are encoded to negative values)
|
||||
];
|
||||
return entry.Id;
|
||||
}
|
||||
SerializeNode(root);
|
||||
return nodeEntries.Values.OrderBy(entry => entry.Id).SelectMany(entry => entry.Data ?? []).ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<OutputType>Exe</OutputType>
|
||||
<RootNamespace>$(ProjectName).Playground</RootNamespace>
|
||||
<AssemblyName>$(RootNamespace)</AssemblyName>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Telegram.Bot" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe\MaigoLabs.NeedLe.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,162 @@
|
||||
using System.Diagnostics;
|
||||
using System.Text.Encodings.Web;
|
||||
using System.Text.Json;
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MaigoLabs.NeedLe.Indexer;
|
||||
using MaigoLabs.NeedLe.Searcher;
|
||||
using Telegram.Bot;
|
||||
using Telegram.Bot.Polling;
|
||||
using Telegram.Bot.Types;
|
||||
using Telegram.Bot.Types.Enums;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Playground;
|
||||
|
||||
public class Program
|
||||
{
|
||||
private static LoadedInvertedIndex _invertedIndex = null!;
|
||||
private static long _targetChatId;
|
||||
|
||||
public static async Task Main(string[] args)
|
||||
{
|
||||
var botToken = Environment.GetEnvironmentVariable("TELEGRAM_BOT_TOKEN")
|
||||
?? throw new InvalidOperationException("Missing environment variable TELEGRAM_BOT_TOKEN");
|
||||
var targetChatIdStr = Environment.GetEnvironmentVariable("TARGET_CHAT_ID")
|
||||
?? throw new InvalidOperationException("Missing environment variable TARGET_CHAT_ID");
|
||||
_targetChatId = long.Parse(targetChatIdStr);
|
||||
|
||||
// Build inverted index
|
||||
var exampleDocuments = File.ReadAllLines("../../example.txt").Where(line => line.Length > 0).ToArray();
|
||||
|
||||
var startBuild = Stopwatch.GetTimestamp();
|
||||
var compressed = InvertedIndexBuilder.BuildInvertedIndex(exampleDocuments);
|
||||
var endBuild = Stopwatch.GetTimestamp();
|
||||
Console.WriteLine($"Built inverted index in {Stopwatch.GetElapsedTime(startBuild, endBuild).TotalMilliseconds}ms");
|
||||
|
||||
var startLoad = Stopwatch.GetTimestamp();
|
||||
_invertedIndex = InvertedIndexLoader.Load(compressed);
|
||||
var endLoad = Stopwatch.GetTimestamp();
|
||||
Console.WriteLine($"Loaded inverted index in {Stopwatch.GetElapsedTime(startLoad, endLoad).TotalMilliseconds}ms");
|
||||
|
||||
// Start bot
|
||||
var bot = new TelegramBotClient(botToken);
|
||||
var me = await bot.GetMe();
|
||||
Console.WriteLine($"Bot logged in as {me.FirstName} (@{me.Username})");
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cts.Cancel(); };
|
||||
|
||||
bot.StartReceiving(
|
||||
updateHandler: HandleUpdateAsync,
|
||||
errorHandler: HandleErrorAsync,
|
||||
receiverOptions: new ReceiverOptions { AllowedUpdates = [UpdateType.Message] },
|
||||
cancellationToken: cts.Token
|
||||
);
|
||||
await Task.Delay(-1, cts.Token).ContinueWith(_ => { });
|
||||
}
|
||||
|
||||
private static async Task HandleUpdateAsync(ITelegramBotClient bot, Update update, CancellationToken ct)
|
||||
{
|
||||
if (update.Message is not { Text: { } text, Chat.Id: var chatId, From: { } from }) return;
|
||||
|
||||
Console.WriteLine($"{chatId}:{from.Id} {JsonSerializer.Serialize(text, JsonSerializerOptions)}");
|
||||
|
||||
if (chatId != _targetChatId) return;
|
||||
|
||||
if (text.StartsWith("/needle "))
|
||||
{
|
||||
var query = text["/needle ".Length..];
|
||||
var response = HandleNeedleCommand(query);
|
||||
await bot.SendMessage(chatId, response, parseMode: ParseMode.Html, cancellationToken: ct);
|
||||
}
|
||||
else if (text.StartsWith("/tokenize "))
|
||||
{
|
||||
var query = text["/tokenize ".Length..];
|
||||
var response = HandleTokenizeCommand(query);
|
||||
await bot.SendMessage(chatId, response, parseMode: ParseMode.Html, cancellationToken: ct);
|
||||
}
|
||||
}
|
||||
|
||||
private static Task HandleErrorAsync(ITelegramBotClient bot, Exception exception, HandleErrorSource source, CancellationToken ct)
|
||||
{
|
||||
Console.WriteLine($"Error: {exception.Message}");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static string HandleNeedleCommand(string query)
|
||||
{
|
||||
var startSearch = Stopwatch.GetTimestamp();
|
||||
var results = InvertedIndexSearcher.Search(_invertedIndex, query);
|
||||
var endSearch = Stopwatch.GetTimestamp();
|
||||
var searchDuration = Stopwatch.GetElapsedTime(startSearch, endSearch).TotalMilliseconds.ToString("F3");
|
||||
|
||||
if (results.Length == 0)
|
||||
return Codify($"No results found after {searchDuration}ms");
|
||||
|
||||
var showingResults = results.Take(5).ToArray();
|
||||
return string.Join('\n',
|
||||
[
|
||||
Codify($"Search completed in {searchDuration}ms, showing {showingResults.Length}/{results.Length} results:\n"),
|
||||
.. showingResults.Select(result => InspectSearchResult(result, true))
|
||||
]).TrimEnd();
|
||||
}
|
||||
|
||||
private static string HandleTokenizeCommand(string query)
|
||||
{
|
||||
var tokenizer = new Tokenizer();
|
||||
var startTokenize = Stopwatch.GetTimestamp();
|
||||
var tokens = tokenizer.Tokenize(query);
|
||||
var tokenDefinitions = tokenizer.Tokens.Values.ToArray();
|
||||
var endTokenize = Stopwatch.GetTimestamp();
|
||||
var tokenizeDuration = Stopwatch.GetElapsedTime(startTokenize, endTokenize).TotalMilliseconds.ToString("F3");
|
||||
if (tokens.Count == 0) return Codify($"No tokens emitted after {tokenizeDuration}ms");
|
||||
|
||||
var codePoints = query.ToCodePoints().ToArray();
|
||||
var lines = new List<string>
|
||||
{
|
||||
$"Tokenization completed in {tokenizeDuration}ms, emitted {tokens.Count} tokens:"
|
||||
};
|
||||
foreach (var token in tokens)
|
||||
{
|
||||
var tokenDef = tokenDefinitions[token.Id];
|
||||
var originalPhrase = codePoints.Skip(token.Start).Take(token.End - token.Start).ToUtf32String();
|
||||
lines.Add($" {tokenDef.Type}: {JsonSerializer.Serialize(tokenDef.Text, JsonSerializerOptions)} <- {JsonSerializer.Serialize(originalPhrase, JsonSerializerOptions)} [{token.Start}, {token.End}]");
|
||||
}
|
||||
return Codify(string.Join('\n', lines));
|
||||
}
|
||||
|
||||
private static string InspectSearchResult(SearchResult result, bool htmlHighlight)
|
||||
{
|
||||
var documentText = result.DocumentText;
|
||||
var documentCodePoints = result.DocumentCodePoints;
|
||||
var tokens = result.Tokens;
|
||||
var rangeCount = result.RangeCount;
|
||||
var matchRatio = result.MatchRatio;
|
||||
var matchRatioLevel = result.MatchRatioLevel;
|
||||
|
||||
var resultText = htmlHighlight
|
||||
? string.Join("", SearchResultHighlighter.Highlight(result).Select(part => !part.IsHighlighted ? EscapeHtml(part.Text) : $"<u><b>{EscapeHtml(part.Text)}</b></u>"))
|
||||
: documentText;
|
||||
var description = $" ({rangeCount} ranges, {Math.Round(matchRatio * 10000) / 10000} => L{matchRatioLevel})";
|
||||
return string.Join('\n',
|
||||
[
|
||||
resultText + (htmlHighlight ? $"<code>{description}</code>" : description),
|
||||
.. tokens.Select(token =>
|
||||
{
|
||||
var escapedTokenText = JsonSerializer.Serialize(token.Definition.Text, JsonSerializerOptions);
|
||||
var escapedDocumentText = JsonSerializer.Serialize(documentCodePoints.Skip(token.DocumentOffset.Start).Take(token.DocumentOffset.Length).ToUtf32String(), JsonSerializerOptions);
|
||||
if (htmlHighlight)
|
||||
{
|
||||
escapedTokenText = EscapeHtml(escapedTokenText);
|
||||
escapedDocumentText = EscapeHtml(escapedDocumentText);
|
||||
}
|
||||
var line = $" {token.Definition.Type}: {escapedTokenText} -> {escapedDocumentText}" + (token.IsTokenPrefixMatching ? " (prefix match)" : "");
|
||||
return htmlHighlight ? $"<code>{line}</code>" : line;
|
||||
}),
|
||||
"",
|
||||
]);
|
||||
}
|
||||
|
||||
private static string Codify(string text) => $"<code>{EscapeHtml(text)}</code>";
|
||||
private static JsonSerializerOptions JsonSerializerOptions => new() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping };
|
||||
private static string EscapeHtml(string text) => text.Replace("&", "&").Replace("<", "<").Replace(">", ">");
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MaigoLabs.NeedLe.Common.Types;
|
||||
using MaigoLabs.NeedLe.Searcher.Trie;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Searcher;
|
||||
|
||||
public class LoadedInvertedIndex
|
||||
{
|
||||
public class TokenDocumentReference
|
||||
{
|
||||
public required int DocumentId { get; set; }
|
||||
public required OffsetSpan[] Offsets { get; set; }
|
||||
}
|
||||
|
||||
public class TokenDefinitionExtended : TokenDefinition
|
||||
{
|
||||
public required TokenDocumentReference[] References { get; set; }
|
||||
}
|
||||
|
||||
public class TypedTries
|
||||
{
|
||||
public required TrieNode Romaji { get; set; }
|
||||
public required TrieNode Kana { get; set; }
|
||||
public required TrieNode Other { get; set; }
|
||||
}
|
||||
|
||||
public required string[] Documents { get; set; }
|
||||
public required int[][] DocumentCodePoints { get; set; }
|
||||
public required TokenDefinitionExtended[] TokenDefinitions { get; set; }
|
||||
public required TypedTries Tries { get; set; }
|
||||
}
|
||||
|
||||
public class InvertedIndexLoader
|
||||
{
|
||||
public static LoadedInvertedIndex Load(CompressedInvertedIndex compressed)
|
||||
{
|
||||
var documents = compressed.documents;
|
||||
var documentCodePoints = documents.Select(document => document.ToCodePoints().ToArray()).ToArray();
|
||||
|
||||
var romajiTrie = TrieDeserializer.Deserialize(compressed.tries.romaji);
|
||||
var kanaTrie = TrieDeserializer.Deserialize(compressed.tries.kana);
|
||||
var otherTrie = TrieDeserializer.Deserialize(compressed.tries.other);
|
||||
|
||||
var tokenCodePoints = romajiTrie.TokenCodePoints.Concat(kanaTrie.TokenCodePoints).Concat(otherTrie.TokenCodePoints)
|
||||
.ToDictionary(entry => entry.Key, entry => entry.Value);
|
||||
var tokenDefinitions = compressed.tokenTypes.Select((type, index) => new LoadedInvertedIndex.TokenDefinitionExtended
|
||||
{
|
||||
Id = index, Type = (TokenType)type, Text = tokenCodePoints[index].ToUtf32String(),
|
||||
CodePointLength = tokenCodePoints[index].Length,
|
||||
References = compressed.tokenReferences[index].Select(data => new LoadedInvertedIndex.TokenDocumentReference
|
||||
{
|
||||
DocumentId = data[0],
|
||||
Offsets = Enumerable.Range(0, data.Length / 2)
|
||||
.Select(i => new OffsetSpan { Start = data[i * 2 + 1], End = data[i * 2 + 2] }).ToArray(),
|
||||
}).ToArray(),
|
||||
}).ToArray();
|
||||
|
||||
return new LoadedInvertedIndex
|
||||
{
|
||||
Documents = documents,
|
||||
DocumentCodePoints = documentCodePoints,
|
||||
TokenDefinitions = tokenDefinitions,
|
||||
Tries = new LoadedInvertedIndex.TypedTries
|
||||
{
|
||||
Romaji = romajiTrie.Root,
|
||||
Kana = kanaTrie.Root,
|
||||
Other = otherTrie.Root,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MaigoLabs.NeedLe.Common.Types;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Searcher;
|
||||
|
||||
public class SearchResultToken
|
||||
{
|
||||
public required TokenDefinition Definition { get; set; }
|
||||
public required OffsetSpan DocumentOffset { get; set; }
|
||||
public required OffsetSpan InputOffset { get; set; }
|
||||
public required bool IsTokenPrefixMatching { get; set; }
|
||||
}
|
||||
|
||||
public class SearchResult
|
||||
{
|
||||
public required int DocumentId { get; set; }
|
||||
public required string DocumentText { get; set; }
|
||||
public required int[] DocumentCodePoints { get; set; }
|
||||
public required SearchResultToken[] Tokens { get; set; }
|
||||
public required int PrefixMatchCount { get; set; }
|
||||
public required int RangeCount { get; set; }
|
||||
public required double MatchRatio { get; set; }
|
||||
public required int MatchRatioLevel { get; set; }
|
||||
}
|
||||
|
||||
public static class InvertedIndexSearcher
|
||||
{
|
||||
public abstract class ComparableStateBase<T> : IComparable<T>
|
||||
where T : ComparableStateBase<T>
|
||||
{
|
||||
protected abstract int GetRangeCount();
|
||||
protected abstract int GetPrefixMatchCount();
|
||||
protected abstract OffsetSpan GetFirstTokenDocumentOffset();
|
||||
protected abstract OffsetSpan GetLastTokenDocumentOffset();
|
||||
protected virtual SearchResultToken? GetLastToken() => null; // Not on intermediate results
|
||||
protected virtual int? GetMatchRatioLevel() => null; // Not on intermediate/candidate results
|
||||
protected abstract double GetMatchRatio();
|
||||
protected virtual int FallbackCompareTo(T other) => 0; // Called when all other comparisons are equal
|
||||
|
||||
public int CompareTo(T other)
|
||||
{
|
||||
// Prefer matches that not relying on end-of-input loose matching (full match over prefix match)
|
||||
SearchResultToken? aLastToken = GetLastToken(), bLastToken = other.GetLastToken();
|
||||
if (aLastToken != null && bLastToken != null)
|
||||
{
|
||||
var aDidPrefixMatchByTokenType = aLastToken.IsTokenPrefixMatching && tokenTypePrefixMatchingPolicy[aLastToken.Definition.Type] == TokenTypePrefixMatchingPolicy.AllowOnlyAtInputEnd;
|
||||
var bDidPrefixMatchByTokenType = bLastToken.IsTokenPrefixMatching && tokenTypePrefixMatchingPolicy[bLastToken.Definition.Type] == TokenTypePrefixMatchingPolicy.AllowOnlyAtInputEnd;
|
||||
if (aDidPrefixMatchByTokenType != bDidPrefixMatchByTokenType) return aDidPrefixMatchByTokenType ? 1 : -1;
|
||||
}
|
||||
|
||||
// Prefer results that matched fewer discontinuous ranges over more
|
||||
int aRangeCount = GetRangeCount(), bRangeCount = other.GetRangeCount();
|
||||
if (aRangeCount != bRangeCount) return aRangeCount - bRangeCount;
|
||||
|
||||
// Prefer results that matches first token in document earlier over later
|
||||
OffsetSpan aFirstTokenDocumentOffset = GetFirstTokenDocumentOffset(), bFirstTokenDocumentOffset = other.GetFirstTokenDocumentOffset();
|
||||
if (aFirstTokenDocumentOffset.Start != bFirstTokenDocumentOffset.Start) return aFirstTokenDocumentOffset.Start - bFirstTokenDocumentOffset.Start;
|
||||
|
||||
// Prefer results that has higher match ratio (but don't distinguish similar ratios, so we introduced `matchRatioLevel`)
|
||||
int? aMatchRatioLevel = GetMatchRatioLevel(), bMatchRatioLevel = other.GetMatchRatioLevel();
|
||||
if (aMatchRatioLevel != null && bMatchRatioLevel != null)
|
||||
{
|
||||
if (aMatchRatioLevel.Value != bMatchRatioLevel.Value) return bMatchRatioLevel.Value - aMatchRatioLevel.Value;
|
||||
}
|
||||
|
||||
// Prefer results that last token occurred earlier (if same, ended earlier) in the document over later
|
||||
OffsetSpan aLastTokenDocumentOffset = GetLastTokenDocumentOffset(), bLastTokenDocumentOffset = other.GetLastTokenDocumentOffset();
|
||||
if (aLastTokenDocumentOffset.Start != bLastTokenDocumentOffset.Start) return aLastTokenDocumentOffset.Start - bLastTokenDocumentOffset.Start;
|
||||
if (aLastTokenDocumentOffset.End != bLastTokenDocumentOffset.End) return aLastTokenDocumentOffset.End - bLastTokenDocumentOffset.End;
|
||||
|
||||
// Prefer results that has higher match ratio (precisely)
|
||||
double aMatchRatio = GetMatchRatio(), bMatchRatio = other.GetMatchRatio();
|
||||
if (aMatchRatio != bMatchRatio) return bMatchRatio < aMatchRatio ? -1 : bMatchRatio > aMatchRatio ? 1 : 0;
|
||||
|
||||
return FallbackCompareTo(other);
|
||||
}
|
||||
}
|
||||
|
||||
public class IntermediateResult : ComparableStateBase<IntermediateResult>
|
||||
{
|
||||
public required IntermediateResult? PreviousState { get; init; }
|
||||
public required OffsetSpan FirstTokenDocumentOffset { get; init; }
|
||||
public required int RangeCount { get; init; }
|
||||
public required int TokenCount { get; init; }
|
||||
public required int PrefixMatchCount { get; init; }
|
||||
public required double MatchedTokenLength { get; init; }
|
||||
public required int TokenId { get; init; }
|
||||
public required OffsetSpan DocumentOffset { get; init; }
|
||||
public required OffsetSpan InputOffset { get; init; }
|
||||
public required bool IsTokenPrefixMatching { get; init; }
|
||||
|
||||
protected override int GetRangeCount() => RangeCount;
|
||||
protected override int GetPrefixMatchCount() => PrefixMatchCount;
|
||||
protected override OffsetSpan GetFirstTokenDocumentOffset() => FirstTokenDocumentOffset;
|
||||
protected override OffsetSpan GetLastTokenDocumentOffset() => DocumentOffset;
|
||||
protected override double GetMatchRatio() => MatchedTokenLength; // No need to divide document length since intermediate results are for same document
|
||||
}
|
||||
|
||||
public class CandidateResult : ComparableStateBase<CandidateResult>
|
||||
{
|
||||
public required SearchResultToken[] Tokens { get; init; }
|
||||
public required int PrefixMatchCount { get; init; }
|
||||
public required double MatchedTokenLength { get; init; }
|
||||
public required int RangeCount { get; init; }
|
||||
|
||||
protected override int GetRangeCount() => RangeCount;
|
||||
protected override int GetPrefixMatchCount() => PrefixMatchCount;
|
||||
protected override OffsetSpan GetFirstTokenDocumentOffset() => Tokens[0].DocumentOffset;
|
||||
protected override OffsetSpan GetLastTokenDocumentOffset() => Tokens[^1].DocumentOffset;
|
||||
protected override SearchResultToken? GetLastToken() => Tokens[^1];
|
||||
protected override double GetMatchRatio() => MatchedTokenLength; // No need to divide document length since intermediate results are for same document
|
||||
}
|
||||
|
||||
public class FinalResult : ComparableStateBase<FinalResult>
|
||||
{
|
||||
public required SearchResult Result { get; init; }
|
||||
|
||||
protected override int GetRangeCount() => Result.RangeCount;
|
||||
protected override int GetPrefixMatchCount() => Result.PrefixMatchCount;
|
||||
protected override OffsetSpan GetFirstTokenDocumentOffset() => Result.Tokens[0].DocumentOffset;
|
||||
protected override OffsetSpan GetLastTokenDocumentOffset() => Result.Tokens[^1].DocumentOffset;
|
||||
protected override SearchResultToken? GetLastToken() => Result.Tokens[^1];
|
||||
protected override double GetMatchRatio() => Result.MatchRatio;
|
||||
protected override int? GetMatchRatioLevel() => Result.MatchRatioLevel;
|
||||
protected override int FallbackCompareTo(FinalResult other) => string.Compare(Result.DocumentText, other.Result.DocumentText, StringComparison.InvariantCulture);
|
||||
}
|
||||
|
||||
private static bool IsIgnorableCodePoint(int codePoint) => CommonUtils.IsWhitespace(codePoint) || codePoint == 0x3099 || codePoint == 0x309A;
|
||||
|
||||
public enum TokenTypePrefixMatchingPolicy {
|
||||
AlwaysAllow,
|
||||
NeverAllow,
|
||||
AllowOnlyAtInputEnd,
|
||||
}
|
||||
|
||||
private static Dictionary<TokenType, TokenTypePrefixMatchingPolicy> tokenTypePrefixMatchingPolicy = new()
|
||||
{
|
||||
[TokenType.Romaji] = TokenTypePrefixMatchingPolicy.NeverAllow,
|
||||
[TokenType.Kana] = TokenTypePrefixMatchingPolicy.AlwaysAllow,
|
||||
// These token types are in an "other" Trie
|
||||
[TokenType.Han] = TokenTypePrefixMatchingPolicy.AllowOnlyAtInputEnd, // No effect because always 1 code point
|
||||
[TokenType.Pinyin] = TokenTypePrefixMatchingPolicy.AllowOnlyAtInputEnd,
|
||||
[TokenType.Raw] = TokenTypePrefixMatchingPolicy.AllowOnlyAtInputEnd, // No effect because always 1 code point
|
||||
};
|
||||
|
||||
private static bool ShouldAllowPrefixMatching(TokenType tokenType, bool isAtInputEnd) =>
|
||||
tokenTypePrefixMatchingPolicy[tokenType] == TokenTypePrefixMatchingPolicy.AlwaysAllow ||
|
||||
(tokenTypePrefixMatchingPolicy[tokenType] != TokenTypePrefixMatchingPolicy.NeverAllow && isAtInputEnd);
|
||||
|
||||
private static bool HasNonEmptyCharacters(int[] documentCodePoints, int start, int end) =>
|
||||
start != end && !documentCodePoints.Skip(start).Take(end - start).All(CommonUtils.IsWhitespace);
|
||||
|
||||
public static SearchResult[] Search(LoadedInvertedIndex invertedIndex, string text)
|
||||
{
|
||||
var documents = invertedIndex.Documents;
|
||||
var documentCodePoints = invertedIndex.DocumentCodePoints;
|
||||
var tokenDefinitions = invertedIndex.TokenDefinitions;
|
||||
var tries = invertedIndex.Tries;
|
||||
|
||||
var codePoints = text.ToCodePoints().Select(CommonNormalization.NormalizeCodePoint).Select(CommonNormalization.ToKatakana).ToArray();
|
||||
// dp[i] = docId => end => IntermediateResult, starts from dp[-1] (l === 0), ends at dp[N - 1] (r === N - 1)
|
||||
var dp = Enumerable.Range(0, codePoints.Length).Select(l => new Dictionary<int, Dictionary<int, IntermediateResult>>()).ToArray();
|
||||
for (var l = 0; l < codePoints.Length; l++)
|
||||
{
|
||||
if (l != 0 && dp[l - 1].Count == 0) continue; // No documents match input from beginning to this position
|
||||
var romajiNode = tries.Romaji;
|
||||
var kanaNode = tries.Kana;
|
||||
var otherNode = tries.Other;
|
||||
for (var r = l; r < codePoints.Length && (romajiNode != null || kanaNode != null || otherNode != null); r++) // [l, r]
|
||||
{
|
||||
var codePoint = codePoints[r];
|
||||
romajiNode = romajiNode.TraverseStep(codePoint, IsIgnorableCodePoint(codePoint));
|
||||
kanaNode = kanaNode.TraverseStep(codePoint, IsIgnorableCodePoint(codePoint));
|
||||
otherNode = otherNode.TraverseStep(codePoint, IsIgnorableCodePoint(codePoint));
|
||||
var reachingInputEnd = r == codePoints.Length - 1;
|
||||
HashSet<int> matchingTokenIds =
|
||||
[
|
||||
// Allow suffix matching of romaji/other tokens if we're at the end of the input
|
||||
.. romajiNode.GetTokenIds(ShouldAllowPrefixMatching(TokenType.Romaji, reachingInputEnd)),
|
||||
.. kanaNode.GetTokenIds(ShouldAllowPrefixMatching(TokenType.Kana, reachingInputEnd)),
|
||||
.. otherNode.GetTokenIds(reachingInputEnd),
|
||||
];
|
||||
foreach (var tokenId in matchingTokenIds) foreach (var reference in tokenDefinitions[tokenId].References)
|
||||
{
|
||||
var isTokenPrefixMatching = !romajiNode.IsTokenExactMatch(tokenId) && !kanaNode.IsTokenExactMatch(tokenId) && !otherNode.IsTokenExactMatch(tokenId);
|
||||
var previousMatchesOfDocument = l != 0 && dp[l - 1].TryGetValue(reference.DocumentId, out var previousMatches) ? previousMatches : null;
|
||||
if (l != 0 && previousMatchesOfDocument == null) continue;
|
||||
foreach (var documentOffset in reference.Offsets)
|
||||
{
|
||||
int currentStart = documentOffset.Start, currentEnd = documentOffset.End;
|
||||
if (l == 0) ContributeNextMatchingState(null);
|
||||
else foreach (var (previousEnd, previousMatch) in previousMatchesOfDocument!) if (currentStart >= previousEnd) ContributeNextMatchingState(previousMatch);
|
||||
void ContributeNextMatchingState(IntermediateResult? previousState)
|
||||
{
|
||||
var nextMatchingMap = dp[r];
|
||||
if (!nextMatchingMap.TryGetValue(reference.DocumentId, out var nextMatches)) nextMatches = nextMatchingMap[reference.DocumentId] = [];
|
||||
var oldResult = nextMatches.TryGetValue(currentEnd, out var result) ? result : null;
|
||||
var inputOffset = new OffsetSpan { Start = l, End = r + 1 };
|
||||
var newResult = new IntermediateResult
|
||||
{
|
||||
PreviousState = previousState,
|
||||
FirstTokenDocumentOffset = previousState?.FirstTokenDocumentOffset ?? documentOffset,
|
||||
RangeCount = previousState == null ? 1 :
|
||||
previousState.RangeCount + (HasNonEmptyCharacters(documentCodePoints[reference.DocumentId], previousState.DocumentOffset.End, currentStart) ? 1 : 0),
|
||||
TokenCount = (previousState?.TokenCount ?? 0) + 1,
|
||||
PrefixMatchCount = (previousState?.PrefixMatchCount ?? 0) + (isTokenPrefixMatching ? 1 : 0),
|
||||
MatchedTokenLength = (previousState?.MatchedTokenLength ?? 0) + documentOffset.Length *
|
||||
Math.Min(isTokenPrefixMatching ? (double)inputOffset.Length / tokenDefinitions[tokenId].CodePointLength : double.PositiveInfinity, 1),
|
||||
TokenId = tokenId,
|
||||
DocumentOffset = documentOffset,
|
||||
InputOffset = inputOffset,
|
||||
IsTokenPrefixMatching = isTokenPrefixMatching,
|
||||
};
|
||||
nextMatches[currentEnd] = oldResult == null || newResult.CompareTo(oldResult) < 0 ? newResult : oldResult;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build search results and sort documents
|
||||
return dp[codePoints.Length - 1].Select(entry =>
|
||||
{
|
||||
var (documentId, matches) = entry;
|
||||
var sortedMatches = matches.Values.Select(match =>
|
||||
{
|
||||
var tokens = new List<SearchResultToken>();
|
||||
// Build token list from backtracking
|
||||
var state = match;
|
||||
while (state != null)
|
||||
{
|
||||
tokens.Add(new SearchResultToken
|
||||
{
|
||||
Definition = tokenDefinitions[state.TokenId],
|
||||
DocumentOffset = state.DocumentOffset, InputOffset = state.InputOffset,
|
||||
IsTokenPrefixMatching = state.IsTokenPrefixMatching,
|
||||
});
|
||||
state = state.PreviousState;
|
||||
}
|
||||
tokens.Reverse();
|
||||
return new CandidateResult
|
||||
{
|
||||
Tokens = tokens.ToArray(),
|
||||
PrefixMatchCount = match.PrefixMatchCount,
|
||||
MatchedTokenLength = match.MatchedTokenLength,
|
||||
RangeCount = match.RangeCount,
|
||||
};
|
||||
}).OrderBy(match => match);
|
||||
var bestMatch = sortedMatches.First();
|
||||
var documentText = documents[documentId];
|
||||
var matchRatio = bestMatch.MatchedTokenLength / documentCodePoints[documentId].Length;
|
||||
var matchRatioLevel = (int)Math.Round(matchRatio * 5);
|
||||
return new FinalResult
|
||||
{
|
||||
Result = new SearchResult
|
||||
{
|
||||
DocumentId = documentId,
|
||||
DocumentText = documentText,
|
||||
DocumentCodePoints = documentCodePoints[documentId],
|
||||
Tokens = bestMatch.Tokens,
|
||||
PrefixMatchCount = bestMatch.PrefixMatchCount,
|
||||
RangeCount = bestMatch.RangeCount,
|
||||
MatchRatio = matchRatio,
|
||||
MatchRatioLevel = matchRatioLevel,
|
||||
}
|
||||
};
|
||||
}).OrderBy(result => result).Select(result => result.Result).ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.0</TargetFramework>
|
||||
<OutputType>Library</OutputType>
|
||||
<RootNamespace>$(ProjectName).Searcher</RootNamespace>
|
||||
<AssemblyName>$(RootNamespace)</AssemblyName>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>$(RootNamespace)</PackageId>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe.Common\MaigoLabs.NeedLe.Common.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="DotNetCampus.LatestCSharpFeatures" PrivateAssets="all" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,37 @@
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MaigoLabs.NeedLe.Common.Types;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Searcher;
|
||||
|
||||
public class HighlightedTextPart
|
||||
{
|
||||
public required string Text { get; init; }
|
||||
public required bool IsHighlighted { get; init; }
|
||||
}
|
||||
|
||||
public static class SearchResultHighlighter
|
||||
{
|
||||
public static List<HighlightedTextPart> Highlight(SearchResult resultDocument)
|
||||
{
|
||||
var result = new List<HighlightedTextPart>();
|
||||
var previousHighlightEnd = 0;
|
||||
foreach (var token in resultDocument.Tokens)
|
||||
{
|
||||
var notHighlightedText = resultDocument.DocumentCodePoints.Skip(previousHighlightEnd).Take(token.DocumentOffset.Start - previousHighlightEnd).ToUtf32String();
|
||||
if (notHighlightedText.Length > 0) result.Add(new HighlightedTextPart { Text = notHighlightedText, IsHighlighted = false });
|
||||
var highlightEnd = token.IsTokenPrefixMatching && token.Definition.Type == TokenType.Kana
|
||||
? token.DocumentOffset.Start + Math.Max(
|
||||
1,
|
||||
(int)Math.Round(
|
||||
token.DocumentOffset.Length *
|
||||
Math.Min(1, (double)token.InputOffset.Length / token.Definition.CodePointLength)
|
||||
)
|
||||
)
|
||||
: token.DocumentOffset.End;
|
||||
result.Add(new HighlightedTextPart { Text = resultDocument.DocumentCodePoints.Skip(token.DocumentOffset.Start).Take(highlightEnd - token.DocumentOffset.Start).ToUtf32String(), IsHighlighted = true });
|
||||
previousHighlightEnd = highlightEnd;
|
||||
}
|
||||
if (previousHighlightEnd < resultDocument.DocumentCodePoints.Length) result.Add(new HighlightedTextPart { Text = resultDocument.DocumentCodePoints.Skip(previousHighlightEnd).ToUtf32String(), IsHighlighted = false });
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Searcher.Trie;
|
||||
|
||||
public class DeserializedTrie
|
||||
{
|
||||
public required TrieNode Root { get; set; }
|
||||
public required Dictionary<int, int[]> TokenCodePoints { get; set; }
|
||||
}
|
||||
|
||||
public static class TrieDeserializer
|
||||
{
|
||||
public static DeserializedTrie Deserialize(int[] data)
|
||||
{
|
||||
var nodes = new List<TrieNode?>();
|
||||
TrieNode GetNode(int id)
|
||||
{
|
||||
if (id > nodes.Count) nodes.AddRange(Enumerable.Repeat<TrieNode?>(null, id - nodes.Count));
|
||||
return nodes[id - 1] ??= new TrieNode { Parent = null, Children = [], TokenIds = [], SubTreeTokenIds = [] };
|
||||
}
|
||||
var currentId = 0;
|
||||
for (var i = 0; i < data.Length; )
|
||||
{
|
||||
var node = GetNode(++currentId);
|
||||
var parentId = data[i++];
|
||||
node.Parent = parentId != 0 ? GetNode(parentId) : null;
|
||||
|
||||
var endOfChildren = i;
|
||||
while (endOfChildren < data.Length && data[endOfChildren] > 0) endOfChildren++;
|
||||
var numberOfChildren = (endOfChildren - i) / 2;
|
||||
for (var j = i; j < i + numberOfChildren; j++)
|
||||
{
|
||||
var codePoint = data[j];
|
||||
var child = GetNode(data[j + numberOfChildren]);
|
||||
node.Children.Add(codePoint, child);
|
||||
}
|
||||
i = endOfChildren;
|
||||
|
||||
if (data[i] == 0) i++; // No token IDs
|
||||
else while (i < data.Length && data[i] < 0) node.TokenIds.Add(-data[i++] - 1);
|
||||
}
|
||||
var root = nodes[0]!;
|
||||
|
||||
// DFS to construct code point paths for each token
|
||||
var tokenCodePoints = new Dictionary<int, int[]>();
|
||||
var currentCodePoints = new List<int>();
|
||||
void DfsCodePoints(TrieNode node)
|
||||
{
|
||||
foreach (var tokenId in node.TokenIds) tokenCodePoints.Add(tokenId, [.. currentCodePoints]);
|
||||
foreach (var (codePoint, child) in node.Children)
|
||||
{
|
||||
if (child.Parent != node) continue; // Skip grafted paths as these are not the canonical representation of the tokens
|
||||
currentCodePoints.Add(codePoint);
|
||||
DfsCodePoints(child);
|
||||
currentCodePoints.RemoveAt(currentCodePoints.Count - 1);
|
||||
}
|
||||
}
|
||||
DfsCodePoints(root);
|
||||
|
||||
// DFS to construct subTreeTokenIds for each node
|
||||
var visitedNodes = new HashSet<TrieNode>();
|
||||
List<int> DfsSubTreeTokenIds(TrieNode node)
|
||||
{
|
||||
if (visitedNodes.Contains(node)) return node.SubTreeTokenIds;
|
||||
visitedNodes.Add(node);
|
||||
node.SubTreeTokenIds = new HashSet<int>(node.TokenIds.Concat(node.Children.Values.SelectMany(DfsSubTreeTokenIds))).ToList();
|
||||
return node.SubTreeTokenIds;
|
||||
};
|
||||
DfsSubTreeTokenIds(root);
|
||||
|
||||
return new DeserializedTrie { Root = root, TokenCodePoints = tokenCodePoints };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.Common;
|
||||
|
||||
#region ToKatakana
|
||||
|
||||
public sealed class ToKatakana_ConvertsHiraganaToKatakanaTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("アイウエオ", CommonNormalization.ToKatakana("あいうえお"));
|
||||
Assert.Equal("カキクケコ", CommonNormalization.ToKatakana("かきくけこ"));
|
||||
Assert.Equal("サシスセソ", CommonNormalization.ToKatakana("さしすせそ"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ToKatakana_KeepsKatakanaUnchangedTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("アイウエオ", CommonNormalization.ToKatakana("アイウエオ"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ToKatakana_KeepsNonKanaUnchangedTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("abc123", CommonNormalization.ToKatakana("abc123"));
|
||||
Assert.Equal("漢字", CommonNormalization.ToKatakana("漢字"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ToKatakana_HandlesMixedInputTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("アアa漢", CommonNormalization.ToKatakana("あアa漢"));
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region NormalizeCodePoint
|
||||
|
||||
public sealed class NormalizeCodePoint_ConvertsFullwidthAsciiToHalfwidthLowercaseTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal('a', CommonNormalization.NormalizeCodePoint('Ａ'));
|
||||
Assert.Equal('b', CommonNormalization.NormalizeCodePoint('Ｂ'));
|
||||
Assert.Equal('c', CommonNormalization.NormalizeCodePoint('Ｃ'));
|
||||
Assert.Equal('1', CommonNormalization.NormalizeCodePoint('１'));
|
||||
Assert.Equal('2', CommonNormalization.NormalizeCodePoint('２'));
|
||||
Assert.Equal('3', CommonNormalization.NormalizeCodePoint('３'));
|
||||
Assert.Equal('!', CommonNormalization.NormalizeCodePoint('！'));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class NormalizeCodePoint_ConvertsFullwidthSpaceToHalfwidthTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal(' ', CommonNormalization.NormalizeCodePoint(' '));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class NormalizeCodePoint_ConvertsHalfwidthKanaToFullwidthTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal('ア', CommonNormalization.NormalizeCodePoint('ｱ'));
|
||||
Assert.Equal('イ', CommonNormalization.NormalizeCodePoint('ｲ'));
|
||||
Assert.Equal('ウ', CommonNormalization.NormalizeCodePoint('ｳ'));
|
||||
Assert.Equal('エ', CommonNormalization.NormalizeCodePoint('ｴ'));
|
||||
Assert.Equal('オ', CommonNormalization.NormalizeCodePoint('ｵ'));
|
||||
Assert.Equal('カ', CommonNormalization.NormalizeCodePoint('ｶ'));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class NormalizeCodePoint_NormalizesVoicedSoundMarksTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal(0x3099, CommonNormalization.NormalizeCodePoint('ﾞ')); // halfwidth voiced -> combining
|
||||
Assert.Equal(0x309A, CommonNormalization.NormalizeCodePoint('ﾟ')); // halfwidth semi-voiced -> combining
|
||||
Assert.Equal(0x3099, CommonNormalization.NormalizeCodePoint('゛')); // fullwidth voiced -> combining
|
||||
Assert.Equal(0x309A, CommonNormalization.NormalizeCodePoint('゜')); // fullwidth semi-voiced -> combining
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class NormalizeCodePoint_ConvertsHalfwidthPunctuationToFullwidthTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal('。', CommonNormalization.NormalizeCodePoint('｡'));
|
||||
Assert.Equal('「', CommonNormalization.NormalizeCodePoint('｢'));
|
||||
Assert.Equal('」', CommonNormalization.NormalizeCodePoint('｣'));
|
||||
Assert.Equal('、', CommonNormalization.NormalizeCodePoint('､'));
|
||||
Assert.Equal('・', CommonNormalization.NormalizeCodePoint('･'));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class NormalizeCodePoint_LowercasesRegularAsciiTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal('a', CommonNormalization.NormalizeCodePoint('A'));
|
||||
Assert.Equal('b', CommonNormalization.NormalizeCodePoint('B'));
|
||||
Assert.Equal('c', CommonNormalization.NormalizeCodePoint('C'));
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
using MaigoLabs.NeedLe.Indexer;
|
||||
using MaigoLabs.NeedLe.Searcher;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.E2E;
|
||||
|
||||
public sealed class Search_MatchesWithMixedSearchQueryTest : NeedleTestBase
|
||||
{
|
||||
private static readonly string[] TestDocuments =
|
||||
[
|
||||
"ミーティア",
|
||||
"エンドマークに希望と涙を添えて",
|
||||
"宵の鳥",
|
||||
"僕の和風本当上手",
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var compressed = InvertedIndexBuilder.BuildInvertedIndex(TestDocuments, TokenizerOptions);
|
||||
var invertedIndex = InvertedIndexLoader.Load(compressed);
|
||||
|
||||
var results = InvertedIndexSearcher.Search(invertedIndex, "bokunoh风じょう");
|
||||
|
||||
// Should have at least one result
|
||||
Assert.NotEmpty(results);
|
||||
|
||||
// The first result should be "僕の和風本当上手"
|
||||
Assert.Equal("僕の和風本当上手", results[0].DocumentText);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class Search_HighlightsSearchResultCorrectlyTest : NeedleTestBase
|
||||
{
|
||||
private static readonly string[] TestDocuments =
|
||||
[
|
||||
"ミーティア",
|
||||
"エンドマークに希望と涙を添えて",
|
||||
"宵の鳥",
|
||||
"僕の和風本当上手",
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var compressed = InvertedIndexBuilder.BuildInvertedIndex(TestDocuments, TokenizerOptions);
|
||||
var invertedIndex = InvertedIndexLoader.Load(compressed);
|
||||
|
||||
var results = InvertedIndexSearcher.Search(invertedIndex, "bokunoh风じょう");
|
||||
Assert.NotEmpty(results);
|
||||
|
||||
var highlighted = SearchResultHighlighter.Highlight(results[0]);
|
||||
|
||||
// Should be a list of parts
|
||||
Assert.NotEmpty(highlighted);
|
||||
|
||||
// Collect highlighted text
|
||||
var highlightedTexts = highlighted.Where(p => p.IsHighlighted).Select(p => p.Text).ToList();
|
||||
var highlightedJoined = string.Join("", highlightedTexts);
|
||||
|
||||
Assert.Contains("僕", highlightedJoined);
|
||||
Assert.Contains("の", highlightedJoined);
|
||||
Assert.Contains("和", highlightedJoined);
|
||||
Assert.Contains("風", highlightedJoined);
|
||||
Assert.Contains("上", highlightedJoined);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class Search_MatchesRomajiInputToKanaDocumentsTest : NeedleTestBase
|
||||
{
|
||||
private static readonly string[] TestDocuments =
|
||||
[
|
||||
"ミーティア",
|
||||
"エンドマークに希望と涙を添えて",
|
||||
"宵の鳥",
|
||||
"僕の和風本当上手",
|
||||
];
|
||||
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var compressed = InvertedIndexBuilder.BuildInvertedIndex(TestDocuments, TokenizerOptions);
|
||||
var invertedIndex = InvertedIndexLoader.Load(compressed);
|
||||
|
||||
// Search for "yoi" should match "宵の鳥"
|
||||
var results = InvertedIndexSearcher.Search(invertedIndex, "yoi");
|
||||
var matchedTexts = results.Select(r => r.DocumentText).ToList();
|
||||
|
||||
Assert.Contains("宵の鳥", matchedTexts);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MaigoLabs.NeedLe.Indexer.Trie;
|
||||
using MaigoLabs.NeedLe.Searcher.Trie;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.E2E;
|
||||
|
||||
#region Trie Building
|
||||
|
||||
public sealed class TrieBuilding_BuildsTrieWithMultipleDifferentTokensTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var trie = TrieBuilder.BuildTrie([
|
||||
(0, "hello".ToCodePoints()),
|
||||
(1, "help".ToCodePoints()),
|
||||
(2, "world".ToCodePoints()),
|
||||
(3, "word".ToCodePoints()),
|
||||
]);
|
||||
|
||||
// Traverse to verify structure
|
||||
var helloNode = trie.Traverse("hello".ToCodePoints().ToArray());
|
||||
var helpNode = trie.Traverse("help".ToCodePoints().ToArray());
|
||||
var worldNode = trie.Traverse("world".ToCodePoints().ToArray());
|
||||
var wordNode = trie.Traverse("word".ToCodePoints().ToArray());
|
||||
|
||||
Assert.NotNull(helloNode);
|
||||
Assert.NotNull(helpNode);
|
||||
Assert.NotNull(worldNode);
|
||||
Assert.NotNull(wordNode);
|
||||
|
||||
// Check token IDs
|
||||
Assert.Contains(0, helloNode!.TokenIds);
|
||||
Assert.Contains(1, helpNode!.TokenIds);
|
||||
Assert.Contains(2, worldNode!.TokenIds);
|
||||
Assert.Contains(3, wordNode!.TokenIds);
|
||||
|
||||
// Check that 'hel' prefix node has both tokens in subTree
|
||||
var helNode = trie.Traverse("hel".ToCodePoints().ToArray());
|
||||
Assert.NotNull(helNode);
|
||||
Assert.Contains(0, helNode!.SubTreeTokenIds);
|
||||
Assert.Contains(1, helNode.SubTreeTokenIds);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class TrieBuilding_HandlesJapaneseTextTokensTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var trie = TrieBuilder.BuildTrie([
|
||||
(0, "さくら".ToCodePoints()),
|
||||
(1, "サクラ".ToCodePoints()),
|
||||
(2, "桜".ToCodePoints()),
|
||||
]);
|
||||
|
||||
Assert.Contains(0, trie.Traverse("さくら".ToCodePoints().ToArray())?.TokenIds ?? []);
|
||||
Assert.Contains(1, trie.Traverse("サクラ".ToCodePoints().ToArray())?.TokenIds ?? []);
|
||||
Assert.Contains(2, trie.Traverse("桜".ToCodePoints().ToArray())?.TokenIds ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Trie Serialization
|
||||
|
||||
public sealed class TrieSerialization_SerializesAndDeserializesCorrectlyTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var originalTrie = TrieBuilder.BuildTrie([
|
||||
(0, "apple".ToCodePoints()),
|
||||
(1, "app".ToCodePoints()),
|
||||
(2, "banana".ToCodePoints()),
|
||||
]);
|
||||
|
||||
// Serialize
|
||||
var serialized = TrieSerializer.Serialize(originalTrie);
|
||||
Assert.True(serialized.Length > 0);
|
||||
|
||||
// Deserialize
|
||||
var deserialized = TrieDeserializer.Deserialize(serialized);
|
||||
var deserializedTrie = deserialized.Root;
|
||||
var tokenCodePoints = deserialized.TokenCodePoints;
|
||||
|
||||
// Verify structure is preserved
|
||||
var appleNode = deserializedTrie.Traverse("apple".ToCodePoints().ToArray());
|
||||
var appNode = deserializedTrie.Traverse("app".ToCodePoints().ToArray());
|
||||
var bananaNode = deserializedTrie.Traverse("banana".ToCodePoints().ToArray());
|
||||
|
||||
Assert.NotNull(appleNode);
|
||||
Assert.NotNull(appNode);
|
||||
Assert.NotNull(bananaNode);
|
||||
|
||||
Assert.Contains(0, appleNode!.TokenIds);
|
||||
Assert.Contains(1, appNode!.TokenIds);
|
||||
Assert.Contains(2, bananaNode!.TokenIds);
|
||||
|
||||
// Verify tokenCodePoints map
|
||||
Assert.Equal("apple", tokenCodePoints[0].ToUtf32String());
|
||||
Assert.Equal("app", tokenCodePoints[1].ToUtf32String());
|
||||
Assert.Equal("banana", tokenCodePoints[2].ToUtf32String());
|
||||
|
||||
// Verify subTreeTokenIds are reconstructed
|
||||
Assert.Contains(0, appNode.SubTreeTokenIds);
|
||||
Assert.Contains(1, appNode.SubTreeTokenIds);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class TrieSerialization_PreservesParentReferencesAfterDeserializationTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var originalTrie = TrieBuilder.BuildTrie([
|
||||
(0, "test".ToCodePoints()),
|
||||
]);
|
||||
|
||||
var serialized = TrieSerializer.Serialize(originalTrie);
|
||||
var deserialized = TrieDeserializer.Deserialize(serialized);
|
||||
var root = deserialized.Root;
|
||||
|
||||
var testNode = root.Traverse("test".ToCodePoints().ToArray());
|
||||
Assert.NotNull(testNode);
|
||||
|
||||
// Walk back to root via parent references
|
||||
TrieNode? node = testNode;
|
||||
var depth = 0;
|
||||
while (node?.Parent != null)
|
||||
{
|
||||
node = node.Parent;
|
||||
depth++;
|
||||
}
|
||||
Assert.Equal(4, depth); // 't' -> 'e' -> 's' -> 't' -> root
|
||||
Assert.Same(root, node);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
using MaigoLabs.NeedLe.Indexer.Han;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
|
||||
|
||||
#region IsHanCharacter
|
||||
|
||||
public sealed class IsHanCharacter_ReturnsTrueForCjkCharactersTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.True(HanVariantProvider.IsHanCharacter('中'));
|
||||
Assert.True(HanVariantProvider.IsHanCharacter('国'));
|
||||
Assert.True(HanVariantProvider.IsHanCharacter('日'));
|
||||
Assert.True(HanVariantProvider.IsHanCharacter('本'));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class IsHanCharacter_ReturnsFalseForNonCjkCharactersTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.False(HanVariantProvider.IsHanCharacter('a'));
|
||||
Assert.False(HanVariantProvider.IsHanCharacter('あ'));
|
||||
Assert.False(HanVariantProvider.IsHanCharacter('ア'));
|
||||
Assert.False(HanVariantProvider.IsHanCharacter('1'));
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region GetHanVariants
|
||||
|
||||
public sealed class GetHanVariants_ReturnsVariantsForSimplifiedTraditionalTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var provider = new HanVariantProvider();
|
||||
// 国 (simplified) and 國 (traditional) should be variants of each other
|
||||
var variants1 = provider.GetHanVariants('国');
|
||||
var variants2 = provider.GetHanVariants('國');
|
||||
Assert.Contains('国', variants1);
|
||||
Assert.Contains('國', variants1);
|
||||
Assert.Contains('国', variants2);
|
||||
Assert.Contains('國', variants2);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GetHanVariants_ReturnsCharacterItselfForNoVariantsTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var provider = new HanVariantProvider();
|
||||
var variants = provider.GetHanVariants('一');
|
||||
Assert.Contains('一', variants);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GetHanVariants_ReturnsEmptyForNonHanCharactersTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var provider = new HanVariantProvider();
|
||||
Assert.Empty(provider.GetHanVariants('a'));
|
||||
Assert.Empty(provider.GetHanVariants('あ'));
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
using MaigoLabs.NeedLe.Indexer.Han;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
|
||||
|
||||
public sealed class GetPinyinCandidates_ReturnsPinyinForHanCharacterTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var candidates = PinyinHelper.GetPinyinCandidates('中').ToList();
|
||||
Assert.Contains("zhong", candidates);
|
||||
Assert.Contains("zh", candidates); // initial
|
||||
Assert.Contains("z", candidates); // first letter
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GetPinyinCandidates_ReturnsMultiplePinyinForPolyphonicTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
// 行 can be "xing" or "hang"
|
||||
var candidates = PinyinHelper.GetPinyinCandidates('行').ToList();
|
||||
Assert.Contains("xing", candidates);
|
||||
Assert.Contains("hang", candidates);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GetPinyinCandidates_IncludesFuzzyPinyinVariantsTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
// 风 is "feng", should also have fuzzy variant "fen"
|
||||
var candidates = PinyinHelper.GetPinyinCandidates('风').ToList();
|
||||
Assert.Contains("feng", candidates);
|
||||
Assert.Contains("fen", candidates); // fuzzy: eng -> en
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GetPinyinCandidates_ReturnsEmptyForNonHanCharactersTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Empty(PinyinHelper.GetPinyinCandidates('a'));
|
||||
Assert.Empty(PinyinHelper.GetPinyinCandidates('あ'));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
using MaigoLabs.NeedLe.Indexer.Han;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
|
||||
|
||||
public sealed class UnionFindSet_FindsSelfAsRootInitiallyTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var ufs = new UnionFindSet();
|
||||
Assert.Equal(1, ufs.Find(1));
|
||||
Assert.Equal(2, ufs.Find(2));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class UnionFindSet_UnionsTwoElementsTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var ufs = new UnionFindSet();
|
||||
ufs.Union(1, 2);
|
||||
Assert.Equal(ufs.Find(1), ufs.Find(2));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class UnionFindSet_UnionsMultipleElementsTransitivelyTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var ufs = new UnionFindSet();
|
||||
ufs.Union(1, 2);
|
||||
ufs.Union(2, 3);
|
||||
ufs.Union(4, 5);
|
||||
Assert.Equal(ufs.Find(1), ufs.Find(3));
|
||||
Assert.NotEqual(ufs.Find(1), ufs.Find(4));
|
||||
ufs.Union(3, 4);
|
||||
Assert.Equal(ufs.Find(1), ufs.Find(5));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class UnionFindSet_IteratesAllKeysTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var ufs = new UnionFindSet();
|
||||
ufs.Union(1, 2);
|
||||
ufs.Union(3, 4);
|
||||
var keys = ufs.Keys.ToList();
|
||||
Assert.Contains(1, keys);
|
||||
Assert.Contains(2, keys);
|
||||
Assert.Contains(3, keys);
|
||||
Assert.Contains(4, keys);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
using MaigoLabs.NeedLe.Indexer.Japanese;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.Indexer.Japanese;
|
||||
|
||||
#region ToRomajiStrictly
|
||||
|
||||
public sealed class ToRomajiStrictly_ConvertsBasicKanaToRomajiTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("a", JapaneseUtils.ToRomajiStrictly("あ"));
|
||||
Assert.Equal("ka", JapaneseUtils.ToRomajiStrictly("か"));
|
||||
Assert.Equal("sakura", JapaneseUtils.ToRomajiStrictly("さくら"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ToRomajiStrictly_ConvertsKatakanaToRomajiTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("a", JapaneseUtils.ToRomajiStrictly("ア"));
|
||||
Assert.Equal("ka", JapaneseUtils.ToRomajiStrictly("カ"));
|
||||
Assert.Equal("sakura", JapaneseUtils.ToRomajiStrictly("サクラ"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ToRomajiStrictly_HandlesLongVowelsTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("ou", JapaneseUtils.ToRomajiStrictly("おう"));
|
||||
Assert.Equal("oo", JapaneseUtils.ToRomajiStrictly("おお"));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ToRomajiStrictly_ReturnsEmptyForInvalidFirstCharacterTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("", JapaneseUtils.ToRomajiStrictly("ー")); // prolonged sound mark cannot be first
|
||||
Assert.Equal("", JapaneseUtils.ToRomajiStrictly("ゃ")); // small ya cannot be first
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ToRomajiStrictly_ReturnsEmptyForInvalidLastCharacterTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("", JapaneseUtils.ToRomajiStrictly("っ")); // small tsu cannot be last
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class ToRomajiStrictly_HandlesGeminationTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
Assert.Equal("katta", JapaneseUtils.ToRomajiStrictly("かった"));
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
using MaigoLabs.NeedLe.Indexer.Japanese;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.Indexer.Japanese;
|
||||
|
||||
public sealed class GetAllKanaReadings_ReturnsKatakanaForPureKanaInputTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var provider = new TranscriptionProvider();
|
||||
var readings = provider.GetAllKanaReadings("あ");
|
||||
Assert.Contains("ア", readings);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GetAllKanaReadings_ReturnsReadingsForKanjiTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var provider = new TranscriptionProvider();
|
||||
var readings = provider.GetAllKanaReadings("僕");
|
||||
Assert.NotEmpty(readings);
|
||||
// 僕 should have reading ボク
|
||||
Assert.Contains("ボク", readings);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GetAllKanaReadings_ReturnsReadingsForCompoundWordsTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var provider = new TranscriptionProvider();
|
||||
var readings = provider.GetAllKanaReadings("和風");
|
||||
Assert.NotEmpty(readings);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
using MaigoLabs.NeedLe.Common.Types;
|
||||
using MaigoLabs.NeedLe.Indexer;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.Indexer;
|
||||
|
||||
public sealed class Tokenizer_TokenizesMixedJapaneseTextTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var tokenizer = new Tokenizer(TokenizerOptions);
|
||||
var tokens = tokenizer.Tokenize("僕の和風本当上手");
|
||||
|
||||
var tokenDefs = tokenizer.Tokens.Values.ToList();
|
||||
|
||||
// Should have tokens of various types
|
||||
var types = tokenDefs.Select(t => t.Type).ToHashSet();
|
||||
Assert.Contains(TokenType.Han, types);
|
||||
Assert.Contains(TokenType.Pinyin, types);
|
||||
Assert.Contains(TokenType.Kana, types);
|
||||
Assert.Contains(TokenType.Romaji, types);
|
||||
|
||||
// Helper to get token texts at a specific position by type
|
||||
List<string> GetTokenTextsAt(int pos, TokenType type) => tokens
|
||||
.Where(t => t.Start <= pos && t.End > pos)
|
||||
.Select(t => tokenDefs.First(d => d.Id == t.Id))
|
||||
.Where(d => d.Type == type)
|
||||
.Select(d => d.Text)
|
||||
.ToList();
|
||||
|
||||
// Position 0: 僕
|
||||
Assert.Contains("僕", GetTokenTextsAt(0, TokenType.Han));
|
||||
Assert.Contains("pu", GetTokenTextsAt(0, TokenType.Pinyin));
|
||||
Assert.Contains("ボク", GetTokenTextsAt(0, TokenType.Kana));
|
||||
Assert.Contains("boku", GetTokenTextsAt(0, TokenType.Romaji));
|
||||
|
||||
// Position 1: の (hiragana, no Han/Pinyin)
|
||||
Assert.Empty(GetTokenTextsAt(1, TokenType.Han));
|
||||
Assert.Empty(GetTokenTextsAt(1, TokenType.Pinyin));
|
||||
Assert.Contains("ノ", GetTokenTextsAt(1, TokenType.Kana));
|
||||
Assert.Contains("no", GetTokenTextsAt(1, TokenType.Romaji));
|
||||
|
||||
// Position 2: 和
|
||||
Assert.Contains("和", GetTokenTextsAt(2, TokenType.Han));
|
||||
Assert.Contains("he", GetTokenTextsAt(2, TokenType.Pinyin));
|
||||
Assert.Contains("ワ", GetTokenTextsAt(2, TokenType.Kana));
|
||||
Assert.Contains("wa", GetTokenTextsAt(2, TokenType.Romaji));
|
||||
|
||||
// Position 3: 風
|
||||
Assert.Contains("風", GetTokenTextsAt(3, TokenType.Han));
|
||||
Assert.Contains("风", GetTokenTextsAt(3, TokenType.Han)); // simplified variant
|
||||
Assert.Contains("feng", GetTokenTextsAt(3, TokenType.Pinyin));
|
||||
Assert.Contains("フウ", GetTokenTextsAt(3, TokenType.Kana));
|
||||
Assert.Contains("fu", GetTokenTextsAt(3, TokenType.Romaji));
|
||||
|
||||
// Position 4: 本
|
||||
Assert.Contains("本", GetTokenTextsAt(4, TokenType.Han));
|
||||
Assert.Contains("ben", GetTokenTextsAt(4, TokenType.Pinyin));
|
||||
Assert.Contains("ホン", GetTokenTextsAt(4, TokenType.Kana));
|
||||
Assert.Contains("hon", GetTokenTextsAt(4, TokenType.Romaji));
|
||||
|
||||
// Position 5: 当
|
||||
Assert.Contains("当", GetTokenTextsAt(5, TokenType.Han));
|
||||
Assert.Contains("當", GetTokenTextsAt(5, TokenType.Han)); // traditional variant
|
||||
Assert.Contains("dang", GetTokenTextsAt(5, TokenType.Pinyin));
|
||||
Assert.Contains("トウ", GetTokenTextsAt(5, TokenType.Kana));
|
||||
Assert.Contains("to", GetTokenTextsAt(5, TokenType.Romaji)); // normalized: tou -> to
|
||||
|
||||
// Position 6: 上
|
||||
Assert.Contains("上", GetTokenTextsAt(6, TokenType.Han));
|
||||
Assert.Contains("shang", GetTokenTextsAt(6, TokenType.Pinyin));
|
||||
Assert.Contains("ジョウ", GetTokenTextsAt(6, TokenType.Kana));
|
||||
Assert.Contains("jo", GetTokenTextsAt(6, TokenType.Romaji)); // normalized: jou -> jo
|
||||
|
||||
// Position 7: 手
|
||||
Assert.Contains("手", GetTokenTextsAt(7, TokenType.Han));
|
||||
Assert.Contains("shou", GetTokenTextsAt(7, TokenType.Pinyin));
|
||||
Assert.Contains("シュ", GetTokenTextsAt(7, TokenType.Kana));
|
||||
Assert.Contains("shu", GetTokenTextsAt(7, TokenType.Romaji));
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class Tokenizer_NoDuplicateTokensTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var tokenizer = new Tokenizer(TokenizerOptions);
|
||||
|
||||
// Tokenize multiple music names that share some characters
|
||||
tokenizer.Tokenize("僕の和風本当上手");
|
||||
tokenizer.Tokenize("僕");
|
||||
tokenizer.Tokenize("和風");
|
||||
|
||||
// Check that there are no duplicate tokens
|
||||
var tokenDefs = tokenizer.Tokens.Values.ToList();
|
||||
var tokenKeys = tokenDefs.Select(t => $"{t.Type}:{t.Text}").ToList();
|
||||
var uniqueKeys = tokenKeys.ToHashSet();
|
||||
|
||||
Assert.Equal(uniqueKeys.Count, tokenKeys.Count);
|
||||
|
||||
// Also check that IDs are unique
|
||||
var ids = tokenDefs.Select(t => t.Id).ToList();
|
||||
var uniqueIds = ids.ToHashSet();
|
||||
Assert.Equal(uniqueIds.Count, ids.Count);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class Tokenizer_HandlesRawTokensForNonCjkTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var tokenizer = new Tokenizer(TokenizerOptions);
|
||||
tokenizer.Tokenize("a-b");
|
||||
|
||||
var tokenDefs = tokenizer.Tokens.Values.ToList();
|
||||
var rawTokenTexts = tokenDefs.Where(t => t.Type == TokenType.Raw).Select(t => t.Text).ToList();
|
||||
|
||||
Assert.Contains("a", rawTokenTexts);
|
||||
Assert.Contains("-", rawTokenTexts);
|
||||
Assert.Contains("b", rawTokenTexts);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class Tokenizer_TokenizesCompoundWordKyouTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var tokenizer = new Tokenizer(TokenizerOptions);
|
||||
var tokens = tokenizer.Tokenize("今日");
|
||||
var tokenDefs = tokenizer.Tokens.Values.ToList();
|
||||
|
||||
// Helper to get tokens with specific type and span
|
||||
List<string> GetTokensWithSpan(TokenType type, int start, int end) => tokens
|
||||
.Where(t => t.Start == start && t.End == end)
|
||||
.Select(t => tokenDefs.First(d => d.Id == t.Id))
|
||||
.Where(d => d.Type == type)
|
||||
.Select(d => d.Text)
|
||||
.ToList();
|
||||
|
||||
// Individual character readings at position 0: 今
|
||||
Assert.Contains("今", GetTokensWithSpan(TokenType.Han, 0, 1));
|
||||
Assert.Contains("jin", GetTokensWithSpan(TokenType.Pinyin, 0, 1));
|
||||
Assert.Contains("コン", GetTokensWithSpan(TokenType.Kana, 0, 1));
|
||||
Assert.Contains("イマ", GetTokensWithSpan(TokenType.Kana, 0, 1));
|
||||
Assert.Contains("kon", GetTokensWithSpan(TokenType.Romaji, 0, 1));
|
||||
Assert.Contains("ima", GetTokensWithSpan(TokenType.Romaji, 0, 1));
|
||||
|
||||
// Individual character readings at position 1: 日
|
||||
Assert.Contains("日", GetTokensWithSpan(TokenType.Han, 1, 2));
|
||||
Assert.Contains("ri", GetTokensWithSpan(TokenType.Pinyin, 1, 2));
|
||||
Assert.Contains("ニチ", GetTokensWithSpan(TokenType.Kana, 1, 2));
|
||||
Assert.Contains("ヒ", GetTokensWithSpan(TokenType.Kana, 1, 2));
|
||||
Assert.Contains("niti", GetTokensWithSpan(TokenType.Romaji, 1, 2));
|
||||
Assert.Contains("hi", GetTokensWithSpan(TokenType.Romaji, 1, 2));
|
||||
|
||||
// Combined reading for "今日" [0, 2] - this is an indivisible compound word
|
||||
Assert.Contains("キョウ", GetTokensWithSpan(TokenType.Kana, 0, 2));
|
||||
Assert.Contains("kyo", GetTokensWithSpan(TokenType.Romaji, 0, 2)); // normalized: kyou -> kyo
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
using MaigoLabs.NeedLe.Common;
|
||||
using MaigoLabs.NeedLe.Common.Extensions;
|
||||
using MaigoLabs.NeedLe.Indexer.Trie;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests.Indexer;
|
||||
|
||||
#region GraftTriePaths
|
||||
|
||||
public sealed class GraftTriePaths_GraftsPathsAccordingToNormalizationRulesTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
// Build a trie with tokens containing normalized forms
|
||||
var trie = TrieBuilder.BuildTrie([
|
||||
(0, "sya".ToCodePoints()), // normalized form of "sha"
|
||||
(1, "tu".ToCodePoints()), // normalized form of "tsu"
|
||||
]);
|
||||
|
||||
// Graft paths so that "sha" -> "sya" and "tsu" -> "tu"
|
||||
TrieBuilder.GraftTriePaths(trie, [
|
||||
("sha".ToCodePoints().ToArray(), "sya".ToCodePoints().ToArray()),
|
||||
("tsu".ToCodePoints().ToArray(), "tu".ToCodePoints().ToArray()),
|
||||
]);
|
||||
|
||||
// Now we should be able to traverse using both the original and grafted paths
|
||||
var syaNode = trie.Traverse("sya".ToCodePoints().ToArray());
|
||||
var shaNode = trie.Traverse("sha".ToCodePoints().ToArray());
|
||||
Assert.NotNull(syaNode);
|
||||
Assert.NotNull(shaNode);
|
||||
Assert.Same(syaNode, shaNode); // Both paths should lead to the same node
|
||||
|
||||
var tuNode = trie.Traverse("tu".ToCodePoints().ToArray());
|
||||
var tsuNode = trie.Traverse("tsu".ToCodePoints().ToArray());
|
||||
Assert.NotNull(tuNode);
|
||||
Assert.NotNull(tsuNode);
|
||||
Assert.Same(tuNode, tsuNode);
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class GraftTriePaths_HandlesChainedGraftRulesTest : NeedleTestBase
|
||||
{
|
||||
[Fact]
|
||||
public void Execute()
|
||||
{
|
||||
var trie = TrieBuilder.BuildTrie([
|
||||
(0, "o".ToCodePoints()), // normalized vowel
|
||||
]);
|
||||
|
||||
// Chain: "ou" -> "o", "oo" -> "o"
|
||||
TrieBuilder.GraftTriePaths(trie, [
|
||||
("ou".ToCodePoints().ToArray(), "o".ToCodePoints().ToArray()),
|
||||
("oo".ToCodePoints().ToArray(), "o".ToCodePoints().ToArray()),
|
||||
]);
|
||||
|
||||
var oNode = trie.Traverse("o".ToCodePoints().ToArray());
|
||||
var ouNode = trie.Traverse("ou".ToCodePoints().ToArray());
|
||||
var ooNode = trie.Traverse("oo".ToCodePoints().ToArray());
|
||||
|
||||
Assert.NotNull(oNode);
|
||||
Assert.Same(oNode, ouNode);
|
||||
Assert.Same(oNode, ooNode);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -0,0 +1,28 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<RootNamespace>$(ProjectName).Tests</RootNamespace>
|
||||
<AssemblyName>$(RootNamespace)</AssemblyName>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" />
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" />
|
||||
<PackageReference Include="xunit" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe.Common\MaigoLabs.NeedLe.Common.csproj" />
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe.Indexer\MaigoLabs.NeedLe.Indexer.csproj" />
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe.Searcher\MaigoLabs.NeedLe.Searcher.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
using MaigoLabs.NeedLe.Indexer;
|
||||
using MaigoLabs.NeedLe.Indexer.Han;
|
||||
using MaigoLabs.NeedLe.Indexer.Japanese;
|
||||
|
||||
namespace MaigoLabs.NeedLe.Tests;
|
||||
|
||||
public abstract class NeedleTestBase
|
||||
{
|
||||
public static HanVariantProvider HanVariantProvider { get; set; } = new();
|
||||
public static TranscriptionProvider TranscriptionProvider { get; set; } = new();
|
||||
public static TokenizerOptions TokenizerOptions => new() { HanVariantProvider = HanVariantProvider, TranscriptionProvider = TranscriptionProvider };
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
<Solution>
|
||||
|
||||
<Configurations>
|
||||
<Platform Name="Any CPU" />
|
||||
<Platform Name="x64" />
|
||||
<Platform Name="x86" />
|
||||
</Configurations>
|
||||
|
||||
<Project Path="MaigoLabs.NeedLe/MaigoLabs.NeedLe.csproj" />
|
||||
<Project Path="MaigoLabs.NeedLe.Common/MaigoLabs.NeedLe.Common.csproj" />
|
||||
<Project Path="MaigoLabs.NeedLe.Indexer/MaigoLabs.NeedLe.Indexer.csproj" />
|
||||
<Project Path="MaigoLabs.NeedLe.Searcher/MaigoLabs.NeedLe.Searcher.csproj" />
|
||||
<Project Path="MaigoLabs.NeedLe.Playground/MaigoLabs.NeedLe.Playground.csproj" />
|
||||
<Project Path="MaigoLabs.NeedLe.Tests/MaigoLabs.NeedLe.Tests.csproj" />
|
||||
|
||||
</Solution>
|
||||
@@ -0,0 +1,35 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Library</OutputType>
|
||||
<RootNamespace>$(ProjectName)</RootNamespace>
|
||||
<AssemblyName>$(RootNamespace)</AssemblyName>
|
||||
</PropertyGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>$(RootNamespace)</PackageId>
|
||||
<IncludeBuildOutput>false</IncludeBuildOutput>
|
||||
<IncludeContentInPack>false</IncludeContentInPack>
|
||||
<NoPackageAnalysis>true</NoPackageAnalysis>
|
||||
<MeCabUseDefaultDictionary>False</MeCabUseDefaultDictionary>
|
||||
<PackageReadmeFile></PackageReadmeFile>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe.Common\MaigoLabs.NeedLe.Common.csproj" PrivateAssets="none" />
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe.Indexer\MaigoLabs.NeedLe.Indexer.csproj" PrivateAssets="none" />
|
||||
<ProjectReference Include="..\MaigoLabs.NeedLe.Searcher\MaigoLabs.NeedLe.Searcher.csproj" PrivateAssets="none" />
|
||||
</ItemGroup>
|
||||
|
||||
<!-- Directly add README to package files -->
|
||||
<Target Name="AddReadmeToPackage" BeforeTargets="GenerateNuspec">
|
||||
<ItemGroup>
|
||||
<_PackageFiles Include="..\README.md" PackagePath="/" />
|
||||
</ItemGroup>
|
||||
<PropertyGroup>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
</PropertyGroup>
|
||||
</Target>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,57 @@
|
||||
# `MaigoLabs.NeedLe`
|
||||
|
||||
Fuzzy search engine for small text pieces, with Chinese/Japanese pronunciation support.
|
||||
|
||||
See also [in-browser demo](https://needle.maigo.dev) (TypeScript version, but the same features as in C#).
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
dotnet add package MaigoLabs.NeedLe
|
||||
```
|
||||
|
||||
Or install sub-packages separately:
|
||||
|
||||
```bash
|
||||
dotnet add package MaigoLabs.NeedLe.Indexer # For building indexes
|
||||
dotnet add package MaigoLabs.NeedLe.Searcher # For searching only
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Indexing
|
||||
|
||||
Indexing requires dictionaries. These are installed as dependencies of the `MaigoLabs.NeedLe.Indexer` package:
|
||||
|
||||
* MeCab.DotNet
|
||||
* OpenccNetLib
|
||||
* hyjiacan.pinyin4net
|
||||
|
||||
```csharp
|
||||
using MaigoLabs.NeedLe.Indexer;
|
||||
|
||||
var documents = new[] { "你好世界", "こんにちは" };
|
||||
var compressedIndex = InvertedIndexBuilder.BuildInvertedIndex(documents);
|
||||
// To customize dictionary paths, pass the second argument `TokenizerOptions` to `BuildInvertedIndex`.
|
||||
|
||||
// The built index could be stored for later use, or sent to frontend to load with TypeScript package `@maigolabs/needle`.
|
||||
// For compatibility with .NET Standard, we don't provide JSON related methods. You can use any JSON library to serialize/deserialize the index in the way you prefer.
|
||||
var json = JsonSerializer.Serialize(compressedIndex);
|
||||
```
|
||||
|
||||
### Searching
|
||||
|
||||
Searching requires a prebuilt index but doesn't require dictionaries. Searcher is a lightweight package without dependencies.
|
||||
|
||||
```csharp
|
||||
using MaigoLabs.NeedLe.Searcher;
|
||||
|
||||
// Index returned by `BuildInvertedIndex`.
|
||||
var index = InvertedIndexLoader.Load(compressedIndex);
|
||||
|
||||
var results = InvertedIndexSearcher.Search(index, "sekai");
|
||||
foreach (var result in results) Console.WriteLine($"{result.DocumentText} ({result.MatchRatio:P0})")
|
||||
// → 你好世界 (50%)
|
||||
```
|
||||
|
||||
To highlight the search result, see also `SearchResultHighlighter`.
|
||||
Reference in New Issue
Block a user