feat: initial commit

2026-01-01 03:40:41 +08:00
commit 631f8ed771
98 changed files with 14776 additions and 0 deletions
@@ -0,0 +1,80 @@
+using MaigoLabs.NeedLe.Common.Extensions;
+using OpenccNetLib;
+
+namespace MaigoLabs.NeedLe.Indexer.Han;
+
+public class HanVariantProvider
+{
+    private readonly Dictionary<int, int[]> EXCHANGE_MAP;
+    public HanVariantProvider(DictWithMaxLength[]? dicts = null)
+    {
+        dicts ??=
+        [
+            DictionaryLib.Provider.hk_variants,
+            DictionaryLib.Provider.hk_variants_rev,
+            DictionaryLib.Provider.jp_variants,
+            DictionaryLib.Provider.jp_variants_rev,
+            DictionaryLib.Provider.st_characters,
+            DictionaryLib.Provider.ts_characters,
+            DictionaryLib.Provider.tw_variants,
+            DictionaryLib.Provider.tw_variants_rev,
+        ];
+        EXCHANGE_MAP = BuildHanExchangeMap(dicts);
+    }
+
+    private Dictionary<int, int[]> BuildHanExchangeMap(DictWithMaxLength[] dicts)
+    {
+        var unionFindSet = new UnionFindSet();
+        foreach (var dict in dicts) foreach (var item in dict.Dict)
+        {
+            var from = item.Key.ToCodePoints().ToArray();
+            var to = item.Value.ToCodePoints().ToArray();
+            if (from.Length != 1 || to.Length != 1) continue;
+            unionFindSet.Union(from[0], to[0]);
+        }
+        var variants = new Dictionary<int, List<int>>();
+        foreach (var x in unionFindSet.Keys)
+        {
+            var parent = unionFindSet.Find(x);
+            if (!variants.TryGetValue(parent, out var list)) variants[parent] = list = [];
+            if (x != parent) variants[x] = list;
+            list.Add(x);
+        }
+        return variants.ToDictionary(item => item.Key, item => item.Value.OrderBy(x => x).ToArray());
+    }
+
+    // https://github.com/google/re2/blob/e7aec5985072c1dbe735add802653ef4b36c231a/re2/unicode_groups.cc#L5590-L5615
+    private static readonly (int Min, int Max)[] RE2_SCRIPT_HAN_RENAGES =
+    [
+        // Han_range16
+        (11904, 11929),
+        (11931, 12019),
+        (12032, 12245),
+        (12293, 12293),
+        (12295, 12295),
+        (12321, 12329),
+        (12344, 12347),
+        (13312, 19903),
+        (19968, 40959),
+        (63744, 64109),
+        (64112, 64217),
+        // Han_range32
+        (94178, 94179),
+        (94192, 94193),
+        (131072, 173791),
+        (173824, 177977),
+        (177984, 178205),
+        (178208, 183969),
+        (183984, 191456),
+        (191472, 192093),
+        (194560, 195101),
+        (196608, 201546),
+        (201552, 205743),
+    ];
+
+    public static bool IsHanCharacter(int codePoint) => RE2_SCRIPT_HAN_RENAGES.Any(range => codePoint >= range.Min && codePoint <= range.Max);
+
+    public int[] GetHanVariants(int codePoint) => EXCHANGE_MAP.TryGetValue(codePoint, out var variants)
+        ? variants
+        : IsHanCharacter(codePoint) ? [codePoint] : [];
+}
@@ -0,0 +1,19 @@
+using hyjiacan.py4n;
+
+namespace MaigoLabs.NeedLe.Indexer.Han;
+
+public static class PinyinHelper
+{
+    private static readonly string[] PINYIN_INITIALS = ["b", "p", "m", "f", "d", "t", "n", "l", "g", "k", "h", "j", "q", "x", "zh", "ch", "sh", "r", "z", "c", "s", "y", "w"];
+    private static readonly Dictionary<string, string> PINYIN_FINALS_FUZZY_MAP = new() { ["ang"] = "an", ["eng"] = "en", ["ing"] = "in" };
+
+    public static IEnumerable<string> GetPinyinCandidates(int codePoint) => codePoint < char.MinValue || codePoint > char.MaxValue || !PinyinUtil.IsHanzi((char)codePoint) ? [] :
+        Pinyin4Net.GetPinyin((char)codePoint, PinyinFormat.LOWERCASE | PinyinFormat.WITHOUT_TONE).Where(pinyin => pinyin.Length > 0).SelectMany(pinyin =>
+        {
+            var initial = PINYIN_INITIALS.FirstOrDefault(initial => pinyin.StartsWith(initial));
+            var initialAlphabet = initial != null ? initial[..1] : pinyin[..1];
+            var fuzzySuffix = pinyin.Length < 3 ? null : pinyin[^3..];
+            var fuzzyPinyin = fuzzySuffix != null && PINYIN_FINALS_FUZZY_MAP.TryGetValue(fuzzySuffix, out var fuzzySuffixTarget) ? pinyin[..^3] + fuzzySuffixTarget : null;
+            return new string?[] { pinyin, initial, initialAlphabet, fuzzyPinyin }.OfType<string>();
+        }).Distinct();
+}
@@ -0,0 +1,33 @@
+namespace MaigoLabs.NeedLe.Indexer.Han;
+
+public class UnionFindSet
+{
+    private Dictionary<int, int> Parent { get; set; } = [];
+    private Dictionary<int, int> Rank { get; set; } = [];
+
+    public IEnumerable<int> Keys => Parent.Keys;
+
+    public int Find(int x)
+    {
+        if (!Parent.TryGetValue(x, out var parent)) return Parent[x] = x;
+        else if (x == parent) return x;
+        else return Parent[x] = Find(parent);
+    }
+
+    public void Union(int x, int y)
+    {
+        x = Find(x);
+        y = Find(y);
+        if (x == y) return;
+        int rankX = GetRank(x), rankY = GetRank(y);
+        if (rankX < rankY) Parent[x] = y;
+        else if (rankX > rankY) Parent[y] = x;
+        else
+        {
+            Parent[y] = x;
+            Rank[x] = rankX + 1;
+        }
+    }
+
+    private int GetRank(int x) => !Rank.TryGetValue(x, out var rank) ? 0 : rank;
+}
@@ -0,0 +1,57 @@
+using MaigoLabs.NeedLe.Common;
+using MaigoLabs.NeedLe.Common.Extensions;
+using MaigoLabs.NeedLe.Common.Types;
+using MaigoLabs.NeedLe.Indexer.Japanese;
+using MaigoLabs.NeedLe.Indexer.Trie;
+
+namespace MaigoLabs.NeedLe.Indexer;
+
+public static class InvertedIndexBuilder
+{
+    private static TrieNode BuildTypedTrie(IEnumerable<TokenDefinition> tokenDefinitions, Func<TokenType, bool> typePredicate) =>
+        TrieBuilder.BuildTrie(tokenDefinitions
+            .Where(token => typePredicate(token.Type))
+            .Select(token => (token.Id, CodePoints: token.Text.ToCodePoints())));
+
+    public static CompressedInvertedIndex BuildInvertedIndex(string[] documents, TokenizerOptions? tokenizerOptions = null)
+    {
+        var tokenizer = new Tokenizer(tokenizerOptions);
+        var documentTokens = documents.Select(tokenizer.Tokenize).ToArray();
+
+        var tokenDefinitions = tokenizer.Tokens.Values;
+        var romajiRoot = BuildTypedTrie(tokenDefinitions, type => type == TokenType.Romaji);
+        var kanaRoot = BuildTypedTrie(tokenDefinitions, type => type == TokenType.Kana);
+        var otherRoot = BuildTypedTrie(tokenDefinitions, type => type != TokenType.Romaji && type != TokenType.Kana);
+        TrieBuilder.GraftTriePaths(romajiRoot, JapaneseNormalization.NORMALIZE_RULES_ROMAJI_CODEPOINTS);
+        TrieBuilder.GraftTriePaths(kanaRoot, JapaneseNormalization.NORMALIZE_RULES_KANA_DAKUTEN_CODEPOINTS);
+
+        var invertedIndex = new CompressedInvertedIndex
+        {
+            documents = documents,
+            tokenTypes = [.. tokenDefinitions.Select(token => (int)token.Type)],
+            tokenReferences = [.. tokenDefinitions.Select(_ => new List<int[]>())],
+            tries = new CompressedInvertedIndexTries
+            {
+                romaji = TrieSerializer.Serialize(romajiRoot),
+                kana = TrieSerializer.Serialize(kanaRoot),
+                other = TrieSerializer.Serialize(otherRoot),
+            },
+        };
+        for (var documentId = 0; documentId < documents.Length; documentId++)
+        {
+            var tokens = documentTokens[documentId];
+            var tokenOccurrences = new Dictionary<int, List<int>>();
+            foreach (var token in tokens)
+            {
+                if (!tokenOccurrences.TryGetValue(token.Id, out var occurrences)) tokenOccurrences[token.Id] = occurrences = [];
+                occurrences.Add(token.Start);
+                occurrences.Add(token.End);
+            }
+            foreach (var (tokenId, occurrences) in tokenOccurrences)
+            {
+                invertedIndex.tokenReferences[tokenId].Add([documentId, .. occurrences]);
+            }
+        }
+        return invertedIndex;
+    }
+}
@@ -0,0 +1,69 @@
+using MaigoLabs.NeedLe.Common.Extensions;
+
+namespace MaigoLabs.NeedLe.Indexer.Japanese;
+
+public static class JapaneseNormalization
+{
+    public delegate string Normalizer(string text);
+
+    public static Normalizer CreateNormalizer(Dictionary<string, string> rules) => text =>
+    {
+        while (true)
+        {
+            var beforeCurrentIteration = text;
+            foreach (var (from, to) in rules) text = text.Replace(from, to);
+            if (text == beforeCurrentIteration) break;
+        }
+        return text;
+    };
+
+    public static IEnumerable<(int[] From, int[] To)> ToCodePointPairs(Dictionary<string, string> rules) =>
+        rules.Select(rule => (From: rule.Key.ToCodePoints().ToArray(), To: rule.Value.ToCodePoints().ToArray()));
+
+    public static readonly Dictionary<string, string> NORMALIZE_RULES_ROMAJI = new()
+    {
+        // Remove all long vowels (sa-ba- -> saba)
+        ["-"] = "",
+        // Collapse consecutive vowels
+        ["aa"] = "a",
+        ["ii"] = "i",
+        ["uu"] = "u",
+        ["ee"] = "e",
+        ["oo"] = "o",
+        ["ou"] = "o",
+        // mb/mp/mm -> nb/np/nm (shimbun -> shinbun)
+        ["mb"] = "nb",
+        ["mp"] = "np",
+        ["mm"] = "nm",
+        // Others
+        ["sha"] = "sya",
+        ["tsu"] = "tu",
+        ["chi"] = "ti",
+        ["shi"] = "si",
+        ["ji"] = "zi",
+    };
+    public static readonly IEnumerable<(int[] From, int[] To)> NORMALIZE_RULES_ROMAJI_CODEPOINTS = ToCodePointPairs(NORMALIZE_RULES_ROMAJI);
+    public static readonly Normalizer NormalizeRomaji = CreateNormalizer(NORMALIZE_RULES_ROMAJI);
+
+    public static readonly Dictionary<string, string> NORMALIZE_RULES_KANA_DAKUTEN = new()
+    {
+        ["う\u3099"] = "ゔ",
+        ["か\u3099"] = "が", ["き\u3099"] = "ぎ", ["く\u3099"] = "ぐ", ["け\u3099"] = "げ", ["こ\u3099"] = "ご",
+        ["さ\u3099"] = "ざ", ["し\u3099"] = "じ", ["す\u3099"] = "ず", ["せ\u3099"] = "ぜ", ["そ\u3099"] = "ぞ",
+        ["た\u3099"] = "だ", ["ち\u3099"] = "ぢ", ["つ\u3099"] = "づ", ["て\u3099"] = "で", ["と\u3099"] = "ど",
+        ["は\u3099"] = "ば", ["ひ\u3099"] = "び", ["ふ\u3099"] = "ぶ", ["へ\u3099"] = "べ", ["ほ\u3099"] = "ぼ",
+        ["は\u309A"] = "ぱ", ["ひ\u309A"] = "ぴ", ["ふ\u309A"] = "ぷ", ["へ\u309A"] = "ぺ", ["ほ\u309A"] = "ぽ",
+        ["ゝ\u3099"] = "ゞ",
+
+        ["ウ\u3099"] = "ヴ",
+        ["カ\u3099"] = "ガ", ["キ\u3099"] = "ギ", ["ク\u3099"] = "グ", ["ケ\u3099"] = "ゲ", ["コ\u3099"] = "ゴ",
+        ["サ\u3099"] = "ザ", ["シ\u3099"] = "ジ", ["ス\u3099"] = "ズ", ["セ\u3099"] = "ゼ", ["ソ\u3099"] = "ゾ",
+        ["タ\u3099"] = "ダ", ["チ\u3099"] = "ヂ", ["ツ\u3099"] = "ヅ", ["テ\u3099"] = "デ", ["ト\u3099"] = "ド",
+        ["ハ\u3099"] = "バ", ["ヒ\u3099"] = "ビ", ["フ\u3099"] = "ブ", ["ヘ\u3099"] = "ベ", ["ホ\u3099"] = "ボ",
+        ["ハ\u309A"] = "パ", ["ヒ\u309A"] = "ピ", ["フ\u309A"] = "プ", ["ヘ\u309A"] = "ペ", ["ホ\u309A"] = "ポ",
+        ["ワ\u3099"] = "ヷ", ["ヰ\u3099"] = "ヸ", ["ヱ\u3099"] = "ヹ", ["ヲ\u3099"] = "ヺ",
+        ["ヽ\u3099"] = "ヾ",
+    };
+    public static readonly IEnumerable<(int[] From, int[] To)> NORMALIZE_RULES_KANA_DAKUTEN_CODEPOINTS = ToCodePointPairs(NORMALIZE_RULES_KANA_DAKUTEN);
+    public static readonly Normalizer NormalizeKanaDakuten = CreateNormalizer(NORMALIZE_RULES_KANA_DAKUTEN);
+}
@@ -0,0 +1,52 @@
+using MaigoLabs.NeedLe.Indexer.Han;
+using MyNihongo.KanaConverter;
+
+namespace MaigoLabs.NeedLe.Indexer.Japanese;
+
+public static class JapaneseUtils
+{
+    public static bool IsMaybeJapanese(int codePoint) =>
+        HanVariantProvider.IsHanCharacter(codePoint) ||
+        IsKana(codePoint) ||
+        IsJapaneseSoundMark(codePoint) ||
+        codePoint == 0x3005 || codePoint == 0x3006 || codePoint == 0x30FC;
+
+    // See also Common/Normalization.cs
+    public static bool IsJapaneseSoundMark(int codePoint) => codePoint == 0x3099 || codePoint == 0x309A;
+    public static string StripJapaneseSoundMarks(string text) => string.Concat(text.Where(codePoint => !IsJapaneseSoundMark(codePoint)));
+
+    public static bool IsKana(int codePoint) => (codePoint >= 0x3041 && codePoint <= 0x309F) || (codePoint >= 0x30A0 && codePoint <= 0x30FF);
+
+    private static readonly int[] KANAS_CANNOT_BE_FIRST =
+    [
+        'ァ', 'ィ', 'ゥ', 'ェ', 'ォ',
+        'ぁ', 'ぃ', 'ぅ', 'ぇ', 'ぉ',
+        'ャ', 'ュ', 'ョ',
+        'ゃ', 'ゅ', 'ょ',
+        'ヮ', 'ゎ',
+        'ㇰ', 'ㇱ', 'ㇲ', 'ㇳ', 'ㇴ', 'ㇵ', 'ㇶ', 'ㇷ', 'ㇸ', 'ㇹ', 'ㇺ', 'ㇻ', 'ㇼ', 'ㇽ', 'ㇾ', 'ㇿ',
+        'ー',
+    ];
+
+    private static readonly int[] KANAS_CANNOT_BE_LAST =
+    [
+        'ッ', 'っ'
+    ];
+
+    public static string ToRomajiStrictly(string kanaText)
+    {
+        if (kanaText.Length == 0) return "";
+        if (KANAS_CANNOT_BE_FIRST.Contains(kanaText[0])) return "";
+        if (KANAS_CANNOT_BE_LAST.Contains(kanaText[^1])) return "";
+        string romaji;
+        try { romaji = kanaText.ToRomaji(); }
+        catch { return ""; }
+        if (!romaji.All(c => c is >= 'a' and <= 'z')) return "";
+        return romaji;
+    }
+
+    public static bool IsValidJapanesePhrase(ReadOnlySpan<int> codePoints, int start, int length) =>
+        // Skip splittings that cause sound marks to occur in the first position of a phrase
+        !IsJapaneseSoundMark(codePoints[start]) && (start + length == codePoints.Length || !IsJapaneseSoundMark(codePoints[start + length]));
+    public static bool IsValidJapanesePhrase(ReadOnlyMemory<int> codePoints, int start, int length) => IsValidJapanesePhrase(codePoints.Span, start, length);
+}
@@ -0,0 +1,105 @@
+using System.Runtime.InteropServices;
+using MaigoLabs.NeedLe.Common;
+using MaigoLabs.NeedLe.Common.Extensions;
+using MeCab;
+using MeCab.Core;
+
+namespace MaigoLabs.NeedLe.Indexer.Japanese;
+
+public class Transcription
+{
+    public required int Start { get; set; }
+    public required int Length { get; set; }
+    public required string[] Transcriptions { get; set; }
+}
+
+public delegate IEnumerable<Transcription> TranscriptionEnumerator(ReadOnlyMemory<int> codePoints);
+public delegate bool IsValidPhraseDelegate(ReadOnlyMemory<int> codePoints, int start, int length);
+public delegate HashSet<string> GetAllTranscriptionsDelegate(string phrase);
+
+public class TranscriptionProvider
+{
+    public MeCabDictionary[] Dictionaries { get; set; }
+
+    public TranscriptionProvider(MeCabDictionary[]? dictionaries = null)
+    {
+        if (dictionaries == null)
+        {
+            var param = new MeCabParam();
+            param.LoadDicRC();
+            var dictionary = new MeCabDictionary();
+            dictionary.Open(Path.Combine(param.DicDir, "sys.dic"));
+            dictionaries = [dictionary];
+        }
+        Dictionaries = dictionaries;
+    }
+
+    public static TranscriptionEnumerator CreateTranscriptionEnumerator(IsValidPhraseDelegate isValidPhrase, GetAllTranscriptionsDelegate getAllTranscriptions) => codePoints =>
+    {
+        var resultMap = new Dictionary<(int Start, int Length), Transcription>();
+        for (int phraseLength = 1; phraseLength <= codePoints.Length; phraseLength++) for (int start = 0; start + phraseLength <= codePoints.Length; start++)
+        {
+            if (!isValidPhrase(codePoints, start, phraseLength)) continue;
+            var phrase = MemoryMarshal.ToEnumerable(codePoints.Slice(start, phraseLength)).ToUtf32String();
+            var atomicTranscriptions = getAllTranscriptions(phrase).Where(transcription => transcription != null).Where(candidateTranscription =>
+            {
+                if (candidateTranscription.Length == 0) return false;
+                // Ensure the transcription is atomic (not a combination of multiple shorter transcriptions, separated by any midpoints)
+                var visitedStates = new HashSet<(int PhrasePosition, int TranscriptionPosition)>();
+                var queue = new Queue<(int PhrasePosition, int TranscriptionPosition)>();
+                queue.Enqueue((0, 0));
+                while (queue.Count > 0)
+                {
+                    var (phrasePosition, transcriptionPosition) = queue.Dequeue();
+                    for (int prefixLength = 1; prefixLength <= phraseLength - phrasePosition; prefixLength++)
+                    {
+                        if (!resultMap.TryGetValue((start + phrasePosition, prefixLength), out var prefixResult)) continue;
+                        foreach (var transcription in prefixResult.Transcriptions) if (string.Compare(candidateTranscription, transcriptionPosition, transcription, 0, transcription.Length) == 0)
+                        {
+                            var nextState = (PhrasePosition: phrasePosition + prefixLength, TranscriptionPosition: transcriptionPosition + transcription.Length);
+                            if (nextState.PhrasePosition == phraseLength && nextState.TranscriptionPosition == candidateTranscription.Length) return false; // Found a valid combination
+                            if (visitedStates.Contains(nextState)) continue;
+                            visitedStates.Add(nextState);
+                            queue.Enqueue(nextState);
+                        }
+                    }
+                }
+                return true;
+            }).ToArray();
+            if (atomicTranscriptions.Length > 0) resultMap[(start, phraseLength)] = new() { Start = start, Length = phraseLength, Transcriptions = atomicTranscriptions };
+        }
+        return resultMap.Values;
+    };
+
+    public HashSet<string> GetAllKanaReadings(string phrase)
+    {
+        var result = new HashSet<string>();
+        var isKana = phrase.All(ch => JapaneseUtils.IsKana(ch));
+        if (isKana) result.Add(CommonNormalization.ToKatakana(phrase));
+        if (isKana && phrase.Length == 1) return result;
+
+        foreach (var dictionary in Dictionaries)
+        {
+            var searchResult = dictionary.ExactMatchSearch(phrase);
+            if (searchResult.Value == -1) continue;
+            var tokens = dictionary.GetToken(searchResult);
+            foreach (var token in tokens)
+            {
+                var feature = dictionary.GetFeature(token.Feature);
+                var parts = feature.Split(',');
+                if (parts.Length > 7) result.Add(CommonNormalization.ToKatakana(parts[7]));
+            }
+        }
+        return result;
+    }
+
+    public HashSet<string> GetAllKanaReadingsWithNormalization(string phrase) =>
+        GetAllKanaReadings(JapaneseUtils.StripJapaneseSoundMarks(JapaneseNormalization.NormalizeKanaDakuten(phrase)));
+
+    public TranscriptionEnumerator EnumerateKanaTranscriptions => CreateTranscriptionEnumerator(
+        JapaneseUtils.IsValidJapanesePhrase,
+        GetAllKanaReadingsWithNormalization);
+    public TranscriptionEnumerator EnumerateRomajiTranscriptions => CreateTranscriptionEnumerator(
+        JapaneseUtils.IsValidJapanesePhrase,
+        phrase => [.. GetAllKanaReadingsWithNormalization(phrase).Select(kana => JapaneseNormalization.NormalizeRomaji(JapaneseUtils.ToRomajiStrictly(kana)))]);
+}
@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <OutputType>Library</OutputType>
+    <RootNamespace>$(ProjectName).Indexer</RootNamespace>
+    <AssemblyName>$(RootNamespace)</AssemblyName>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <IsPackable>true</IsPackable>
+    <PackageId>$(RootNamespace)</PackageId>
+    <!-- Don't include MeCab dictionaries in this package; let MeCab.DotNet provide them to end users -->
+    <MeCabUseDefaultDictionary>False</MeCabUseDefaultDictionary>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\MaigoLabs.NeedLe.Common\MaigoLabs.NeedLe.Common.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="DotNetCampus.LatestCSharpFeatures" PrivateAssets="all" />
+    <PackageReference Include="hyjiacan.pinyin4net" />
+    <PackageReference Include="MeCab.DotNet" PrivateAssets="analyzers" />
+    <PackageReference Include="MyNihongo.KanaConverter" />
+    <PackageReference Include="OpenccNetLib" PrivateAssets="analyzers" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,104 @@
+using MaigoLabs.NeedLe.Common;
+using MaigoLabs.NeedLe.Common.Extensions;
+using MaigoLabs.NeedLe.Common.Types;
+using MaigoLabs.NeedLe.Indexer.Han;
+using MaigoLabs.NeedLe.Indexer.Japanese;
+
+namespace MaigoLabs.NeedLe.Indexer;
+
+public class TokenizerOptions
+{
+    public HanVariantProvider? HanVariantProvider { get; set; }
+    public TranscriptionProvider? TranscriptionProvider { get; set; }
+}
+
+public class Tokenizer(TokenizerOptions? options = null)
+{
+    public HanVariantProvider HanVariantProvider { get; set; } = options?.HanVariantProvider ?? new HanVariantProvider();
+    public TranscriptionProvider TranscriptionProvider { get; set; } = options?.TranscriptionProvider ?? new TranscriptionProvider();
+
+    public class Token
+    {
+        public required int Id { get; set; }
+        public required int Start { get; set; }
+        public required int End { get; set; }
+    }
+
+    public Dictionary<(TokenType Type, string Text), TokenDefinition> Tokens { get; } = [];
+    private TokenDefinition EnsureToken(TokenType type, string text)
+    {
+        var key = (type, text);
+        if (Tokens.TryGetValue(key, out var tokenDefinition)) return tokenDefinition;
+        tokenDefinition = new TokenDefinition { Id = Tokens.Count, Type = type, Text = text, CodePointLength = text.ToCodePoints().Count() };
+        Tokens.Add(key, tokenDefinition);
+        return tokenDefinition;
+    }
+
+    public List<Token> Tokenize(string text)
+    {
+        var codePoints = text.ToCodePoints().Select(CommonNormalization.NormalizeCodePoint).ToArray();
+        var results = new List<Token>();
+        Action<TokenType /* tokenType */, string /* text */> Emitter(int start, int end) =>
+            (tokenType, codePoints) => results.Add(new Token { Id = EnsureToken(tokenType, codePoints).Id, Start = start, End = end });
+
+        void EmitMaybeJapanese(ReadOnlyMemory<int> codePoints, int offset)
+        {
+            foreach (var combination in TranscriptionProvider.EnumerateKanaTranscriptions(codePoints))
+            {
+                var emit = Emitter(offset + combination.Start, offset + combination.Start + combination.Length);
+                foreach (var transcription in combination.Transcriptions) emit(TokenType.Kana, transcription);
+            }
+            foreach (var combination in TranscriptionProvider.EnumerateRomajiTranscriptions(codePoints))
+            {
+                var emit = Emitter(offset + combination.Start, offset + combination.Start + combination.Length);
+                foreach (var transcription in combination.Transcriptions) emit(TokenType.Romaji, transcription);
+            }
+            for (int i = 0; i < codePoints.Length; i++)
+            {
+                // Single character may have not only kana readings, but also Chinese pronunciations or Simplified/Traditional/Japanese variants.
+                var hanAlternates = HanVariantProvider.GetHanVariants(codePoints.Span[i]); // All possible variant characters (Simplified/Traditional/Japanese)
+                var pinyinAlternates = hanAlternates.SelectMany(PinyinHelper.GetPinyinCandidates).Distinct();
+                var emit = Emitter(offset + i, offset + i + 1);
+                foreach (var han in hanAlternates) emit(TokenType.Han, char.ConvertFromUtf32(han));
+                foreach (var pinyin in pinyinAlternates) emit(TokenType.Pinyin, pinyin);
+            }
+        }
+
+        var consequentCharsets = new (Func<int, bool> Is, Action<ReadOnlyMemory<int>, int> Emit)[]
+        {
+            (Is: JapaneseUtils.IsMaybeJapanese, Emit: EmitMaybeJapanese),
+        };
+
+        void EmitRaw(int codePoint, int offset) => Emitter(offset, offset + 1)(TokenType.Raw, char.ConvertFromUtf32(codePoint));
+
+        for (int start = 0; start < codePoints.Length; )
+        {
+            var codePoint = codePoints[start];
+            var emitted = false;
+            foreach (var (Is, Emit) in consequentCharsets)
+            {
+                var length = 0;
+                while (start + length < codePoints.Length && Is(codePoints[start + length])) length++;
+                if (length > 0)
+                {
+                    Emit(new Memory<int>(codePoints, start, length), start);
+                    start += length;
+                    emitted = true;
+                    break;
+                }
+            }
+            if (emitted) continue;
+
+            // Skip whitespaces
+            if (CommonUtils.IsWhitespace(codePoint))
+            {
+                start++;
+                continue;
+            }
+
+            EmitRaw(codePoint, start);
+            start++;
+        }
+        return results;
+    }
+}
@@ -0,0 +1,93 @@
+using MaigoLabs.NeedLe.Common;
+
+namespace MaigoLabs.NeedLe.Indexer.Trie;
+
+public static class TrieBuilder
+{
+    private static TrieNode NewNode(TrieNode? parent) => new() { Parent = parent, Children = [], TokenIds = [], SubTreeTokenIds = [] };
+
+    public static TrieNode BuildTrie(IEnumerable<(int Id, IEnumerable<int> CodePoints)> tokens)
+    {
+        var root = NewNode(null);
+        foreach (var (id, codePoints) in tokens)
+        {
+            var node = root;
+            foreach (var codePoint in codePoints)
+            {
+                node.Children.TryGetValue(codePoint, out var childNode);
+                if (childNode == null) node.Children[codePoint] = childNode = NewNode(node);
+                node = childNode;
+                node.SubTreeTokenIds.Add(id);
+            }
+            node.TokenIds.Add(id);
+        }
+        return root;
+    }
+
+    public static void GraftTriePaths(TrieNode root, IEnumerable<(int[] From, int[] To)> rules)
+    {
+        foreach (var (inputPhrase, graftTo) in rules) if (graftTo.Length > inputPhrase.Length) throw new ArgumentException($"Graft rule {inputPhrase} -> {graftTo} maps to longer string and may cause infinite loop");
+        var visitedNodes = new HashSet<TrieNode>();
+        void GraftFromNode(TrieNode node, bool recursiveChildren)
+        {
+            if (!visitedNodes.Add(node)) return;
+            if (recursiveChildren) foreach (var child in node.Children.Values) GraftFromNode(child, true);
+            while (true)
+            {
+                var nodesWithNewGraftedChildren = new Dictionary<TrieNode, /* depth from initial node */ int>();
+                foreach (var (inputPhrase, graftTo) in rules)
+                {
+                    var targetNode = node.Traverse(graftTo);
+                    if (targetNode == null) continue;
+                    var graftedPath = new TrieNode[inputPhrase.Length - 1];
+                    var isGrafted = false;
+                    var currentNode = node;
+                    for (var i = 0; i < inputPhrase.Length; i++)
+                    {
+                        var codePoint = inputPhrase[i];
+                        currentNode.Children.TryGetValue(codePoint, out var childNode);
+                        if (i == inputPhrase.Length - 1)
+                        {
+                            if (childNode != null)
+                            {
+                                if (childNode != targetNode) throw new ArgumentException($"Grafted path {inputPhrase} conflicts with existing path");
+                                // Already grafted
+                            }
+                            else
+                            {
+                                currentNode.Children[codePoint] = childNode = targetNode;
+                                isGrafted = true;
+                            }
+                        }
+                        else
+                        {
+                            if (childNode == null)
+                            {
+                                childNode = NewNode(currentNode);
+                                childNode.SubTreeTokenIds = targetNode.SubTreeTokenIds;
+                                currentNode.Children[codePoint] = childNode;
+                            }
+                            else
+                            {
+                                // Part of another grafted path?
+                                childNode.SubTreeTokenIds = new HashSet<int>(childNode.SubTreeTokenIds.Concat(targetNode.SubTreeTokenIds)).ToList();
+                            }
+                            graftedPath[i] = currentNode = childNode;
+                        }
+                    }
+                    if (isGrafted) for (var i = 0; i < graftedPath.Length; i++) nodesWithNewGraftedChildren[graftedPath[i]!] = i + 1;
+                }
+                if (nodesWithNewGraftedChildren.Count > 0)
+                {
+                    // Re-check graft rules on the newly grafted path
+                    // 1. No need to recursive other children (not on this path) since their children are not affected
+                    // 2. No need to consider ancestors of this node since they're handled later (we run in DFS order)
+                    var sortedNodes = nodesWithNewGraftedChildren.OrderByDescending(x => x.Value);
+                    foreach (var (changedNode, _) in sortedNodes) GraftFromNode(changedNode, false);
+                }
+                else break; // No new grafts applied
+            }
+        }
+        GraftFromNode(root, true);
+    }
+}
@@ -0,0 +1,41 @@
+using MaigoLabs.NeedLe.Common;
+
+namespace MaigoLabs.NeedLe.Indexer.Trie;
+
+public static class TrieSerializer
+{
+    private class NodeEntry
+    {
+        public int Id { get; set; }
+        public bool Visited { get; set; }
+        public int[]? Data { get; set; }
+    }
+
+    public static int[] Serialize(TrieNode root)
+    {
+        var nodeEntries = new Dictionary<TrieNode, NodeEntry>();
+        var currentId = 0;
+        NodeEntry GetNodeEntry(TrieNode node) => nodeEntries.TryGetValue(node, out var nodeEntry) ? nodeEntry :
+            nodeEntries[node] = new NodeEntry { Id = ++currentId, Visited = false, Data = null };
+        int SerializeNode(TrieNode node)
+        {
+            var entry = GetNodeEntry(node);
+            if (entry.Visited) return entry.Id;
+            entry.Visited = true;
+            var children = node.Children.Select(child => (CodePoint: child.Key, ChildId: SerializeNode(child.Value))).ToArray();
+            entry.Data =
+            [
+                node.Parent != null ? GetNodeEntry(node.Parent).Id : 0,
+                .. children.Select(child => child.CodePoint),
+                .. children.Select(child => child.ChildId),
+                // End of children list (<= 0 are not valid code points nor node IDs)
+                .. node.TokenIds.Count > 0
+                    ? node.TokenIds.Select(tokenId => -(tokenId + 1)) // Use the negative value of (tokenId + 1)
+                    : [0], // End of children list, no token IDs (token IDs are encoded to negative values)
+            ];
+            return entry.Id;
+        }
+        SerializeNode(root);
+        return nodeEntries.Values.OrderBy(entry => entry.Id).SelectMany(entry => entry.Data ?? []).ToArray();
+    }
+}