feat: initial commit

dotnet/MaigoLabs.NeedLe.Tests/Common/CommonNormalizationTests.cs

@@ -0,0 +1,126 @@
+using MaigoLabs.NeedLe.Common;
+
+namespace MaigoLabs.NeedLe.Tests.Common;
+
+#region ToKatakana
+
+public sealed class ToKatakana_ConvertsHiraganaToKatakanaTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("アイウエオ", CommonNormalization.ToKatakana("あいうえお"));
+        Assert.Equal("カキクケコ", CommonNormalization.ToKatakana("かきくけこ"));
+        Assert.Equal("サシスセソ", CommonNormalization.ToKatakana("さしすせそ"));
+    }
+}
+
+public sealed class ToKatakana_KeepsKatakanaUnchangedTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("アイウエオ", CommonNormalization.ToKatakana("アイウエオ"));
+    }
+}
+
+public sealed class ToKatakana_KeepsNonKanaUnchangedTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("abc123", CommonNormalization.ToKatakana("abc123"));
+        Assert.Equal("漢字", CommonNormalization.ToKatakana("漢字"));
+    }
+}
+
+public sealed class ToKatakana_HandlesMixedInputTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("アアa漢", CommonNormalization.ToKatakana("あアa漢"));
+    }
+}
+
+#endregion
+
+#region NormalizeCodePoint
+
+public sealed class NormalizeCodePoint_ConvertsFullwidthAsciiToHalfwidthLowercaseTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal('a', CommonNormalization.NormalizeCodePoint('Ａ'));
+        Assert.Equal('b', CommonNormalization.NormalizeCodePoint('Ｂ'));
+        Assert.Equal('c', CommonNormalization.NormalizeCodePoint('Ｃ'));
+        Assert.Equal('1', CommonNormalization.NormalizeCodePoint('１'));
+        Assert.Equal('2', CommonNormalization.NormalizeCodePoint('２'));
+        Assert.Equal('3', CommonNormalization.NormalizeCodePoint('３'));
+        Assert.Equal('!', CommonNormalization.NormalizeCodePoint('！'));
+    }
+}
+
+public sealed class NormalizeCodePoint_ConvertsFullwidthSpaceToHalfwidthTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal(' ', CommonNormalization.NormalizeCodePoint('　'));
+    }
+}
+
+public sealed class NormalizeCodePoint_ConvertsHalfwidthKanaToFullwidthTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal('ア', CommonNormalization.NormalizeCodePoint('ｱ'));
+        Assert.Equal('イ', CommonNormalization.NormalizeCodePoint('ｲ'));
+        Assert.Equal('ウ', CommonNormalization.NormalizeCodePoint('ｳ'));
+        Assert.Equal('エ', CommonNormalization.NormalizeCodePoint('ｴ'));
+        Assert.Equal('オ', CommonNormalization.NormalizeCodePoint('ｵ'));
+        Assert.Equal('カ', CommonNormalization.NormalizeCodePoint('ｶ'));
+    }
+}
+
+public sealed class NormalizeCodePoint_NormalizesVoicedSoundMarksTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal(0x3099, CommonNormalization.NormalizeCodePoint('ﾞ')); // halfwidth voiced -> combining
+        Assert.Equal(0x309A, CommonNormalization.NormalizeCodePoint('ﾟ')); // halfwidth semi-voiced -> combining
+        Assert.Equal(0x3099, CommonNormalization.NormalizeCodePoint('゛')); // fullwidth voiced -> combining
+        Assert.Equal(0x309A, CommonNormalization.NormalizeCodePoint('゜')); // fullwidth semi-voiced -> combining
+    }
+}
+
+public sealed class NormalizeCodePoint_ConvertsHalfwidthPunctuationToFullwidthTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal('。', CommonNormalization.NormalizeCodePoint('｡'));
+        Assert.Equal('「', CommonNormalization.NormalizeCodePoint('｢'));
+        Assert.Equal('」', CommonNormalization.NormalizeCodePoint('｣'));
+        Assert.Equal('、', CommonNormalization.NormalizeCodePoint('､'));
+        Assert.Equal('・', CommonNormalization.NormalizeCodePoint('･'));
+    }
+}
+
+public sealed class NormalizeCodePoint_LowercasesRegularAsciiTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal('a', CommonNormalization.NormalizeCodePoint('A'));
+        Assert.Equal('b', CommonNormalization.NormalizeCodePoint('B'));
+        Assert.Equal('c', CommonNormalization.NormalizeCodePoint('C'));
+    }
+}
+
+#endregion
+
+

dotnet/MaigoLabs.NeedLe.Tests/E2E/SearchTests.cs

@@ -0,0 +1,91 @@
+using MaigoLabs.NeedLe.Indexer;
+using MaigoLabs.NeedLe.Searcher;
+
+namespace MaigoLabs.NeedLe.Tests.E2E;
+
+public sealed class Search_MatchesWithMixedSearchQueryTest : NeedleTestBase
+{
+    private static readonly string[] TestDocuments =
+    [
+        "ミーティア",
+        "エンドマークに希望と涙を添えて",
+        "宵の鳥",
+        "僕の和風本当上手",
+    ];
+
+    [Fact]
+    public void Execute()
+    {
+        var compressed = InvertedIndexBuilder.BuildInvertedIndex(TestDocuments, TokenizerOptions);
+        var invertedIndex = InvertedIndexLoader.Load(compressed);
+
+        var results = InvertedIndexSearcher.Search(invertedIndex, "bokunoh风じょう");
+
+        // Should have at least one result
+        Assert.NotEmpty(results);
+
+        // The first result should be "僕の和風本当上手"
+        Assert.Equal("僕の和風本当上手", results[0].DocumentText);
+    }
+}
+
+public sealed class Search_HighlightsSearchResultCorrectlyTest : NeedleTestBase
+{
+    private static readonly string[] TestDocuments =
+    [
+        "ミーティア",
+        "エンドマークに希望と涙を添えて",
+        "宵の鳥",
+        "僕の和風本当上手",
+    ];
+
+    [Fact]
+    public void Execute()
+    {
+        var compressed = InvertedIndexBuilder.BuildInvertedIndex(TestDocuments, TokenizerOptions);
+        var invertedIndex = InvertedIndexLoader.Load(compressed);
+
+        var results = InvertedIndexSearcher.Search(invertedIndex, "bokunoh风じょう");
+        Assert.NotEmpty(results);
+
+        var highlighted = SearchResultHighlighter.Highlight(results[0]);
+
+        // Should be a list of parts
+        Assert.NotEmpty(highlighted);
+
+        // Collect highlighted text
+        var highlightedTexts = highlighted.Where(p => p.IsHighlighted).Select(p => p.Text).ToList();
+        var highlightedJoined = string.Join("", highlightedTexts);
+
+        Assert.Contains("僕", highlightedJoined);
+        Assert.Contains("の", highlightedJoined);
+        Assert.Contains("和", highlightedJoined);
+        Assert.Contains("風", highlightedJoined);
+        Assert.Contains("上", highlightedJoined);
+    }
+}
+
+public sealed class Search_MatchesRomajiInputToKanaDocumentsTest : NeedleTestBase
+{
+    private static readonly string[] TestDocuments =
+    [
+        "ミーティア",
+        "エンドマークに希望と涙を添えて",
+        "宵の鳥",
+        "僕の和風本当上手",
+    ];
+
+    [Fact]
+    public void Execute()
+    {
+        var compressed = InvertedIndexBuilder.BuildInvertedIndex(TestDocuments, TokenizerOptions);
+        var invertedIndex = InvertedIndexLoader.Load(compressed);
+
+        // Search for "yoi" should match "宵の鳥"
+        var results = InvertedIndexSearcher.Search(invertedIndex, "yoi");
+        var matchedTexts = results.Select(r => r.DocumentText).ToList();
+
+        Assert.Contains("宵の鳥", matchedTexts);
+    }
+}
+

dotnet/MaigoLabs.NeedLe.Tests/E2E/TrieSerializationTests.cs

@@ -0,0 +1,143 @@
+using MaigoLabs.NeedLe.Common;
+using MaigoLabs.NeedLe.Common.Extensions;
+using MaigoLabs.NeedLe.Indexer.Trie;
+using MaigoLabs.NeedLe.Searcher.Trie;
+
+namespace MaigoLabs.NeedLe.Tests.E2E;
+
+#region Trie Building
+
+public sealed class TrieBuilding_BuildsTrieWithMultipleDifferentTokensTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var trie = TrieBuilder.BuildTrie([
+            (0, "hello".ToCodePoints()),
+            (1, "help".ToCodePoints()),
+            (2, "world".ToCodePoints()),
+            (3, "word".ToCodePoints()),
+        ]);
+
+        // Traverse to verify structure
+        var helloNode = trie.Traverse("hello".ToCodePoints().ToArray());
+        var helpNode = trie.Traverse("help".ToCodePoints().ToArray());
+        var worldNode = trie.Traverse("world".ToCodePoints().ToArray());
+        var wordNode = trie.Traverse("word".ToCodePoints().ToArray());
+
+        Assert.NotNull(helloNode);
+        Assert.NotNull(helpNode);
+        Assert.NotNull(worldNode);
+        Assert.NotNull(wordNode);
+
+        // Check token IDs
+        Assert.Contains(0, helloNode!.TokenIds);
+        Assert.Contains(1, helpNode!.TokenIds);
+        Assert.Contains(2, worldNode!.TokenIds);
+        Assert.Contains(3, wordNode!.TokenIds);
+
+        // Check that 'hel' prefix node has both tokens in subTree
+        var helNode = trie.Traverse("hel".ToCodePoints().ToArray());
+        Assert.NotNull(helNode);
+        Assert.Contains(0, helNode!.SubTreeTokenIds);
+        Assert.Contains(1, helNode.SubTreeTokenIds);
+    }
+}
+
+public sealed class TrieBuilding_HandlesJapaneseTextTokensTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var trie = TrieBuilder.BuildTrie([
+            (0, "さくら".ToCodePoints()),
+            (1, "サクラ".ToCodePoints()),
+            (2, "桜".ToCodePoints()),
+        ]);
+
+        Assert.Contains(0, trie.Traverse("さくら".ToCodePoints().ToArray())?.TokenIds ?? []);
+        Assert.Contains(1, trie.Traverse("サクラ".ToCodePoints().ToArray())?.TokenIds ?? []);
+        Assert.Contains(2, trie.Traverse("桜".ToCodePoints().ToArray())?.TokenIds ?? []);
+    }
+}
+
+#endregion
+
+#region Trie Serialization
+
+public sealed class TrieSerialization_SerializesAndDeserializesCorrectlyTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var originalTrie = TrieBuilder.BuildTrie([
+            (0, "apple".ToCodePoints()),
+            (1, "app".ToCodePoints()),
+            (2, "banana".ToCodePoints()),
+        ]);
+
+        // Serialize
+        var serialized = TrieSerializer.Serialize(originalTrie);
+        Assert.True(serialized.Length > 0);
+
+        // Deserialize
+        var deserialized = TrieDeserializer.Deserialize(serialized);
+        var deserializedTrie = deserialized.Root;
+        var tokenCodePoints = deserialized.TokenCodePoints;
+
+        // Verify structure is preserved
+        var appleNode = deserializedTrie.Traverse("apple".ToCodePoints().ToArray());
+        var appNode = deserializedTrie.Traverse("app".ToCodePoints().ToArray());
+        var bananaNode = deserializedTrie.Traverse("banana".ToCodePoints().ToArray());
+
+        Assert.NotNull(appleNode);
+        Assert.NotNull(appNode);
+        Assert.NotNull(bananaNode);
+
+        Assert.Contains(0, appleNode!.TokenIds);
+        Assert.Contains(1, appNode!.TokenIds);
+        Assert.Contains(2, bananaNode!.TokenIds);
+
+        // Verify tokenCodePoints map
+        Assert.Equal("apple", tokenCodePoints[0].ToUtf32String());
+        Assert.Equal("app", tokenCodePoints[1].ToUtf32String());
+        Assert.Equal("banana", tokenCodePoints[2].ToUtf32String());
+
+        // Verify subTreeTokenIds are reconstructed
+        Assert.Contains(0, appNode.SubTreeTokenIds);
+        Assert.Contains(1, appNode.SubTreeTokenIds);
+    }
+}
+
+public sealed class TrieSerialization_PreservesParentReferencesAfterDeserializationTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var originalTrie = TrieBuilder.BuildTrie([
+            (0, "test".ToCodePoints()),
+        ]);
+
+        var serialized = TrieSerializer.Serialize(originalTrie);
+        var deserialized = TrieDeserializer.Deserialize(serialized);
+        var root = deserialized.Root;
+
+        var testNode = root.Traverse("test".ToCodePoints().ToArray());
+        Assert.NotNull(testNode);
+
+        // Walk back to root via parent references
+        TrieNode? node = testNode;
+        var depth = 0;
+        while (node?.Parent != null)
+        {
+            node = node.Parent;
+            depth++;
+        }
+        Assert.Equal(4, depth); // 't' -> 'e' -> 's' -> 't' -> root
+        Assert.Same(root, node);
+    }
+}
+
+#endregion
+
+

dotnet/MaigoLabs.NeedLe.Tests/Indexer/Han/HanVariantProviderTests.cs

@@ -0,0 +1,75 @@
+using MaigoLabs.NeedLe.Indexer.Han;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
+
+#region IsHanCharacter
+
+public sealed class IsHanCharacter_ReturnsTrueForCjkCharactersTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.True(HanVariantProvider.IsHanCharacter('中'));
+        Assert.True(HanVariantProvider.IsHanCharacter('国'));
+        Assert.True(HanVariantProvider.IsHanCharacter('日'));
+        Assert.True(HanVariantProvider.IsHanCharacter('本'));
+    }
+}
+
+public sealed class IsHanCharacter_ReturnsFalseForNonCjkCharactersTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.False(HanVariantProvider.IsHanCharacter('a'));
+        Assert.False(HanVariantProvider.IsHanCharacter('あ'));
+        Assert.False(HanVariantProvider.IsHanCharacter('ア'));
+        Assert.False(HanVariantProvider.IsHanCharacter('1'));
+    }
+}
+
+#endregion
+
+#region GetHanVariants
+
+public sealed class GetHanVariants_ReturnsVariantsForSimplifiedTraditionalTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new HanVariantProvider();
+        // 国 (simplified) and 國 (traditional) should be variants of each other
+        var variants1 = provider.GetHanVariants('国');
+        var variants2 = provider.GetHanVariants('國');
+        Assert.Contains('国', variants1);
+        Assert.Contains('國', variants1);
+        Assert.Contains('国', variants2);
+        Assert.Contains('國', variants2);
+    }
+}
+
+public sealed class GetHanVariants_ReturnsCharacterItselfForNoVariantsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new HanVariantProvider();
+        var variants = provider.GetHanVariants('一');
+        Assert.Contains('一', variants);
+    }
+}
+
+public sealed class GetHanVariants_ReturnsEmptyForNonHanCharactersTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new HanVariantProvider();
+        Assert.Empty(provider.GetHanVariants('a'));
+        Assert.Empty(provider.GetHanVariants('あ'));
+    }
+}
+
+#endregion
+
+

dotnet/MaigoLabs.NeedLe.Tests/Indexer/Han/PinyinHelperTests.cs

@@ -0,0 +1,51 @@
+using MaigoLabs.NeedLe.Indexer.Han;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
+
+public sealed class GetPinyinCandidates_ReturnsPinyinForHanCharacterTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var candidates = PinyinHelper.GetPinyinCandidates('中').ToList();
+        Assert.Contains("zhong", candidates);
+        Assert.Contains("zh", candidates); // initial
+        Assert.Contains("z", candidates); // first letter
+    }
+}
+
+public sealed class GetPinyinCandidates_ReturnsMultiplePinyinForPolyphonicTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        // 行 can be "xing" or "hang"
+        var candidates = PinyinHelper.GetPinyinCandidates('行').ToList();
+        Assert.Contains("xing", candidates);
+        Assert.Contains("hang", candidates);
+    }
+}
+
+public sealed class GetPinyinCandidates_IncludesFuzzyPinyinVariantsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        // 风 is "feng", should also have fuzzy variant "fen"
+        var candidates = PinyinHelper.GetPinyinCandidates('风').ToList();
+        Assert.Contains("feng", candidates);
+        Assert.Contains("fen", candidates); // fuzzy: eng -> en
+    }
+}
+
+public sealed class GetPinyinCandidates_ReturnsEmptyForNonHanCharactersTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Empty(PinyinHelper.GetPinyinCandidates('a'));
+        Assert.Empty(PinyinHelper.GetPinyinCandidates('あ'));
+    }
+}
+
+

dotnet/MaigoLabs.NeedLe.Tests/Indexer/Han/UnionFindSetTests.cs

@@ -0,0 +1,59 @@
+using MaigoLabs.NeedLe.Indexer.Han;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
+
+public sealed class UnionFindSet_FindsSelfAsRootInitiallyTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var ufs = new UnionFindSet();
+        Assert.Equal(1, ufs.Find(1));
+        Assert.Equal(2, ufs.Find(2));
+    }
+}
+
+public sealed class UnionFindSet_UnionsTwoElementsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var ufs = new UnionFindSet();
+        ufs.Union(1, 2);
+        Assert.Equal(ufs.Find(1), ufs.Find(2));
+    }
+}
+
+public sealed class UnionFindSet_UnionsMultipleElementsTransitivelyTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var ufs = new UnionFindSet();
+        ufs.Union(1, 2);
+        ufs.Union(2, 3);
+        ufs.Union(4, 5);
+        Assert.Equal(ufs.Find(1), ufs.Find(3));
+        Assert.NotEqual(ufs.Find(1), ufs.Find(4));
+        ufs.Union(3, 4);
+        Assert.Equal(ufs.Find(1), ufs.Find(5));
+    }
+}
+
+public sealed class UnionFindSet_IteratesAllKeysTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var ufs = new UnionFindSet();
+        ufs.Union(1, 2);
+        ufs.Union(3, 4);
+        var keys = ufs.Keys.ToList();
+        Assert.Contains(1, keys);
+        Assert.Contains(2, keys);
+        Assert.Contains(3, keys);
+        Assert.Contains(4, keys);
+    }
+}
+
+

dotnet/MaigoLabs.NeedLe.Tests/Indexer/Japanese/JapaneseUtilsTests.cs

@@ -0,0 +1,69 @@
+using MaigoLabs.NeedLe.Indexer.Japanese;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Japanese;
+
+#region ToRomajiStrictly
+
+public sealed class ToRomajiStrictly_ConvertsBasicKanaToRomajiTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("a", JapaneseUtils.ToRomajiStrictly("あ"));
+        Assert.Equal("ka", JapaneseUtils.ToRomajiStrictly("か"));
+        Assert.Equal("sakura", JapaneseUtils.ToRomajiStrictly("さくら"));
+    }
+}
+
+public sealed class ToRomajiStrictly_ConvertsKatakanaToRomajiTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("a", JapaneseUtils.ToRomajiStrictly("ア"));
+        Assert.Equal("ka", JapaneseUtils.ToRomajiStrictly("カ"));
+        Assert.Equal("sakura", JapaneseUtils.ToRomajiStrictly("サクラ"));
+    }
+}
+
+public sealed class ToRomajiStrictly_HandlesLongVowelsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("ou", JapaneseUtils.ToRomajiStrictly("おう"));
+        Assert.Equal("oo", JapaneseUtils.ToRomajiStrictly("おお"));
+    }
+}
+
+public sealed class ToRomajiStrictly_ReturnsEmptyForInvalidFirstCharacterTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("", JapaneseUtils.ToRomajiStrictly("ー")); // prolonged sound mark cannot be first
+        Assert.Equal("", JapaneseUtils.ToRomajiStrictly("ゃ")); // small ya cannot be first
+    }
+}
+
+public sealed class ToRomajiStrictly_ReturnsEmptyForInvalidLastCharacterTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("", JapaneseUtils.ToRomajiStrictly("っ")); // small tsu cannot be last
+    }
+}
+
+public sealed class ToRomajiStrictly_HandlesGeminationTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("katta", JapaneseUtils.ToRomajiStrictly("かった"));
+    }
+}
+
+#endregion
+
+

dotnet/MaigoLabs.NeedLe.Tests/Indexer/Japanese/TranscriptionProviderTests.cs

@@ -0,0 +1,40 @@
+using MaigoLabs.NeedLe.Indexer.Japanese;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Japanese;
+
+public sealed class GetAllKanaReadings_ReturnsKatakanaForPureKanaInputTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new TranscriptionProvider();
+        var readings = provider.GetAllKanaReadings("あ");
+        Assert.Contains("ア", readings);
+    }
+}
+
+public sealed class GetAllKanaReadings_ReturnsReadingsForKanjiTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new TranscriptionProvider();
+        var readings = provider.GetAllKanaReadings("僕");
+        Assert.NotEmpty(readings);
+        // 僕 should have reading ボク
+        Assert.Contains("ボク", readings);
+    }
+}
+
+public sealed class GetAllKanaReadings_ReturnsReadingsForCompoundWordsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new TranscriptionProvider();
+        var readings = provider.GetAllKanaReadings("和風");
+        Assert.NotEmpty(readings);
+    }
+}
+
+

dotnet/MaigoLabs.NeedLe.Tests/Indexer/TokenizerTests.cs

@@ -0,0 +1,165 @@
+using MaigoLabs.NeedLe.Common.Types;
+using MaigoLabs.NeedLe.Indexer;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer;
+
+public sealed class Tokenizer_TokenizesMixedJapaneseTextTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var tokenizer = new Tokenizer(TokenizerOptions);
+        var tokens = tokenizer.Tokenize("僕の和風本当上手");
+
+        var tokenDefs = tokenizer.Tokens.Values.ToList();
+
+        // Should have tokens of various types
+        var types = tokenDefs.Select(t => t.Type).ToHashSet();
+        Assert.Contains(TokenType.Han, types);
+        Assert.Contains(TokenType.Pinyin, types);
+        Assert.Contains(TokenType.Kana, types);
+        Assert.Contains(TokenType.Romaji, types);
+
+        // Helper to get token texts at a specific position by type
+        List<string> GetTokenTextsAt(int pos, TokenType type) => tokens
+            .Where(t => t.Start <= pos && t.End > pos)
+            .Select(t => tokenDefs.First(d => d.Id == t.Id))
+            .Where(d => d.Type == type)
+            .Select(d => d.Text)
+            .ToList();
+
+        // Position 0: 僕
+        Assert.Contains("僕", GetTokenTextsAt(0, TokenType.Han));
+        Assert.Contains("pu", GetTokenTextsAt(0, TokenType.Pinyin));
+        Assert.Contains("ボク", GetTokenTextsAt(0, TokenType.Kana));
+        Assert.Contains("boku", GetTokenTextsAt(0, TokenType.Romaji));
+
+        // Position 1: の (hiragana, no Han/Pinyin)
+        Assert.Empty(GetTokenTextsAt(1, TokenType.Han));
+        Assert.Empty(GetTokenTextsAt(1, TokenType.Pinyin));
+        Assert.Contains("ノ", GetTokenTextsAt(1, TokenType.Kana));
+        Assert.Contains("no", GetTokenTextsAt(1, TokenType.Romaji));
+
+        // Position 2: 和
+        Assert.Contains("和", GetTokenTextsAt(2, TokenType.Han));
+        Assert.Contains("he", GetTokenTextsAt(2, TokenType.Pinyin));
+        Assert.Contains("ワ", GetTokenTextsAt(2, TokenType.Kana));
+        Assert.Contains("wa", GetTokenTextsAt(2, TokenType.Romaji));
+
+        // Position 3: 風
+        Assert.Contains("風", GetTokenTextsAt(3, TokenType.Han));
+        Assert.Contains("风", GetTokenTextsAt(3, TokenType.Han)); // simplified variant
+        Assert.Contains("feng", GetTokenTextsAt(3, TokenType.Pinyin));
+        Assert.Contains("フウ", GetTokenTextsAt(3, TokenType.Kana));
+        Assert.Contains("fu", GetTokenTextsAt(3, TokenType.Romaji));
+
+        // Position 4: 本
+        Assert.Contains("本", GetTokenTextsAt(4, TokenType.Han));
+        Assert.Contains("ben", GetTokenTextsAt(4, TokenType.Pinyin));
+        Assert.Contains("ホン", GetTokenTextsAt(4, TokenType.Kana));
+        Assert.Contains("hon", GetTokenTextsAt(4, TokenType.Romaji));
+
+        // Position 5: 当
+        Assert.Contains("当", GetTokenTextsAt(5, TokenType.Han));
+        Assert.Contains("當", GetTokenTextsAt(5, TokenType.Han)); // traditional variant
+        Assert.Contains("dang", GetTokenTextsAt(5, TokenType.Pinyin));
+        Assert.Contains("トウ", GetTokenTextsAt(5, TokenType.Kana));
+        Assert.Contains("to", GetTokenTextsAt(5, TokenType.Romaji)); // normalized: tou -> to
+
+        // Position 6: 上
+        Assert.Contains("上", GetTokenTextsAt(6, TokenType.Han));
+        Assert.Contains("shang", GetTokenTextsAt(6, TokenType.Pinyin));
+        Assert.Contains("ジョウ", GetTokenTextsAt(6, TokenType.Kana));
+        Assert.Contains("jo", GetTokenTextsAt(6, TokenType.Romaji)); // normalized: jou -> jo
+
+        // Position 7: 手
+        Assert.Contains("手", GetTokenTextsAt(7, TokenType.Han));
+        Assert.Contains("shou", GetTokenTextsAt(7, TokenType.Pinyin));
+        Assert.Contains("シュ", GetTokenTextsAt(7, TokenType.Kana));
+        Assert.Contains("shu", GetTokenTextsAt(7, TokenType.Romaji));
+    }
+}
+
+public sealed class Tokenizer_NoDuplicateTokensTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var tokenizer = new Tokenizer(TokenizerOptions);
+
+        // Tokenize multiple music names that share some characters
+        tokenizer.Tokenize("僕の和風本当上手");
+        tokenizer.Tokenize("僕");
+        tokenizer.Tokenize("和風");
+
+        // Check that there are no duplicate tokens
+        var tokenDefs = tokenizer.Tokens.Values.ToList();
+        var tokenKeys = tokenDefs.Select(t => $"{t.Type}:{t.Text}").ToList();
+        var uniqueKeys = tokenKeys.ToHashSet();
+
+        Assert.Equal(uniqueKeys.Count, tokenKeys.Count);
+
+        // Also check that IDs are unique
+        var ids = tokenDefs.Select(t => t.Id).ToList();
+        var uniqueIds = ids.ToHashSet();
+        Assert.Equal(uniqueIds.Count, ids.Count);
+    }
+}
+
+public sealed class Tokenizer_HandlesRawTokensForNonCjkTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var tokenizer = new Tokenizer(TokenizerOptions);
+        tokenizer.Tokenize("a-b");
+
+        var tokenDefs = tokenizer.Tokens.Values.ToList();
+        var rawTokenTexts = tokenDefs.Where(t => t.Type == TokenType.Raw).Select(t => t.Text).ToList();
+
+        Assert.Contains("a", rawTokenTexts);
+        Assert.Contains("-", rawTokenTexts);
+        Assert.Contains("b", rawTokenTexts);
+    }
+}
+
+public sealed class Tokenizer_TokenizesCompoundWordKyouTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var tokenizer = new Tokenizer(TokenizerOptions);
+        var tokens = tokenizer.Tokenize("今日");
+        var tokenDefs = tokenizer.Tokens.Values.ToList();
+
+        // Helper to get tokens with specific type and span
+        List<string> GetTokensWithSpan(TokenType type, int start, int end) => tokens
+            .Where(t => t.Start == start && t.End == end)
+            .Select(t => tokenDefs.First(d => d.Id == t.Id))
+            .Where(d => d.Type == type)
+            .Select(d => d.Text)
+            .ToList();
+
+        // Individual character readings at position 0: 今
+        Assert.Contains("今", GetTokensWithSpan(TokenType.Han, 0, 1));
+        Assert.Contains("jin", GetTokensWithSpan(TokenType.Pinyin, 0, 1));
+        Assert.Contains("コン", GetTokensWithSpan(TokenType.Kana, 0, 1));
+        Assert.Contains("イマ", GetTokensWithSpan(TokenType.Kana, 0, 1));
+        Assert.Contains("kon", GetTokensWithSpan(TokenType.Romaji, 0, 1));
+        Assert.Contains("ima", GetTokensWithSpan(TokenType.Romaji, 0, 1));
+
+        // Individual character readings at position 1: 日
+        Assert.Contains("日", GetTokensWithSpan(TokenType.Han, 1, 2));
+        Assert.Contains("ri", GetTokensWithSpan(TokenType.Pinyin, 1, 2));
+        Assert.Contains("ニチ", GetTokensWithSpan(TokenType.Kana, 1, 2));
+        Assert.Contains("ヒ", GetTokensWithSpan(TokenType.Kana, 1, 2));
+        Assert.Contains("niti", GetTokensWithSpan(TokenType.Romaji, 1, 2));
+        Assert.Contains("hi", GetTokensWithSpan(TokenType.Romaji, 1, 2));
+
+        // Combined reading for "今日" [0, 2] - this is an indivisible compound word
+        Assert.Contains("キョウ", GetTokensWithSpan(TokenType.Kana, 0, 2));
+        Assert.Contains("kyo", GetTokensWithSpan(TokenType.Romaji, 0, 2)); // normalized: kyou -> kyo
+    }
+}
+
+

dotnet/MaigoLabs.NeedLe.Tests/Indexer/TrieTests.cs

@@ -0,0 +1,66 @@
+using MaigoLabs.NeedLe.Common;
+using MaigoLabs.NeedLe.Common.Extensions;
+using MaigoLabs.NeedLe.Indexer.Trie;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer;
+
+#region GraftTriePaths
+
+public sealed class GraftTriePaths_GraftsPathsAccordingToNormalizationRulesTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        // Build a trie with tokens containing normalized forms
+        var trie = TrieBuilder.BuildTrie([
+            (0, "sya".ToCodePoints()), // normalized form of "sha"
+            (1, "tu".ToCodePoints()),  // normalized form of "tsu"
+        ]);
+
+        // Graft paths so that "sha" -> "sya" and "tsu" -> "tu"
+        TrieBuilder.GraftTriePaths(trie, [
+            ("sha".ToCodePoints().ToArray(), "sya".ToCodePoints().ToArray()),
+            ("tsu".ToCodePoints().ToArray(), "tu".ToCodePoints().ToArray()),
+        ]);
+
+        // Now we should be able to traverse using both the original and grafted paths
+        var syaNode = trie.Traverse("sya".ToCodePoints().ToArray());
+        var shaNode = trie.Traverse("sha".ToCodePoints().ToArray());
+        Assert.NotNull(syaNode);
+        Assert.NotNull(shaNode);
+        Assert.Same(syaNode, shaNode); // Both paths should lead to the same node
+
+        var tuNode = trie.Traverse("tu".ToCodePoints().ToArray());
+        var tsuNode = trie.Traverse("tsu".ToCodePoints().ToArray());
+        Assert.NotNull(tuNode);
+        Assert.NotNull(tsuNode);
+        Assert.Same(tuNode, tsuNode);
+    }
+}
+
+public sealed class GraftTriePaths_HandlesChainedGraftRulesTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var trie = TrieBuilder.BuildTrie([
+            (0, "o".ToCodePoints()), // normalized vowel
+        ]);
+
+        // Chain: "ou" -> "o", "oo" -> "o"
+        TrieBuilder.GraftTriePaths(trie, [
+            ("ou".ToCodePoints().ToArray(), "o".ToCodePoints().ToArray()),
+            ("oo".ToCodePoints().ToArray(), "o".ToCodePoints().ToArray()),
+        ]);
+
+        var oNode = trie.Traverse("o".ToCodePoints().ToArray());
+        var ouNode = trie.Traverse("ou".ToCodePoints().ToArray());
+        var ooNode = trie.Traverse("oo".ToCodePoints().ToArray());
+
+        Assert.NotNull(oNode);
+        Assert.Same(oNode, ouNode);
+        Assert.Same(oNode, ooNode);
+    }
+}
+
+#endregion

dotnet/MaigoLabs.NeedLe.Tests/MaigoLabs.NeedLe.Tests.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <RootNamespace>$(ProjectName).Tests</RootNamespace>
+    <AssemblyName>$(RootNamespace)</AssemblyName>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\MaigoLabs.NeedLe.Common\MaigoLabs.NeedLe.Common.csproj" />
+    <ProjectReference Include="..\MaigoLabs.NeedLe.Indexer\MaigoLabs.NeedLe.Indexer.csproj" />
+    <ProjectReference Include="..\MaigoLabs.NeedLe.Searcher\MaigoLabs.NeedLe.Searcher.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="Xunit" />
+  </ItemGroup>
+
+</Project>
+

dotnet/MaigoLabs.NeedLe.Tests/NeedleTestBase.cs

@@ -0,0 +1,12 @@
+using MaigoLabs.NeedLe.Indexer;
+using MaigoLabs.NeedLe.Indexer.Han;
+using MaigoLabs.NeedLe.Indexer.Japanese;
+
+namespace MaigoLabs.NeedLe.Tests;
+
+public abstract class NeedleTestBase
+{
+    public static HanVariantProvider HanVariantProvider { get; set; } = new();
+    public static TranscriptionProvider TranscriptionProvider { get; set; } = new();
+    public static TokenizerOptions TokenizerOptions => new() { HanVariantProvider = HanVariantProvider, TranscriptionProvider = TranscriptionProvider };
+}