feat: initial commit

2026-01-01 03:40:41 +08:00
commit 631f8ed771
98 changed files with 14776 additions and 0 deletions
@@ -0,0 +1,75 @@
+using MaigoLabs.NeedLe.Indexer.Han;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
+
+#region IsHanCharacter
+
+public sealed class IsHanCharacter_ReturnsTrueForCjkCharactersTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.True(HanVariantProvider.IsHanCharacter('中'));
+        Assert.True(HanVariantProvider.IsHanCharacter('国'));
+        Assert.True(HanVariantProvider.IsHanCharacter('日'));
+        Assert.True(HanVariantProvider.IsHanCharacter('本'));
+    }
+}
+
+public sealed class IsHanCharacter_ReturnsFalseForNonCjkCharactersTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.False(HanVariantProvider.IsHanCharacter('a'));
+        Assert.False(HanVariantProvider.IsHanCharacter('あ'));
+        Assert.False(HanVariantProvider.IsHanCharacter('ア'));
+        Assert.False(HanVariantProvider.IsHanCharacter('1'));
+    }
+}
+
+#endregion
+
+#region GetHanVariants
+
+public sealed class GetHanVariants_ReturnsVariantsForSimplifiedTraditionalTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new HanVariantProvider();
+        // 国 (simplified) and 國 (traditional) should be variants of each other
+        var variants1 = provider.GetHanVariants('国');
+        var variants2 = provider.GetHanVariants('國');
+        Assert.Contains('国', variants1);
+        Assert.Contains('國', variants1);
+        Assert.Contains('国', variants2);
+        Assert.Contains('國', variants2);
+    }
+}
+
+public sealed class GetHanVariants_ReturnsCharacterItselfForNoVariantsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new HanVariantProvider();
+        var variants = provider.GetHanVariants('一');
+        Assert.Contains('一', variants);
+    }
+}
+
+public sealed class GetHanVariants_ReturnsEmptyForNonHanCharactersTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new HanVariantProvider();
+        Assert.Empty(provider.GetHanVariants('a'));
+        Assert.Empty(provider.GetHanVariants('あ'));
+    }
+}
+
+#endregion
+
+
@@ -0,0 +1,51 @@
+using MaigoLabs.NeedLe.Indexer.Han;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
+
+public sealed class GetPinyinCandidates_ReturnsPinyinForHanCharacterTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var candidates = PinyinHelper.GetPinyinCandidates('中').ToList();
+        Assert.Contains("zhong", candidates);
+        Assert.Contains("zh", candidates); // initial
+        Assert.Contains("z", candidates); // first letter
+    }
+}
+
+public sealed class GetPinyinCandidates_ReturnsMultiplePinyinForPolyphonicTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        // 行 can be "xing" or "hang"
+        var candidates = PinyinHelper.GetPinyinCandidates('行').ToList();
+        Assert.Contains("xing", candidates);
+        Assert.Contains("hang", candidates);
+    }
+}
+
+public sealed class GetPinyinCandidates_IncludesFuzzyPinyinVariantsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        // 风 is "feng", should also have fuzzy variant "fen"
+        var candidates = PinyinHelper.GetPinyinCandidates('风').ToList();
+        Assert.Contains("feng", candidates);
+        Assert.Contains("fen", candidates); // fuzzy: eng -> en
+    }
+}
+
+public sealed class GetPinyinCandidates_ReturnsEmptyForNonHanCharactersTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Empty(PinyinHelper.GetPinyinCandidates('a'));
+        Assert.Empty(PinyinHelper.GetPinyinCandidates('あ'));
+    }
+}
+
+
@@ -0,0 +1,59 @@
+using MaigoLabs.NeedLe.Indexer.Han;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Han;
+
+public sealed class UnionFindSet_FindsSelfAsRootInitiallyTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var ufs = new UnionFindSet();
+        Assert.Equal(1, ufs.Find(1));
+        Assert.Equal(2, ufs.Find(2));
+    }
+}
+
+public sealed class UnionFindSet_UnionsTwoElementsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var ufs = new UnionFindSet();
+        ufs.Union(1, 2);
+        Assert.Equal(ufs.Find(1), ufs.Find(2));
+    }
+}
+
+public sealed class UnionFindSet_UnionsMultipleElementsTransitivelyTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var ufs = new UnionFindSet();
+        ufs.Union(1, 2);
+        ufs.Union(2, 3);
+        ufs.Union(4, 5);
+        Assert.Equal(ufs.Find(1), ufs.Find(3));
+        Assert.NotEqual(ufs.Find(1), ufs.Find(4));
+        ufs.Union(3, 4);
+        Assert.Equal(ufs.Find(1), ufs.Find(5));
+    }
+}
+
+public sealed class UnionFindSet_IteratesAllKeysTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var ufs = new UnionFindSet();
+        ufs.Union(1, 2);
+        ufs.Union(3, 4);
+        var keys = ufs.Keys.ToList();
+        Assert.Contains(1, keys);
+        Assert.Contains(2, keys);
+        Assert.Contains(3, keys);
+        Assert.Contains(4, keys);
+    }
+}
+
+
@@ -0,0 +1,69 @@
+using MaigoLabs.NeedLe.Indexer.Japanese;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Japanese;
+
+#region ToRomajiStrictly
+
+public sealed class ToRomajiStrictly_ConvertsBasicKanaToRomajiTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("a", JapaneseUtils.ToRomajiStrictly("あ"));
+        Assert.Equal("ka", JapaneseUtils.ToRomajiStrictly("か"));
+        Assert.Equal("sakura", JapaneseUtils.ToRomajiStrictly("さくら"));
+    }
+}
+
+public sealed class ToRomajiStrictly_ConvertsKatakanaToRomajiTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("a", JapaneseUtils.ToRomajiStrictly("ア"));
+        Assert.Equal("ka", JapaneseUtils.ToRomajiStrictly("カ"));
+        Assert.Equal("sakura", JapaneseUtils.ToRomajiStrictly("サクラ"));
+    }
+}
+
+public sealed class ToRomajiStrictly_HandlesLongVowelsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("ou", JapaneseUtils.ToRomajiStrictly("おう"));
+        Assert.Equal("oo", JapaneseUtils.ToRomajiStrictly("おお"));
+    }
+}
+
+public sealed class ToRomajiStrictly_ReturnsEmptyForInvalidFirstCharacterTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("", JapaneseUtils.ToRomajiStrictly("ー")); // prolonged sound mark cannot be first
+        Assert.Equal("", JapaneseUtils.ToRomajiStrictly("ゃ")); // small ya cannot be first
+    }
+}
+
+public sealed class ToRomajiStrictly_ReturnsEmptyForInvalidLastCharacterTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("", JapaneseUtils.ToRomajiStrictly("っ")); // small tsu cannot be last
+    }
+}
+
+public sealed class ToRomajiStrictly_HandlesGeminationTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        Assert.Equal("katta", JapaneseUtils.ToRomajiStrictly("かった"));
+    }
+}
+
+#endregion
+
+
@@ -0,0 +1,40 @@
+using MaigoLabs.NeedLe.Indexer.Japanese;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer.Japanese;
+
+public sealed class GetAllKanaReadings_ReturnsKatakanaForPureKanaInputTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new TranscriptionProvider();
+        var readings = provider.GetAllKanaReadings("あ");
+        Assert.Contains("ア", readings);
+    }
+}
+
+public sealed class GetAllKanaReadings_ReturnsReadingsForKanjiTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new TranscriptionProvider();
+        var readings = provider.GetAllKanaReadings("僕");
+        Assert.NotEmpty(readings);
+        // 僕 should have reading ボク
+        Assert.Contains("ボク", readings);
+    }
+}
+
+public sealed class GetAllKanaReadings_ReturnsReadingsForCompoundWordsTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var provider = new TranscriptionProvider();
+        var readings = provider.GetAllKanaReadings("和風");
+        Assert.NotEmpty(readings);
+    }
+}
+
+
@@ -0,0 +1,165 @@
+using MaigoLabs.NeedLe.Common.Types;
+using MaigoLabs.NeedLe.Indexer;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer;
+
+public sealed class Tokenizer_TokenizesMixedJapaneseTextTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var tokenizer = new Tokenizer(TokenizerOptions);
+        var tokens = tokenizer.Tokenize("僕の和風本当上手");
+
+        var tokenDefs = tokenizer.Tokens.Values.ToList();
+
+        // Should have tokens of various types
+        var types = tokenDefs.Select(t => t.Type).ToHashSet();
+        Assert.Contains(TokenType.Han, types);
+        Assert.Contains(TokenType.Pinyin, types);
+        Assert.Contains(TokenType.Kana, types);
+        Assert.Contains(TokenType.Romaji, types);
+
+        // Helper to get token texts at a specific position by type
+        List<string> GetTokenTextsAt(int pos, TokenType type) => tokens
+            .Where(t => t.Start <= pos && t.End > pos)
+            .Select(t => tokenDefs.First(d => d.Id == t.Id))
+            .Where(d => d.Type == type)
+            .Select(d => d.Text)
+            .ToList();
+
+        // Position 0: 僕
+        Assert.Contains("僕", GetTokenTextsAt(0, TokenType.Han));
+        Assert.Contains("pu", GetTokenTextsAt(0, TokenType.Pinyin));
+        Assert.Contains("ボク", GetTokenTextsAt(0, TokenType.Kana));
+        Assert.Contains("boku", GetTokenTextsAt(0, TokenType.Romaji));
+
+        // Position 1: の (hiragana, no Han/Pinyin)
+        Assert.Empty(GetTokenTextsAt(1, TokenType.Han));
+        Assert.Empty(GetTokenTextsAt(1, TokenType.Pinyin));
+        Assert.Contains("ノ", GetTokenTextsAt(1, TokenType.Kana));
+        Assert.Contains("no", GetTokenTextsAt(1, TokenType.Romaji));
+
+        // Position 2: 和
+        Assert.Contains("和", GetTokenTextsAt(2, TokenType.Han));
+        Assert.Contains("he", GetTokenTextsAt(2, TokenType.Pinyin));
+        Assert.Contains("ワ", GetTokenTextsAt(2, TokenType.Kana));
+        Assert.Contains("wa", GetTokenTextsAt(2, TokenType.Romaji));
+
+        // Position 3: 風
+        Assert.Contains("風", GetTokenTextsAt(3, TokenType.Han));
+        Assert.Contains("风", GetTokenTextsAt(3, TokenType.Han)); // simplified variant
+        Assert.Contains("feng", GetTokenTextsAt(3, TokenType.Pinyin));
+        Assert.Contains("フウ", GetTokenTextsAt(3, TokenType.Kana));
+        Assert.Contains("fu", GetTokenTextsAt(3, TokenType.Romaji));
+
+        // Position 4: 本
+        Assert.Contains("本", GetTokenTextsAt(4, TokenType.Han));
+        Assert.Contains("ben", GetTokenTextsAt(4, TokenType.Pinyin));
+        Assert.Contains("ホン", GetTokenTextsAt(4, TokenType.Kana));
+        Assert.Contains("hon", GetTokenTextsAt(4, TokenType.Romaji));
+
+        // Position 5: 当
+        Assert.Contains("当", GetTokenTextsAt(5, TokenType.Han));
+        Assert.Contains("當", GetTokenTextsAt(5, TokenType.Han)); // traditional variant
+        Assert.Contains("dang", GetTokenTextsAt(5, TokenType.Pinyin));
+        Assert.Contains("トウ", GetTokenTextsAt(5, TokenType.Kana));
+        Assert.Contains("to", GetTokenTextsAt(5, TokenType.Romaji)); // normalized: tou -> to
+
+        // Position 6: 上
+        Assert.Contains("上", GetTokenTextsAt(6, TokenType.Han));
+        Assert.Contains("shang", GetTokenTextsAt(6, TokenType.Pinyin));
+        Assert.Contains("ジョウ", GetTokenTextsAt(6, TokenType.Kana));
+        Assert.Contains("jo", GetTokenTextsAt(6, TokenType.Romaji)); // normalized: jou -> jo
+
+        // Position 7: 手
+        Assert.Contains("手", GetTokenTextsAt(7, TokenType.Han));
+        Assert.Contains("shou", GetTokenTextsAt(7, TokenType.Pinyin));
+        Assert.Contains("シュ", GetTokenTextsAt(7, TokenType.Kana));
+        Assert.Contains("shu", GetTokenTextsAt(7, TokenType.Romaji));
+    }
+}
+
+public sealed class Tokenizer_NoDuplicateTokensTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var tokenizer = new Tokenizer(TokenizerOptions);
+
+        // Tokenize multiple music names that share some characters
+        tokenizer.Tokenize("僕の和風本当上手");
+        tokenizer.Tokenize("僕");
+        tokenizer.Tokenize("和風");
+
+        // Check that there are no duplicate tokens
+        var tokenDefs = tokenizer.Tokens.Values.ToList();
+        var tokenKeys = tokenDefs.Select(t => $"{t.Type}:{t.Text}").ToList();
+        var uniqueKeys = tokenKeys.ToHashSet();
+
+        Assert.Equal(uniqueKeys.Count, tokenKeys.Count);
+
+        // Also check that IDs are unique
+        var ids = tokenDefs.Select(t => t.Id).ToList();
+        var uniqueIds = ids.ToHashSet();
+        Assert.Equal(uniqueIds.Count, ids.Count);
+    }
+}
+
+public sealed class Tokenizer_HandlesRawTokensForNonCjkTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var tokenizer = new Tokenizer(TokenizerOptions);
+        tokenizer.Tokenize("a-b");
+
+        var tokenDefs = tokenizer.Tokens.Values.ToList();
+        var rawTokenTexts = tokenDefs.Where(t => t.Type == TokenType.Raw).Select(t => t.Text).ToList();
+
+        Assert.Contains("a", rawTokenTexts);
+        Assert.Contains("-", rawTokenTexts);
+        Assert.Contains("b", rawTokenTexts);
+    }
+}
+
+public sealed class Tokenizer_TokenizesCompoundWordKyouTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var tokenizer = new Tokenizer(TokenizerOptions);
+        var tokens = tokenizer.Tokenize("今日");
+        var tokenDefs = tokenizer.Tokens.Values.ToList();
+
+        // Helper to get tokens with specific type and span
+        List<string> GetTokensWithSpan(TokenType type, int start, int end) => tokens
+            .Where(t => t.Start == start && t.End == end)
+            .Select(t => tokenDefs.First(d => d.Id == t.Id))
+            .Where(d => d.Type == type)
+            .Select(d => d.Text)
+            .ToList();
+
+        // Individual character readings at position 0: 今
+        Assert.Contains("今", GetTokensWithSpan(TokenType.Han, 0, 1));
+        Assert.Contains("jin", GetTokensWithSpan(TokenType.Pinyin, 0, 1));
+        Assert.Contains("コン", GetTokensWithSpan(TokenType.Kana, 0, 1));
+        Assert.Contains("イマ", GetTokensWithSpan(TokenType.Kana, 0, 1));
+        Assert.Contains("kon", GetTokensWithSpan(TokenType.Romaji, 0, 1));
+        Assert.Contains("ima", GetTokensWithSpan(TokenType.Romaji, 0, 1));
+
+        // Individual character readings at position 1: 日
+        Assert.Contains("日", GetTokensWithSpan(TokenType.Han, 1, 2));
+        Assert.Contains("ri", GetTokensWithSpan(TokenType.Pinyin, 1, 2));
+        Assert.Contains("ニチ", GetTokensWithSpan(TokenType.Kana, 1, 2));
+        Assert.Contains("ヒ", GetTokensWithSpan(TokenType.Kana, 1, 2));
+        Assert.Contains("niti", GetTokensWithSpan(TokenType.Romaji, 1, 2));
+        Assert.Contains("hi", GetTokensWithSpan(TokenType.Romaji, 1, 2));
+
+        // Combined reading for "今日" [0, 2] - this is an indivisible compound word
+        Assert.Contains("キョウ", GetTokensWithSpan(TokenType.Kana, 0, 2));
+        Assert.Contains("kyo", GetTokensWithSpan(TokenType.Romaji, 0, 2)); // normalized: kyou -> kyo
+    }
+}
+
+
@@ -0,0 +1,66 @@
+using MaigoLabs.NeedLe.Common;
+using MaigoLabs.NeedLe.Common.Extensions;
+using MaigoLabs.NeedLe.Indexer.Trie;
+
+namespace MaigoLabs.NeedLe.Tests.Indexer;
+
+#region GraftTriePaths
+
+public sealed class GraftTriePaths_GraftsPathsAccordingToNormalizationRulesTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        // Build a trie with tokens containing normalized forms
+        var trie = TrieBuilder.BuildTrie([
+            (0, "sya".ToCodePoints()), // normalized form of "sha"
+            (1, "tu".ToCodePoints()),  // normalized form of "tsu"
+        ]);
+
+        // Graft paths so that "sha" -> "sya" and "tsu" -> "tu"
+        TrieBuilder.GraftTriePaths(trie, [
+            ("sha".ToCodePoints().ToArray(), "sya".ToCodePoints().ToArray()),
+            ("tsu".ToCodePoints().ToArray(), "tu".ToCodePoints().ToArray()),
+        ]);
+
+        // Now we should be able to traverse using both the original and grafted paths
+        var syaNode = trie.Traverse("sya".ToCodePoints().ToArray());
+        var shaNode = trie.Traverse("sha".ToCodePoints().ToArray());
+        Assert.NotNull(syaNode);
+        Assert.NotNull(shaNode);
+        Assert.Same(syaNode, shaNode); // Both paths should lead to the same node
+
+        var tuNode = trie.Traverse("tu".ToCodePoints().ToArray());
+        var tsuNode = trie.Traverse("tsu".ToCodePoints().ToArray());
+        Assert.NotNull(tuNode);
+        Assert.NotNull(tsuNode);
+        Assert.Same(tuNode, tsuNode);
+    }
+}
+
+public sealed class GraftTriePaths_HandlesChainedGraftRulesTest : NeedleTestBase
+{
+    [Fact]
+    public void Execute()
+    {
+        var trie = TrieBuilder.BuildTrie([
+            (0, "o".ToCodePoints()), // normalized vowel
+        ]);
+
+        // Chain: "ou" -> "o", "oo" -> "o"
+        TrieBuilder.GraftTriePaths(trie, [
+            ("ou".ToCodePoints().ToArray(), "o".ToCodePoints().ToArray()),
+            ("oo".ToCodePoints().ToArray(), "o".ToCodePoints().ToArray()),
+        ]);
+
+        var oNode = trie.Traverse("o".ToCodePoints().ToArray());
+        var ouNode = trie.Traverse("ou".ToCodePoints().ToArray());
+        var ooNode = trie.Traverse("oo".ToCodePoints().ToArray());
+
+        Assert.NotNull(oNode);
+        Assert.Same(oNode, ouNode);
+        Assert.Same(oNode, ooNode);
+    }
+}
+
+#endregion