feat: initial commit

2026-01-01 03:40:41 +08:00
commit 631f8ed771
98 changed files with 14776 additions and 0 deletions
@@ -0,0 +1,45 @@
+namespace MaigoLabs.NeedLe.Common;
+
+// This is for global normalization for any input and documents.
+public static class CommonNormalization
+{
+    public static int NormalizeCodePoint(int codePoint)
+    {
+        // Fullwidth ASCII -> Halfwidth ASCII
+        if (codePoint >= 0xFF01 && codePoint <= 0xFF5E) return ToLowerCaseAscii(codePoint - 0xFEE0);
+        // Fullwidth space -> Halfwidth space
+        else if (codePoint == /* '　' */ 0x3000) return ' ';
+        // Halfwidth kana (U+FF66 - U+FF9D) -> Fullwidth kana
+        else if (codePoint >= 0xFF66 && codePoint <= 0xFF9D) return HALF_TO_FULL_KANA.TryGetValue(codePoint, out var value) ? value : codePoint;
+        else if (codePoint == /* '｡' */ 0xFF61) return '。';
+        else if (codePoint == /* '｢' */ 0xFF62) return '「';
+        else if (codePoint == /* '｣' */ 0xFF63) return '」';
+        else if (codePoint == /* '､' */ 0xFF64) return '、';
+        else if (codePoint == /* '･' */ 0xFF65) return '・';
+        else if (codePoint == /* 'ﾞ' */ 0xFF9E || codePoint == /* '゛' */ 0x309B) return 0x3099; // -> COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK
+        else if (codePoint == /* 'ﾟ' */ 0xFF9F || codePoint == /* '゜' */ 0x309C) return 0x309A; // -> COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK
+        else return ToLowerCaseAscii(codePoint);
+    }
+
+    private static readonly Dictionary<int, int> HALF_TO_FULL_KANA = new Dictionary<int, int> {
+        ['ｦ'] = 'ヲ', ['ｧ'] = 'ァ', ['ｨ'] = 'ィ', ['ｩ'] = 'ゥ', ['ｪ'] = 'ェ', ['ｫ'] = 'ォ',
+        ['ｬ'] = 'ャ', ['ｭ'] = 'ュ', ['ｮ'] = 'ョ', ['ｯ'] = 'ッ',
+        ['ｰ'] = 'ー',
+        ['ｱ'] = 'ア', ['ｲ'] = 'イ', ['ｳ'] = 'ウ', ['ｴ'] = 'エ', ['ｵ'] = 'オ',
+        ['ｶ'] = 'カ', ['ｷ'] = 'キ', ['ｸ'] = 'ク', ['ｹ'] = 'ケ', ['ｺ'] = 'コ',
+        ['ｻ'] = 'サ', ['ｼ'] = 'シ', ['ｽ'] = 'ス', ['ｾ'] = 'セ', ['ｿ'] = 'ソ',
+        ['ﾀ'] = 'タ', ['ﾁ'] = 'チ', ['ﾂ'] = 'ツ', ['ﾃ'] = 'テ', ['ﾄ'] = 'ト',
+        ['ﾅ'] = 'ナ', ['ﾆ'] = 'ニ', ['ﾇ'] = 'ヌ', ['ﾈ'] = 'ネ', ['ﾉ'] = 'ノ',
+        ['ﾊ'] = 'ハ', ['ﾋ'] = 'ヒ', ['ﾌ'] = 'フ', ['ﾍ'] = 'ヘ', ['ﾎ'] = 'ホ',
+        ['ﾏ'] = 'マ', ['ﾐ'] = 'ミ', ['ﾑ'] = 'ム', ['ﾒ'] = 'メ', ['ﾓ'] = 'モ',
+        ['ﾔ'] = 'ヤ', ['ﾕ'] = 'ユ', ['ﾖ'] = 'ヨ',
+        ['ﾗ'] = 'ラ', ['ﾘ'] = 'リ', ['ﾙ'] = 'ル', ['ﾚ'] = 'レ', ['ﾛ'] = 'ロ',
+        ['ﾜ'] = 'ワ', ['ﾝ'] = 'ン',
+    };
+
+    public static int ToLowerCaseAscii(int codePoint) => codePoint >= 0x41 && codePoint <= 0x5A ? codePoint + 0x20 : codePoint;
+
+    public static bool IsHiraganaRange(int codePoint) => (codePoint >= 0x3041 && codePoint <= 0x3096) || (codePoint >= 0x309D && codePoint <= 0x309E);
+    public static int ToKatakana(int codePoint) => IsHiraganaRange(codePoint) ? codePoint + 0x60 : codePoint;
+    public static string ToKatakana(string text) => string.Concat(text.Select(c => (char)ToKatakana(c)));
+}
@@ -0,0 +1,21 @@
+namespace MaigoLabs.NeedLe.Common;
+
+public static class CommonUtils
+{
+    public static bool IsWhitespace(int codePoint) =>
+        codePoint == 0x0009 /* \t */ ||
+        codePoint == 0x000A /* \n */ ||
+        codePoint == 0x000B /* Vertical Tab */ ||
+        codePoint == 0x000C /* \f */ ||
+        codePoint == 0x000D /* \r */ ||
+        codePoint == 0x0020 /* Space */ ||
+        codePoint == 0x0085 /* Next Line (NEL) */ ||
+        codePoint == 0x00A0 /* No-Break Space */ ||
+        codePoint == 0x1680 /* Ogham Space Mark */ ||
+        codePoint >= 0x2000 && codePoint <= 0x200A ||
+        codePoint == 0x2028 /* Line Separator */ ||
+        codePoint == 0x2029 /* Paragraph Separator */ ||
+        codePoint == 0x202F /* Narrow No-Break Space */ ||
+        codePoint == 0x205F /* Medium Mathematical Space */ ||
+        codePoint == 0x3000 /* Ideographic Space */;
+}
@@ -0,0 +1,25 @@
+using System.Text;
+
+namespace MaigoLabs.NeedLe.Common.Extensions;
+
+public static class UnicodeExtensions
+{
+    public static IEnumerable<int> ToCodePoints(this string s)
+    {
+        for (int i = 0; i < s.Length; i++)
+        {
+            int codePoint = char.ConvertToUtf32(s, i);
+            if (codePoint > 0xffff) i++;
+            yield return codePoint;
+        }
+    }
+
+    public static StringBuilder ToUtf32StringBuilder(this IEnumerable<int> codePoints)
+    {
+        var sb = new StringBuilder();
+        foreach (var codePoint in codePoints) sb.Append(char.ConvertFromUtf32(codePoint));
+        return sb;
+    }
+
+    public static string ToUtf32String(this IEnumerable<int> codePoints) => ToUtf32StringBuilder(codePoints).ToString();
+}
@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <OutputType>Library</OutputType>
+    <RootNamespace>$(ProjectName).Common</RootNamespace>
+    <AssemblyName>$(RootNamespace)</AssemblyName>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <IsPackable>true</IsPackable>
+    <PackageId>$(RootNamespace)</PackageId>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="DotNetCampus.LatestCSharpFeatures" PrivateAssets="all" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,33 @@
+namespace MaigoLabs.NeedLe.Common;
+
+public class TrieNode
+{
+    public required TrieNode? Parent { get; set; }
+    public required Dictionary<int, TrieNode> Children { get; set; } // Unicode code point -> child node
+    public required List<int> TokenIds { get; set; }
+    public required List<int> SubTreeTokenIds { get; set; } // Empty on root.
+}
+
+public static class TrieNodeExtensions
+{
+    public static TrieNode? TraverseStep(this TrieNode? node, int codePoint, bool isIgnorable = false) =>
+        (node?.Children.TryGetValue(codePoint, out var child) ?? false)
+            ? child
+            : isIgnorable ? node : null;
+
+    public static TrieNode? Traverse(this TrieNode? node, int[] codePoints, bool isIgnorable = false)
+    {
+        if (node == null) return null;
+        foreach (var codePoint in codePoints)
+        {
+            node = node?.TraverseStep(codePoint, isIgnorable);
+            if (node == null) return null;
+        }
+        return node;
+    }
+
+    public static List<int> GetTokenIds(this TrieNode? node, bool includeSubTree = false) =>
+        (includeSubTree ? node?.SubTreeTokenIds : node?.TokenIds) ?? [];
+
+    public static bool IsTokenExactMatch(this TrieNode? node, int tokenId) => node?.TokenIds.Contains(tokenId) ?? false;
+}
@@ -0,0 +1,20 @@
+namespace MaigoLabs.NeedLe.Common.Types;
+
+#pragma warning disable IDE1006 // Naming rule violation
+
+// For compatibility with TypeScript, we use camelCase property names here.
+
+public class CompressedInvertedIndex
+{
+    public required string[] documents { get; set; }
+    public required int[] tokenTypes { get; set; } // Use int values here instead of TokenType enum to avoid JSON serialization issues.
+    public required List<int[]>[] tokenReferences { get; set; } // tokenId -> [documentId, start1, end1, start2, end2, ...]
+    public required CompressedInvertedIndexTries tries { get; set; }
+}
+
+public class CompressedInvertedIndexTries
+{
+    public required int[] romaji { get; set; }
+    public required int[] kana { get; set; }
+    public required int[] other { get; set; }
+}
@@ -0,0 +1,9 @@
+namespace MaigoLabs.NeedLe.Common.Types;
+
+public class OffsetSpan
+{
+    public required int Start { get; init; }
+    public required int End { get; init; }
+
+    public int Length => End - Start;
+}
@@ -0,0 +1,9 @@
+namespace MaigoLabs.NeedLe.Common.Types;
+
+public class TokenDefinition
+{
+    public required int Id { get; set; }
+    public required TokenType Type { get; set; }
+    public required string Text { get; set; }
+    public required int CodePointLength { get; set; }
+}
@@ -0,0 +1,10 @@
+namespace MaigoLabs.NeedLe.Common.Types;
+
+public enum TokenType
+{
+    Raw,
+    Kana,
+    Romaji,
+    Han,
+    Pinyin,
+}