MaigoLabs/needLe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: initial commit

Browse Source
This commit is contained in:
Menci
2026-01-01 03:40:41 +08:00
commit 631f8ed771
98 changed files with 14776 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dotnet/MaigoLabs.NeedLe.Indexer/Japanese/JapaneseNormalization.cs
+69
View File
@@ -0,0 +1,69 @@
using MaigoLabs.NeedLe.Common.Extensions;
namespace MaigoLabs.NeedLe.Indexer.Japanese;
public static class JapaneseNormalization
{
public delegate string Normalizer(string text);
public static Normalizer CreateNormalizer(Dictionary<string, string> rules) => text =>
{
while (true)
{
var beforeCurrentIteration = text;
foreach (var (from, to) in rules) text = text.Replace(from, to);
if (text == beforeCurrentIteration) break;
}
return text;
};
public static IEnumerable<(int[] From, int[] To)> ToCodePointPairs(Dictionary<string, string> rules) =>
rules.Select(rule => (From: rule.Key.ToCodePoints().ToArray(), To: rule.Value.ToCodePoints().ToArray()));
public static readonly Dictionary<string, string> NORMALIZE_RULES_ROMAJI = new()
{
// Remove all long vowels (sa-ba- -> saba)
["-"] = "",
// Collapse consecutive vowels
["aa"] = "a",
["ii"] = "i",
["uu"] = "u",
["ee"] = "e",
["oo"] = "o",
["ou"] = "o",
// mb/mp/mm -> nb/np/nm (shimbun -> shinbun)
["mb"] = "nb",
["mp"] = "np",
["mm"] = "nm",
// Others
["sha"] = "sya",
["tsu"] = "tu",
["chi"] = "ti",
["shi"] = "si",
["ji"] = "zi",
};
public static readonly IEnumerable<(int[] From, int[] To)> NORMALIZE_RULES_ROMAJI_CODEPOINTS = ToCodePointPairs(NORMALIZE_RULES_ROMAJI);
public static readonly Normalizer NormalizeRomaji = CreateNormalizer(NORMALIZE_RULES_ROMAJI);
public static readonly Dictionary<string, string> NORMALIZE_RULES_KANA_DAKUTEN = new()
{
["う\u3099"] = "ゔ",
["か\u3099"] = "が", ["き\u3099"] = "ぎ", ["く\u3099"] = "ぐ", ["け\u3099"] = "げ", ["こ\u3099"] = "ご",
["さ\u3099"] = "ざ", ["し\u3099"] = "じ", ["す\u3099"] = "ず", ["せ\u3099"] = "ぜ", ["そ\u3099"] = "ぞ",
["た\u3099"] = "だ", ["ち\u3099"] = "ぢ", ["つ\u3099"] = "づ", ["て\u3099"] = "で", ["と\u3099"] = "ど",
["は\u3099"] = "ば", ["ひ\u3099"] = "び", ["ふ\u3099"] = "ぶ", ["へ\u3099"] = "べ", ["ほ\u3099"] = "ぼ",
["は\u309A"] = "ぱ", ["ひ\u309A"] = "ぴ", ["ふ\u309A"] = "ぷ", ["へ\u309A"] = "ぺ", ["ほ\u309A"] = "ぽ",
["ゝ\u3099"] = "ゞ",
["ウ\u3099"] = "ヴ",
["カ\u3099"] = "ガ", ["キ\u3099"] = "ギ", ["ク\u3099"] = "グ", ["ケ\u3099"] = "ゲ", ["コ\u3099"] = "ゴ",
["サ\u3099"] = "ザ", ["シ\u3099"] = "ジ", ["ス\u3099"] = "ズ", ["セ\u3099"] = "ゼ", ["ソ\u3099"] = "ゾ",
["タ\u3099"] = "ダ", ["チ\u3099"] = "ヂ", ["ツ\u3099"] = "ヅ", ["テ\u3099"] = "デ", ["ト\u3099"] = "ド",
["ハ\u3099"] = "バ", ["ヒ\u3099"] = "ビ", ["フ\u3099"] = "ブ", ["ヘ\u3099"] = "ベ", ["ホ\u3099"] = "ボ",
["ハ\u309A"] = "パ", ["ヒ\u309A"] = "ピ", ["フ\u309A"] = "プ", ["ヘ\u309A"] = "ペ", ["ホ\u309A"] = "ポ",
["ワ\u3099"] = "ヷ", ["ヰ\u3099"] = "ヸ", ["ヱ\u3099"] = "ヹ", ["ヲ\u3099"] = "ヺ",
["ヽ\u3099"] = "ヾ",
};
public static readonly IEnumerable<(int[] From, int[] To)> NORMALIZE_RULES_KANA_DAKUTEN_CODEPOINTS = ToCodePointPairs(NORMALIZE_RULES_KANA_DAKUTEN);
public static readonly Normalizer NormalizeKanaDakuten = CreateNormalizer(NORMALIZE_RULES_KANA_DAKUTEN);
}
dotnet/MaigoLabs.NeedLe.Indexer/Japanese/JapaneseUtils.cs
+52
View File
@@ -0,0 +1,52 @@
using MaigoLabs.NeedLe.Indexer.Han;
using MyNihongo.KanaConverter;
namespace MaigoLabs.NeedLe.Indexer.Japanese;
public static class JapaneseUtils
{
public static bool IsMaybeJapanese(int codePoint) =>
HanVariantProvider.IsHanCharacter(codePoint) ||
IsKana(codePoint) ||
IsJapaneseSoundMark(codePoint) ||
codePoint == 0x3005 || codePoint == 0x3006 || codePoint == 0x30FC;
// See also Common/Normalization.cs
public static bool IsJapaneseSoundMark(int codePoint) => codePoint == 0x3099 || codePoint == 0x309A;
public static string StripJapaneseSoundMarks(string text) => string.Concat(text.Where(codePoint => !IsJapaneseSoundMark(codePoint)));
public static bool IsKana(int codePoint) => (codePoint >= 0x3041 && codePoint <= 0x309F) || (codePoint >= 0x30A0 && codePoint <= 0x30FF);
private static readonly int[] KANAS_CANNOT_BE_FIRST =
[
'ァ', 'ィ', 'ゥ', 'ェ', 'ォ',
'ぁ', 'ぃ', 'ぅ', 'ぇ', 'ぉ',
'ャ', 'ュ', 'ョ',
'ゃ', 'ゅ', 'ょ',
'ヮ', 'ゎ',
'ㇰ', 'ㇱ', 'ㇲ', 'ㇳ', 'ㇴ', 'ㇵ', 'ㇶ', 'ㇷ', 'ㇸ', 'ㇹ', 'ㇺ', 'ㇻ', 'ㇼ', 'ㇽ', 'ㇾ', 'ㇿ',
'ー',
];
private static readonly int[] KANAS_CANNOT_BE_LAST =
[
'ッ', 'っ'
];
public static string ToRomajiStrictly(string kanaText)
{
if (kanaText.Length == 0) return "";
if (KANAS_CANNOT_BE_FIRST.Contains(kanaText[0])) return "";
if (KANAS_CANNOT_BE_LAST.Contains(kanaText[^1])) return "";
string romaji;
try { romaji = kanaText.ToRomaji(); }
catch { return ""; }
if (!romaji.All(c => c is >= 'a' and <= 'z')) return "";
return romaji;
}
public static bool IsValidJapanesePhrase(ReadOnlySpan<int> codePoints, int start, int length) =>
// Skip splittings that cause sound marks to occur in the first position of a phrase
!IsJapaneseSoundMark(codePoints[start]) && (start + length == codePoints.Length || !IsJapaneseSoundMark(codePoints[start + length]));
public static bool IsValidJapanesePhrase(ReadOnlyMemory<int> codePoints, int start, int length) => IsValidJapanesePhrase(codePoints.Span, start, length);
}
dotnet/MaigoLabs.NeedLe.Indexer/Japanese/TranscriptionProvider.cs
+105
View File
@@ -0,0 +1,105 @@
using System.Runtime.InteropServices;
using MaigoLabs.NeedLe.Common;
using MaigoLabs.NeedLe.Common.Extensions;
using MeCab;
using MeCab.Core;
namespace MaigoLabs.NeedLe.Indexer.Japanese;
public class Transcription
{
public required int Start { get; set; }
public required int Length { get; set; }
public required string[] Transcriptions { get; set; }
}
public delegate IEnumerable<Transcription> TranscriptionEnumerator(ReadOnlyMemory<int> codePoints);
public delegate bool IsValidPhraseDelegate(ReadOnlyMemory<int> codePoints, int start, int length);
public delegate HashSet<string> GetAllTranscriptionsDelegate(string phrase);
public class TranscriptionProvider
{
public MeCabDictionary[] Dictionaries { get; set; }
public TranscriptionProvider(MeCabDictionary[]? dictionaries = null)
{
if (dictionaries == null)
{
var param = new MeCabParam();
param.LoadDicRC();
var dictionary = new MeCabDictionary();
dictionary.Open(Path.Combine(param.DicDir, "sys.dic"));
dictionaries = [dictionary];
}
Dictionaries = dictionaries;
}
public static TranscriptionEnumerator CreateTranscriptionEnumerator(IsValidPhraseDelegate isValidPhrase, GetAllTranscriptionsDelegate getAllTranscriptions) => codePoints =>
{
var resultMap = new Dictionary<(int Start, int Length), Transcription>();
for (int phraseLength = 1; phraseLength <= codePoints.Length; phraseLength++) for (int start = 0; start + phraseLength <= codePoints.Length; start++)
{
if (!isValidPhrase(codePoints, start, phraseLength)) continue;
var phrase = MemoryMarshal.ToEnumerable(codePoints.Slice(start, phraseLength)).ToUtf32String();
var atomicTranscriptions = getAllTranscriptions(phrase).Where(transcription => transcription != null).Where(candidateTranscription =>
{
if (candidateTranscription.Length == 0) return false;
// Ensure the transcription is atomic (not a combination of multiple shorter transcriptions, separated by any midpoints)
var visitedStates = new HashSet<(int PhrasePosition, int TranscriptionPosition)>();
var queue = new Queue<(int PhrasePosition, int TranscriptionPosition)>();
queue.Enqueue((0, 0));
while (queue.Count > 0)
{
var (phrasePosition, transcriptionPosition) = queue.Dequeue();
for (int prefixLength = 1; prefixLength <= phraseLength - phrasePosition; prefixLength++)
{
if (!resultMap.TryGetValue((start + phrasePosition, prefixLength), out var prefixResult)) continue;
foreach (var transcription in prefixResult.Transcriptions) if (string.Compare(candidateTranscription, transcriptionPosition, transcription, 0, transcription.Length) == 0)
{
var nextState = (PhrasePosition: phrasePosition + prefixLength, TranscriptionPosition: transcriptionPosition + transcription.Length);
if (nextState.PhrasePosition == phraseLength && nextState.TranscriptionPosition == candidateTranscription.Length) return false; // Found a valid combination
if (visitedStates.Contains(nextState)) continue;
visitedStates.Add(nextState);
queue.Enqueue(nextState);
}
}
}
return true;
}).ToArray();
if (atomicTranscriptions.Length > 0) resultMap[(start, phraseLength)] = new() { Start = start, Length = phraseLength, Transcriptions = atomicTranscriptions };
}
return resultMap.Values;
};
public HashSet<string> GetAllKanaReadings(string phrase)
{
var result = new HashSet<string>();
var isKana = phrase.All(ch => JapaneseUtils.IsKana(ch));
if (isKana) result.Add(CommonNormalization.ToKatakana(phrase));
if (isKana && phrase.Length == 1) return result;
foreach (var dictionary in Dictionaries)
{
var searchResult = dictionary.ExactMatchSearch(phrase);
if (searchResult.Value == -1) continue;
var tokens = dictionary.GetToken(searchResult);
foreach (var token in tokens)
{
var feature = dictionary.GetFeature(token.Feature);
var parts = feature.Split(',');
if (parts.Length > 7) result.Add(CommonNormalization.ToKatakana(parts[7]));
}
}
return result;
}
public HashSet<string> GetAllKanaReadingsWithNormalization(string phrase) =>
GetAllKanaReadings(JapaneseUtils.StripJapaneseSoundMarks(JapaneseNormalization.NormalizeKanaDakuten(phrase)));
public TranscriptionEnumerator EnumerateKanaTranscriptions => CreateTranscriptionEnumerator(
JapaneseUtils.IsValidJapanesePhrase,
GetAllKanaReadingsWithNormalization);
public TranscriptionEnumerator EnumerateRomajiTranscriptions => CreateTranscriptionEnumerator(
JapaneseUtils.IsValidJapanesePhrase,
phrase => [.. GetAllKanaReadingsWithNormalization(phrase).Select(kana => JapaneseNormalization.NormalizeRomaji(JapaneseUtils.ToRomajiStrictly(kana)))]);
}