feat: initial commit

packages/needle/src/common/trie.ts

@@ -0,0 +1,17 @@
+export interface TrieNode {
+  parent: TrieNode | undefined;
+  children: Map<number, TrieNode>; // Unicode code point -> child node
+  tokenIds: number[];
+  subTreeTokenIds: number[]; // Empty on root. Will Uint16Array be faster?
+}
+
+export const traverseTrieStep = (node: TrieNode | undefined, codePoint: string, ignorableCodePoints?: RegExp) =>
+  node?.children.get(codePoint.codePointAt(0)!) ?? (ignorableCodePoints?.test(codePoint) ? node : undefined);
+export const traverseTrie = (node: TrieNode | undefined, text: string, ignorableCodePoints?: RegExp) => {
+  if (!node) return;
+  for (const codePoint of text) {
+    node = traverseTrieStep(node, codePoint, ignorableCodePoints);
+    if (!node) return;
+  }
+  return node;
+};