fix(tokenizers): use mandarin instead of japanese in tokenizer language (#904)

undefined-moe · web-flow · commit 968b4a8786de · 2025-03-06T10:05:30.000+01:00
diff --git a/packages/tokenizers/src/index.ts b/packages/tokenizers/src/index.ts
@@ -1,5 +1,7 @@
 import { createTokenizer as createJapaneseTokenizer } from "./japanese.js";
+import { createTokenizer as createMandarinTokenizer } from "./mandarin.js";
 
 export default {
     japanese: createJapaneseTokenizer,
+    mandarin: createMandarinTokenizer,
 }
diff --git a/packages/tokenizers/src/mandarin.ts b/packages/tokenizers/src/mandarin.ts
@@ -1,15 +1,15 @@
 import type { DefaultTokenizer, DefaultTokenizerConfig } from "@orama/orama";
 import { normalizeToken } from "@orama/orama/internals";
 
-const tokenizerLanguage = "japanese";
+const tokenizerLanguage = "mandarin";
 
 type TLanguage = typeof tokenizerLanguage;
 
-type JapaneseTokenizerConfig = DefaultTokenizerConfig & {
+type MandarinTokenizerConfig = DefaultTokenizerConfig & {
   language: TLanguage;
 };
 
-const defaultConfig: JapaneseTokenizerConfig = {
+const defaultConfig: MandarinTokenizerConfig = {
   language: tokenizerLanguage,
 };
 
@@ -68,7 +68,7 @@ function tokenizeInternal(
 }
 
 export function createTokenizer(
-  config: JapaneseTokenizerConfig = defaultConfig,
+  config: MandarinTokenizerConfig = defaultConfig,
 ): DefaultTokenizer {
   const tokenizerConfig = {
     tokenize: tokenizeInternal,

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,7 @@`
`1`	`1`	`import { createTokenizer as createJapaneseTokenizer } from "./japanese.js";`
	`2`	`+import { createTokenizer as createMandarinTokenizer } from "./mandarin.js";`
`2`	`3`
`3`	`4`	`export default {`
`4`	`5`	`japanese: createJapaneseTokenizer,`
	`6`	`+ mandarin: createMandarinTokenizer,`
`5`	`7`	`}`