Improve Language Processing Logic

pipeline/aggregate/blocks/language/LanguageStats.ts

@@ -1,3 +1,5 @@
+import { isUndefined } from "util";
+
 import { BlockDescription, BlockFn } from "@pipeline/aggregate/Blocks";
 import { IndexEntry } from "@pipeline/aggregate/Common";
 import { filterMessages } from "@pipeline/aggregate/Helpers";
@@ -37,13 +39,18 @@ const fn: BlockFn<LanguageStats> = (database, filters, common, args) => {
     filterMessages(processMessage, database, filters);
 
     // lang
-    const langThreshold = Math.max(1, totalWithLang * 0.03); // at least 3% to be reliable
+    const langThreshold = Math.max(1, totalWithLang * 0.001); // at least 0.1% to be reliable
     const allLanguages = languagesCount.map((count, index) => ({ index, value: count }));
     const totalUnreliable = allLanguages
         .filter((lang) => lang.value < langThreshold)
         .reduce((sum, lang) => sum + lang.value, 0);
     const languageList = allLanguages.filter((lang) => lang.value >= langThreshold);
-    languageList.push({ index: 0, value: totalUnreliable });
+    const UnreliableToDetectIndex = languageList.findIndex((item) => item.index == 0);
+    if (UnreliableToDetectIndex < 0) {
+        languageList.push({ index: 0, value: totalUnreliable }); // if the index didn't exist, push the value
+    } else {
+        languageList[UnreliableToDetectIndex].value += totalUnreliable; // append cutoff languages to unreliable languages count
+    }
     languageList.sort((a, b) => b.value - a.value);
 
     return {

pipeline/process/DatabaseBuilder.ts

@@ -207,8 +207,8 @@ export class DatabaseBuilder {
         return (
             this.langCounts
                 .map((count, index) => ({ code: LanguageCodes[index], ratio: count / totalWithLang }))
-                // we need AT LEAST 3% to consider reliable
-                .filter((l) => l.ratio >= 0.03)
+                // we need AT LEAST 0.1% to consider reliable
+                .filter((l) => l.ratio >= 0.001)
                 // sort most used
                 .sort((a, b) => b.ratio - a.ratio)
                 // only keep the code

pipeline/process/MessageProcessor.ts

@@ -49,9 +49,18 @@ export class MessageProcessor {
 
         // detect language in the whole group text
         // this yields better accuracy
-        let langIndex: number | undefined;
-        if (allText.length > 0) {
-            langIndex = this.langPredictModel!.identifyLanguage(allText).iso639index;
+        let langIndex: number = 0;
+        let result;
+        const accuracy_threshold = 0.7; // language model accuracy must be at least this high (0-1)
+        const word_count_threshold = 1; // must have at least this many words for language detection
+        let word_count: number = tokenizations
+            .map((msg) => msg.reduce((sum, token) => (token.tag == "word" ? sum + 1 : sum), 0)) // sum the number of words per message
+            .reduce((sum, len) => sum + len, 0); // sum the number of words in all messages in the group
+        if (word_count >= word_count_threshold) {
+            result = this.langPredictModel!.identifyLanguage(allText);
+            if (result.accuracy >= accuracy_threshold) {
+                langIndex = result.iso639index;
+            }
         }
 
         return group.map((message, index) => this.processMessage(message, tokenizations[index], langIndex));

report/components/cards/language/LanguageStatsTable.tsx

@@ -40,7 +40,7 @@ const LanguageStatsTable = () => {
                     depth: 1,
                     tooltip:
                         language.index === 0
-                            ? "Messages that did not have enough text to reliable detect the language"
+                            ? "Messages that did not have enough text to reliably detect the language"
                             : undefined,
                 } as Line)
         ) ?? []),