Improve Language Processing Logic

pipeline/aggregate/blocks/language/LanguageStats.ts

@@ -43,7 +43,12 @@ const fn: BlockFn<LanguageStats> = (database, filters, common, args) => {
         .filter((lang) => lang.value < langThreshold)
         .reduce((sum, lang) => sum + lang.value, 0);
     const languageList = allLanguages.filter((lang) => lang.value >= langThreshold);
-    languageList.push({ index: 0, value: totalUnreliable });
+    const UnreliableToDetectIndex = languageList.findIndex((item) => item.index == 0);
+    if (UnreliableToDetectIndex < 0) {
+        languageList.push({ index: 0, value: totalUnreliable }); // if the index didn't exist, push the value
+    } else {
+        languageList[UnreliableToDetectIndex].value += totalUnreliable; // append cutoff languages to unreliable languages count
+    }
     languageList.sort((a, b) => b.value - a.value);
 
     return {

pipeline/process/MessageProcessor.ts

@@ -49,9 +49,18 @@ export class MessageProcessor {
 
         // detect language in the whole group text
         // this yields better accuracy
-        let langIndex: number | undefined;
-        if (allText.length > 0) {
-            langIndex = this.langPredictModel!.identifyLanguage(allText).iso639index;
+        let langIndex: number = 0;
+        let result;
+        const accuracy_threshold = 0; // language model accuracy must be at least this high (0-1)
+        const word_count_threshold = 1; // must have at least this many words for language detection
+        let word_count: number = tokenizations
+            .map((msg) => msg.reduce((sum, token) => (token.tag == "word" ? sum + 1 : sum), 0)) // sum the number of words per message
+            .reduce((sum, len) => sum + len, 0); // sum the number of words in all messages in the group
+        if (word_count >= word_count_threshold) {
+            result = this.langPredictModel!.identifyLanguage(allText);
+            if (result.accuracy >= accuracy_threshold) {
+                langIndex = result.iso639index;
+            }
         }
 
         return group.map((message, index) => this.processMessage(message, tokenizations[index], langIndex));

report/components/cards/language/LanguageStatsTable.tsx

@@ -40,7 +40,7 @@ const LanguageStatsTable = () => {
                     depth: 1,
                     tooltip:
                         language.index === 0
-                            ? "Messages that did not have enough text to reliable detect the language"
+                            ? "Messages that did not have enough text to reliably detect the language"
                             : undefined,
                 } as Line)
         ) ?? []),