Language Detection

Detect languages in extracted text using whatlang — supports 60+ languages with ISO 639-3 codes. Set detect_multiple: true to chunk the text into 200-character segments and return all detected languages sorted by prevalence.

Set min_confidence (0.0–1.0, default 0.8) to the lowest whatlang confidence a detection must reach to be reported. In single-language mode, the primary detection is dropped and no language is returned when it scores below the threshold. In detect_multiple mode, the threshold is applied to each 200-character chunk; chunks below it are discarded, and if no chunk clears it, detection falls back to single-language mode. Per-chunk confidence runs lower than whole-document confidence, so a high threshold can suppress multi-language results.

Configuration

import asyncio
from xberg import ExtractInput, ExtractionConfig, LanguageDetectionConfig, extract

async def main() -> None:
    config: ExtractionConfig = ExtractionConfig(
        language_detection=LanguageDetectionConfig(
            enabled=True,
            min_confidence=0.85,
            detect_multiple=False
        )
    )
    result = await extract(ExtractInput.from_uri("document.pdf"), config)
    if result.detected_languages:
        print(f"Primary language: {result.detected_languages[0]}")
    print(f"Content length: {len(result.results[0].content)} chars")

asyncio.run(main())

import { extract } from "@xberg-io/xberg";

const config = {
  languageDetection: {
    enabled: true,
    minConfidence: 0.8,
    detectMultiple: false,
  },
};

const result = await extract({ kind: "uri", uri: "document.pdf" }, config);
if (result.detectedLanguages) {
  console.log(`Detected languages: ${result.detectedLanguages.join(", ")}`);
}

use xberg::{ExtractionConfig, LanguageDetectionConfig};

let config = ExtractionConfig {
    language_detection: Some(LanguageDetectionConfig {
        enabled: true,
        min_confidence: 0.8,
        detect_multiple: false,
    }),
    ..Default::default()
};

package main

import (
  "fmt"

  "github.com/xberg-io/xberg/packages/go"
)

func main() {
  minConfidence := 0.8
  config := &xberg.ExtractionConfig{
    LanguageDetection: &xberg.LanguageDetectionConfig{
      Enabled:        true,
      MinConfidence:  &minConfidence,
      DetectMultiple: false,
    },
  }

  fmt.Printf("Language detection enabled: %v\n", config.LanguageDetection.Enabled)
  fmt.Printf("Min confidence: %f\n", *config.LanguageDetection.MinConfidence)
}

import io.xberg.ExtractionConfig;
import io.xberg.LanguageDetectionConfig;

ExtractionConfig config = ExtractionConfig.builder()
    .languageDetection(LanguageDetectionConfig.builder()
        .enabled(true)
        .minConfidence(0.8)
        .build())
    .build();

using Xberg;

class Program
{
    static async Task Main()
    {
        var config = new ExtractionConfig
        {
            LanguageDetection = new LanguageDetectionConfig
            {
                Enabled = true,
                MinConfidence = 0.8m,
                DetectMultiple = false
            }
        };

        try
        {
            var result = (await XbergConverter.ExtractAsync(ExtractInput.FromUri("document.pdf"), config)).Results[0];

            if (result.DetectedLanguages?.Count > 0)
            {
                Console.WriteLine($"Detected Language: {result.DetectedLanguages[0]}");
            }
            else
            {
                Console.WriteLine("No language detected");
            }

            Console.WriteLine($"Content length: {result.Content.Length} characters");
        }
        catch (XbergException ex)
        {
            Console.WriteLine($"Extraction failed: {ex.Message}");
        }
    }
}

require 'xberg'

config = Xberg::ExtractionConfig.new(
  language_detection: Xberg::LanguageDetectionConfig.new(
    enabled: true,
    min_confidence: 0.8,
    detect_multiple: false
  )
)

Multilingual Example

import asyncio
from xberg import ExtractInput, extract, ExtractionConfig, LanguageDetectionConfig

async def main() -> None:
    config: ExtractionConfig = ExtractionConfig(
        language_detection=LanguageDetectionConfig(
            enabled=True,
            min_confidence=0.7,
            detect_multiple=True
        )
    )
    result = await extract(ExtractInput.from_uri("multilingual_document.pdf"), config)
    languages: list[str] = result.detected_languages or []
    print(f"Detected {len(languages)} languages: {languages}")

asyncio.run(main())

import { extract } from "@xberg-io/xberg";

const config = {
  languageDetection: {
    enabled: true,
    minConfidence: 0.8,
    detectMultiple: true,
  },
};

const result = await extract({ kind: "uri", uri: "multilingual_document.pdf" }, config);
if (result.detectedLanguages) {
  console.log(`Detected languages: ${result.detectedLanguages.join(", ")}`);
}

use xberg::{extract, ExtractionConfig, ExtractInput, LanguageDetectionConfig};

let config = ExtractionConfig {
    language_detection: Some(LanguageDetectionConfig {
        enabled: true,
        min_confidence: 0.8,
        detect_multiple: true,
    }),
    ..Default::default()
};

let output = extract(ExtractInput::from_uri("multilingual_document.pdf"), &config).await?;

println!("Detected languages: {:?}", output.results[0].detected_languages);

package main

import (
  "fmt"
  "log"
  "strings"

  "github.com/xberg-io/xberg/packages/go"
)

func main() {
  enabled := true
  detectMultiple := true
  minConfidence := 0.8

  cfg := xberg.ExtractionConfig{
    LanguageDetection: &xberg.LanguageDetectionConfig{
      Enabled:        &enabled,
      MinConfidence:  &minConfidence,
      DetectMultiple: &detectMultiple,
    },
  }

  input := xberg.ExtractInputFromURI("multilingual_document.pdf")
  result, err := xberg.Extract(*input, cfg)
  if err != nil {
    log.Fatalf("Processing failed: %v", err)
  }

  languages := result.Results[0].DetectedLanguages
  if len(languages) > 0 {
    fmt.Printf("Detected %d language(s): %s\n", len(languages), strings.Join(languages, ", "))
  } else {
    fmt.Println("No languages detected")
  }

  fmt.Printf("Total content: %d characters\n", len(result.Results[0].Content))
  fmt.Printf("MIME type: %s\n", result.Results[0].MimeType)
}

import io.xberg.Xberg;
import io.xberg.ExtractInputKind;
import io.xberg.ExtractionResult;
import io.xberg.ExtractedDocument;
import io.xberg.ExtractionConfig;
import io.xberg.ExtractInput;
import io.xberg.LanguageDetectionConfig;
import java.math.BigDecimal;
import java.util.List;

ExtractionConfig config = ExtractionConfig.builder()
    .languageDetection(LanguageDetectionConfig.builder()
        .enabled(true)
        .minConfidence(new BigDecimal("0.8"))
        .detectMultiple(true)
        .build())
    .build();
try {
    ExtractionResult output = Xberg.extract(
        ExtractInput.builder().withKind(ExtractInputKind.Uri).withUri("multilingual_document.pdf").build(),
        config
    );
    ExtractedDocument result = output.results().get(0);
    List<String> languages = result.detectedLanguages() != null
        ? result.detectedLanguages()
        : List.of();
    if (!languages.isEmpty()) {
        System.out.println("Detected " + languages.size() + " language(s): " + String.join(", ", languages));
    } else {
        System.out.println("No languages detected");
    }
    System.out.println("Total content: " + result.content().length() + " characters");
    System.out.println("MIME type: " + result.mimeType());
} catch (Exception ex) {
    System.err.println("Processing failed: " + ex.getMessage());
}

using Xberg;

class Program
{
    static async Task Main()
    {
        var config = new ExtractionConfig
        {
            LanguageDetection = new LanguageDetectionConfig
            {
                Enabled = true,
                MinConfidence = 0.8m,
                DetectMultiple = true
            }
        };

        try
        {
            var result = (await XbergConverter.ExtractAsync(ExtractInput.FromUri("multilingual_document.pdf"), config)).Results[0];

            var languages = result.DetectedLanguages ?? new List<string>();

            if (languages.Count > 0)
            {
                Console.WriteLine($"Detected {languages.Count} language(s): {string.Join(", ", languages)}");
            }
            else
            {
                Console.WriteLine("No languages detected");
            }

            Console.WriteLine($"Total content: {result.Content.Length} characters");
            Console.WriteLine($"MIME type: {result.MimeType}");
        }
        catch (XbergException ex)
        {
            Console.WriteLine($"Processing failed: {ex.Message}");
        }
    }
}

require 'xberg'

config = Xberg::ExtractionConfig.new(
  language_detection: Xberg::LanguageDetectionConfig.new(
    enabled: true,
    min_confidence: 0.8,
    detect_multiple: true
  )
)

input = Xberg::ExtractInput.new(uri: 'multilingual_document.pdf')
result = Xberg.extract(input, config)
first_result = result.results.first

languages = first_result.detected_languages || []

if languages.any?
  puts "Detected #{languages.length} language(s): #{languages.join(', ')}"
else
  puts "No languages detected"
end

puts "Total content: #{first_result.content.length} characters"
puts "MIME type: #{first_result.mime_type}"

Language Detection

Configuration

Multilingual Example

See also