Token Reduction

Reduce token count while preserving meaning for LLM pipelines.

Set the reduction mode on token_reduction (a TokenReductionOptions).

Mode	Effect
off	No reduction; text returned as-is.
light	Remove only the most common stopwords.
moderate	Balanced stopword removal and redundancy filtering.
aggressive	Aggressive filtering; may remove less common content words.
maximum	Maximum compression; prioritizes brevity over completeness.

Configuration

Python

Python

from xberg import ExtractionConfig, TokenReductionConfig

config: ExtractionConfig = ExtractionConfig(
    token_reduction=TokenReductionConfig(
        mode="moderate",
        preserve_important_words=True,
    )
)

TypeScript

TypeScript

import { extract } from "@xberg-io/xberg";

const config = {
  tokenReduction: {
    mode: "moderate",
    preserveImportantWords: true,
  },
};

const result = await extract({ kind: "uri", uri: "document.pdf" }, config);
console.log(result.content);

Rust

Rust

use xberg::{ExtractionConfig, TokenReductionConfig};

let config = ExtractionConfig {
    token_reduction: Some(TokenReductionConfig {
        mode: "moderate".to_string(),
        preserve_markdown: true,
        preserve_code: true,
        language_hint: Some("eng".to_string()),
        ..Default::default()
    }),
    ..Default::default()
};

Go

Go

package main

import (
  "fmt"

  "github.com/xberg-io/xberg/packages/go"
)

func main() {
  preserveImportant := true
  config := xberg.ExtractionConfig{
    TokenReduction: &xberg.TokenReductionOptions{
      Mode:                   "moderate",
      PreserveImportantWords: &preserveImportant,
    },
  }

  fmt.Printf("Mode: %s, Preserve Important Words: %v\n",
    config.TokenReduction.Mode,
    *config.TokenReduction.PreserveImportantWords)
}

Java

Java

import io.xberg.ExtractionConfig;
import io.xberg.TokenReductionConfig;

ExtractionConfig config = ExtractionConfig.builder()
    .tokenReduction(TokenReductionConfig.builder()
        .mode("moderate")
        .preserveImportantWords(true)
        .build())
    .build();

C#

C#

using Xberg;

var config = new ExtractionConfig
{
    TokenReduction = new TokenReductionConfig
    {
        Mode = "moderate",              // "off", "moderate", or "aggressive"
        PreserveMarkdown = true,
        PreserveCode = true,
        LanguageHint = "eng"
    }
};

Ruby

Ruby

require 'xberg'

config = Xberg::ExtractionConfig.new(
  token_reduction: Xberg::TokenReductionConfig.new(
    mode: 'moderate',
    preserve_markdown: true,
    preserve_code: true,
    language_hint: 'eng'
  )
)

Example

Python

Python

import asyncio
from xberg import ExtractInput, extract, ExtractionConfig, TokenReductionConfig

async def main() -> None:
    config: ExtractionConfig = ExtractionConfig(
        token_reduction=TokenReductionConfig(
            mode="moderate", preserve_important_words=True
        )
    )
    result = await extract(ExtractInput.from_uri("verbose_document.pdf"), config)
    original: int = result.results[0].metadata.get("original_token_count", 0)
    reduced: int = result.results[0].metadata.get("token_count", 0)
    ratio: float = result.results[0].metadata.get("token_reduction_ratio", 0.0)
    print(f"Reduced from {original} to {reduced} tokens")
    print(f"Reduction: {ratio * 100:.1f}%")

asyncio.run(main())

TypeScript

TypeScript

import { extract } from "@xberg-io/xberg";

const config = {
  tokenReduction: {
    mode: "moderate",
    preserveImportantWords: true,
  },
};

const result = await extract({ kind: "uri", uri: "verbose_document.pdf" }, config);
console.log(`Content length: ${result.content.length}`);
console.log(`Metadata: ${JSON.stringify(result.metadata)}`);

Rust

Rust

use xberg::{extract, ExtractionConfig, ExtractInput, TokenReductionConfig};

let config = ExtractionConfig {
    token_reduction: Some(TokenReductionConfig {
        mode: "moderate".to_string(),
        preserve_markdown: true,
        ..Default::default()
    }),
    ..Default::default()
};

let output = extract(ExtractInput::from_uri("verbose_document.pdf"), &config).await?;
let result = &output.results[0];

if let Some(original) = result.original_token_count {
    println!("Original tokens: {}", original);
}
if let Some(reduced) = result.reduced_token_count {
    println!("Reduced tokens: {}", reduced);
}

Go

Go

package main

import (
  "fmt"
  "log"

  "github.com/xberg-io/xberg/packages/go"
)

func main() {
  preserveMarkdown := true
  mode := "moderate"

  cfg := xberg.ExtractionConfig{
    TokenReduction: &xberg.TokenReductionConfig{
      Mode:             &mode,
      PreserveMarkdown: &preserveMarkdown,
    },
  }

  input := xberg.ExtractInputFromURI("verbose_document.pdf")
  result, err := xberg.Extract(*input, cfg)
  if err != nil {
    log.Fatalf("extraction failed: %v", err)
  }

  original := 0
  reduced := 0
  ratio := 0.0

  if val, ok := result.Results[0].Metadata["original_token_count"]; ok {
    original = val.(int)
  }

  if val, ok := result.Results[0].Metadata["token_count"]; ok {
    reduced = val.(int)
  }

  if val, ok := result.Results[0].Metadata["token_reduction_ratio"]; ok {
    ratio = val.(float64)
  }

  fmt.Printf("Reduced from %d to %d tokens\n", original, reduced)
  fmt.Printf("Reduction: %.1f%%\n", ratio*100)
}

Java

Java

import io.xberg.Xberg;
import io.xberg.ExtractInputKind;
import io.xberg.ExtractionResult;
import io.xberg.ExtractedDocument;
import io.xberg.ExtractionConfig;
import io.xberg.ExtractInput;
import io.xberg.TokenReductionConfig;
import java.util.Map;

ExtractionConfig config = ExtractionConfig.builder()
    .tokenReduction(TokenReductionConfig.builder()
        .mode("moderate")
        .preserveMarkdown(true)
        .build())
    .build();
ExtractionResult output = Xberg.extract(
    ExtractInput.builder().withKind(ExtractInputKind.Uri).withUri("verbose_document.pdf").build(),
    config
);
ExtractedDocument result = output.results().get(0);
Map<String, Object> metadata = result.metadata() != null ? result.metadata() : Map.of();
int original = metadata.containsKey("original_token_count")
    ? ((Number) metadata.get("original_token_count")).intValue()
    : 0;
int reduced = metadata.containsKey("token_count")
    ? ((Number) metadata.get("token_count")).intValue()
    : 0;
double ratio = metadata.containsKey("token_reduction_ratio")
    ? ((Number) metadata.get("token_reduction_ratio")).doubleValue()
    : 0.0;
System.out.println("Reduced from " + original + " to " + reduced + " tokens");
System.out.println(String.format("Reduction: %.1f%%", ratio * 100));

C#

C#

using Xberg;

var config = new ExtractionConfig
{
    TokenReduction = new TokenReductionConfig
    {
        Mode = "moderate",
        PreserveMarkdown = true
    }
};

var result = (await XbergConverter.ExtractAsync(ExtractInput.FromUri(
    "verbose_document.pdf"), config
)).Results[0];

var original = result.Metadata.ContainsKey("original_token_count")
    ? (int)result.Metadata["original_token_count"]
    : 0;

var reduced = result.Metadata.ContainsKey("token_count")
    ? (int)result.Metadata["token_count"]
    : 0;

var ratio = result.Metadata.ContainsKey("token_reduction_ratio")
    ? (double)result.Metadata["token_reduction_ratio"]
    : 0.0;

Console.WriteLine($"Reduced from {original} to {reduced} tokens");
Console.WriteLine($"Reduction: {ratio * 100:F1}%");

Ruby

Ruby

require 'xberg'

config = Xberg::ExtractionConfig.new(
  token_reduction: Xberg::TokenReductionConfig.new(
    mode: 'moderate',
    preserve_markdown: true
  )
)

input = Xberg::ExtractInput.new(uri: 'verbose_document.pdf')
result = Xberg.extract(input, config)
first_result = result.results.first

original_tokens = first_result.metadata&.dig('original_token_count') || 0
reduced_tokens = first_result.metadata&.dig('token_count') || 0
reduction_ratio = first_result.metadata&.dig('token_reduction_ratio') || 0.0

puts "Reduced from #{original_tokens} to #{reduced_tokens} tokens"
puts "Reduction: #{(reduction_ratio * 100).round(1)}%"

See also

• Configuration Reference — all reduction options
• LLM Integration — use token reduction with LLM pipelines