Token Reduction

Reduce token count while preserving meaning for LLM pipelines.

Set the reduction mode on token_reduction (a TokenReductionOptions).

Mode	Effect
off	No reduction; text returned as-is.
light	Remove only the most common stopwords.
moderate	Balanced stopword removal and redundancy filtering.
aggressive	Aggressive filtering; may remove less common content words.
maximum	Maximum compression; prioritizes brevity over completeness.

Configuration

Python

Python

from xberg import ExtractionConfig, TokenReductionOptions

config: ExtractionConfig = ExtractionConfig(
    token_reduction=TokenReductionOptions(
        mode="moderate",
        preserve_important_words=True,
    )
)

TypeScript

TypeScript

import { extract } from "@xberg-io/xberg";

const config = {
  tokenReduction: {
    level: "Moderate",
    preserveImportantWords: true,
  },
};

const output = await extract({ kind: "uri", uri: "document.pdf" }, config);
console.log(output.results[0].content);

Rust

Rust

use xberg::{ExtractionConfig, TokenReductionOptions};

let config = ExtractionConfig {
    token_reduction: Some(TokenReductionOptions {
        mode: "moderate".to_string(),
        preserve_important_words: true,
    }),
    ..Default::default()
};

Go

Go

package main

import (
  "fmt"

  "github.com/xberg-io/xberg/packages/go"
)

func main() {
  preserveImportant := true
  config := xberg.ExtractionConfig{
    TokenReduction: &xberg.TokenReductionOptions{
      Mode:                   "moderate",
      PreserveImportantWords: &preserveImportant,
    },
  }

  fmt.Printf("Mode: %s, Preserve Important Words: %v\n",
    config.TokenReduction.Mode,
    *config.TokenReduction.PreserveImportantWords)
}

Java

Java

import io.xberg.ExtractionConfig;
import io.xberg.TokenReductionOptions;

ExtractionConfig config = ExtractionConfig.builder()
    .withTokenReduction(TokenReductionOptions.builder()
        .withMode("moderate")
        .withPreserveImportantWords(true)
        .build())
    .build();

C#

C#

using Xberg;

var config = new ExtractionConfig
{
    TokenReduction = new TokenReductionOptions
    {
        Mode = "moderate",              // "off", "light", "moderate", "aggressive", or "maximum"
        PreserveImportantWords = true
    }
};

Ruby

Ruby

require 'xberg'

config = Xberg::ExtractionConfig.new(
  token_reduction: Xberg::TokenReductionConfig.new(
    mode: 'moderate',
    preserve_markdown: true,
    preserve_code: true,
    language_hint: 'eng'
  )
)

Example

Python

Python

import asyncio
from xberg import ExtractInput, extract, ExtractionConfig, TokenReductionOptions

async def main() -> None:
    config: ExtractionConfig = ExtractionConfig(
        token_reduction=TokenReductionOptions(
            mode="moderate", preserve_important_words=True
        )
    )
    result = await extract(ExtractInput(uri="verbose_document.pdf"), config)
    print(f"Reduced content length: {len(result.results[0].content)} chars")

asyncio.run(main())

TypeScript

TypeScript

import { extract } from "@xberg-io/xberg";

const config = {
  tokenReduction: {
    level: "Moderate",
    preserveImportantWords: true,
  },
};

const output = await extract({ kind: "uri", uri: "verbose_document.pdf" }, config);
const result = output.results[0];
console.log(`Content length: ${result.content.length}`);
console.log(`Metadata: ${JSON.stringify(result.metadata)}`);

Rust

Rust

use xberg::{extract, ExtractionConfig, ExtractInput, TokenReductionOptions};

#[tokio::main]
async fn main() -> xberg::Result<()> {
    let config = ExtractionConfig {
        token_reduction: Some(TokenReductionOptions {
            mode: "moderate".to_string(),
            preserve_important_words: true,
        }),
        ..Default::default()
    };

    let output = extract(ExtractInput::from_uri("verbose_document.pdf"), &config).await?;
    let result = &output.results[0];

    println!("Reduced content length: {} chars", result.content.len());
    Ok(())
}

Go

Go

package main

import (
  "fmt"
  "log"

  "github.com/xberg-io/xberg/packages/go"
)

func main() {
  preserveMarkdown := true
  mode := "moderate"

  cfg := xberg.ExtractionConfig{
    TokenReduction: &xberg.TokenReductionConfig{
      Mode:             &mode,
      PreserveMarkdown: &preserveMarkdown,
    },
  }

  input := xberg.ExtractInputFromURI("verbose_document.pdf")
  result, err := xberg.Extract(*input, cfg)
  if err != nil {
    log.Fatalf("extraction failed: %v", err)
  }

  original := 0
  reduced := 0
  ratio := 0.0

  if val, ok := result.Results[0].Metadata["original_token_count"]; ok {
    original = val.(int)
  }

  if val, ok := result.Results[0].Metadata["token_count"]; ok {
    reduced = val.(int)
  }

  if val, ok := result.Results[0].Metadata["token_reduction_ratio"]; ok {
    ratio = val.(float64)
  }

  fmt.Printf("Reduced from %d to %d tokens\n", original, reduced)
  fmt.Printf("Reduction: %.1f%%\n", ratio*100)
}

Java

Java

import io.xberg.Xberg;
import io.xberg.ExtractInputKind;
import io.xberg.ExtractionResult;
import io.xberg.ExtractedDocument;
import io.xberg.ExtractionConfig;
import io.xberg.ExtractInput;
import io.xberg.TokenReductionOptions;
import java.util.Map;

ExtractionConfig config = ExtractionConfig.builder()
    .withTokenReduction(TokenReductionOptions.builder()
        .withMode("moderate")
        .withPreserveImportantWords(true)
        .build())
    .build();
ExtractionResult output = Xberg.extract(
    ExtractInput.builder().withKind(ExtractInputKind.Uri).withUri("verbose_document.pdf").build(),
    config
);
ExtractedDocument result = output.results().get(0);
Map<String, Object> metadata = result.metadata() != null && result.metadata().additional() != null
    ? result.metadata().additional()
    : Map.of();
int original = metadata.containsKey("original_token_count")
    ? ((Number) metadata.get("original_token_count")).intValue()
    : 0;
int reduced = metadata.containsKey("token_count")
    ? ((Number) metadata.get("token_count")).intValue()
    : 0;
double ratio = metadata.containsKey("token_reduction_ratio")
    ? ((Number) metadata.get("token_reduction_ratio")).doubleValue()
    : 0.0;
System.out.println("Reduced from " + original + " to " + reduced + " tokens");
System.out.println(String.format("Reduction: %.1f%%", ratio * 100));

C#

C#

using Xberg;

var config = new ExtractionConfig
{
    TokenReduction = new TokenReductionOptions
    {
        Mode = "moderate",
        PreserveImportantWords = true
    }
};

var result = (await XbergConverter.ExtractAsync(ExtractInput.FromUri(
    "verbose_document.pdf"), config
)).Results[0];

Console.WriteLine($"Reduced content length: {result.Content.Length} chars");

Ruby

Ruby

require 'xberg'

config = Xberg::ExtractionConfig.new(
  token_reduction: Xberg::TokenReductionConfig.new(
    mode: 'moderate',
    preserve_markdown: true
  )
)

input = Xberg::ExtractInput.new(uri: 'verbose_document.pdf')
result = Xberg.extract(input, config)
first_result = result.results.first

original_tokens = first_result.metadata&.dig('original_token_count') || 0
reduced_tokens = first_result.metadata&.dig('token_count') || 0
reduction_ratio = first_result.metadata&.dig('token_reduction_ratio') || 0.0

puts "Reduced from #{original_tokens} to #{reduced_tokens} tokens"
puts "Reduction: #{(reduction_ratio * 100).round(1)}%"

See also

• Configuration Reference — all reduction options
• LLM Integration — use token reduction with LLM pipelines