Embeddings

Turn extracted text into vectors for semantic search and RAG, using local ONNX models or a registered backend — no external API calls. Enable the embeddings feature to use in-process embedding backends.

Preset	Model	Dimensions	Chunk Size (chars)	Use Case
`fast`	all-MiniLM-L6-v2 (quantized)	384	512	Quick prototyping, development, resource-constrained
`balanced`	BGE-base-en-v1.5	768	1024	General-purpose RAG, production deployments, English
`quality`	BGE-large-en-v1.5	1024	2000	Complex documents, maximum accuracy, sufficient compute
`multilingual`	multilingual-e5-base	768	1024	International documents, mixed-language content

The chunk-size column is the preset’s target chunk size in characters (its chunk_size), not a token limit.

Model Types

Select the embedding source via the model field of EmbeddingConfig. EmbeddingModelType has four variants, tagged by type:

preset — a bundled ONNX model configuration by name (see the preset table above). Recommended default.
custom — any ONNX embedding model from HuggingFace, given its model_id and output dimensions.
llm — a provider-hosted embedding model through liter-llm, configured by a nested llm (LlmConfig), e.g. openai/text-embedding-3-small.
plugin — an in-process backend registered by name via the plugin system (see below).

In-Process Embedding Backends (Plugin Variant)

Plug a caller-managed embedder (e.g. llama-cpp-python, sentence-transformers) into Xberg via the Plugin variant of EmbeddingModelType — Xberg calls back into the registered backend instead of running its own ONNX model.

Register the backend once at startup via xberg::plugins::register_embedding_backend(Arc::new(MyEmbedder)). The backend implements EmbeddingBackend (a Plugin-inheriting async trait with dimensions() and embed(texts) -> Vec<Vec<f32>>).
Reference it by name in EmbeddingConfig: { "model": { "type": "plugin", "name": "my-embedder" } }.
Optional: set EmbeddingConfig.max_embed_duration_secs (default 60) to bound the wait on a hung backend; None disables the timeout.

Rust extraction configuration and XBERG_EMBEDDING_PLUGIN_NAME accept the Plugin variant once a backend is registered.

Fork-safety: Python callers running under multiprocessing, gunicorn’s prefork worker, or Celery prefork must re-register the backend in each child process — native-backed embedders (including llama-cpp-python) aren’t fork-safe. Use os.register_at_fork(after_in_child=reregister_fn) to automate the re-registration.

Configuration

from xberg import (
    ExtractionConfig,
    ChunkingConfig,
    EmbeddingConfig,
    EmbeddingModelType,
)

config: ExtractionConfig = ExtractionConfig(
    chunking=ChunkingConfig(
        max_chars=1024,
        max_overlap=100,
        embedding=EmbeddingConfig(
            model=EmbeddingModelType.preset("balanced"),
            normalize=True,
            batch_size=32,
            show_download_progress=False,
        ),
    )
)

import { extract } from "@xberg-io/xberg";

const config = {
  chunking: {
    maxChars: 1024,
    maxOverlap: 100,
    embedding: {
      preset: "balanced",
    },
  },
};

const result = await extract({ kind: "uri", uri: "document.pdf" }, config);
console.log(`Chunks: ${result.chunks?.length ?? 0}`);

use xberg::{ExtractionConfig, ChunkingConfig, EmbeddingConfig};

let config = ExtractionConfig {
    chunking: Some(ChunkingConfig {
        max_characters: 1024,
        overlap: 100,
        embedding: Some(EmbeddingConfig {
            model: "balanced".to_string(),
            normalize: true,
            batch_size: 32,
            show_download_progress: false,
            ..Default::default()
        }),
        ..Default::default()
    }),
    ..Default::default()
};

package main

import (
  "fmt"

  "github.com/xberg-io/xberg/packages/go"
)

func main() {
  maxChars := 512
  maxOverlap := 50
  normalize := true
  batchSize := int32(32)
  showProgress := false

  cfg := xberg.ExtractionConfig{
    Chunking: &xberg.ChunkingConfig{
      MaxChars:   &maxChars,
      MaxOverlap: &maxOverlap,
      Embedding: &xberg.EmbeddingConfig{
        Model:                xberg.EmbeddingModelType_Preset("balanced"),
        Normalize:            &normalize,
        BatchSize:            &batchSize,
        ShowDownloadProgress: &showProgress,
      },
    },
  }

  input := xberg.ExtractInputFromURI("document.pdf")
  result, err := xberg.Extract(*input, cfg)
  if err != nil {
    fmt.Printf("Error: %v\n", err)
    return
  }

  for index, chunk := range result.Results[0].Chunks {
    chunkID := fmt.Sprintf("doc_chunk_%d", index)
    content := chunk.Content
    if len(content) > 50 {
      content = content[:50]
    }
    fmt.Printf("Chunk %s: %s\n", chunkID, content)

    if chunk.Embedding != nil && len(chunk.Embedding) > 0 {
      fmt.Printf("  Embedding dimensions: %d\n", len(chunk.Embedding))
    }
  }
}

import io.xberg.Xberg;
import io.xberg.ExtractInputKind;
import io.xberg.ExtractionResult;
import io.xberg.ExtractedDocument;
import io.xberg.ExtractionConfig;
import io.xberg.ExtractInput;
import io.xberg.ChunkingConfig;
import io.xberg.EmbeddingConfig;
import io.xberg.EmbeddingModelType;
import java.util.List;

ExtractionConfig config = ExtractionConfig.builder()
    .chunking(ChunkingConfig.builder()
        .maxChars(512)
        .maxOverlap(50)
        .embedding(EmbeddingConfig.builder()
            .model(EmbeddingModelType.preset("balanced"))
            .normalize(true)
            .batchSize(32)
            .showDownloadProgress(false)
            .build())
        .build())
    .build();
ExtractionResult output = Xberg.extract(
    ExtractInput.builder().withKind(ExtractInputKind.Uri).withUri("document.pdf").build(),
    config
);
ExtractedDocument result = output.results().get(0);
List<Object> chunks = result.chunks() != null ? result.chunks() : List.of();
for (int index = 0; index < chunks.size(); index++) {
    Object chunk = chunks.get(index);
    String chunkId = "doc_chunk_" + index;
    System.out.println("Chunk " + chunkId + ": " + chunk.toString().substring(0, Math.min(50, chunk.toString().length())));
    if (chunk instanceof java.util.Map) {
        Object embedding = ((java.util.Map<String, Object>) chunk).get("embedding");
        if (embedding != null) {
            System.out.println("  Embedding dimensions: " + ((float[]) embedding).length);
        }
    }
}

using Xberg;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;

var config = new ExtractionConfig
{
    Chunking = new ChunkingConfig
    {
        MaxChars = 512,
        MaxOverlap = 50,
        Embedding = new EmbeddingConfig
        {
            Model = EmbeddingModelType.Preset("balanced"),
            Normalize = true,
            BatchSize = 32,
            ShowDownloadProgress = false
        }
    }
};

var result = await Xberg.ExtractAsync("document.pdf", config);

var chunks = result.Chunks ?? new List<Chunk>();
foreach (var (index, chunk) in chunks.WithIndex())
{
    var chunkId = $"doc_chunk_{index}";
    Console.WriteLine($"Chunk {chunkId}: {chunk.Content[..Math.Min(50, chunk.Content.Length)]}");

    if (chunk.Embedding != null)
    {
        Console.WriteLine($"  Embedding dimensions: {chunk.Embedding.Length}");
    }
}

internal static class EnumerableExtensions
{
    public static IEnumerable<(int Index, T Item)> WithIndex<T>(
        this IEnumerable<T> items)
    {
        var index = 0;
        foreach (var item in items)
        {
            yield return (index++, item);
        }
    }
}

require 'xberg'

config = Xberg::ExtractionConfig.new(
  chunking: Xberg::ChunkingConfig.new(
    max_characters: 512,
    overlap: 50,
    embedding: Xberg::EmbeddingConfig.new(
      model: Xberg::EmbeddingModelType.new(
        type: 'preset',
        name: 'balanced'
      ),
      normalize: true,
      batch_size: 32,
      show_download_progress: false
    )
  )
)

input = Xberg::ExtractInput.new(uri: 'document.pdf')
result = Xberg.extract(input, config)

chunks = result.results.first.chunks || []
chunks.each_with_index do |chunk, idx|
  chunk_id = "doc_chunk_#{idx}"
  puts "Chunk #{chunk_id}: #{chunk.content[0...50]}"

  if chunk.embedding
    puts "  Embedding dimensions: #{chunk.embedding.length}"
  end
end

Standalone Embedding

Call embed_texts (or embed_texts_async) to embed a list of strings directly with an EmbeddingConfig, bypassing extraction and chunking. Each input string maps to one output vector.

from xberg import embed_sync, embed, EmbeddingConfig, EmbeddingModelType

# Synchronous
embeddings = embed_sync(
    ["Hello, world!", "Xberg is fast"],
    config=EmbeddingConfig(model=EmbeddingModelType.preset("balanced"), normalize=True),
)
assert len(embeddings) == 2
assert len(embeddings[0]) == 768

# Asynchronous
async def main():
    embeddings = await embed(
        ["Hello, world!", "Xberg is fast"],
        config=EmbeddingConfig(model=EmbeddingModelType.preset("balanced"), normalize=True),
    )
    assert len(embeddings) == 2

import { embed, embedSync } from "@xberg-io/xberg";
import type { EmbeddingConfig } from "@xberg-io/xberg";

const config: EmbeddingConfig = {
  model: { type: "preset", name: "balanced" },
  normalize: true,
};

// Synchronous
const embeddings = embedSync(["Hello, world!", "Xberg is fast"], config);
console.log(embeddings.length); // 2
console.log(embeddings[0].length); // 768

// Asynchronous (preferred)
const asyncEmbeddings = await embed(["Hello, world!", "Xberg is fast"], config);
console.log(asyncEmbeddings[0].length); // 768

use xberg::{EmbeddingConfig, EmbeddingModelType, embed_texts};

let config = EmbeddingConfig {
    model: EmbeddingModelType::Preset { name: "balanced".to_string() },
    normalize: true,
    ..Default::default()
};

let texts = vec!["Hello, world!", "Xberg is fast"];
let embeddings = embed_texts(&texts, &config)?;

assert_eq!(embeddings.len(), 2);
assert_eq!(embeddings[0].len(), 768);

package main

import (
  "fmt"

  "github.com/xberg-io/xberg/packages/go"
)

func main() {
  preset := "balanced"
  normalize := true
  config := xberg.EmbeddingConfig{
    Model: xberg.EmbeddingModelType{
      Type: "preset",
      Name: &preset,
    },
    Normalize: &normalize,
  }

  // Synchronous
  embeddings, err := xberg.EmbedTexts([]string{"Hello, world!", "Xberg is fast"}, config)
  if err != nil {
    panic(err)
  }
  fmt.Println(len(embeddings))    // 2
  fmt.Println(len(embeddings[0])) // 768

  // Asynchronous
  embeddings, err = xberg.EmbedTextsAsync([]string{"Hello, world!"}, config)
  if err != nil {
    panic(err)
  }
  fmt.Println(len(embeddings[0])) // 768
}

import io.xberg.Xberg;
import io.xberg.EmbeddingConfig;

// Embed with default config
float[][] embeddings = Xberg.embed(List.of("Hello world", "How are you?"), null);

// Embed with specific preset
EmbeddingConfig config = EmbeddingConfig.withPreset("fast");
float[][] fastEmbeddings = Xberg.embed(List.of("Hello world"), config);

// Async variant
CompletableFuture<float[][]> future = Xberg.embedAsync(texts, null);

using Xberg;

var client = new XbergLib();

var config = new EmbeddingConfig { Model = EmbeddingModelType.Preset("balanced"), Normalize = true };
var texts = new[] { "Hello, world!", "Xberg is fast" };

// Synchronous
var embeddings = client.EmbedSync(texts, config).ToList();
Console.WriteLine(embeddings.Count);       // 2
Console.WriteLine(embeddings[0].Length);   // 768

// Asynchronous
var asyncEmbeddings = await client.EmbedAsync(texts, config);
Console.WriteLine(asyncEmbeddings.First().Length); // 768

require "xberg"

config = { model: { type: "preset", name: "balanced" }, normalize: true }
texts = ["Hello, world!", "Xberg is fast"]

# Synchronous
embeddings = Xberg.embed_sync(texts: texts, config: config)
puts embeddings.length    # 2
puts embeddings[0].length # 768

# Async variant (uses same thread, returns when done)
embeddings = Xberg.embed(texts: texts, config: config)
puts embeddings[0].length # 768

Vector Database Integration

import asyncio
from xberg import (
    ExtractInput,
    extract,
    ExtractionConfig,
    ChunkingConfig,
    EmbeddingConfig,
    EmbeddingModelType,
)

async def main() -> None:
    config: ExtractionConfig = ExtractionConfig(
        chunking=ChunkingConfig(
            max_chars=512,
            max_overlap=50,
            embedding=EmbeddingConfig(
                model=EmbeddingModelType.preset("balanced"), normalize=True
            ),
        )
    )
    result = await extract(ExtractInput.from_uri("document.pdf"), config)
    chunks = result.chunks or []
    for i, chunk in enumerate(chunks):
        chunk_id: str = f"doc_chunk_{i}"
        print(f"Chunk {chunk_id}: {chunk.content[:50]}")

asyncio.run(main())

import { extract } from "@xberg-io/xberg";

const config = {
  chunking: {
    maxChars: 512,
    maxOverlap: 50,
    embedding: {
      preset: "balanced",
    },
  },
};

const result = await extract({ kind: "uri", uri: "document.pdf" }, config);

if (result.chunks) {
  for (const chunk of result.chunks) {
    console.log(`Chunk: ${chunk.content.slice(0, 100)}...`);
    if (chunk.embedding) {
      console.log(`Embedding dims: ${chunk.embedding.length}`);
    }
  }
}

use xberg::{extract, ExtractionConfig, ExtractInput, ChunkingConfig, EmbeddingConfig};

struct VectorRecord {
    id: String,
    content: String,
    embedding: Vec<f32>,
    metadata: std::collections::HashMap<String, String>,
}

async fn extract_and_vectorize(
    document_path: &str,
    document_id: &str,
) -> Result<Vec<VectorRecord>, Box<dyn std::error::Error>> {
    let config = ExtractionConfig {
        chunking: Some(ChunkingConfig {
            max_characters: 512,
            overlap: 50,
            embedding: Some(EmbeddingConfig {
                model: xberg::EmbeddingModelType::Preset {
                    name: "balanced".to_string(),
                },
                normalize: true,
                batch_size: 32,
                ..Default::default()
            }),
            ..Default::default()
        }),
        ..Default::default()
    };

    let output = extract(ExtractInput::from_uri(document_path), &config).await?;
    let result = &output.results[0];

    let mut records = Vec::new();
    if let Some(chunks) = &result.chunks {
        for (index, chunk) in chunks.iter().enumerate() {
            if let Some(embedding) = &chunk.embedding {
                let mut metadata = std::collections::HashMap::new();
                metadata.insert("document_id".to_string(), document_id.to_string());
                metadata.insert("chunk_index".to_string(), index.to_string());
                metadata.insert("content_length".to_string(), chunk.content.len().to_string());

                records.push(VectorRecord {
                    id: format!("{}_chunk_{}", document_id, index),
                    content: chunk.content.clone(),
                    embedding: embedding.clone(),
                    metadata,
                });
            }
        }
    }

    Ok(records)
}

package main

import (
  "fmt"

  "github.com/xberg-io/xberg/packages/go"
)

type VectorRecord struct {
  ID        string
  Embedding []float32
  Content   string
  Metadata  map[string]string
}

func extractAndVectorize(documentPath string, documentID string) ([]VectorRecord, error) {
  maxChars := 512
  maxOverlap := 50
  normalize := true
  batchSize := int32(32)

  cfg := xberg.ExtractionConfig{
    Chunking: &xberg.ChunkingConfig{
      MaxChars:   &maxChars,
      MaxOverlap: &maxOverlap,
      Embedding: &xberg.EmbeddingConfig{
        Model:     xberg.EmbeddingModelType_Preset("balanced"),
        Normalize: &normalize,
        BatchSize: &batchSize,
      },
    },
  }

  input := xberg.ExtractInputFromURI(documentPath)
  result, err := xberg.Extract(*input, cfg)
  if err != nil {
    return nil, err
  }

  var vectorRecords []VectorRecord
  for index, chunk := range result.Results[0].Chunks {
    record := VectorRecord{
      ID:        fmt.Sprintf("%s_chunk_%d", documentID, index),
      Content:   chunk.Content,
      Embedding: chunk.Embedding,
      Metadata: map[string]string{
        "document_id":  documentID,
        "chunk_index":  fmt.Sprintf("%d", index),
        "content_length": fmt.Sprintf("%d", len(chunk.Content)),
      },
    }
    vectorRecords = append(vectorRecords, record)
  }

  storeInVectorDatabase(vectorRecords)
  return vectorRecords, nil
}

func storeInVectorDatabase(records []VectorRecord) {
  for _, record := range records {
    if len(record.Embedding) > 0 {
      fmt.Printf("Storing %s: %d chars, %d dims\n",
        record.ID, len(record.Content), len(record.Embedding))
    }
  }
}

import io.xberg.Xberg;
import io.xberg.ExtractInputKind;
import io.xberg.ExtractInput;
import io.xberg.ExtractionResult;
import io.xberg.ExtractedDocument;
import io.xberg.ExtractionConfig;
import io.xberg.ChunkingConfig;
import io.xberg.EmbeddingConfig;
import io.xberg.EmbeddingModelType;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class VectorDatabaseIntegration {
    public static class VectorRecord {
        public String id;
        public float[] embedding;
        public String content;
        public Map<String, String> metadata;
    }
    public static List<VectorRecord> extractAndVectorize(String documentPath, String documentId) throws Exception {
        ExtractionConfig config = ExtractionConfig.builder()
            .chunking(ChunkingConfig.builder()
                .maxChars(512)
                .maxOverlap(50)
                .embedding(EmbeddingConfig.builder()
                    .model(EmbeddingModelType.preset("balanced"))
                    .normalize(true)
                    .batchSize(32)
                    .build())
                .build())
            .build();
        ExtractionResult output = Xberg.extract(ExtractInput.builder().withKind(ExtractInputKind.Uri).withUri(documentPath).build(), config);
        ExtractedDocument result = output.results().get(0);
        List<Object> chunks = result.chunks() != null ? result.chunks() : List.of();
        List<VectorRecord> vectorRecords = new java.util.ArrayList<>();
        for (int index = 0; index < chunks.size(); index++) {
            Object chunk = chunks.get(index);
            VectorRecord record = new VectorRecord();
            record.id = documentId + "_chunk_" + index;
            record.metadata = new HashMap<>();
            record.metadata.put("document_id", documentId);
            record.metadata.put("chunk_index", String.valueOf(index));
            if (chunk instanceof java.util.Map) {
                Map<String, Object> chunkMap = (Map<String, Object>) chunk;
                record.content = (String) chunkMap.get("content");
                record.embedding = (float[]) chunkMap.get("embedding");
                record.metadata.put("content_length", String.valueOf(record.content.length()));
            }
            vectorRecords.add(record);
        }
        storeInVectorDatabase(vectorRecords);
        return vectorRecords;
    }

    private static void storeInVectorDatabase(List<VectorRecord> records) {
        for (VectorRecord record : records) {
            if (record.embedding != null && record.embedding.length > 0) {
                System.out.println("Storing " + record.id + ": " + record.content.length()
                    + " chars, " + record.embedding.length + " dims");
            }
        }
    }
}

using Xberg;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;

public class VectorDatabaseIntegration
{
    public class VectorRecord
    {
        public string Id { get; set; }
        public float[] Embedding { get; set; }
        public string Content { get; set; }
        public Dictionary<string, string> Metadata { get; set; }
    }

    public async Task<List<VectorRecord>> ExtractAndVectorize(
        string documentPath,
        string documentId)
    {
        var config = new ExtractionConfig
        {
            Chunking = new ChunkingConfig
            {
                MaxChars = 512,
                MaxOverlap = 50,
                Embedding = new EmbeddingConfig
                {
                    Model = EmbeddingModelType.Preset("balanced"),
                    Normalize = true,
                    BatchSize = 32
                }
            }
        };

        var result = await Xberg.ExtractAsync(documentPath, config);
        var chunks = result.Chunks ?? new List<Chunk>();

        var vectorRecords = chunks
            .Select((chunk, index) => new VectorRecord
            {
                Id = $"{documentId}_chunk_{index}",
                Content = chunk.Content,
                Embedding = chunk.Embedding,
                Metadata = new Dictionary<string, string>
                {
                    { "document_id", documentId },
                    { "chunk_index", index.ToString() },
                    { "content_length", chunk.Content.Length.ToString() }
                }
            })
            .ToList();

        await StoreInVectorDatabase(vectorRecords);
        return vectorRecords;
    }

    private async Task StoreInVectorDatabase(List<VectorRecord> records)
    {
        foreach (var record in records)
        {
            if (record.Embedding != null && record.Embedding.Length > 0)
            {
                Console.WriteLine(
                    $"Storing {record.Id}: {record.Content.Length} chars, " +
                    $"{record.Embedding.Length} dims");
            }
        }

        await Task.CompletedTask;
    }
}

require 'xberg'

class VectorDatabaseIntegration
  VectorRecord = Struct.new(:id, :embedding, :content, :metadata, keyword_init: true)

  def extract_and_vectorize(document_path, document_id)
    config = Xberg::ExtractionConfig.new(
      chunking: Xberg::ChunkingConfig.new(
        max_characters: 512,
        overlap: 50,
        embedding: Xberg::EmbeddingConfig.new(
          model: Xberg::EmbeddingModelType.new(
            type: 'preset',
            name: 'balanced'
          ),
          normalize: true,
          batch_size: 32
        )
      )
    )

    output = Xberg.extract(Xberg::ExtractInput.new(kind: "uri", uri: document_path), config)
result = output.results.first
    chunks = result.chunks || []

    vector_records = chunks.map.with_index do |chunk, idx|
      VectorRecord.new(
        id: "#{document_id}_chunk_#{idx}",
        content: chunk.content,
        embedding: chunk.embedding,
        metadata: {
          document_id: document_id,
          chunk_index: idx,
          content_length: chunk.content.length
        }
      )
    end

    store_in_vector_database(vector_records)
    vector_records
  end

  private

  def store_in_vector_database(records)
    records.each do |record|
      if record.embedding&.any?
        puts "Storing #{record.id}: #{record.content.length} chars, #{record.embedding.length} dims"
      end
    end
  end
end