Keyword Extraction

Extract ranked keywords and key phrases from document text for search indexing, topic detection, and content summarization. Choose between YAKE (best for single terms and multilingual content) or RAKE (best for multi-word phrases in technical documents).

Algorithm	Scoring	Best for
YAKE	Higher score = more relevant (0.0–1.0)	General documents, single terms, multilingual
RAKE	Higher score = more relevant (0.0–1.0)	Multi-word phrases, technical docs

Quick Start

Tests keyword extraction via YAKE algorithm

import asyncio
from xberg import extract, ExtractInput, ExtractInputKind
from xberg._xberg import ExtractionConfig

async def main() -> None:
    input = ExtractInput(kind=ExtractInputKind("uri"), uri="https://example.com/pdf/fake_memo.pdf")
    config = ExtractionConfig.from_json("{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}")
    result = await extract(input, config)
    for keyword in result.results[0].extracted_keywords or []:
        print(keyword.text)
        print(keyword.score)

asyncio.run(main())

Tests keyword extraction via YAKE algorithm

import { ExtractInput, ExtractInputKind, ExtractionConfig, KeywordAlgorithm, extract } from "@xberg-io/xberg";
async function main() {
  const input: ExtractInput = { kind: ExtractInputKind.Uri, uri: "https://example.com/pdf/fake_memo.pdf" };
  const config: ExtractionConfig = { keywords: { algorithm: KeywordAlgorithm.Yake, maxKeywords: 10 } };
  const result = await extract(input, config);
  const [first] = result.results ?? [];
  for (const keyword of first?.extractedKeywords ?? []) {
    console.log(keyword.text);
    console.log(keyword.score);
  }
}

void main();

Tests keyword extraction via YAKE algorithm

import { WasmExtractInput, WasmExtractInputKind, extract } from "@xberg-io/xberg-wasm";
async function main() {
  const input: WasmExtractInput = (() => { const _u0 = WasmExtractInput.default(); _u0.kind = WasmExtractInputKind.Uri; _u0.uri = "https://example.com/pdf/fake_memo.pdf"; return _u0; })();
  const result = await extract(input, { keywords: { algorithm: "yake", maxKeywords: 10 } });
  const [first] = result.results ?? [];
  for (const keyword of first?.extractedKeywords ?? []) {
    console.log(keyword.text);
    console.log(keyword.score);
  }
}

void main();

Tests keyword extraction via YAKE algorithm

use xberg::extract;
use xberg::ExtractInput;

#[tokio::main]
async fn main() {
    let input_json: serde_json::Value = serde_json::from_str(r#"{"kind":"uri","uri":"https://example.com/pdf/fake_memo.pdf"}"#).unwrap();
    let input = serde_json::from_value::<ExtractInput>(input_json).unwrap();
    let config_json: serde_json::Value = serde_json::from_str(r#"{"keywords":{"algorithm":"yake","max_keywords":10}}"#).unwrap();
    let config = serde_json::from_value(config_json).unwrap();
    let result = extract(input, &config).await.expect("call failed");
    for keyword in result.results[0].extracted_keywords.iter().flatten() {
        println!("{}", keyword.text);
        println!("{}", keyword.score);
    }
}

Tests keyword extraction via YAKE algorithm

package main

import (
  "fmt"
  xberg "github.com/xberg-io/xberg/packages/go"
)

func ptr[T any](value T) *T { return &value }
func main() {
  input := xberg.ExtractInput{
    Kind: ptr(xberg.ExtractInputKindURI),
    URI:  ptr(`https://example.com/pdf/fake_memo.pdf`),
  }
  config := xberg.ExtractionConfig{
    Keywords: &xberg.KeywordConfig{
    Algorithm:   xberg.KeywordAlgorithmYake,
    MaxKeywords: ptr(uint(10)),
  },
  }
  result, err := xberg.Extract(input, config)
  if err != nil {
    panic(err)
  }
  for _, keyword := range result.Results[0].ExtractedKeywords {
    fmt.Printf("%v\n", keyword.Text)
    fmt.Printf("%v\n", keyword.Score)
  }
}

Tests keyword extraction via YAKE algorithm

import io.xberg.*;

public final class Example {
    public static void main(String[] args) throws Exception {
        var inputJson = "{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}";
        var input = JsonUtil.fromJson(inputJson, ExtractInput.class);
        var configJson = "{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}";
        var config = JsonUtil.fromJson(configJson, ExtractionConfig.class);
        var result = Xberg.extract(input, config);
        for (var keyword : result.results().get(0).extractedKeywords()) {
            System.out.println(keyword.text());
            System.out.println(keyword.score());
        }
    }
}

Tests keyword extraction via YAKE algorithm

import io.xberg.*
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper

fun main() = kotlinx.coroutines.runBlocking {
    val mapper = jacksonObjectMapper().setPropertyNamingStrategy(com.fasterxml.jackson.databind.PropertyNamingStrategies.SNAKE_CASE)
    val input = mapper.readValue("{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}", ExtractInput::class.java)
    val config = mapper.readValue("{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10},\"url\":{\"crawl\":{\"ssrf\":{}}}}", ExtractionConfig::class.java)
    val result = Xberg.extract(input, config)
    for (keyword in result.results.first().extractedKeywords.orEmpty()) {
        println(keyword.text)
        println(keyword.score)
    }
}

Tests keyword extraction via YAKE algorithm

using System;
using System.Text.Json;
using Xberg;

var ConfigOptions = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
var result = await XbergConverter.ExtractAsync(new ExtractInput { Kind = JsonSerializer.Deserialize<ExtractInputKind>("\"uri\"", ConfigOptions)!, Uri = "https://example.com/pdf/fake_memo.pdf" }, new ExtractionConfig { Keywords = new KeywordConfig { Algorithm = JsonSerializer.Deserialize<KeywordAlgorithm>("\"yake\"", ConfigOptions)!, MaxKeywords = 10 } });
foreach (var keyword in result.Results[0].ExtractedKeywords!)
{
    Console.WriteLine(keyword.Text);
    Console.WriteLine(keyword.Score);
}

Tests keyword extraction via YAKE algorithm

import Xberg

let result = try await Xberg.extract("{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}", "{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}")
print(result)

Tests keyword extraction via YAKE algorithm

require "xberg"
result = Xberg.extract(Xberg::ExtractInput.new(kind: 'uri', uri: 'https://example.com/pdf/fake_memo.pdf'), { 'keywords' => { 'algorithm' => 'yake', 'max_keywords' => 10 } })
(result.results[0].extracted_keywords || []).each do |keyword|
  puts keyword.text
  puts keyword.score
end

Tests keyword extraction via YAKE algorithm

<?php

declare(strict_types=1);

require_once __DIR__ . '/vendor/autoload.php';

use Xberg\Xberg;
use Xberg\ExtractInput;
$input = \Xberg\ExtractInput::from_json(json_encode(["kind" => "uri", "uri" => "https://example.com/pdf/fake_memo.pdf"]));
$result = Xberg::extract($input, ["keywords" => ["algorithm" => "yake", "max_keywords" => 10]]);
foreach ($result->getResults()[0]->getExtractedKeywords() ?? [] as $keyword) {
    echo $keyword->text, PHP_EOL;
    echo $keyword->score, PHP_EOL;
}

Tests keyword extraction via YAKE algorithm

input_value = %Xberg.ExtractInput{kind: "uri", uri: "https://example.com/pdf/fake_memo.pdf"}
result = Xberg.extract_async(input_value, "{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}")
Enum.each(Enum.at(result.results, 0).extracted_keywords || [], fn keyword ->
  IO.puts(keyword.text)
  IO.puts(keyword.score)
end)

Tests keyword extraction via YAKE algorithm

import 'dart:io';
import 'package:xberg/xberg.dart';
import 'package:xberg/src/xberg_bridge_generated/frb_generated.dart' show RustLib;
Future<void> main() async {
  await RustLib.init();
  try {
    final input = await createExtractInputFromJson(json: '{"kind":"uri","uri":"https://example.com/pdf/fake_memo.pdf"}');
    final config = await createExtractionConfigFromJson(json: '{"keywords":{"algorithm":"yake","max_keywords":10}}');
    final result = await XbergBridge.extract(input, config: config);
    for (final keyword in result.results[0].extractedKeywords ?? []) {
      stdout.writeln(keyword.text);
      stdout.writeln(keyword.score);
    }
  } finally {
    RustLib.dispose();
  }
}

Tests keyword extraction via YAKE algorithm

const std = @import("std");
const xberg = @import("xberg");

pub fn main() !void {
    const _result_json = try xberg.extract("{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}", "{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}");
    defer std.heap.c_allocator.free(_result_json);
    std.debug.print("{s}\n", .{_result_json});

}

Tests keyword extraction via YAKE algorithm

#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "xberg.h"

int main(void) {
    XBERGAlefHandle input_handle = xberg_extract_input_from_json("{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}");
    XBERGAlefHandle config_handle = xberg_extraction_config_from_json("{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}");
    XBERGAlefHandle result = xberg_extract(input_handle, config_handle);
    xberg_extract_input_free(input_handle);
    xberg_extraction_config_free(config_handle);
    xberg_extraction_result_free(result);
    return EXIT_SUCCESS;
}

Keywords are returned in result.extracted_keywords as objects with text and score fields.

Configuration

See KeywordConfig reference for all configuration options.

Tests keyword extraction via YAKE algorithm

import asyncio
from xberg import extract, ExtractInput, ExtractInputKind
from xberg._xberg import ExtractionConfig

async def main() -> None:
    input = ExtractInput(kind=ExtractInputKind("uri"), uri="https://example.com/pdf/fake_memo.pdf")
    config = ExtractionConfig.from_json("{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}")
    result = await extract(input, config)
    for keyword in result.results[0].extracted_keywords or []:
        print(keyword.text)
        print(keyword.score)

asyncio.run(main())

Tests keyword extraction via YAKE algorithm

import { ExtractInput, ExtractInputKind, ExtractionConfig, KeywordAlgorithm, extract } from "@xberg-io/xberg";
async function main() {
  const input: ExtractInput = { kind: ExtractInputKind.Uri, uri: "https://example.com/pdf/fake_memo.pdf" };
  const config: ExtractionConfig = { keywords: { algorithm: KeywordAlgorithm.Yake, maxKeywords: 10 } };
  const result = await extract(input, config);
  const [first] = result.results ?? [];
  for (const keyword of first?.extractedKeywords ?? []) {
    console.log(keyword.text);
    console.log(keyword.score);
  }
}

void main();

Tests keyword extraction via YAKE algorithm

import { WasmExtractInput, WasmExtractInputKind, extract } from "@xberg-io/xberg-wasm";
async function main() {
  const input: WasmExtractInput = (() => { const _u0 = WasmExtractInput.default(); _u0.kind = WasmExtractInputKind.Uri; _u0.uri = "https://example.com/pdf/fake_memo.pdf"; return _u0; })();
  const result = await extract(input, { keywords: { algorithm: "yake", maxKeywords: 10 } });
  const [first] = result.results ?? [];
  for (const keyword of first?.extractedKeywords ?? []) {
    console.log(keyword.text);
    console.log(keyword.score);
  }
}

void main();

Tests keyword extraction via YAKE algorithm

use xberg::extract;
use xberg::ExtractInput;

#[tokio::main]
async fn main() {
    let input_json: serde_json::Value = serde_json::from_str(r#"{"kind":"uri","uri":"https://example.com/pdf/fake_memo.pdf"}"#).unwrap();
    let input = serde_json::from_value::<ExtractInput>(input_json).unwrap();
    let config_json: serde_json::Value = serde_json::from_str(r#"{"keywords":{"algorithm":"yake","max_keywords":10}}"#).unwrap();
    let config = serde_json::from_value(config_json).unwrap();
    let result = extract(input, &config).await.expect("call failed");
    for keyword in result.results[0].extracted_keywords.iter().flatten() {
        println!("{}", keyword.text);
        println!("{}", keyword.score);
    }
}

Tests keyword extraction via YAKE algorithm

package main

import (
  "fmt"
  xberg "github.com/xberg-io/xberg/packages/go"
)

func ptr[T any](value T) *T { return &value }
func main() {
  input := xberg.ExtractInput{
    Kind: ptr(xberg.ExtractInputKindURI),
    URI:  ptr(`https://example.com/pdf/fake_memo.pdf`),
  }
  config := xberg.ExtractionConfig{
    Keywords: &xberg.KeywordConfig{
    Algorithm:   xberg.KeywordAlgorithmYake,
    MaxKeywords: ptr(uint(10)),
  },
  }
  result, err := xberg.Extract(input, config)
  if err != nil {
    panic(err)
  }
  for _, keyword := range result.Results[0].ExtractedKeywords {
    fmt.Printf("%v\n", keyword.Text)
    fmt.Printf("%v\n", keyword.Score)
  }
}

Tests keyword extraction via YAKE algorithm

import io.xberg.*;

public final class Example {
    public static void main(String[] args) throws Exception {
        var inputJson = "{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}";
        var input = JsonUtil.fromJson(inputJson, ExtractInput.class);
        var configJson = "{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}";
        var config = JsonUtil.fromJson(configJson, ExtractionConfig.class);
        var result = Xberg.extract(input, config);
        for (var keyword : result.results().get(0).extractedKeywords()) {
            System.out.println(keyword.text());
            System.out.println(keyword.score());
        }
    }
}

Tests keyword extraction via YAKE algorithm

import io.xberg.*
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper

fun main() = kotlinx.coroutines.runBlocking {
    val mapper = jacksonObjectMapper().setPropertyNamingStrategy(com.fasterxml.jackson.databind.PropertyNamingStrategies.SNAKE_CASE)
    val input = mapper.readValue("{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}", ExtractInput::class.java)
    val config = mapper.readValue("{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10},\"url\":{\"crawl\":{\"ssrf\":{}}}}", ExtractionConfig::class.java)
    val result = Xberg.extract(input, config)
    for (keyword in result.results.first().extractedKeywords.orEmpty()) {
        println(keyword.text)
        println(keyword.score)
    }
}

Tests keyword extraction via YAKE algorithm

using System;
using System.Text.Json;
using Xberg;

var ConfigOptions = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
var result = await XbergConverter.ExtractAsync(new ExtractInput { Kind = JsonSerializer.Deserialize<ExtractInputKind>("\"uri\"", ConfigOptions)!, Uri = "https://example.com/pdf/fake_memo.pdf" }, new ExtractionConfig { Keywords = new KeywordConfig { Algorithm = JsonSerializer.Deserialize<KeywordAlgorithm>("\"yake\"", ConfigOptions)!, MaxKeywords = 10 } });
foreach (var keyword in result.Results[0].ExtractedKeywords!)
{
    Console.WriteLine(keyword.Text);
    Console.WriteLine(keyword.Score);
}

Tests keyword extraction via YAKE algorithm

import Xberg

let result = try await Xberg.extract("{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}", "{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}")
print(result)

Tests keyword extraction via YAKE algorithm

require "xberg"
result = Xberg.extract(Xberg::ExtractInput.new(kind: 'uri', uri: 'https://example.com/pdf/fake_memo.pdf'), { 'keywords' => { 'algorithm' => 'yake', 'max_keywords' => 10 } })
(result.results[0].extracted_keywords || []).each do |keyword|
  puts keyword.text
  puts keyword.score
end

Tests keyword extraction via YAKE algorithm

<?php

declare(strict_types=1);

require_once __DIR__ . '/vendor/autoload.php';

use Xberg\Xberg;
use Xberg\ExtractInput;
$input = \Xberg\ExtractInput::from_json(json_encode(["kind" => "uri", "uri" => "https://example.com/pdf/fake_memo.pdf"]));
$result = Xberg::extract($input, ["keywords" => ["algorithm" => "yake", "max_keywords" => 10]]);
foreach ($result->getResults()[0]->getExtractedKeywords() ?? [] as $keyword) {
    echo $keyword->text, PHP_EOL;
    echo $keyword->score, PHP_EOL;
}

Tests keyword extraction via YAKE algorithm

input_value = %Xberg.ExtractInput{kind: "uri", uri: "https://example.com/pdf/fake_memo.pdf"}
result = Xberg.extract_async(input_value, "{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}")
Enum.each(Enum.at(result.results, 0).extracted_keywords || [], fn keyword ->
  IO.puts(keyword.text)
  IO.puts(keyword.score)
end)

Tests keyword extraction via YAKE algorithm

import 'dart:io';
import 'package:xberg/xberg.dart';
import 'package:xberg/src/xberg_bridge_generated/frb_generated.dart' show RustLib;
Future<void> main() async {
  await RustLib.init();
  try {
    final input = await createExtractInputFromJson(json: '{"kind":"uri","uri":"https://example.com/pdf/fake_memo.pdf"}');
    final config = await createExtractionConfigFromJson(json: '{"keywords":{"algorithm":"yake","max_keywords":10}}');
    final result = await XbergBridge.extract(input, config: config);
    for (final keyword in result.results[0].extractedKeywords ?? []) {
      stdout.writeln(keyword.text);
      stdout.writeln(keyword.score);
    }
  } finally {
    RustLib.dispose();
  }
}

Tests keyword extraction via YAKE algorithm

const std = @import("std");
const xberg = @import("xberg");

pub fn main() !void {
    const _result_json = try xberg.extract("{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}", "{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}");
    defer std.heap.c_allocator.free(_result_json);
    std.debug.print("{s}\n", .{_result_json});

}

Tests keyword extraction via YAKE algorithm

#include <assert.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "xberg.h"

int main(void) {
    XBERGAlefHandle input_handle = xberg_extract_input_from_json("{\"kind\":\"uri\",\"uri\":\"https://example.com/pdf/fake_memo.pdf\"}");
    XBERGAlefHandle config_handle = xberg_extraction_config_from_json("{\"keywords\":{\"algorithm\":\"yake\",\"max_keywords\":10}}");
    XBERGAlefHandle result = xberg_extract(input_handle, config_handle);
    xberg_extract_input_free(input_handle);
    xberg_extraction_config_free(config_handle);
    xberg_extraction_result_free(result);
    return EXIT_SUCCESS;
}

YAKE Score Tuning

Use min_score as a lower-bound cutoff. Higher YAKE scores = higher relevance:

`min_score`	Effect
`0.1`	Keeps most keywords
`0.3`	Main topics only
`0.5`	Core concepts only

yake_params.window_size controls co-occurrence context: 1–2 for narrow domains, 2–3 for general (default: 2), 3–4 for discussion-heavy content.

RAKE Score Tuning

Use min_score as a lower-bound cutoff. Higher RAKE scores = higher relevance:

`min_score`	Effect
`0.1`	Keeps most keywords
`0.3`	Main phrases only
`0.5`	Only highly specific phrases

rake_params.min_word_length (default: 1) and rake_params.max_words_per_phrase (default: 3) control phrase boundaries.

Troubleshooting

Too few keywords — Lower min_score, check result.content is non-empty, set language to match the document or None to disable stopword filtering
Too many irrelevant keywords — Raise min_score, set language for stopword filtering, reduce ngram_range upper bound
Multi-word phrases missing (YAKE) — Switch to RAKE or confirm ngram_range upper bound is >= 2
Keywords don’t match content — Verify text was extracted (result.content) and language matches the document

See the KeywordConfig reference for the full parameter list.