VLM Parsers & Inference

{
  "engines": {
    "local-vlm": {
      "openai-vlm-parser": {
        "baseUrl": "http://127.0.0.1:8000",
        "model": "mock-vlm",
        "apiKey": "${env:BOOKSHELF_VLM_KEY}",
        "timeoutMillis": 600000,
        "maxTokens": 4096
      }
    }
  },
  "text-recognizers": [ { "engine": "local-vlm" } ]
}

org.apache.tika.exception.TikaConfigException: environment variable BOOKSHELF_VLM_KEY is not set;
referenced at /engines/local-vlm/openai-vlm-parser/apiKey

import java.nio.file.Path;

import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.RecursiveParserWrapperHandler;

public class VlmIntake {
    public static void main(String[] args) throws Exception {
        TikaLoader loader = TikaLoader.load(Path.of(args[0]));              // tika-config.json
        RecursiveParserWrapper parser = new RecursiveParserWrapper(loader.loadAutoDetectParser());
        ParseContext context = loader.loadParseContext();
        RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
                new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.MARKDOWN, -1));
        try (TikaInputStream tis = TikaInputStream.get(Path.of(args[1]))) {  // book-cover.jpg
            parser.parse(tis, handler, new Metadata(), context);
        }
        for (Metadata m : handler.getMetadataList()) {
            System.out.println("Content-Type          : " + m.get("Content-Type"));
            System.out.println("tk:parsed-by          : " + String.join(", ", m.getValues("tk:parsed-by")));
            System.out.println("vlm:model             : " + m.get("vlm:model"));
            System.out.println("vlm:prompt-tokens     : " + m.get("vlm:prompt-tokens"));
            System.out.println("vlm:completion-tokens : " + m.get("vlm:completion-tokens"));
            System.out.println("tk:content            :\n" + m.get("tk:content"));
        }
    }
}

$ export BOOKSHELF_VLM_KEY=...        # any value for a local server; never commit it
$ java -cp target/classes:$(cat cp.txt) VlmIntake tika-config.json book-cover.jpg
Content-Type          : image/jpeg
tk:parsed-by          : org.apache.tika.parser.DefaultParser, org.apache.tika.parser.image.JpegParser, org.apache.tika.parser.vlm.OpenAIVLMParser
vlm:model             : mock-vlm
vlm:prompt-tokens     : 812
vlm:completion-tokens : 41
tk:content            :
# The Lighthouse Keeper

A novel by **A. Writer**

|Edition|Year|
|---|---|
|First|2026|

{
  "engines": {
    "claude": {
      "claude-vlm-parser": {
        "apiKey": "${env:ANTHROPIC_API_KEY}",
        "model": "claude-sonnet-4-20250514",
        "maxTokens": 4096,
        "timeoutMillis": 300000
      }
    }
  },
  "text-recognizers": [ { "engine": "claude" } ]
}

{
  "engines": {
    "gemini": {
      "gemini-vlm-parser": {
        "apiKey": "${env:GEMINI_API_KEY}",
        "model": "gemini-2.5-flash",
        "maxTokens": 4096
      }
    }
  },
  "text-recognizers": [ { "engine": "gemini" } ]
}

Unrecognized field "apiKeyHeaderName" (class org.apache.tika.parser.vlm.VLMOCRConfig), not marked as ignorable
(15 known properties: "inlineContent", "maxRetries", "prompt", "apiKey", "timeoutMillis", "model", "textRecognizer",
"allowRuntimePrompt", "minFileSizeToOcr", "completionsPath", "baseUrl", "skipOcr", "maxFileSizeToOcr",
"maxImagePixels", "maxTokens")

{
  "text-recognizers": [
    { "engine": "local-vlm", "_mime-exclude": ["image/gif"], "_min-width": 200, "_min-height": 200 }
  ]
}

{
  "engines": {
    "local-vlm": {
      "openai-vlm-parser": {
        "baseUrl": "http://127.0.0.1:8000",
        "model": "mock-vlm",
        "apiKey": "${env:BOOKSHELF_VLM_KEY}",
        "timeoutMillis": 600000
      }
    }
  },
  "text-recognizers": [ { "engine": "local-vlm" } ],
  "parsers": [
    { "default-parser": {} },
    { "pdf-parser": { "pages": { "text": "OCR", "render": { "dpi": 150, "imageType": "RGB" } } } }
  ]
}

<body><div class="page"><div class="ocr"><h1>The Lighthouse Keeper</h1>
<p>A novel by <b>A. Writer</b></p>
<table><thead><tr><th>Edition</th><th>Year</th></tr></thead>
<tbody><tr><td>First</td><td>2026</td></tr></tbody></table>
</div></div></body>

tk:parsed-by=org.apache.tika.parser.CompositeParser, org.apache.tika.parser.pdf.PDFParser
tk:parsed-by-full-set=org.apache.tika.parser.CompositeParser, org.apache.tika.parser.pdf.PDFParser, org.apache.tika.parser.vlm.OpenAIVLMParser
pdf:ocr-page-count=1

{
  "parsers": [
    { "default-parser": { "exclude": ["pdf-parser"] } },
    { "claude-vlm-parser": { "apiKey": "${env:ANTHROPIC_API_KEY}", "model": "claude-sonnet-4-20250514" } }
  ]
}

import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.vlm.VLMOCRConfig;

public final class CoverCaptionContext {
    private CoverCaptionContext() {
    }

    /** A captioning prompt for the cover workflow: not a transcription, so textRecognizer is false. */
    public static ParseContext forCovers() throws TikaConfigException {
        VLMOCRConfig override = new VLMOCRConfig();
        override.setBaseUrl("http://127.0.0.1:8000");
        override.setModel("mock-vlm");
        override.setPrompt("Describe this book cover in one sentence. Return Markdown.");
        override.setTextRecognizer(false);
        override.setMaxTokens(256);
        ParseContext context = new ParseContext();
        context.set(VLMOCRConfig.class, override);
        return context;
    }
}

import java.nio.file.Path;

import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.sax.ToMarkdownContentHandler;

public class PerRequest {
    public static void main(String[] args) throws Exception {
        TikaLoader loader = TikaLoader.load(Path.of(args[0]));
        Parser parser = loader.loadAutoDetectParser();
        String[] overrides = {
            "{\"maxTokens\": 1024}", "{\"maxTokens\": 9000}", "{\"model\": \"another-model\"}",
            "{\"prompt\": \"Describe the cover.\"}", "{\"skipOcr\": true}"
        };
        for (String json : overrides) {
            ParseContext context = loader.loadParseContext();
            context.setJsonConfig("openai-vlm-parser", json);     // untrusted, as from a request
            Metadata metadata = new Metadata();
            try (TikaInputStream tis = TikaInputStream.get(Path.of(args[1]))) {
                parser.parse(tis, new ToMarkdownContentHandler(), metadata, context);
                System.out.println(json + " -> OK, vlm:model=" + metadata.get("vlm:model"));
            } catch (Exception e) {
                Throwable root = e;
                while (root.getCause() != null) {
                    root = root.getCause();
                }
                System.out.println(json + " -> " + root.getMessage());
            }
        }
    }
}

{"maxTokens": 1024} -> OK, vlm:model=mock-vlm
{"maxTokens": 9000} -> Cannot increase maxTokens beyond the init-time value (4096) at runtime. Requested: 9000
{"model": "another-model"} -> Cannot modify model at runtime. Models must be configured at initialization time. If you need a different model, configure a separate parser instance.
{"prompt": "Describe the cover."} -> Cannot modify prompt at runtime. Set allowRuntimePrompt=true at initialization time to permit per-request prompt overrides.
{"skipOcr": true} -> OK, vlm:model=null

curl -X POST http://localhost:9998/rmeta/config/md \
  -F "file=@scanned-loan-form.pdf" \
  -F "config={\"openai-vlm-parser\":{\"maxTokens\":1024}};type=application/json"

{
  "presets": {
    "no-enrichment": { "text-recognizers": { "enabled": false }, "inference": { "enabled": false } }
  }
}

curl -T manuscript.docx http://localhost:9998/rmeta/preset/no-enrichment/md

{
  "engines": {
    "embedder": {
      "openai-embedding-engine": {
        "baseUrl": "http://127.0.0.1:8000",
        "model": "mock-embed",
        "apiKey": "${env:BOOKSHELF_EMBED_KEY}",
        "maxBatchSize": 16,
        "requestParameters": { "dimensions": 4 }
      }
    }
  },
  "inference": [
    { "id": "text-vectors", "engine": "embedder", "input": "TEXT", "tasks": ["embed"],
      "chunker": { "markdown-chunker": { "maxChunkChars": 1500, "overlapChars": 200 } },
      "maxChunks": 500 },
    { "id": "page-vectors", "engine": "embedder", "input": "PAGES", "tasks": ["embed"], "maxChunks": 50 }
  ],
  "parsers": [
    { "default-parser": {} },
    { "pdf-parser": { "pages": { "inference": ["TEXT", "PAGES"],
                                 "render": { "dpi": 96, "imageType": "RGB" } } } }
  ]
}

import java.nio.file.Path;
import java.util.List;

import org.apache.tika.config.loader.TikaLoader;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.parser.inference.EngineRegistry;
import org.apache.tika.parser.inference.InferenceDispatcher;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.RecursiveParserWrapperHandler;

public class BookshelfVectors {
    public static void main(String[] args) throws Exception {
        TikaLoader loader = TikaLoader.load(Path.of(args[0]));               // tika-config.json
        try (EngineRegistry engines = loader.get(EngineRegistry.class)) {     // closes the HTTP clients
            RecursiveParserWrapper parser = new RecursiveParserWrapper(loader.loadAutoDetectParser());
            ParseContext context = loader.loadParseContext();
            RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
                    new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.MARKDOWN, -1));
            try (TikaInputStream tis = TikaInputStream.get(Path.of(args[1]))) {
                parser.parse(tis, handler, new Metadata(), context);          // IMAGES, PAGES, MEDIA flush here
            }
            List<Metadata> metadataList = handler.getMetadataList();
            InferenceDispatcher inference = loader.get(InferenceDispatcher.class);
            if (inference != null) {
                inference.text(metadataList, context);                        // the TEXT stage
            }
            loader.loadMetadataFilters().filter(metadataList);                // filters run last
            for (Metadata m : metadataList) {
                System.out.println(m.get("Content-Type") + " -> tk:chunks = " + m.get("tk:chunks"));
            }
        }
    }
}

$ export BOOKSHELF_EMBED_KEY=...
$ java -cp target/classes:$(cat cp.txt) BookshelfVectors tika-config.json catalogue.pdf
application/pdf -> tk:chunks = [{"vector":"PczMzT5MzM0+mZmaPszMzQ==","producer":"page-vectors","modality":"visual",
  "locators":{"paginated":[{"page":1}]}},{"text":"Spring catalogue  \n\nBookshelf spring catalogue Forty titles from
  independent publishers.","vector":"PczMzT5MzM0+mZmaPszMzQ==","producer":"text-vectors","modality":"text",
  "locators":{"text":[{"start_offset":0,"end_offset":88}]}}]

$ java -jar tika-app-4.1.0.jar --config=tika-config.json -J catalogue.pdf

{
  "engines": {
    "jina": { "openai-embedding-engine": {
      "baseUrl": "https://api.jina.ai", "apiKey": "${env:JINA_API_KEY}",
      "model": "jina-embeddings-v5-omni-small", "maxBatchSize": 16,
      "requestParameters": { "task": "retrieval.passage", "dimensions": 1024, "normalized": true } } }
  },
  "inference": [
    { "id": "pictures", "engine": "jina", "input": "IMAGES", "tasks": ["embed"], "maxBytes": 5000000 },
    { "id": "pages",    "engine": "jina", "input": "PAGES",  "tasks": ["embed"], "maxBytes": 5000000 },
    { "id": "text",     "engine": "jina", "input": "TEXT",   "tasks": ["embed"],
      "chunker": { "markdown-chunker": { "maxChunkChars": 1500, "overlapChars": 200 } } }
  ],
  "parsers": [
    { "default-parser": {} },
    { "pdf-parser": { "pages": { "inference": ["TEXT", "PAGES"], "render": { "dpi": 96, "imageType": "RGB" } } } }
  ]
}

{
  "engines": {
    "azure": { "openai-embedding-engine": {
      "baseUrl": "https://bookshelf-search.openai.azure.com",
      "embeddingsPath": "/openai/deployments/text-embedding-3-large/embeddings?api-version=2024-10-21",
      "apiKeyHeaderName": "api-key", "apiKeyPrefix": "", "apiKey": "${env:AZURE_OPENAI_KEY}",
      "requestParameters": { "dimensions": 1024 } } }
  }
}

{ "parse-context": { "inference": { "bindings": ["page-vectors"], "enabled": true } } }