Testing Tika Integrations

package bookshelf;

import java.nio.file.Path;
import java.util.List;

import org.apache.tika.io.TikaInputStream;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
import org.apache.tika.parser.RecursiveParserWrapper;
import org.apache.tika.sax.BasicContentHandlerFactory;
import org.apache.tika.sax.BodyContentHandler;
import org.apache.tika.sax.RecursiveParserWrapperHandler;

/** Small helpers shared by the Bookshelf intake tests. */
final class Fixtures {

    static final Path DIR = Path.of("src/test/resources/fixtures");

    // AutoDetectParser is thread-safe and expensive to build: one per test JVM
    static final Parser PARSER = new AutoDetectParser();

    private Fixtures() {
    }

    static Path fixture(String name) {
        return DIR.resolve(name);
    }

    /** Container text and metadata, in-process, no write limit. */
    static Parsed parse(Path file) throws Exception {
        Metadata metadata = new Metadata();
        BodyContentHandler handler = new BodyContentHandler(-1);
        try (TikaInputStream tis = TikaInputStream.get(file, metadata)) {
            PARSER.parse(tis, handler, metadata, new ParseContext());
        }
        return new Parsed(handler.toString(), metadata);
    }

    /** One Metadata per document (container first), each with its text in tk:content. No write limit. */
    static List<Metadata> parseRecursive(Path file) throws Exception {
        return parseRecursive(file, -1);
    }

    /**
     * Same, with a total write limit that truncates and flags instead of throwing. The wrapper takes its limit from
     * the handler factory, not from OutputLimits in the ParseContext.
     */
    static List<Metadata> parseRecursive(Path file, int writeLimit) throws Exception {
        ParseContext context = new ParseContext();
        RecursiveParserWrapper wrapper = new RecursiveParserWrapper(PARSER);
        RecursiveParserWrapperHandler handler = new RecursiveParserWrapperHandler(
                new BasicContentHandlerFactory(BasicContentHandlerFactory.HANDLER_TYPE.TEXT, writeLimit, false, context));
        Metadata metadata = new Metadata();
        try (TikaInputStream tis = TikaInputStream.get(file, metadata)) {
            wrapper.parse(tis, handler, metadata, context);
        }
        return handler.getMetadataList();
    }

    record Parsed(String text, Metadata metadata) {
    }
}

package bookshelf;

import static bookshelf.Fixtures.fixture;
import static org.assertj.core.api.Assertions.assertThat;

import java.util.List;

import org.apache.tika.Tika;
import org.apache.tika.metadata.HttpHeaders;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.assertj.core.api.SoftAssertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;

class FixtureExtractionTest {

    private static final Tika TIKA = new Tika();

    @Test
    void manuscriptTextAndMetadata() throws Exception {
        Fixtures.Parsed parsed = Fixtures.parse(fixture("manuscript.docx"));

        assertThat(parsed.text())
                .contains("Chapter one begins at dusk.")
                .doesNotContain("�");                     // no replacement characters

        SoftAssertions softly = new SoftAssertions();          // report every mismatch, not just the first
        softly.assertThat(parsed.metadata().get(TikaCoreProperties.TITLE)).isEqualTo("The Lighthouse Keeper");
        softly.assertThat(parsed.metadata().get(TikaCoreProperties.CREATOR)).isEqualTo("Ana Torres");
        softly.assertThat(parsed.metadata().get(HttpHeaders.CONTENT_TYPE))
                .isEqualTo("application/vnd.openxmlformats-officedocument.wordprocessingml.document");
        softly.assertAll();
    }

    @ParameterizedTest(name = "{0} is {1}")
    @CsvSource({
            "manuscript.docx,     application/vnd.openxmlformats-officedocument.wordprocessingml.document",
            "catalogue.pdf,       application/pdf",
            "renamed-invoice.png, application/pdf",           // a PDF with the wrong extension
            "book-cover.jpg,      image/jpeg",
            "novel.epub,          application/epub+zip",
            "publisher-email.eml, message/rfc822",
            "reviews-es.html,     text/html",
            "blob.bin,            application/octet-stream"
    })
    void detectsByContentNotByName(String file, String expectedType) throws Exception {
        assertThat(TIKA.detect(fixture(file))).isEqualTo(expectedType);
    }

    @Test
    void legacyCatalogueIsDecodedAsWindows1252() throws Exception {
        Fixtures.Parsed parsed = Fixtures.parse(fixture("legacy-catalogue.txt"));

        assertThat(parsed.metadata().get(HttpHeaders.CONTENT_TYPE)).isEqualTo("text/plain; charset=windows-1252");
        assertThat(parsed.text()).contains("Catálogo de primavera", "12€", "librería");
    }

    @Test
    void emailAttachmentsAreParsedRecursively() throws Exception {
        List<Metadata> docs = Fixtures.parseRecursive(fixture("publisher-email.eml"));

        assertThat(docs).hasSize(6);  // the message, the zip, three zip entries and the figure inside the docx
        assertThat(docs.get(0).get(TikaCoreProperties.TITLE)).isEqualTo("Spring submissions: samples attached");
        assertThat(docs)
                .extracting(m -> m.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH))
                .contains("/samples.zip", "/samples.zip/catalogue.pdf", "/samples.zip/chapter-one.docx",
                        "/samples.zip/chapter-one.docx/figure1.png");
        assertThat(docs)
                .allSatisfy(m -> assertThat(m.get(TikaCoreProperties.CONTAINER_EXCEPTION)).isNull())
                .allSatisfy(m -> assertThat(m.getValues(TikaCoreProperties.EMBEDDED_EXCEPTION)).isEmpty());
    }
}

[INFO] Tests run: 11, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 18.91 s -- in bookshelf.FixtureExtractionTest

package bookshelf;

import static org.assertj.core.api.Assertions.assertThat;

import java.util.List;
import java.util.Set;

import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.junit.jupiter.api.Test;

/** Uses Tika's own test base class from the tika-core tests jar; files come from src/test/resources/test-documents. */
class ManuscriptTikaTest extends TikaTest {

    @Test
    void xhtmlOutputIsWellFormedAndHasTheTitle() throws Exception {
        XMLResult result = getXML("manuscript.docx");

        assertContains("<title>The Lighthouse Keeper</title>", result.xml);
        assertThat(result.metadata.get(TikaCoreProperties.CREATOR)).isEqualTo("Ana Torres");
    }

    @Test
    void recursiveMetadataIsStableAcrossRuns() throws Exception {
        List<Metadata> first = getRecursiveMetadata("manuscript.docx");
        List<Metadata> second = getRecursiveMetadata("manuscript.docx");

        assertMetadataListEquals(first, second, Set.of("tk:parse-time-millis"));
    }
}

[INFO] Tests run: 2, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 9.171 s -- in bookshelf.ManuscriptTikaTest

package bookshelf;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

import java.nio.file.Path;
import java.util.List;

import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.junit.jupiter.api.Test;

/** MockParser comes from the tika-core tests jar (test scope) and handles any XML file with a mock root. */
class MockParserInProcessTest {

    private static final Path MOCK = Path.of("src/test/resources/mock");

    @Test
    void mockFileIsRoutedToMockParser() throws Exception {
        Fixtures.Parsed parsed = Fixtures.parse(MOCK.resolve("ok.xml"));

        assertThat(parsed.text()).contains("It was a dark and stormy night.");
        assertThat(parsed.metadata().getValues(TikaCoreProperties.TIKA_PARSED_BY))
                .contains("org.apache.tika.parser.mock.MockParser");
    }

    @Test
    void parserBugSurfacesAsTikaExceptionWithTheCause() {
        assertThatThrownBy(() -> Fixtures.parse(MOCK.resolve("npe.xml")))
                .isInstanceOf(TikaException.class)
                .hasRootCauseInstanceOf(NullPointerException.class)
                .hasRootCauseMessage("simulated parser bug");
    }

    @Test
    void outOfMemoryErrorIsNotWrapped() {
        // An Error escapes the parse: in-process code cannot "handle" it, which is why production forks
        assertThatThrownBy(() -> Fixtures.parse(MOCK.resolve("fake-oom.xml")))
                .isInstanceOf(OutOfMemoryError.class)
                .hasMessage("simulated heap exhaustion");
    }

    @Test
    void hugeOutputIsTruncatedAndFlagged() throws Exception {
        List<Metadata> docs = Fixtures.parseRecursive(MOCK.resolve("big.xml"), 1_000);

        assertThat(docs.get(0).get(TikaCoreProperties.WRITE_LIMIT_REACHED)).isEqualTo("true");
        assertThat(docs.get(0).get(TikaCoreProperties.TIKA_CONTENT)).hasSizeLessThanOrEqualTo(1_100);
    }
}

[INFO] Tests run: 4, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.029 s -- in bookshelf.MockParserInProcessTest

package bookshelf;

import static org.assertj.core.api.Assertions.assertThat;

import java.nio.file.Path;

import org.apache.tika.config.TimeoutLimits;
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.pipes.api.PipesResult;
import org.apache.tika.pipes.fork.PipesForkParser;
import org.apache.tika.pipes.fork.PipesForkParserConfig;
import org.apache.tika.pipes.fork.PipesForkResult;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;

@Tag("slow")      // starts forked JVMs: about 5 s per fork start
@Timeout(120)
class ForkedFailureModesTest {

    private static final Path MOCK = Path.of("src/test/resources/mock");
    private static PipesForkParser parser;

    @BeforeAll
    static void startParser() throws Exception {
        parser = new PipesForkParser(new PipesForkParserConfig()
                .setTimeoutLimits(new TimeoutLimits(30_000, 3_000))   // 30 s per document, 3 s without progress
                .addJvmArg("-Xmx256m"));                               // small heap: oom.xml fails fast
    }

    @AfterAll
    static void stopParser() throws Exception {
        parser.close();                                                // kills the forks
    }

    @ParameterizedTest(name = "{0} -> {1}, then the next file still parses")
    @CsvSource({
            "hang.xml,  TIMEOUT",
            "oom.xml,   OOM",
            "crash.xml, UNSPECIFIED_CRASH"
    })
    void survivesTheFailureAndRecovers(String file, PipesResult.RESULT_STATUS expected) throws Exception {
        try (TikaInputStream bad = TikaInputStream.get(MOCK.resolve(file))) {
            PipesForkResult result = parser.parse(bad);
            assertThat(result.getStatus()).isEqualTo(expected);
            assertThat(result.isProcessCrash()).isTrue();
            assertThat(result.isSuccess()).isFalse();
        }
        try (TikaInputStream good = TikaInputStream.get(MOCK.resolve("ok.xml"))) {
            PipesForkResult result = parser.parse(good);                 // served by a fresh fork
            assertThat(result.getStatus()).isEqualTo(PipesResult.RESULT_STATUS.PARSE_SUCCESS);
            assertThat(result.getContent()).contains("It was a dark and stormy night.");
        }
    }
}

[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 42.38 s -- in bookshelf.ForkedFailureModesTest

package bookshelf;

import static bookshelf.Fixtures.fixture;
import static org.assertj.core.api.Assertions.assertThat;

import java.io.StringReader;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpRequest.BodyPublishers;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandlers;
import java.time.Duration;
import java.util.List;

import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.serialization.JsonMetadataList;
import org.junit.jupiter.api.Test;

/** The HTTP behaviour the Bookshelf intake service relies on. Subclasses say where the server is. */
abstract class TikaServerContract {

    private static final HttpClient HTTP = HttpClient.newBuilder()
            .connectTimeout(Duration.ofSeconds(5))
            .build();

    /** For example http://localhost:32791 */
    abstract URI baseUri();

    @Test
    void reportsItsVersion() throws Exception {
        HttpResponse<String> response = send(HttpRequest.newBuilder(uri("/version")).GET());

        assertThat(response.statusCode()).isEqualTo(200);
        assertThat(response.body()).startsWith("Apache Tika 4.1");
    }

    @Test
    void detectsTheRenamedInvoiceAsPdf() throws Exception {
        HttpResponse<String> response = put("/detect", "renamed-invoice.png", "text/plain");

        assertThat(response.statusCode()).isEqualTo(200);
        assertThat(response.body()).isEqualTo("application/pdf");
    }

    @Test
    void extractsCatalogueText() throws Exception {
        HttpResponse<String> response = put("/tika/text", "catalogue.pdf", "text/plain");

        assertThat(response.statusCode()).isEqualTo(200);
        assertThat(response.body()).contains("Forty titles from independent publishers.");
    }

    @Test
    void returnsEveryEmbeddedDocumentFromRmeta() throws Exception {
        HttpResponse<String> response = put("/rmeta/text", "publisher-email.eml", "application/json");

        assertThat(response.statusCode()).isEqualTo(200);
        List<Metadata> docs = JsonMetadataList.fromJson(new StringReader(response.body()));
        assertThat(docs).hasSize(6);
        assertThat(docs).extracting(m -> m.get(TikaCoreProperties.EMBEDDED_RESOURCE_PATH))
                .contains("/samples.zip/catalogue.pdf");
    }

    @Test
    void encryptedPdfIsUnprocessableOnTheRawEndpoint() throws Exception {
        HttpResponse<String> response = put("/tika/text", "members-only.pdf", "text/plain");

        assertThat(response.statusCode()).isEqualTo(422);
    }

    @Test
    void encryptedPdfIsReportedInsideRmeta() throws Exception {
        HttpResponse<String> response = put("/rmeta", "members-only.pdf", "application/json");

        assertThat(response.statusCode()).isEqualTo(200);
        List<Metadata> docs = JsonMetadataList.fromJson(new StringReader(response.body()));
        assertThat(docs.get(0).get(TikaCoreProperties.CONTAINER_EXCEPTION))
                .contains("EncryptedDocumentException");
    }

    @Test
    void perRequestConfigIsDisabledByDefault() throws Exception {
        HttpResponse<String> response = send(HttpRequest.newBuilder(uri("/tika/config"))
                .header("Content-Type", "multipart/form-data; boundary=x")
                .POST(BodyPublishers.ofString("--x--\r\n")));

        assertThat(response.statusCode()).isEqualTo(403);
    }

    @Test
    void unknownPathsAndMethodsAreRejected() throws Exception {
        assertThat(send(HttpRequest.newBuilder(uri("/nonexistent")).GET()).statusCode()).isEqualTo(404);
        assertThat(send(HttpRequest.newBuilder(uri("/tika/text")).DELETE()).statusCode()).isEqualTo(405);
    }

    private HttpResponse<String> put(String path, String fixtureName, String accept) throws Exception {
        return send(HttpRequest.newBuilder(uri(path))
                .header("Accept", accept)
                .PUT(BodyPublishers.ofFile(fixture(fixtureName))));
    }

    private HttpResponse<String> send(HttpRequest.Builder request) throws Exception {
        return HTTP.send(request.timeout(Duration.ofSeconds(60)).build(), BodyHandlers.ofString());
    }

    private URI uri(String path) {
        return baseUri().resolve(path);
    }
}

package bookshelf;

import java.net.URI;
import java.time.Duration;

import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;

@Testcontainers(disabledWithoutDocker = true)   // skipped, not failed, on a machine without Docker
class TikaServerContainerIT extends TikaServerContract {

    @Container
    static final GenericContainer<?> TIKA = new GenericContainer<>(DockerImageName.parse("apache/tika:4.1.0-1"))
            .withExposedPorts(9998)
            .waitingFor(Wait.forHttp("/version").forStatusCode(200))
            .withStartupTimeout(Duration.ofMinutes(2));

    @Override
    URI baseUri() {
        return URI.create("http://" + TIKA.getHost() + ":" + TIKA.getMappedPort(9998));
    }
}

package bookshelf;

import java.net.URI;

import org.junit.jupiter.api.condition.EnabledIfSystemProperty;

/** The same contract against a server you started yourself: mvn verify -Dtika.url=http://localhost:9998 */
@EnabledIfSystemProperty(named = "tika.url", matches = "https?://.+")
class ExternalTikaServerIT extends TikaServerContract {

    @Override
    URI baseUri() {
        return URI.create(System.getProperty("tika.url"));
    }
}

unzip -q tika-server-standard-4.1.0.zip -d tika-server && cd tika-server
java -jar tika-server-standard-4.1.0.jar -p 19998 -h 127.0.0.1 &
cd ../bookshelf-intake-tests
mvn verify -Dtika.url=http://127.0.0.1:19998

[INFO] Tests run: 8, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 24.59 s -- in bookshelf.ExternalTikaServerIT
ERROR [main] org.testcontainers.dockerclient.DockerClientProviderStrategy Could not find a valid Docker environment. ...
[WARNING] Tests run: 8, Failures: 0, Errors: 0, Skipped: 8, Time elapsed: 0.007 s -- in bookshelf.TikaServerContainerIT
[WARNING] Tests run: 16, Failures: 0, Errors: 0, Skipped: 8