This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4847-eval-crash-join in repository https://gitbox.apache.org/repos/asf/tika.git
commit 46a226c0c80ea051824403ee0c6f923a37d78d07 Author: tallison <[email protected]> AuthorDate: Thu Aug 27 15:37:55 2026 -0400 TIKA-4847: tika-eval joins the pipes jsonl crash ledger and run-info provenance --- .skills/devs/tika-eval-compare/SKILL.md | 23 ++ .skills/devs/tika-eval-compare/run-batch.sh | 138 ++++++++++++ CHANGES.txt | 6 + .../integration-testing/tika-eval-regression.adoc | 44 ++++ .../org/apache/tika/eval/app/ExtractComparer.java | 16 +- .../tika/eval/app/ExtractComparerRunner.java | 49 +++- .../apache/tika/eval/app/ExtractProfileRunner.java | 34 ++- .../org/apache/tika/eval/app/ExtractProfiler.java | 12 +- .../org/apache/tika/eval/app/ProfilerBase.java | 15 ++ .../java/org/apache/tika/eval/app/RunInfo.java | 247 +++++++++++++++++++++ .../java/org/apache/tika/eval/app/db/Cols.java | 4 + .../org/apache/tika/eval/app/io/PipesReport.java | 114 ++++++++++ .../eval/app/reports/MarkdownSummaryWriter.java | 57 +++++ .../src/main/resources/comparison-reports.xml | 125 +++++++++++ .../src/main/resources/profile-reports.xml | 63 ++++++ .../apache/tika/eval/app/ComparerBatchTest.java | 104 +++++++++ .../tika/eval/app/ComparerOneSidedLedgerTest.java | 83 +++++++ .../apache/tika/eval/app/ProfilerBatchTest.java | 37 ++- .../apache/tika/eval/app/io/PipesReportTest.java | 69 ++++++ .../extractsB/.run-info/crashes-run-b1.jsonl | 3 + .../extractsB/.run-info/run-info-run-b1.json | 4 + .../test-dirs/pipes-reports/crashes-run-a1.jsonl | 6 + .../test-dirs/pipes-reports/crashes-run-b1.jsonl | 3 + .../test-dirs/pipes-reports/run-info-run-a1.json | 6 + .../test-dirs/pipes-reports/run-info-run-b1.json | 4 + 25 files changed, 1256 insertions(+), 10 deletions(-) diff --git a/.skills/devs/tika-eval-compare/SKILL.md b/.skills/devs/tika-eval-compare/SKILL.md index e1206be371..7ac78bcc06 100644 --- a/.skills/devs/tika-eval-compare/SKILL.md +++ b/.skills/devs/tika-eval-compare/SKILL.md @@ -91,6 +91,21 @@ Each run walks the input directory recursively and writes one equivalent to `tika-app -J`). The directory structure mirrors the input. +### Provenance + crash ledger (preferred) + +`run-batch.sh` (next to this file) wraps the same invocation and records what +ran, so an extract set can be tied to a build after the fact and a crashed +file is distinguishable from one that parsed to nothing: + +```bash +.skills/devs/tika-eval-compare/run-batch.sh --app <before> --input <input-dir> --extracts <extracts-a-dir> --note baseline +.skills/devs/tika-eval-compare/run-batch.sh --app <after> --input <input-dir> --extracts <extracts-b-dir> --note candidate [--config cfg.json] +``` + +It writes `run-info-<run.id>.json` and (4.1+ only) `crashes-<run.id>.jsonl` +into `<extracts>/.run-info/`; Compare picks them up from there by default +(`-ra/-rb`, `-pa/-pb` override). A 3.x baseline gets run-info but no ledger. + ### Notes - Do NOT pass `-n <N>` as a trailing argument — it confuses the @@ -123,6 +138,14 @@ java -jar <tika-eval>/tika-eval-app-*.jar Compare \ | `-rd` | Reports output directory (default: `reports`) | | `-z` | Gzip the H2 db (`<db>.mv.db.gz`) after Compare for transfer; requires `-d` (no-op + warning for a temp db). Combine with `-r` to package both. | | `-n` | Number of worker threads | +| `-pa`/`-pb` | jsonl ledger for A/B (default: `<extracts>/.run-info/crashes-*.jsonl`); fills `containers.pipes_status_a/b` | +| `-ra`/`-rb` | run-info json for A/B (default: `<extracts>/.run-info/run-info-*.json`); lands in `run_info_a/b`. Refused if its `run.id` is not in the matching `-pa`/`-pb` file name | + +With `-pa`/`-pb`, `summary.md` and `exceptions/extract_exceptions_by_pipes_status_*.xlsx` +split `NO_EXTRACT_FILE` into `CRASH` (OOM/TIMEOUT), `LOST_TO_WORKER_RESTART` +(shared-server collateral; retry once, alone), `NO_PIPES_RECORD`, and +`NO_PIPES_REPORT_SUPPLIED`. A crash status *with* an extract present is a +success whose status was lost — listed separately, not a failure. ## Step 3 — Review Results diff --git a/.skills/devs/tika-eval-compare/run-batch.sh b/.skills/devs/tika-eval-compare/run-batch.sh new file mode 100755 index 0000000000..061fcb4f21 --- /dev/null +++ b/.skills/devs/tika-eval-compare/run-batch.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Runs a tika-app batch extraction with provenance and a crash ledger that +# tika-eval can join back in (Profile --pipesReport/--runInfo, +# Compare --pipesReportA/B --runInfoA/B). +# +# run-batch.sh --app <tika-app dir or jar> --input <dir> --extracts <dir> \ +# [--config <tika-config.json>] [--note "text"] [-- <extra tika-app args>] +# +# Writes to <extracts>/.run-info/ (tika-eval skips that dir when crawling and picks these up by default): +# run-info-<run.id>.json what ran (jar/lib/plugin sha256s, config sha256, jvm, git commit, host, time) +# crashes-<run.id>.jsonl one line per non-success result (file-system-jsonl-reporter; 4.1+ only) +# tika-config-<run.id>.json the config actually passed to tika-app +set -euo pipefail + +APP=""; INPUT=""; EXTRACTS=""; CONFIG=""; NOTE="" +while [[ $# -gt 0 ]]; do + case "$1" in + --app) APP="$2"; shift 2;; + --input) INPUT="$2"; shift 2;; + --extracts) EXTRACTS="$2"; shift 2;; + --config) CONFIG="$2"; shift 2;; + --note) NOTE="$2"; shift 2;; + --) shift; break;; + *) echo "unknown arg: $1" >&2; exit 2;; + esac +done +[[ -n "$APP" && -n "$INPUT" && -n "$EXTRACTS" ]] || { sed -n '17,26p' "$0" >&2; exit 2; } + +if [[ -d "$APP" ]]; then + JAR=$(ls "$APP"/tika-app-*.jar | head -1); APP_DIR="$APP" +else + JAR="$APP"; APP_DIR=$(dirname "$JAR") +fi +[[ -f "$JAR" ]] || { echo "no tika-app jar under $APP" >&2; exit 2; } + +sha() { sha256sum "$1" | cut -d' ' -f1; } +manifest() { unzip -p "$JAR" META-INF/MANIFEST.MF 2>/dev/null | tr -d '\r' | awk -F': ' -v k="$1" '$1==k{print $2}'; } + +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')" +mkdir -p "$EXTRACTS" +OUT_DIR="$(readlink -f "$EXTRACTS")/.run-info" +mkdir -p "$OUT_DIR" +RUN_INFO="$OUT_DIR/run-info-$RUN_ID.json" +LEDGER="$OUT_DIR/crashes-$RUN_ID.jsonl" + +TIKA_VERSION=$(manifest Implementation-Version || true) +[[ -n "$TIKA_VERSION" ]] || TIKA_VERSION=$(basename "$JAR" | sed -E 's/^tika-app-(.*)\.jar$/\1/') +GIT_COMMIT=$(manifest Git-Commit || manifest Implementation-Build || true) + +LIB_SHA="" +if [[ -d "$APP_DIR/lib" ]]; then + LIB_SHA=$(cd "$APP_DIR/lib" && for f in $(ls *.jar | sort); do echo "$f $(sha "$f")"; done | sha256sum | cut -d' ' -f1) +fi +PLUGINS_JSON="{}" +if [[ -d "$APP_DIR/plugins" ]]; then + PLUGINS_JSON=$(cd "$APP_DIR/plugins" && for f in $(ls *.zip 2>/dev/null | sort); do printf '"%s":"%s",' "$f" "$(sha "$f")"; done | sed 's/,$//' | sed 's/^/{/;s/$/}/') +fi +CONFIG_SHA=""; CONFIG_ABS="" +if [[ -n "$CONFIG" ]]; then CONFIG_SHA=$(sha "$CONFIG"); CONFIG_ABS=$(readlink -f "$CONFIG"); fi + +# 4.1+ tika-app ships the file-system plugin with the jsonl reporter; older versions get run-info only. +USE_LEDGER=false +case "$TIKA_VERSION" in 4.[1-9]*|[5-9].*) USE_LEDGER=true;; esac + +EFFECTIVE_CONFIG="$CONFIG" +if $USE_LEDGER; then + EFFECTIVE_CONFIG="$OUT_DIR/tika-config-$RUN_ID.json" + python3 - "$CONFIG" "$EFFECTIVE_CONFIG" "$LEDGER" <<'PY' +import json, sys +src, dst, ledger = sys.argv[1:] +cfg = json.load(open(src)) if src else {} +cfg.setdefault("pipes-reporters", {})["file-system-jsonl-reporter"] = { + "path": ledger, + "includes": ["OOM", "TIMEOUT", "UNSPECIFIED_CRASH", "FAILED_TO_INITIALIZE", "PAYLOAD_LIMIT_EXCEEDED", "EMIT_EXCEPTION", "FETCH_EXCEPTION"], + "onExists": "EXCEPTION", + "maxMessageLength": 10000, +} +json.dump(cfg, open(dst, "w"), indent=2) +PY +fi + +JVM_ARGS="${JAVA_OPTS:-}" +# values go to python via the environment; no shell-into-json quoting +RB_RUN_ID="$RUN_ID" RB_NOTE="$NOTE" RB_START="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ +RB_JAR="$(readlink -f "$JAR")" RB_JAR_SHA="$(sha "$JAR")" RB_VERSION="$TIKA_VERSION" RB_COMMIT="$GIT_COMMIT" \ +RB_LIB_SHA="$LIB_SHA" RB_PLUGINS="$PLUGINS_JSON" RB_CONFIG="$CONFIG_ABS" RB_CONFIG_SHA="$CONFIG_SHA" \ +RB_EFFECTIVE_CONFIG="$( [[ -n "$EFFECTIVE_CONFIG" ]] && readlink -f "$EFFECTIVE_CONFIG" || true )" \ +RB_INPUT="$(readlink -f "$INPUT")" RB_EXTRACTS="$(readlink -f "$EXTRACTS")" \ +RB_JAVA="$(java -version 2>&1 | head -1)" RB_JVM_ARGS="$JVM_ARGS" RB_LEDGER="$( $USE_LEDGER && echo "$LEDGER" || true )" \ +python3 - "$RUN_INFO" <<'PY' +import json, os, socket, sys +e = os.environ.get +json.dump({ + "run": {"id": e("RB_RUN_ID"), "note": e("RB_NOTE"), "start": e("RB_START"), "host": socket.gethostname(), "user": e("USER", "")}, + "tika": {"app_path": e("RB_JAR"), "app_sha256": e("RB_JAR_SHA"), "version": e("RB_VERSION"), "git_commit": e("RB_COMMIT"), + "lib_sha256": e("RB_LIB_SHA"), "plugins": json.loads(e("RB_PLUGINS") or "{}")}, + "config": {"path": e("RB_CONFIG"), "sha256": e("RB_CONFIG_SHA"), "effective_path": e("RB_EFFECTIVE_CONFIG")}, + "input": {"path": e("RB_INPUT")}, + "extracts": {"path": e("RB_EXTRACTS")}, + "jvm": {"version": e("RB_JAVA"), "args": e("RB_JVM_ARGS")}, + "ledger": {"path": e("RB_LEDGER")}, +}, open(sys.argv[1], "w"), indent=2) +PY +echo "run.id=$RUN_ID run-info=$RUN_INFO" >&2 + +set +e +if [[ -n "$EFFECTIVE_CONFIG" ]]; then + java $JVM_ARGS -jar "$JAR" --config="$EFFECTIVE_CONFIG" "$@" "$INPUT" "$EXTRACTS" +else + java $JVM_ARGS -jar "$JAR" "$@" "$INPUT" "$EXTRACTS" +fi +RC=$? +set -e + +python3 - "$RUN_INFO" "$RC" <<'PY' +import json, sys, datetime +p, rc = sys.argv[1], int(sys.argv[2]) +d = json.load(open(p)) +d["run"]["end"] = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +d["run"]["exit_code"] = rc +json.dump(d, open(p, "w"), indent=2) +PY +exit $RC diff --git a/CHANGES.txt b/CHANGES.txt index 3b336d2e81..14ec3a74fa 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,11 @@ Release 4.1.0 - unreleased + * tika-eval: Profile/Compare accept the batch run's jsonl crash ledger + (--pipesReport, -pa/-pb) and a run-info json (--runInfo, -ra/-rb). + containers gains pipes_status/pipes_message; a new run_info table records + eval and batch provenance; reports and summary.md split NO_EXTRACT_FILE + into CRASH / LOST_TO_WORKER_RESTART / NO_PIPES_RECORD (TIKA-4847). + * Stop spooling OLE2 objects whose header over-reserves BAT capacity (TIKA-4845). diff --git a/docs/modules/ROOT/pages/advanced/integration-testing/tika-eval-regression.adoc b/docs/modules/ROOT/pages/advanced/integration-testing/tika-eval-regression.adoc index abbe57f4f3..d5b359759f 100644 --- a/docs/modules/ROOT/pages/advanced/integration-testing/tika-eval-regression.adoc +++ b/docs/modules/ROOT/pages/advanced/integration-testing/tika-eval-regression.adoc @@ -238,6 +238,50 @@ The reports directory contains subdirectories: Open the `.xlsx` files directly, or query the H2 database for custom counts and joins (see <<h2>>). +== Joining the batch crash ledger and run provenance + +A crash result (`OOM`, `TIMEOUT`, `UNSPECIFIED_CRASH`) leaves nothing in the +extract set, so tika-eval alone reports it as `NO_EXTRACT_FILE` -- the same as +a file that parsed to nothing. Two inputs close that gap: + +* the per-document JSONL written during the batch run by the + xref:pipes/plugins/filesystem.adoc#file-system-jsonl-reporter[`file-system-jsonl-reporter`] + (`--pipesReport` / `-pr` on Profile, `--pipesReportA`/`-pa` and `-pb` on Compare) +* a `run-info.json` describing what ran (`--runInfo` / `-ri`; `-ra`/`-rb` on Compare) + +The reporter line's `id` is the source-relative path (filesystem iterator), which is +also `containers.file_path`; both are normalized to `/` at the join. The result lands in +`containers.pipes_status` / `pipes_message` (`_a`/`_b` on Compare), and the reports +reclassify extract-file problems: + +[cols="1,3"] +|=== +|Classification |Meaning + +|`CRASH` |`OOM`/`TIMEOUT` recorded, or `UNSPECIFIED_CRASH` without the shared-mode marker +|`LOST_TO_WORKER_RESTART` |`UNSPECIFIED_CRASH` whose message says `outcome unknown`: a shared-server crash while this request was in flight. Retry once, alone, before blaming the file. +|`NO_PIPES_RECORD` |A report was supplied but has no line for this file +|`NO_PIPES_REPORT_SUPPLIED` |No `--pipesReport` given +|=== + +A crash status with an extract present is a success whose status was lost (the +document emitted before the worker died); it is listed in +`exceptions/crash_status_extract_present*.xlsx`, not counted as a failure. + +`run-info.json` is loaded generically: every key is flattened with dots under +`batch.` into the `run_info` table (`run_info_a`/`run_info_b` on Compare), alongside +`eval.*` (tika-eval version, jar sha256, host, args, start/end) and `extracts.*` +(count and a sha256 fingerprint of sorted name+size). If `run-info.json` carries +`run.id`, the pipes report's file name must contain it, or tika-eval refuses to +start -- rerunning a batch into the same extracts dir is otherwise invisible. +`summary.md` opens with the `run_info` tables. + +The batch-side script that produces both files, `.skills/devs/tika-eval-compare/run-batch.sh`, +writes them to `<extracts>/.run-info/`. tika-eval skips that directory when crawling and +excludes it from the fingerprint, and uses its contents by default when the flags above are +absent -- refusing if it holds more than one run-info, since that means the extracts dir was +written by more than one run. + == Tips * *Keep the digester identical between A and B.* tika-eval uses the diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparer.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparer.java index 3e7cfcbfc0..80523c0a01 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparer.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparer.java @@ -38,6 +38,7 @@ import org.apache.tika.eval.app.db.TableInfo; import org.apache.tika.eval.app.io.ExtractReader; import org.apache.tika.eval.app.io.ExtractReaderException; import org.apache.tika.eval.app.io.IDBWriter; +import org.apache.tika.eval.app.io.PipesReport; import org.apache.tika.eval.core.textstats.BasicTokenCountStatsCalculator; import org.apache.tika.eval.core.tokens.ContrastStatistics; import org.apache.tika.eval.core.tokens.TokenContraster; @@ -57,7 +58,9 @@ public class ExtractComparer extends ProfilerBase { public static TableInfo COMPARISON_CONTAINERS = new TableInfo("containers", new ColInfo(Cols.CONTAINER_ID, Types.INTEGER, "PRIMARY KEY"), new ColInfo(Cols.FILE_PATH, Types.VARCHAR, FILE_PATH_MAX_LEN), new ColInfo(Cols.FILE_EXTENSION, Types.VARCHAR, 12), new ColInfo(Cols.LENGTH, Types.BIGINT), new ColInfo(Cols.EXTRACT_FILE_LENGTH_A, Types.BIGINT), - new ColInfo(Cols.EXTRACT_FILE_LENGTH_B, Types.BIGINT)); + new ColInfo(Cols.EXTRACT_FILE_LENGTH_B, Types.BIGINT), + new ColInfo(Cols.PIPES_STATUS_A, Types.VARCHAR, PIPES_STATUS_MAX_LEN), new ColInfo(Cols.PIPES_MESSAGE_A, Types.VARCHAR, PIPES_MESSAGE_MAX_LEN), + new ColInfo(Cols.PIPES_STATUS_B, Types.VARCHAR, PIPES_STATUS_MAX_LEN), new ColInfo(Cols.PIPES_MESSAGE_B, Types.VARCHAR, PIPES_MESSAGE_MAX_LEN)); public static TableInfo CONTENT_COMPARISONS = new TableInfo("content_comparisons", new ColInfo(Cols.ID, Types.INTEGER, "PRIMARY KEY"), new ColInfo(Cols.TOP_10_UNIQUE_TOKEN_DIFFS_A, Types.VARCHAR, 1024), new ColInfo(Cols.TOP_10_UNIQUE_TOKEN_DIFFS_B, Types.VARCHAR, 1024), new ColInfo(Cols.TOP_10_MORE_IN_A, Types.VARCHAR, 1024), @@ -87,13 +90,22 @@ public class ExtractComparer extends ProfilerBase { private final Path extractsB; private final TokenContraster tokenContraster = new TokenContraster(); private final ExtractReader extractReader; + private final PipesReport pipesReportA; + private final PipesReport pipesReportB; public ExtractComparer(Path inputDir, Path extractsA, Path extractsB, ExtractReader extractReader, IDBWriter writer) { + this(inputDir, extractsA, extractsB, extractReader, writer, null, null); + } + + public ExtractComparer(Path inputDir, Path extractsA, Path extractsB, ExtractReader extractReader, IDBWriter writer, + PipesReport pipesReportA, PipesReport pipesReportB) { super(writer); this.inputDir = inputDir; this.extractsA = extractsA; this.extractsB = extractsB; this.extractReader = extractReader; + this.pipesReportA = pipesReportA; + this.pipesReportB = pipesReportB; } public static void USAGE() throws IOException { @@ -172,6 +184,8 @@ public class ExtractComparer extends ProfilerBase { long extractFileLengthB = getFileLength(fpsB.getExtractFile()); contData.put(Cols.EXTRACT_FILE_LENGTH_B, extractFileLengthB > NON_EXISTENT_FILE_LENGTH ? Long.toString(extractFileLengthB) : ""); + putPipesResult(contData, pipesReportA, fpsA, Cols.PIPES_STATUS_A, Cols.PIPES_MESSAGE_A); + putPipesResult(contData, pipesReportB, fpsB, Cols.PIPES_STATUS_B, Cols.PIPES_MESSAGE_B); writer.writeRow(COMPARISON_CONTAINERS, contData); if (extractExceptionA != null) { diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java index 9b48f06bf7..afd3ed58e2 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractComparerRunner.java @@ -29,6 +29,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; @@ -62,6 +63,7 @@ import org.apache.tika.eval.app.io.DBWriter; import org.apache.tika.eval.app.io.ExtractReader; import org.apache.tika.eval.app.io.ExtractReaderException; import org.apache.tika.eval.app.io.IDBWriter; +import org.apache.tika.eval.app.io.PipesReport; import org.apache.tika.eval.app.reports.ResultsReporter; import org.apache.tika.exception.TikaConfigException; import org.apache.tika.mime.MimeTypes; @@ -92,6 +94,14 @@ public class ExtractComparerRunner { .addOption(Option.builder("m").longOpt("maxExtractLength").hasArg().desc("maximum extract length").get()) .addOption(Option.builder("r").longOpt("report").desc("automatically run Report and tgz after Compare").get()) .addOption(Option.builder("rd").longOpt("reportsDir").hasArg().desc("directory for reports (default: 'reports')").get()) + .addOption(Option.builder("pa").longOpt("pipesReportA").hasArg() + .desc("optional: jsonl from tika-pipes' file-system-jsonl-reporter for A; default: <extractsA>/.run-info/crashes-*.jsonl").get()) + .addOption(Option.builder("pb").longOpt("pipesReportB").hasArg() + .desc("optional: same, for B").get()) + .addOption(Option.builder("ra").longOpt("runInfoA").hasArg() + .desc("optional: run-info json from run-batch.sh for A; default: <extractsA>/.run-info/run-info-*.json").get()) + .addOption(Option.builder("rb").longOpt("runInfoB").hasArg() + .desc("optional: same, for B").get()) .addOption(Option.builder("z").longOpt("gzip").desc("gzip the H2 db file (<db>.mv.db.gz) after Compare for transfer; requires -d").get()) ; } @@ -122,9 +132,26 @@ public class ExtractComparerRunner { evalConfig.setMaxExtractLength(Long.parseLong(commandLine.getOptionValue('m'))); } + Path prA = commandLine.hasOption("pa") ? Paths.get(commandLine.getOptionValue("pa")) : RunInfo.discoverPipesReport(extractsADir); + Path prB = commandLine.hasOption("pb") ? Paths.get(commandLine.getOptionValue("pb")) : RunInfo.discoverPipesReport(extractsBDir); + Path riA = commandLine.hasOption("ra") ? Paths.get(commandLine.getOptionValue("ra")) : RunInfo.discoverRunInfo(extractsADir); + Path riB = commandLine.hasOption("rb") ? Paths.get(commandLine.getOptionValue("rb")) : RunInfo.discoverRunInfo(extractsBDir); + PipesReport pipesReportA = prA == null ? null : PipesReport.load(prA); + PipesReport pipesReportB = prB == null ? null : PipesReport.load(prB); + Map<String, String> batchInfoA = riA == null ? Map.of() : RunInfo.loadBatch(riA); + Map<String, String> batchInfoB = riB == null ? Map.of() : RunInfo.loadBatch(riB); + RunInfo.checkRunId(batchInfoA, pipesReportA == null ? null : pipesReportA.getPath()); + RunInfo.checkRunId(batchInfoB, pipesReportB == null ? null : pipesReportB.getPath()); try { String jdbcString = getJdbcConnectionString(dbPath); - execute(inputDir, extractsADir, extractsBDir, jdbcString, evalConfig); + Map<String, String> runInfo = RunInfo.evalInfo(args, evalConfig, jdbcString, inputDir); + Map<String, String> runInfoA = new LinkedHashMap<>(batchInfoA); + runInfoA.putAll(RunInfo.pipesReportInfo(pipesReportA)); + runInfoA.putAll(RunInfo.extractsInfo(extractsADir)); + Map<String, String> runInfoB = new LinkedHashMap<>(batchInfoB); + runInfoB.putAll(RunInfo.pipesReportInfo(pipesReportB)); + runInfoB.putAll(RunInfo.extractsInfo(extractsBDir)); + execute(inputDir, extractsADir, extractsBDir, jdbcString, evalConfig, pipesReportA, pipesReportB, runInfo, runInfoA, runInfoB); if (commandLine.hasOption('r')) { String reportsDir = commandLine.getOptionValue("rd", "reports"); @@ -163,7 +190,9 @@ public class ExtractComparerRunner { } - private static void execute(Path inputDir, Path extractsA, Path extractsB, String dbPath, EvalConfig evalConfig) throws SQLException, IOException { + private static void execute(Path inputDir, Path extractsA, Path extractsB, String dbPath, EvalConfig evalConfig, PipesReport pipesReportA, + PipesReport pipesReportB, Map<String, String> runInfo, Map<String, String> runInfoA, Map<String, String> runInfoB) + throws SQLException, IOException { //parameterize this? if necesssary try { @@ -176,6 +205,12 @@ public class ExtractComparerRunner { ExtractComparerBuilder builder = new ExtractComparerBuilder(); MimeBuffer mimeBuffer = initTables(jdbcUtil, builder, dbPath, evalConfig); builder.populateRefTables(jdbcUtil, mimeBuffer); + // before workers start, so an aborted run still says what it was + IDBWriter runInfoWriter = builder.getDBWriter(List.of(RunInfo.RUN_INFO_TABLE, RunInfo.RUN_INFO_TABLE_A, RunInfo.RUN_INFO_TABLE_B), jdbcUtil, mimeBuffer); + RunInfo.write(runInfoWriter, RunInfo.RUN_INFO_TABLE, runInfo); + RunInfo.write(runInfoWriter, RunInfo.RUN_INFO_TABLE_A, runInfoA); + RunInfo.write(runInfoWriter, RunInfo.RUN_INFO_TABLE_B, runInfoB); + runInfoWriter.close(); AtomicInteger enqueued = new AtomicInteger(0); AtomicInteger processed = new AtomicInteger(0); @@ -195,7 +230,7 @@ public class ExtractComparerRunner { for (int i = 0; i < evalConfig.getNumWorkers(); i++) { ExtractReader extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.AS_IS, evalConfig.getMinExtractLength(), evalConfig.getMaxExtractLength()); ExtractComparer extractComparer = new ExtractComparer(inputDir, extractsA, extractsB, extractReader, - builder.getDBWriter(builder.getNonRefTableInfos(), jdbcUtil, mimeBuffer)); + builder.getDBWriter(builder.getNonRefTableInfos(), jdbcUtil, mimeBuffer), pipesReportA, pipesReportB); executorCompletionService.submit(new ComparerWorker(queue, extractComparer, processed)); } @@ -221,6 +256,8 @@ public class ExtractComparerRunner { } catch (ExecutionException e) { throw new RuntimeException(e); } finally { + RunInfo.writeEnd(runInfoWriter, RunInfo.RUN_INFO_TABLE); + runInfoWriter.close(); mimeBuffer.close(); executorService.shutdownNow(); try { @@ -379,6 +416,9 @@ public class ExtractComparerRunner { queue.put(PipesIterator.COMPLETED_SEMAPHORE); return COMPARER_WORKER_COMPLETED_VALUE; } + if (RunInfo.isRunInfoPath(t.getFetchKey().getFetchKey())) { + continue; + } extractComparer.processFileResource(t.getFetchKey()); processed.incrementAndGet(); } @@ -414,6 +454,9 @@ public class ExtractComparerRunner { tableInfosAandB.add(ExtractComparer.COMPARISON_CONTAINERS); tableInfosAandB.add(ExtractComparer.CONTENT_COMPARISONS); tableInfosAandB.add(ProfilerBase.MIME_TABLE); + tableInfosAandB.add(RunInfo.RUN_INFO_TABLE); + tableInfosAandB.add(RunInfo.RUN_INFO_TABLE_A); + tableInfosAandB.add(RunInfo.RUN_INFO_TABLE_B); List<TableInfo> refTableInfos = new ArrayList<>(); refTableInfos.add(ExtractComparer.REF_PAIR_NAMES); diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java index fca4c61473..92dcdd1bc1 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfileRunner.java @@ -27,6 +27,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ArrayBlockingQueue; @@ -56,6 +57,7 @@ import org.apache.tika.eval.app.io.DBWriter; import org.apache.tika.eval.app.io.ExtractReader; import org.apache.tika.eval.app.io.ExtractReaderException; import org.apache.tika.eval.app.io.IDBWriter; +import org.apache.tika.eval.app.io.PipesReport; import org.apache.tika.exception.TikaConfigException; import org.apache.tika.mime.MimeTypes; import org.apache.tika.pipes.api.FetchEmitTuple; @@ -82,6 +84,10 @@ public class ExtractProfileRunner { .addOption(Option.builder("c").longOpt("config").hasArg().desc("tika-eval json config file").get()) .addOption(Option.builder("n").longOpt("numWorkers").hasArg().desc("number of worker threads").get()) .addOption(Option.builder("m").longOpt("maxExtractLength").hasArg().desc("maximum extract length").get()) + .addOption(Option.builder("pr").longOpt("pipesReport").hasArg() + .desc("optional: jsonl from tika-pipes' file-system-jsonl-reporter; default: <extracts>/.run-info/crashes-*.jsonl").get()) + .addOption(Option.builder("ri").longOpt("runInfo").hasArg() + .desc("optional: run-info json written by run-batch.sh; default: <extracts>/.run-info/run-info-*.json").get()) ; } @@ -100,7 +106,17 @@ public class ExtractProfileRunner { if (commandLine.hasOption('m')) { evalConfig.setMaxExtractLength(Long.parseLong(commandLine.getOptionValue('m'))); } - execute(inputDir, extractsDir, jdbcString, evalConfig); + Path pipesReportPath = commandLine.hasOption("pr") ? Paths.get(commandLine.getOptionValue("pr")) : RunInfo.discoverPipesReport(extractsDir); + Path runInfoPath = commandLine.hasOption("ri") ? Paths.get(commandLine.getOptionValue("ri")) : RunInfo.discoverRunInfo(extractsDir); + PipesReport pipesReport = pipesReportPath == null ? null : PipesReport.load(pipesReportPath); + Map<String, String> batchInfo = runInfoPath == null ? Map.of() : RunInfo.loadBatch(runInfoPath); + RunInfo.checkRunId(batchInfo, pipesReport == null ? null : pipesReport.getPath()); + + Map<String, String> runInfo = new LinkedHashMap<>(RunInfo.evalInfo(args, evalConfig, jdbcString, inputDir)); + runInfo.putAll(batchInfo); + runInfo.putAll(RunInfo.pipesReportInfo(pipesReport)); + runInfo.putAll(RunInfo.extractsInfo(extractsDir)); + execute(inputDir, extractsDir, jdbcString, evalConfig, pipesReport, runInfo); } private static String getJdbcConnectionString(String dbPath) { @@ -113,7 +129,8 @@ public class ExtractProfileRunner { } - private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig) throws SQLException, IOException { + private static void execute(Path inputDir, Path extractsDir, String dbPath, EvalConfig evalConfig, PipesReport pipesReport, + Map<String, String> runInfo) throws SQLException, IOException { //parameterize this? if necesssary try { @@ -126,6 +143,10 @@ public class ExtractProfileRunner { ExtractProfilerBuilder builder = new ExtractProfilerBuilder(); MimeBuffer mimeBuffer = initTables(jdbcUtil, builder, dbPath, evalConfig); builder.populateRefTables(jdbcUtil, mimeBuffer); + // before workers start, so an aborted run still says what it was + IDBWriter runInfoWriter = builder.getDBWriter(List.of(RunInfo.RUN_INFO_TABLE), jdbcUtil, mimeBuffer); + RunInfo.write(runInfoWriter, RunInfo.RUN_INFO_TABLE, runInfo); + runInfoWriter.close(); AtomicInteger processed = new AtomicInteger(0); AtomicInteger activeWorkers = new AtomicInteger(evalConfig.getNumWorkers()); @@ -143,7 +164,8 @@ public class ExtractProfileRunner { executorCompletionService.submit(pipesIterator); for (int i = 0; i < evalConfig.getNumWorkers(); i++) { ExtractReader extractReader = new ExtractReader(ExtractReader.ALTER_METADATA_LIST.AS_IS, evalConfig.getMinExtractLength(), evalConfig.getMaxExtractLength()); - ExtractProfiler extractProfiler = new ExtractProfiler(inputDir, extractsDir, extractReader, builder.getDBWriter(builder.tableInfos, jdbcUtil, mimeBuffer)); + ExtractProfiler extractProfiler = new ExtractProfiler(inputDir, extractsDir, extractReader, builder.getDBWriter(builder.tableInfos, jdbcUtil, mimeBuffer), + pipesReport); executorCompletionService.submit(new ProfileWorker(queue, extractProfiler, processed)); } @@ -169,6 +191,8 @@ public class ExtractProfileRunner { } catch (ExecutionException e) { throw new RuntimeException(e); } finally { + RunInfo.writeEnd(runInfoWriter, RunInfo.RUN_INFO_TABLE); + runInfoWriter.close(); mimeBuffer.close(); executorService.shutdownNow(); } @@ -238,6 +262,9 @@ public class ExtractProfileRunner { queue.put(PipesIterator.COMPLETED_SEMAPHORE); return PROFILE_WORKER_COMPLETED_VALUE; } + if (RunInfo.isRunInfoPath(t.getFetchKey().getFetchKey())) { + continue; + } extractProfiler.processFileResource(t.getFetchKey()); processed.incrementAndGet(); } @@ -259,6 +286,7 @@ public class ExtractProfileRunner { tableInfos.add(ExtractProfiler.ENCODINGS_TABLE); tableInfos.add(ExtractProfiler.TAGS_TABLE); tableInfos.add(ExtractProfiler.EMBEDDED_FILE_PATH_TABLE); + tableInfos.add(RunInfo.RUN_INFO_TABLE); this.tableInfos = Collections.unmodifiableList(tableInfos); List<TableInfo> refTableInfos = new ArrayList<>(); diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java index c2dedff1d1..171124bc1c 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ExtractProfiler.java @@ -31,6 +31,7 @@ import org.apache.tika.eval.app.db.TableInfo; import org.apache.tika.eval.app.io.ExtractReader; import org.apache.tika.eval.app.io.ExtractReaderException; import org.apache.tika.eval.app.io.IDBWriter; +import org.apache.tika.eval.app.io.PipesReport; import org.apache.tika.eval.core.util.ContentTags; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; @@ -47,7 +48,8 @@ public class ExtractProfiler extends ProfilerBase { new ColInfo(Cols.SORT_STACK_TRACE, Types.VARCHAR, 8192), new ColInfo(Cols.PARSE_EXCEPTION_ID, Types.INTEGER)); public static TableInfo CONTAINER_TABLE = new TableInfo("containers", new ColInfo(Cols.CONTAINER_ID, Types.INTEGER, "PRIMARY KEY"), new ColInfo(Cols.FILE_PATH, Types.VARCHAR, FILE_PATH_MAX_LEN), - new ColInfo(Cols.LENGTH, Types.BIGINT), new ColInfo(Cols.EXTRACT_FILE_LENGTH, Types.BIGINT)); + new ColInfo(Cols.LENGTH, Types.BIGINT), new ColInfo(Cols.EXTRACT_FILE_LENGTH, Types.BIGINT), + new ColInfo(Cols.PIPES_STATUS, Types.VARCHAR, PIPES_STATUS_MAX_LEN), new ColInfo(Cols.PIPES_MESSAGE, Types.VARCHAR, PIPES_MESSAGE_MAX_LEN)); public static TableInfo PROFILE_TABLE = new TableInfo("profiles", new ColInfo(Cols.ID, Types.INTEGER, "PRIMARY KEY"), new ColInfo(Cols.CONTAINER_ID, Types.INTEGER), new ColInfo(Cols.FILE_NAME, Types.VARCHAR, 256), new ColInfo(Cols.MD5, Types.CHAR, 32), new ColInfo(Cols.LENGTH, Types.BIGINT), new ColInfo(Cols.IS_EMBEDDED, Types.BOOLEAN), new ColInfo(Cols.EMBEDDED_DEPTH, Types.INTEGER), new ColInfo(Cols.EMBEDDED_FILE_PATH, Types.VARCHAR, 1024), @@ -85,13 +87,18 @@ public class ExtractProfiler extends ProfilerBase { private final Path inputDir; private final Path extracts; private final ExtractReader extractReader; - + private final PipesReport pipesReport; ExtractProfiler(Path inputDir, Path extracts, ExtractReader extractReader, IDBWriter dbWriter) { + this(inputDir, extracts, extractReader, dbWriter, null); + } + + ExtractProfiler(Path inputDir, Path extracts, ExtractReader extractReader, IDBWriter dbWriter, PipesReport pipesReport) { super(dbWriter); this.inputDir = inputDir; this.extracts = extracts; this.extractReader = extractReader; + this.pipesReport = pipesReport; } @@ -128,6 +135,7 @@ public class ExtractProfiler extends ProfilerBase { if (fps.getExtractFileLength() > 0) { contOutput.put(Cols.EXTRACT_FILE_LENGTH, (fps.getExtractFile() == null) ? "" : Long.toString(fps.getExtractFileLength())); } + putPipesResult(contOutput, pipesReport, fps, Cols.PIPES_STATUS, Cols.PIPES_MESSAGE); try { writer.writeRow(CONTAINER_TABLE, contOutput); } catch (IOException e) { diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java index 3989b0732f..b51cd53a48 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/ProfilerBase.java @@ -45,6 +45,7 @@ import org.apache.tika.eval.app.db.Cols; import org.apache.tika.eval.app.db.TableInfo; import org.apache.tika.eval.app.io.ExtractReaderException; import org.apache.tika.eval.app.io.IDBWriter; +import org.apache.tika.eval.app.io.PipesReport; import org.apache.tika.eval.core.langid.LanguageIDWrapper; import org.apache.tika.eval.core.metadata.TikaEvalMetadataFilter; import org.apache.tika.eval.core.textstats.BasicTokenCountStatsCalculator; @@ -86,6 +87,8 @@ public abstract class ProfilerBase { protected static final AtomicInteger ID = new AtomicInteger(); static final long NON_EXISTENT_FILE_LENGTH = -1l; final static int FILE_PATH_MAX_LEN = 1024;//max len for varchar for file_path + final static int PIPES_STATUS_MAX_LEN = 64; + final static int PIPES_MESSAGE_MAX_LEN = 4096; //Container exception key from the 1.x branch; read-only lookup against legacy extract JSON, so a plain String key suffices. private static final String CONTAINER_EXCEPTION_1X = "X-TIKA" + ":EXCEPTION:runtime"; private static final Logger LOG = LoggerFactory.getLogger(ProfilerBase.class); @@ -362,6 +365,18 @@ public abstract class ProfilerBase { initAnalyzersAndTokenCounter(maxTokens, new LanguageIDWrapper()); } + protected static void putPipesResult(Map<Cols, String> row, PipesReport report, EvalFilePaths fps, Cols statusCol, Cols messageCol) { + if (report == null) { + return; + } + PipesReport.Row r = report.get(fps.getRelativeSourceFilePath()); + if (r == null) { + return; + } + row.put(statusCol, r.status()); + row.put(messageCol, r.message()); + } + protected void writeExtractException(TableInfo extractExceptionTable, String containerId, String filePath, ExtractReaderException.TYPE type) throws IOException { Map<Cols, String> data = new HashMap<>(); data.put(Cols.CONTAINER_ID, containerId); diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/RunInfo.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/RunInfo.java new file mode 100644 index 0000000000..3992f197db --- /dev/null +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/RunInfo.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.app; + +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.CodeSource; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.Types; +import java.time.Instant; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.tika.Tika; +import org.apache.tika.eval.app.db.ColInfo; +import org.apache.tika.eval.app.db.Cols; +import org.apache.tika.eval.app.db.TableInfo; +import org.apache.tika.eval.app.io.IDBWriter; +import org.apache.tika.eval.app.io.PipesReport; + +/** + * Provenance for an eval run: what tika-eval this is, what it was pointed at, and (via + * {@code --runInfo}) what batch run produced the extracts. Lands in {@code run_info} + * ({@code run_info_a}/{@code run_info_b} for the two sides of a comparison) as key/value rows. + */ +public class RunInfo { + + private static final Logger LOG = LoggerFactory.getLogger(RunInfo.class); + + public static final TableInfo RUN_INFO_TABLE = new TableInfo("run_info", + new ColInfo(Cols.RUN_KEY, Types.VARCHAR, 256), new ColInfo(Cols.RUN_VALUE, Types.VARCHAR, 4096)); + public static final TableInfo RUN_INFO_TABLE_A = new TableInfo("run_info_a", RUN_INFO_TABLE.getColInfos()); + public static final TableInfo RUN_INFO_TABLE_B = new TableInfo("run_info_b", RUN_INFO_TABLE.getColInfos()); + + public static final String BATCH_PREFIX = "batch."; + public static final String RUN_ID_KEY = BATCH_PREFIX + "run.id"; + public static final String PIPES_REPORT_PATH_KEY = "pipes_report.path"; + /** Written by run-batch.sh inside the extracts dir; skipped by the crawl and the fingerprint. */ + public static final String RUN_INFO_DIR = ".run-info"; + + private RunInfo() { + } + + /** + * The batch script leaves its files in {@code <extracts>/.run-info/}. Used when the CLI flags are + * absent; exactly one run-info there is required, since two means the extracts were written by two runs. + * @return run-info json path or null if the dir has none + */ + public static Path discoverRunInfo(Path extracts) throws IOException { + return discover(extracts, "run-info-", ".json"); + } + + public static Path discoverPipesReport(Path extracts) throws IOException { + return discover(extracts, "crashes-", ".jsonl"); + } + + private static Path discover(Path extracts, String prefix, String suffix) throws IOException { + Path dir = extracts.resolve(RUN_INFO_DIR); + if (!Files.isDirectory(dir)) { + return null; + } + List<Path> hits; + try (Stream<Path> s = Files.list(dir)) { + hits = s.filter(p -> p.getFileName().toString().startsWith(prefix) && p.getFileName().toString().endsWith(suffix)).sorted().toList(); + } + if (hits.size() > 1) { + throw new IllegalArgumentException(hits.size() + " " + prefix + "*" + suffix + " files in " + dir + + ": the extracts were written by more than one run. Pass the one you mean explicitly."); + } + return hits.isEmpty() ? null : hits.get(0); + } + + public static boolean isRunInfoPath(String relativePath) { + String p = PipesReport.normalize(relativePath); + return p.equals(RUN_INFO_DIR) || p.startsWith(RUN_INFO_DIR + "/"); + } + + /** Flattens the batch script's run-info json into dotted keys under {@code batch.}. */ + public static Map<String, String> loadBatch(Path json) throws IOException { + JsonNode root = new ObjectMapper().readTree(Files.readString(json, StandardCharsets.UTF_8)); + Map<String, String> m = new LinkedHashMap<>(); + flatten(BATCH_PREFIX, root, m); + m.put(BATCH_PREFIX + "run_info.path", json.toAbsolutePath().toString()); + return m; + } + + private static void flatten(String prefix, JsonNode n, Map<String, String> m) { + if (n.isObject()) { + for (Iterator<Map.Entry<String, JsonNode>> it = n.fields(); it.hasNext(); ) { + Map.Entry<String, JsonNode> e = it.next(); + flatten(prefix + e.getKey() + ".", e.getValue(), m); + } + } else { + m.put(prefix.substring(0, prefix.length() - 1), n.isValueNode() ? n.asText() : n.toString()); + } + } + + /** + * A pipes report produced by a different batch run than the run-info describes is the + * failure mode nobody can see after the fact; refuse it. The script names the ledger + * {@code crashes-<run.id>.jsonl}. + */ + public static void checkRunId(Map<String, String> batch, Path pipesReport) { + if (batch == null || pipesReport == null) { + return; + } + String runId = batch.get(RUN_ID_KEY); + if (runId != null && !pipesReport.getFileName().toString().contains(runId)) { + throw new IllegalArgumentException("run.id mismatch: run-info says '" + runId + "' but the pipes report is " + pipesReport); + } + } + + public static Map<String, String> pipesReportInfo(PipesReport report) { + Map<String, String> m = new LinkedHashMap<>(); + if (report == null) { + return m; + } + m.put(PIPES_REPORT_PATH_KEY, report.getPath().toAbsolutePath().toString()); + m.put("pipes_report.rows", Integer.toString(report.size())); + m.put("pipes_report.errors", Integer.toString(report.getErrors().size())); + if (!report.getErrors().isEmpty()) { + m.put("pipes_report.last_error", report.getErrors().get(report.getErrors().size() - 1)); + } + return m; + } + + /** Count and a sha256 over the sorted "relpath size" lines of the extract set. */ + public static Map<String, String> extractsInfo(Path extracts) throws IOException { + Map<String, String> m = new LinkedHashMap<>(); + m.put("extracts.path", extracts.toAbsolutePath().toString()); + long start = System.currentTimeMillis(); + long count = 0; + MessageDigest md = sha256(); + try (Stream<Path> s = Files.walk(extracts)) { + Iterator<Path> it = s.filter(Files::isRegularFile).map(extracts::relativize) + .filter(p -> !isRunInfoPath(p.toString())).sorted().iterator(); + while (it.hasNext()) { + Path p = it.next(); + md.update((PipesReport.normalize(p.toString()) + " " + Files.size(extracts.resolve(p)) + "\n").getBytes(StandardCharsets.UTF_8)); + count++; + } + } + m.put("extracts.count", Long.toString(count)); + m.put("extracts.fingerprint", HexFormat.of().formatHex(md.digest())); + LOG.info("fingerprinted {} extracts under {} in {} ms", count, extracts, System.currentTimeMillis() - start); + return m; + } + + public static Map<String, String> evalInfo(String[] args, EvalConfig config, String jdbcString, Path inputDir) { + Map<String, String> m = new LinkedHashMap<>(); + m.put("eval.tika_version", Tika.getString()); + m.put("eval.jar_sha256", ownJarSha256()); + m.put("eval.start", Instant.now().toString()); + m.put("eval.host", hostName()); + m.put("eval.user", System.getProperty("user.name")); + m.put("eval.java", System.getProperty("java.vendor") + " " + System.getProperty("java.version")); + m.put("eval.args", String.join(" ", args)); + m.put("eval.config", config.toString()); + m.put("db.path", jdbcString); + m.put("input.path", inputDir.toAbsolutePath().toString()); + return m; + } + + public static void write(IDBWriter writer, TableInfo table, Map<String, String> info) throws IOException { + for (Map.Entry<String, String> e : info.entrySet()) { + Map<Cols, String> row = new HashMap<>(); + row.put(Cols.RUN_KEY, e.getKey()); + row.put(Cols.RUN_VALUE, e.getValue()); + writer.writeRow(table, row); + } + } + + public static void writeEnd(IDBWriter writer, TableInfo table) throws IOException { + write(writer, table, Map.of("eval.end", Instant.now().toString())); + } + + private static String ownJarSha256() { + try { + CodeSource cs = RunInfo.class.getProtectionDomain().getCodeSource(); + if (cs == null || cs.getLocation() == null) { + return ""; + } + Path p = Path.of(cs.getLocation().toURI()); + if (!Files.isRegularFile(p)) { + return ""; // running from target/classes + } + MessageDigest md = sha256(); + try (InputStream is = Files.newInputStream(p)) { + byte[] buf = new byte[65536]; + int n; + while ((n = is.read(buf)) > 0) { + md.update(buf, 0, n); + } + } + return HexFormat.of().formatHex(md.digest()); + } catch (IOException | URISyntaxException | RuntimeException e) { + LOG.warn("couldn't hash own jar", e); + return ""; + } + } + + private static String hostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (IOException e) { + return ""; + } + } + + private static MessageDigest sha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/Cols.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/Cols.java index 9d3ea0ec1c..829e8c752c 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/Cols.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/db/Cols.java @@ -19,6 +19,10 @@ package org.apache.tika.eval.app.db; public enum Cols { //container table CONTAINER_ID, FILE_PATH, EXTRACT_FILE_LENGTH, + PIPES_STATUS, PIPES_MESSAGE, //from the batch run's jsonl reporter, joined on file_path + PIPES_STATUS_A, PIPES_MESSAGE_A, PIPES_STATUS_B, PIPES_MESSAGE_B, + //run_info table + RUN_KEY, RUN_VALUE, EXTRACT_FILE_LENGTH_A, //for comparisons EXTRACT_FILE_LENGTH_B, diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/PipesReport.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/PipesReport.java new file mode 100644 index 0000000000..6d7d57f926 --- /dev/null +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/io/PipesReport.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.app.io; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The per-tuple ledger written by tika-pipes' {@code file-system-jsonl-reporter} during a + * batch run, keyed by {@code FetchEmitTuple.id}. For the filesystem iterator the id is the + * source-relative path, so it joins to {@code containers.file_path}; both sides are + * normalized to '/' here. + */ +public class PipesReport { + + private static final Logger LOG = LoggerFactory.getLogger(PipesReport.class); + + public record Row(String status, String message, long elapsedMs) { + } + + private final Path path; + private final Map<String, Row> rows; + private final List<String> errors; + + private PipesReport(Path path, Map<String, Row> rows, List<String> errors) { + this.path = path; + this.rows = rows; + this.errors = errors; + } + + public static PipesReport load(Path path) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + Map<String, Row> rows = new HashMap<>(); + List<String> errors = new ArrayList<>(); + int lineNo = 0; + try (BufferedReader r = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { + String line; + while ((line = r.readLine()) != null) { + lineNo++; + if (line.isBlank()) { + continue; + } + JsonNode n; + try { + n = mapper.readTree(line); + } catch (IOException e) { + throw new IOException("bad json at " + path + ":" + lineNo, e); + } + if (n.hasNonNull("error")) { + errors.add(n.get("error").asText()); + continue; + } + if (!n.hasNonNull("id") || !n.hasNonNull("status")) { + throw new IOException("missing id/status at " + path + ":" + lineNo); + } + String id = normalize(n.get("id").asText()); + Row row = new Row(n.get("status").asText(), n.hasNonNull("message") ? n.get("message").asText() : null, + n.hasNonNull("elapsedMs") ? n.get("elapsedMs").asLong() : -1); + // a retried tuple reports more than once; the last word wins + rows.put(id, row); + } + } + LOG.info("loaded {} pipes report rows ({} pipeline error lines) from {}", rows.size(), errors.size(), path); + return new PipesReport(path, rows, errors); + } + + public static String normalize(String id) { + return id.replace('\\', '/'); + } + + public Row get(Path relativeSourcePath) { + return rows.get(normalize(relativeSourcePath.toString())); + } + + public Path getPath() { + return path; + } + + public int size() { + return rows.size(); + } + + /** Final {@code {"error":...}} lines: the pipeline died before the run finished. */ + public List<String> getErrors() { + return Collections.unmodifiableList(errors); + } +} diff --git a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/reports/MarkdownSummaryWriter.java b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/reports/MarkdownSummaryWriter.java index 75bc7d4d4b..d4bb8f54a1 100644 --- a/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/reports/MarkdownSummaryWriter.java +++ b/tika-eval/tika-eval-app/src/main/java/org/apache/tika/eval/app/reports/MarkdownSummaryWriter.java @@ -56,8 +56,10 @@ public class MarkdownSummaryWriter { try (BufferedWriter w = Files.newBufferedWriter(summaryPath)) { w.write("# Tika Eval Comparison Summary\n\n"); + writeRunInfo(c, w); writeOverview(c, w); writeExtractExceptionSummary(c, w); + writePipesCrashSummary(c, w); writeExceptionSummary(c, w); writeContentQualitySummary(c, w); writeOovComparison(c, w); @@ -76,6 +78,61 @@ public class MarkdownSummaryWriter { LOG.info("Wrote markdown summary to {}", summaryPath); } + private static void writeRunInfo(Connection c, BufferedWriter w) + throws IOException, SQLException { + w.write("## Run Info\n\n"); + w.write("### Eval\n\n"); + writeQueryAsTable(c, w, "select run_key as RUN_KEY, run_value as RUN_VALUE from run_info order by run_key"); + w.write("\n### Batch A\n\n"); + writeQueryAsTable(c, w, "select run_key as RUN_KEY, run_value as RUN_VALUE from run_info_a order by run_key"); + w.write("\n### Batch B\n\n"); + writeQueryAsTable(c, w, "select run_key as RUN_KEY, run_value as RUN_VALUE from run_info_b order by run_key"); + w.write("\n"); + } + + /** Splits NO_EXTRACT_FILE etc. by what the batch run's jsonl reporter said about the container. */ + private static void writePipesCrashSummary(Connection c, BufferedWriter w) + throws IOException, SQLException { + w.write("## Extract File Issues by Pipes Status\n\n"); + w.write("CRASH = OOM/TIMEOUT/UNSPECIFIED_CRASH recorded for this file; LOST_TO_WORKER_RESTART = " + + "shared-server crash while in flight, outcome unknown (retry once, alone); NO_PIPES_RECORD = " + + "reporter ran but has no line for this file; NO_PIPES_REPORT_SUPPLIED = no --pipesReport given.\n\n"); + for (String side : new String[]{"a", "b"}) { + w.write("### Extract " + side.toUpperCase(java.util.Locale.ROOT) + "\n\n"); + writeQueryAsTable(c, w, + "select classification as CLASSIFICATION, extract_exception_description as TYPE, count(1) as COUNT from (" + + "select t.extract_exception_description, " + pipesClassification(side) + + " from extract_exceptions_" + side + " e " + + "join containers c on c.container_id = e.container_id " + + "join ref_extract_exception_types t on t.extract_exception_id = e.extract_exception_id) " + + "group by classification, extract_exception_description order by COUNT desc"); + w.write("\n"); + } + w.write("### Crash status but extract present (success, status lost)\n\n"); + writeQueryAsTable(c, w, + "select 'A' as SIDE, c.file_path as FILE, c.pipes_status_a as STATUS from containers c " + + "where c.pipes_status_a in ('OOM', 'TIMEOUT', 'UNSPECIFIED_CRASH') " + + "and not exists (select 1 from extract_exceptions_a e where e.container_id = c.container_id) " + + "union all " + + "select 'B', c.file_path, c.pipes_status_b from containers c " + + "where c.pipes_status_b in ('OOM', 'TIMEOUT', 'UNSPECIFIED_CRASH') " + + "and not exists (select 1 from extract_exceptions_b e where e.container_id = c.container_id) " + + "order by 1, 2 limit " + TOP_N); + w.write("\n"); + } + + // keep in sync with the same case expression in comparison-reports.xml / profile-reports.xml + private static String pipesClassification(String side) { + String st = "c.pipes_status_" + side; + String msg = "c.pipes_message_" + side; + return "case when " + st + " in ('OOM', 'TIMEOUT') then 'CRASH' " + + "when " + st + " = 'UNSPECIFIED_CRASH' and " + msg + " like '%outcome unknown%' then 'LOST_TO_WORKER_RESTART' " + + "when " + st + " = 'UNSPECIFIED_CRASH' then 'CRASH' " + + "when " + st + " is not null then " + st + " " + + "when exists (select 1 from run_info_" + side + " where run_key = 'pipes_report.path') then 'NO_PIPES_RECORD' " + + "else 'NO_PIPES_REPORT_SUPPLIED' end as classification"; + } + private static void writeOverview(Connection c, BufferedWriter w) throws IOException, SQLException { w.write("## Overview\n\n"); diff --git a/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml b/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml index 9955413bc0..475140605b 100644 --- a/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml +++ b/tika-eval/tika-eval-app/src/main/resources/comparison-reports.xml @@ -1090,6 +1090,131 @@ on e.extract_exception_id=t.extract_exception_id </sql> </report> + + <!-- PIPES CRASH LEDGER (pipesReport A option) --> + <report reportName="Extract Exceptions by Pipes Status A" + reportFilename="exceptions/extract_exceptions_by_pipes_status_a.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select classification, extract_exception_description, count(1) cnt + from ( + select t.extract_exception_description, + case + when c.pipes_status_a in ('OOM', 'TIMEOUT') then 'CRASH' + when c.pipes_status_a = 'UNSPECIFIED_CRASH' and c.pipes_message_a like '%outcome unknown%' then 'LOST_TO_WORKER_RESTART' + when c.pipes_status_a = 'UNSPECIFIED_CRASH' then 'CRASH' + when c.pipes_status_a is not null then c.pipes_status_a + when exists (select 1 from run_info_a where run_key = 'pipes_report.path') then 'NO_PIPES_RECORD' + else 'NO_PIPES_REPORT_SUPPLIED' + end as classification + from extract_exceptions_a e + join containers c on c.container_id = e.container_id + join ref_extract_exception_types t on t.extract_exception_id = e.extract_exception_id + ) + group by classification, extract_exception_description + order by cnt desc + </sql> + </report> + + <report reportName="Extract Exceptions with Pipes Status Details A" + reportFilename="exceptions/extract_exceptions_pipes_details_a.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select c.file_path, t.extract_exception_description, c.pipes_status_a as pipes_status, + case + when c.pipes_status_a in ('OOM', 'TIMEOUT') then 'CRASH' + when c.pipes_status_a = 'UNSPECIFIED_CRASH' and c.pipes_message_a like '%outcome unknown%' then 'LOST_TO_WORKER_RESTART' + when c.pipes_status_a = 'UNSPECIFIED_CRASH' then 'CRASH' + when c.pipes_status_a is not null then c.pipes_status_a + when exists (select 1 from run_info_a where run_key = 'pipes_report.path') then 'NO_PIPES_RECORD' + else 'NO_PIPES_REPORT_SUPPLIED' + end as classification, + c.pipes_message_a as pipes_message + from extract_exceptions_a e + join containers c on c.container_id = e.container_id + join ref_extract_exception_types t on t.extract_exception_id = e.extract_exception_id + order by classification, c.file_path + </sql> + </report> + + <!-- a crash status with an extract present is a success whose status was lost, not a failure --> + <report reportName="Crash Status but Extract Present A" + reportFilename="exceptions/crash_status_extract_present_a.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select c.file_path, c.pipes_status_a as pipes_status, c.pipes_message_a as pipes_message + from containers c + where c.pipes_status_a in ('OOM', 'TIMEOUT', 'UNSPECIFIED_CRASH') + and not exists (select 1 from extract_exceptions_a e where e.container_id = c.container_id) + order by c.file_path + </sql> + </report> + + <!-- PIPES CRASH LEDGER (pipesReport B option) --> + <report reportName="Extract Exceptions by Pipes Status B" + reportFilename="exceptions/extract_exceptions_by_pipes_status_b.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select classification, extract_exception_description, count(1) cnt + from ( + select t.extract_exception_description, + case + when c.pipes_status_b in ('OOM', 'TIMEOUT') then 'CRASH' + when c.pipes_status_b = 'UNSPECIFIED_CRASH' and c.pipes_message_b like '%outcome unknown%' then 'LOST_TO_WORKER_RESTART' + when c.pipes_status_b = 'UNSPECIFIED_CRASH' then 'CRASH' + when c.pipes_status_b is not null then c.pipes_status_b + when exists (select 1 from run_info_b where run_key = 'pipes_report.path') then 'NO_PIPES_RECORD' + else 'NO_PIPES_REPORT_SUPPLIED' + end as classification + from extract_exceptions_b e + join containers c on c.container_id = e.container_id + join ref_extract_exception_types t on t.extract_exception_id = e.extract_exception_id + ) + group by classification, extract_exception_description + order by cnt desc + </sql> + </report> + + <report reportName="Extract Exceptions with Pipes Status Details B" + reportFilename="exceptions/extract_exceptions_pipes_details_b.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select c.file_path, t.extract_exception_description, c.pipes_status_b as pipes_status, + case + when c.pipes_status_b in ('OOM', 'TIMEOUT') then 'CRASH' + when c.pipes_status_b = 'UNSPECIFIED_CRASH' and c.pipes_message_b like '%outcome unknown%' then 'LOST_TO_WORKER_RESTART' + when c.pipes_status_b = 'UNSPECIFIED_CRASH' then 'CRASH' + when c.pipes_status_b is not null then c.pipes_status_b + when exists (select 1 from run_info_b where run_key = 'pipes_report.path') then 'NO_PIPES_RECORD' + else 'NO_PIPES_REPORT_SUPPLIED' + end as classification, + c.pipes_message_b as pipes_message + from extract_exceptions_b e + join containers c on c.container_id = e.container_id + join ref_extract_exception_types t on t.extract_exception_id = e.extract_exception_id + order by classification, c.file_path + </sql> + </report> + + <!-- a crash status with an extract present is a success whose status was lost, not a failure --> + <report reportName="Crash Status but Extract Present B" + reportFilename="exceptions/crash_status_extract_present_b.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select c.file_path, c.pipes_status_b as pipes_status, c.pipes_message_b as pipes_message + from containers c + where c.pipes_status_b in ('OOM', 'TIMEOUT', 'UNSPECIFIED_CRASH') + and not exists (select 1 from extract_exceptions_b e where e.container_id = c.container_id) + order by c.file_path + </sql> + </report> + <report reportName="fixedCatastrophicExtractExceptions" reportFilename="exceptions/fixed_catastrophic_exceptions_in_b.xlsx" format="xlsx" diff --git a/tika-eval/tika-eval-app/src/main/resources/profile-reports.xml b/tika-eval/tika-eval-app/src/main/resources/profile-reports.xml index f239e92ba1..13b6bc6c46 100644 --- a/tika-eval/tika-eval-app/src/main/resources/profile-reports.xml +++ b/tika-eval/tika-eval-app/src/main/resources/profile-reports.xml @@ -271,6 +271,69 @@ CONTAINER_LENGTH asc </sql> </report> + + <!-- PIPES CRASH LEDGER (pipesReport option) --> + <report reportName="Extract Exceptions by Pipes Status" + reportFilename="exceptions/extract_exceptions_by_pipes_status.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select classification, extract_exception_description, count(1) cnt + from ( + select t.extract_exception_description, + case + when c.pipes_status in ('OOM', 'TIMEOUT') then 'CRASH' + when c.pipes_status = 'UNSPECIFIED_CRASH' and c.pipes_message like '%outcome unknown%' then 'LOST_TO_WORKER_RESTART' + when c.pipes_status = 'UNSPECIFIED_CRASH' then 'CRASH' + when c.pipes_status is not null then c.pipes_status + when exists (select 1 from run_info where run_key = 'pipes_report.path') then 'NO_PIPES_RECORD' + else 'NO_PIPES_REPORT_SUPPLIED' + end as classification + from extract_exceptions e + join containers c on c.container_id = e.container_id + join ref_extract_exception_types t on t.extract_exception_id = e.extract_exception_id + ) + group by classification, extract_exception_description + order by cnt desc + </sql> + </report> + + <report reportName="Extract Exceptions with Pipes Status Details" + reportFilename="exceptions/extract_exceptions_pipes_details.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select c.file_path, t.extract_exception_description, c.pipes_status as pipes_status, + case + when c.pipes_status in ('OOM', 'TIMEOUT') then 'CRASH' + when c.pipes_status = 'UNSPECIFIED_CRASH' and c.pipes_message like '%outcome unknown%' then 'LOST_TO_WORKER_RESTART' + when c.pipes_status = 'UNSPECIFIED_CRASH' then 'CRASH' + when c.pipes_status is not null then c.pipes_status + when exists (select 1 from run_info where run_key = 'pipes_report.path') then 'NO_PIPES_RECORD' + else 'NO_PIPES_REPORT_SUPPLIED' + end as classification, + c.pipes_message as pipes_message + from extract_exceptions e + join containers c on c.container_id = e.container_id + join ref_extract_exception_types t on t.extract_exception_id = e.extract_exception_id + order by classification, c.file_path + </sql> + </report> + + <!-- a crash status with an extract present is a success whose status was lost, not a failure --> + <report reportName="Crash Status but Extract Present" + reportFilename="exceptions/crash_status_extract_present.xlsx" + format="xlsx" + includeSql="true"> + <sql> + select c.file_path, c.pipes_status as pipes_status, c.pipes_message as pipes_message + from containers c + where c.pipes_status in ('OOM', 'TIMEOUT', 'UNSPECIFIED_CRASH') + and not exists (select 1 from extract_exceptions e where e.container_id = c.container_id) + order by c.file_path + </sql> + </report> + <after> <!--<sql>drop index on x</sql> diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerBatchTest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerBatchTest.java new file mode 100644 index 0000000000..55342c6233 --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerBatchTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.app; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import org.apache.tika.eval.app.db.H2Util; + +/** End-to-end Compare + Report with the pipes crash ledger and run-info joined in. */ +public class ComparerBatchTest { + + private static Path DB_DIR; + private static Path DB; + private static Path REPORTS; + + @BeforeAll + public static void setUp() throws Exception { + DB_DIR = Files.createTempDirectory("comparer-batch-test"); + DB = DB_DIR.resolve("mydb"); + REPORTS = DB_DIR.resolve("reports"); + Path testDirs = Paths.get(ComparerBatchTest.class.getResource("/test-dirs").toURI()); + Path pr = testDirs.resolve("pipes-reports"); + ExtractComparerRunner.main(new String[]{ + "-i", testDirs.resolve("raw_input").toString(), + "-a", testDirs.resolve("extractsA").toString(), + "-b", testDirs.resolve("extractsB").toString(), + "-d", DB.toAbsolutePath().toString(), + "-pa", pr.resolve("crashes-run-a1.jsonl").toString(), + "-pb", pr.resolve("crashes-run-b1.jsonl").toString(), + "-ra", pr.resolve("run-info-run-a1.json").toString(), + "-rb", pr.resolve("run-info-run-b1.json").toString(), + "-r", "-rd", REPORTS.toString() + }); + } + + @AfterAll + public static void tearDown() throws Exception { + FileUtils.deleteDirectory(DB_DIR.toFile()); + } + + @Test + public void testPipesColumns() throws Exception { + assertEquals("OOM", one("select pipes_status_a from containers where file_path='file9_noextract.txt'")); + assertEquals("TIMEOUT", one("select pipes_status_b from containers where file_path='file9_noextract.txt'")); + assertNull(one("select pipes_status_a from containers where file_path='file2_attachANotB.doc'")); + assertTrue(one("select pipes_message_a from containers where file_path='file12_es.txt'").contains("outcome unknown")); + } + + @Test + public void testRunInfoTables() throws Exception { + assertEquals("run-a1", one("select run_value from run_info_a where run_key='batch.run.id'")); + assertEquals("run-b1", one("select run_value from run_info_b where run_key='batch.run.id'")); + assertEquals("16", one("select run_value from run_info_a where run_key='extracts.count'")); + assertEquals("1", one("select count(1) from run_info where run_key='eval.end'")); + } + + @Test + public void testReportsAndSummary() throws Exception { + assertTrue(Files.isRegularFile(REPORTS.resolve("exceptions/extract_exceptions_by_pipes_status_a.xlsx"))); + assertTrue(Files.isRegularFile(REPORTS.resolve("exceptions/crash_status_extract_present_b.xlsx"))); + String summary = Files.readString(REPORTS.resolve("summary.md"), StandardCharsets.UTF_8); + assertTrue(summary.contains("## Run Info"), summary); + assertTrue(summary.contains("| batch.run.id | run-a1 |"), summary); + assertTrue(summary.contains("| CRASH | NO_EXTRACT_FILE |"), summary); + // file12_es.txt has an extract in A but a crash status: success with status lost, not a failure + assertTrue(summary.contains("| A | file12_es.txt | UNSPECIFIED_CRASH |"), summary); + } + + private static String one(String sql) throws Exception { + try (Connection c = new H2Util(DB).getConnection(); Statement st = c.createStatement(); ResultSet rs = st.executeQuery(sql)) { + assertTrue(rs.next(), "no row for: " + sql); + return rs.getString(1); + } + } +} diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerOneSidedLedgerTest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerOneSidedLedgerTest.java new file mode 100644 index 0000000000..431d6b315e --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ComparerOneSidedLedgerTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.app; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +import org.apache.tika.eval.app.db.H2Util; + +/** A 3.x baseline has no jsonl ledger; B's is discovered from extractsB/.run-info without flags. */ +public class ComparerOneSidedLedgerTest { + + @Test + public void testOnlyB() throws Exception { + Path dir = Files.createTempDirectory("comparer-one-sided"); + try { + Path db = dir.resolve("db"); + Path reports = dir.resolve("reports"); + Path testDirs = Paths.get(getClass().getResource("/test-dirs").toURI()); + ExtractComparerRunner.main(new String[]{ + "-i", testDirs.resolve("raw_input").toString(), + "-a", testDirs.resolve("extractsA").toString(), + "-b", testDirs.resolve("extractsB").toString(), + "-d", db.toAbsolutePath().toString(), + "-r", "-rd", reports.toString() + }); + try (Connection c = new H2Util(db).getConnection(); Statement st = c.createStatement()) { + try (ResultSet rs = st.executeQuery("select pipes_status_a, pipes_status_b from containers where file_path='file9_noextract.txt'")) { + assertTrue(rs.next()); + assertNull(rs.getString(1)); + assertEquals("TIMEOUT", rs.getString(2)); + } + try (ResultSet rs = st.executeQuery("select run_value from run_info_b where run_key='batch.run.id'")) { + assertTrue(rs.next()); + assertEquals("run-b1", rs.getString(1)); + } + try (ResultSet rs = st.executeQuery("select run_value from run_info_b where run_key='extracts.count'")) { + assertTrue(rs.next()); + assertEquals("14", rs.getString(1), ".run-info excluded from the fingerprint walk"); + } + try (ResultSet rs = st.executeQuery("select count(1) from containers where file_path like '.run-info%'")) { + assertTrue(rs.next()); + assertEquals(0, rs.getInt(1), ".run-info skipped by the crawl"); + } + try (ResultSet rs = st.executeQuery("select count(1) from run_info_a")) { + assertTrue(rs.next()); + assertEquals(3, rs.getInt(1), "extracts.* only"); + } + } + String summary = Files.readString(reports.resolve("summary.md"), StandardCharsets.UTF_8); + assertTrue(summary.contains("| NO_PIPES_REPORT_SUPPLIED | NO_EXTRACT_FILE |"), summary); + assertTrue(summary.contains("| CRASH | NO_EXTRACT_FILE |"), summary); + } finally { + FileUtils.deleteDirectory(dir.toFile()); + } + } +} diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ProfilerBatchTest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ProfilerBatchTest.java index 9697abaf69..b21051afeb 100644 --- a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ProfilerBatchTest.java +++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/ProfilerBatchTest.java @@ -17,6 +17,7 @@ package org.apache.tika.eval.app; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -41,6 +42,7 @@ import org.junit.jupiter.api.Test; import org.apache.tika.eval.app.db.Cols; import org.apache.tika.eval.app.db.H2Util; import org.apache.tika.eval.app.db.TableInfo; +import org.apache.tika.eval.app.reports.ResultsReporter; public class ProfilerBatchTest { @@ -60,10 +62,13 @@ public class ProfilerBatchTest { .toURI()); DB = DB_DIR.resolve("mydb"); + Path pipesReports = extractsRoot.resolveSibling("pipes-reports"); String[] args = new String[]{ "-i", inputRoot.toAbsolutePath().toString(), "-e", extractsRoot.toAbsolutePath().toString(), - "-d", "jdbc:h2:file:" + DB.toAbsolutePath().toString() + "-d", "jdbc:h2:file:" + DB.toAbsolutePath().toString(), + "-pr", pipesReports.resolve("crashes-run-a1.jsonl").toString(), + "-ri", pipesReports.resolve("run-info-run-a1.json").toString() }; ExtractProfileRunner.main(args); @@ -124,6 +129,36 @@ public class ProfilerBatchTest { assertTrue(fNameList.contains("file9_noextract.txt"), "file9_noextract.txt"); } + @Test + public void testPipesStatusJoin() throws Exception { + assertEquals("OOM", getSingleResult("select pipes_status from containers where file_path='file9_noextract.txt'")); + assertEquals("EMIT_SUCCESS", getSingleResult("select pipes_status from containers where file_path='file1.pdf'")); + assertEquals(null, getSingleResult("select pipes_status from containers where file_path='file2_attachANotB.doc'")); + assertEquals("CRASH", getSingleResult( + "select case when c.pipes_status in ('OOM','TIMEOUT') then 'CRASH' else 'OTHER' end from extract_exceptions e " + + "join containers c on c.container_id = e.container_id where c.file_path='file9_noextract.txt'")); + } + + @Test + public void testRunInfo() throws Exception { + assertEquals("run-a1", getSingleResult("select run_value from run_info where run_key='batch.run.id'")); + assertEquals("16", getSingleResult("select run_value from run_info where run_key='extracts.count'")); + assertEquals("5", getSingleResult("select run_value from run_info where run_key='pipes_report.rows'")); + assertEquals("1", getSingleResult("select run_value from run_info where run_key='pipes_report.errors'")); + assertEquals("64", getSingleResult("select length(run_value) from run_info where run_key='extracts.fingerprint'")); + for (String k : new String[]{"eval.start", "eval.end", "eval.tika_version", "eval.args", "pipes_report.path"}) { + assertNotNull(getSingleResult("select run_value from run_info where run_key='" + k + "'"), k); + } + } + + @Test + public void testReportsRun() throws Exception { + Path reportsDir = DB_DIR.resolve("reports"); + ResultsReporter.main(new String[]{"-db", DB.toAbsolutePath().toString(), "-rd", reportsDir.toString()}); + assertTrue(Files.isRegularFile(reportsDir.resolve("exceptions/extract_exceptions_by_pipes_status.xlsx"))); + assertTrue(Files.isRegularFile(reportsDir.resolve("exceptions/crash_status_extract_present.xlsx"))); + } + @Test public void testExtractErrors() throws Exception { String sql = diff --git a/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/PipesReportTest.java b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/PipesReportTest.java new file mode 100644 index 0000000000..e030d47a5c --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/java/org/apache/tika/eval/app/io/PipesReportTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.tika.eval.app.io; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import org.apache.tika.eval.app.RunInfo; + +public class PipesReportTest { + + private static Path res(String name) throws Exception { + return Paths.get(PipesReportTest.class.getResource("/test-dirs/pipes-reports/" + name).toURI()); + } + + @Test + public void testLoadAndJoin() throws Exception { + PipesReport r = PipesReport.load(res("crashes-run-a1.jsonl")); + assertEquals(5, r.size()); + assertEquals(1, r.getErrors().size()); + assertTrue(r.getErrors().get(0).contains("pipeline died")); + assertEquals("OOM", r.get(Paths.get("file9_noextract.txt")).status()); + assertEquals("EMIT_SUCCESS", r.get(Paths.get("file1.pdf")).status()); + assertNull(r.get(Paths.get("file1.pdf")).message()); + assertNull(r.get(Paths.get("nope.pdf"))); + } + + @Test + public void testBackslashIdsJoinOnSlash() throws Exception { + PipesReport r = PipesReport.load(res("crashes-run-b1.jsonl")); + assertEquals("TIMEOUT", r.get(Paths.get("sub", "file9_noextract.txt")).status()); + assertEquals("TIMEOUT", r.get(Paths.get("sub\\file9_noextract.txt")).status()); + } + + @Test + public void testRunInfoFlattenAndRunId() throws Exception { + Map<String, String> batch = RunInfo.loadBatch(res("run-info-run-a1.json")); + assertEquals("run-a1", batch.get("batch.run.id")); + assertEquals("4.0.0", batch.get("batch.tika.version")); + assertEquals("[\"-Xmx4g\",\"-XX:+UseG1GC\"]", batch.get("batch.jvm.args")); + + RunInfo.checkRunId(batch, res("crashes-run-a1.jsonl")); + RunInfo.checkRunId(batch, null); + RunInfo.checkRunId(Map.of(), res("crashes-run-b1.jsonl")); + assertThrows(IllegalArgumentException.class, () -> RunInfo.checkRunId(batch, res("crashes-run-b1.jsonl"))); + } +} diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsB/.run-info/crashes-run-b1.jsonl b/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsB/.run-info/crashes-run-b1.jsonl new file mode 100644 index 0000000000..ee8b02a240 --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsB/.run-info/crashes-run-b1.jsonl @@ -0,0 +1,3 @@ +{"id":"file1.pdf","status":"EMIT_SUCCESS","category":"SUCCESS","message":null,"elapsedMs":10,"timestamp":"2026-08-27T11:00:00Z"} +{"id":"sub\\file9_noextract.txt","status":"TIMEOUT","category":"PROCESS_CRASH","message":"timed out","elapsedMs":60000,"timestamp":"2026-08-27T11:00:01Z"} +{"id":"file9_noextract.txt","status":"TIMEOUT","category":"PROCESS_CRASH","message":"timed out after 60000ms","elapsedMs":60000,"timestamp":"2026-08-27T11:01:01Z"} diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsB/.run-info/run-info-run-b1.json b/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsB/.run-info/run-info-run-b1.json new file mode 100644 index 0000000000..de404acc1b --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/extractsB/.run-info/run-info-run-b1.json @@ -0,0 +1,4 @@ +{ + "run": {"id": "run-b1", "note": "candidate", "start": "2026-08-27T10:59:00Z", "host": "box-b", "user": "tester"}, + "tika": {"app_path": "/opt/tika/tika-app-4.1.0-SNAPSHOT.jar", "app_sha256": "def456", "version": "4.1.0-SNAPSHOT"} +} diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/crashes-run-a1.jsonl b/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/crashes-run-a1.jsonl new file mode 100644 index 0000000000..8c4538f99a --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/crashes-run-a1.jsonl @@ -0,0 +1,6 @@ +{"id":"file1.pdf","status":"EMIT_SUCCESS","category":"SUCCESS","message":null,"elapsedMs":12,"timestamp":"2026-08-27T10:00:00Z"} +{"id":"file9_noextract.txt","status":"OOM","category":"PROCESS_CRASH","message":"java.lang.OutOfMemoryError: Java heap space","elapsedMs":3000,"timestamp":"2026-08-27T10:00:01Z"} +{"id":"file10_permahang.txt","status":"TIMEOUT","category":"PROCESS_CRASH","message":"timed out after 60000ms","elapsedMs":60000,"timestamp":"2026-08-27T10:01:01Z"} +{"id":"file11_oom.txt","status":"OOM","category":"PROCESS_CRASH","message":"java.lang.OutOfMemoryError: Java heap space","elapsedMs":900,"timestamp":"2026-08-27T10:01:02Z"} +{"id":"file12_es.txt","status":"UNSPECIFIED_CRASH","category":"PROCESS_CRASH","message":"shared server died while this request was in flight (generation 3); outcome unknown","elapsedMs":5,"timestamp":"2026-08-27T10:01:03Z"} +{"error":"java.lang.IllegalStateException: pipeline died","timestamp":"2026-08-27T10:01:04Z"} diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/crashes-run-b1.jsonl b/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/crashes-run-b1.jsonl new file mode 100644 index 0000000000..ee8b02a240 --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/crashes-run-b1.jsonl @@ -0,0 +1,3 @@ +{"id":"file1.pdf","status":"EMIT_SUCCESS","category":"SUCCESS","message":null,"elapsedMs":10,"timestamp":"2026-08-27T11:00:00Z"} +{"id":"sub\\file9_noextract.txt","status":"TIMEOUT","category":"PROCESS_CRASH","message":"timed out","elapsedMs":60000,"timestamp":"2026-08-27T11:00:01Z"} +{"id":"file9_noextract.txt","status":"TIMEOUT","category":"PROCESS_CRASH","message":"timed out after 60000ms","elapsedMs":60000,"timestamp":"2026-08-27T11:01:01Z"} diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/run-info-run-a1.json b/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/run-info-run-a1.json new file mode 100644 index 0000000000..c2d35bbbae --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/run-info-run-a1.json @@ -0,0 +1,6 @@ +{ + "run": {"id": "run-a1", "note": "baseline", "start": "2026-08-27T09:59:00Z", "host": "box-a", "user": "tester"}, + "tika": {"app_path": "/opt/tika/tika-app-4.0.0.jar", "app_sha256": "abc123", "version": "4.0.0", "git_commit": "deadbeef"}, + "config": {"path": "/opt/tika/tika-config.json", "sha256": "cfg123"}, + "jvm": {"version": "17.0.9", "args": ["-Xmx4g", "-XX:+UseG1GC"]} +} diff --git a/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/run-info-run-b1.json b/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/run-info-run-b1.json new file mode 100644 index 0000000000..de404acc1b --- /dev/null +++ b/tika-eval/tika-eval-app/src/test/resources/test-dirs/pipes-reports/run-info-run-b1.json @@ -0,0 +1,4 @@ +{ + "run": {"id": "run-b1", "note": "candidate", "start": "2026-08-27T10:59:00Z", "host": "box-b", "user": "tester"}, + "tika": {"app_path": "/opt/tika/tika-app-4.1.0-SNAPSHOT.jar", "app_sha256": "def456", "version": "4.1.0-SNAPSHOT"} +}
