This is an automated email from the ASF dual-hosted git repository. tballison pushed a commit to branch TIKA-4809-stage-6 in repository https://gitbox.apache.org/repos/asf/tika.git
commit 288651e8263ab82bd85bd37f9c9071d4fedee8f3 Author: tallison <[email protected]> AuthorDate: Mon Aug 10 10:47:17 2026 -0400 TIKA-4809: Derive numClients from cores when unset --- docs/modules/ROOT/pages/pipes/cpu-sizing.adoc | 18 ++++++++++++++++++ .../tika/pipes/core/PerClientServerManager.java | 9 ++++----- .../java/org/apache/tika/pipes/core/PipesConfig.java | 20 ++++++++++++++++++-- .../apache/tika/pipes/core/server/PipesServer.java | 19 +++++++++++++++++++ .../apache/tika/server/core/TikaServerProcess.java | 2 +- 5 files changed, 60 insertions(+), 8 deletions(-) diff --git a/docs/modules/ROOT/pages/pipes/cpu-sizing.adoc b/docs/modules/ROOT/pages/pipes/cpu-sizing.adoc index b1721b0cdb..6af1dbc253 100644 --- a/docs/modules/ROOT/pages/pipes/cpu-sizing.adoc +++ b/docs/modules/ROOT/pages/pipes/cpu-sizing.adoc @@ -173,6 +173,24 @@ not set `-Xmx` (or `-XX:MaxRAMPercentage`/`-XX:MaxRAMFraction`) yourself, Tika i `-XX:MaxRAMPercentage=75/numClients`, leaving the remainder for the parent JVM and the OS. The `pipes-cpu-sizing` summary line reports the decision as `heap=...`. +[IMPORTANT] +==== +**`numClients` is sized against CPU, not memory.** The rule above — +`numClients × 2 + 2 ≤ hostCores` — considers cores only. Memory is then divided among +however many workers that produced. On a host with many cores relative to its RAM, a +`numClients` that is correct for CPU can leave each fork with too little heap to parse +reliably. + +Tika cannot reconcile the two automatically: the parent sizes forks as a *percentage* of +memory and has no portable way to resolve that to bytes. Each forked JVM therefore checks +its own heap at startup and logs a `WARN` if it came up under 256 MB — the point below which +ordinary documents, not just pathological ones, begin to fail. If you see that warning, +lower `numClients`, raise the container memory limit, or set `-Xmx` explicitly. + +Cross-check both constraints yourself when sizing: `numClients × 2 + 2 ≤ hostCores` **and** +`numClients × per-worker-heap ≤ 75% of memory`. +==== + Set `-Xmx` explicitly when you know your workload: the auto-slice is a safe default, not a tuned one, and a fork that legitimately needs more than its slice will OOM where an untuned JVM might have grown into spare memory. diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PerClientServerManager.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PerClientServerManager.java index 073994271e..0424640a5b 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PerClientServerManager.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PerClientServerManager.java @@ -65,12 +65,10 @@ public class PerClientServerManager implements ServerManager { * formula could otherwise produce slice=1. */ private static final int MIN_AUTO_CAP_SLICE = 2; - /** Share of host/container memory the forks may collectively claim; the remainder - * is left for the parent JVM and the OS. */ + /** Share of host/container memory the forks may collectively claim; the remainder is + * left for the parent JVM, the OS, and page cache for spooled input. */ private static final int FORK_HEAP_BUDGET_PERCENT = 75; - /** Never hand a fork a smaller slice than this, however high numClients goes. */ - private static final int MIN_FORK_HEAP_PERCENT = 5; private static boolean userSetHeap(List<String> args) { return args.stream().anyMatch(a -> a.startsWith("-Xmx") @@ -79,9 +77,10 @@ public class PerClientServerManager implements ServerManager { } private static int forkHeapPercentage(int numClients) { - return Math.max(MIN_FORK_HEAP_PERCENT, FORK_HEAP_BUDGET_PERCENT / numClients); + return Math.max(1, FORK_HEAP_BUDGET_PERCENT / numClients); } + private final PipesConfig pipesConfig; private final Path tikaConfigPath; private final int clientId; diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesConfig.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesConfig.java index b3a42a368b..cbf44731e8 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesConfig.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/PipesConfig.java @@ -32,7 +32,23 @@ public class PipesConfig { public static final long DEFAULT_SHUTDOWN_CLIENT_AFTER_MILLS = 300000; - public static final int DEFAULT_NUM_CLIENTS = 4; + /** Past this, worker count becomes a memory decision, and memory is not visible here. */ + public static final int MAX_AUTO_NUM_CLIENTS = 4; + + private static final int PARENT_RESERVED_CORES = 2; + private static final int MIN_CORES_PER_CLIENT = 2; + + /** + * Worker count when the operator has not chosen one. CPU-derived, so the default + * satisfies Tika's own sizing rule on any host; a fixed 4 needs 10 cores and would + * warn about itself on smaller ones. Memory cannot participate -- no Java SE API + * exposes container memory -- so each fork checks its own heap at startup instead. + */ + public static int defaultNumClients() { + int hostCores = Runtime.getRuntime().availableProcessors(); + int byCores = (hostCores - PARENT_RESERVED_CORES) / MIN_CORES_PER_CLIENT; + return Math.max(1, Math.min(byCores, MAX_AUTO_NUM_CLIENTS)); + } public static final int DEFAULT_MAX_FILES_PROCESSED_PER_PROCESS = 10000; @@ -66,7 +82,7 @@ public class PipesConfig { private long heartbeatIntervalMs = DEFAULT_HEARTBEAT_INTERVAL_MS; private long shutdownClientAfterMillis = DEFAULT_SHUTDOWN_CLIENT_AFTER_MILLS; - private int numClients = DEFAULT_NUM_CLIENTS; + private int numClients = defaultNumClients(); private long maxWaitForClientMillis = DEFAULT_MAX_WAIT_FOR_CLIENT_MS; private int maxFilesProcessedPerProcess = DEFAULT_MAX_FILES_PROCESSED_PER_PROCESS; diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java index 5bd291cbba..15fee9cc58 100644 --- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java +++ b/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/server/PipesServer.java @@ -572,8 +572,27 @@ public class PipesServer implements AutoCloseable { LOG.info("watching parent pid {} for exit", parentPid); } + /** Below this, ordinary documents -- not just pathological ones -- start OOMing. */ + private static final long MIN_USABLE_HEAP_BYTES = 256L * 1024 * 1024; + + /** Checked here, not in the parent: the parent sizes forks by percentage and has no + * portable way to resolve that to bytes. The child knows what it actually got. */ + private static void checkUsableHeap() { + long maxHeapMb = Runtime.getRuntime().maxMemory() / (1024 * 1024); + LOG.info("forked JVM max heap: {} MB", maxHeapMb); + if (maxHeapMb < MIN_USABLE_HEAP_BYTES / (1024 * 1024)) { + LOG.warn("forked JVM max heap is {} MB, below the {} MB needed to parse " + + "reliably. Lower pipes.numClients, raise the container memory " + + "limit, or set -Xmx explicitly in forkedJvmArgs; otherwise " + + "ordinary documents will fail with OOM.", + maxHeapMb, MIN_USABLE_HEAP_BYTES / (1024 * 1024)); + } + } + protected void initializeResources() throws TikaException, IOException, SAXException { + checkUsableHeap(); + TikaJsonConfig tikaJsonConfig = tikaLoader.getConfig(); TikaPluginManager tikaPluginManager = TikaPluginManager.load(tikaJsonConfig); diff --git a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java index ac6473c37d..ab9ff3df14 100644 --- a/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java +++ b/tika-server/tika-server-core/src/main/java/org/apache/tika/server/core/TikaServerProcess.java @@ -642,7 +642,7 @@ public class TikaServerProcess { // Only set default pipes config if there's no existing config // This allows user-provided config to specify their own numClients, etc. if (existingConfigPath == null || !Files.exists(existingConfigPath)) { - builder.setPipesConfig(4, null); + builder.setPipesConfig(PipesConfig.defaultNumClients(), null); } // Add unpack emitter if /unpack endpoint is enabled
