This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch branch_9_7
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9_7 by this push:
new 734f5330478 SOLR-17394: Check response status in IndexFetcher (#2621)
734f5330478 is described below
commit 734f53304786872e483189e566eef03b5687733c
Author: Jason Gerlowski <[email protected]>
AuthorDate: Thu Aug 8 08:32:32 2024 -0400
SOLR-17394: Check response status in IndexFetcher (#2621)
Prior to this commit, IndexFetcher would mis-read 404 responses from
other nodes. It eventually catches this mistake and retries, but not
before it wastefully allocates a massive (> 1GB) byte array.
These allocations were relatively infrequent, as they require out of
date leader information to occur. But they are big enough to impact
performance in production, and were observed to cause OOM's when running
tests.
This commit avoids this by checking the HTTP status code of all
IndexFetcher requests/responses, even those that utilize
InputStreamResponseParser.
---
solr/CHANGES.txt | 2 ++
.../java/org/apache/solr/handler/IndexFetcher.java | 29 +++++++++++++++++++---
2 files changed, 28 insertions(+), 3 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6902dfdae38..8af26f776d6 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -132,6 +132,8 @@ Bug Fixes
* SOLR-17337: Display all custom distributed stages in debug output. (Torsten
Bøgh Köster, Christine Poerschke)
+* SOLR-17394: Detect and handle non-200 HTTP status codes for requests made
by IndexFetcher (Jason Gerlowski)
+
Dependency Upgrades
---------------------
* PR#2512: Update dependency com.carrotsearch:hppc to v0.10.0 (solrbot)
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index 662a94943c9..d439f7196cd 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -71,6 +71,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@@ -1988,16 +1989,38 @@ public class IndexFetcher {
req.setBasePath(leaderBaseUrl);
if (useExternalCompression) req.addHeader("Accept-Encoding", "gzip");
response = solrClient.request(req, leaderCoreName);
+ final var responseStatus = (Integer) response.get("responseStatus");
is = (InputStream) response.get("stream");
+
+ if (responseStatus != 200) {
+ final var errorMsg =
+ String.format(
+ Locale.ROOT,
+ "Unexpected status code [%d] when downloading file [%s].",
+ responseStatus,
+ fileName);
+ closeStreamAndThrowIOE(is, errorMsg, Optional.empty());
+ }
+
if (useInternalCompression) {
is = new InflaterInputStream(is);
}
return new FastInputStream(is);
} catch (Exception e) {
- // close stream on error
- IOUtils.closeQuietly(is);
- throw new IOException("Could not download file '" + fileName + "'", e);
+ closeStreamAndThrowIOE(is, "Could not download file '" + fileName +
"'", Optional.of(e));
+ }
+
+ // Unreachable b/c closeStreamAndThrowIOE will always throw
+ return null;
+ }
+
+ private void closeStreamAndThrowIOE(
+ InputStream is, String exceptionMessage, Optional<Exception> e) throws
IOException {
+ IOUtils.closeQuietly(is);
+ if (e.isPresent()) {
+ throw new IOException(exceptionMessage, e.get());
}
+ throw new IOException(exceptionMessage);
}
}
