This is an automated email from the ASF dual-hosted git repository.
vvivekiyer pushed a commit to branch hotfix-array-agg
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/hotfix-array-agg by this push:
new 316a5a7b19 Fix httpsSegmentFetcher to use the configurable timeouts
(#15485) (#15499)
316a5a7b19 is described below
commit 316a5a7b19d249afe9fb7eb7f95ce7a6b0a4967a
Author: Vivek Iyer Vaidyanathan <[email protected]>
AuthorDate: Tue Apr 8 13:16:11 2025 -0700
Fix httpsSegmentFetcher to use the configurable timeouts (#15485) (#15499)
---
.../pinot/common/utils/fetcher/HttpSegmentFetcher.java | 15 +++++++++++++--
.../common/utils/fetcher/HttpsSegmentFetcher.java | 1 +
.../common/utils/fetcher/HttpSegmentFetcherTest.java | 18 ++++++++++++++++--
3 files changed, 30 insertions(+), 4 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcher.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcher.java
index 6a77611aa6..010b43eb77 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcher.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcher.java
@@ -44,9 +44,9 @@ import org.apache.pinot.spi.utils.retry.RetryPolicies;
public class HttpSegmentFetcher extends BaseSegmentFetcher {
protected FileUploadDownloadClient _httpClient;
- private static final String CONNECTION_REQUEST_TIMEOUT_CONFIG_KEY =
+ public static final String CONNECTION_REQUEST_TIMEOUT_CONFIG_KEY =
"http.request.connectionRequestTimeoutMs";
- private static final String SOCKET_TIMEOUT_CONFIG_KEY =
"http.request.socketTimeoutMs";
+ public static final String SOCKET_TIMEOUT_CONFIG_KEY =
"http.request.socketTimeoutMs";
private int _connectionRequestTimeoutMs;
private int _socketTimeoutMs;
@@ -55,6 +55,17 @@ public class HttpSegmentFetcher extends BaseSegmentFetcher {
_httpClient = httpClient;
}
+ @VisibleForTesting
+ public int getConnectionRequestTimeoutMs() {
+ return _connectionRequestTimeoutMs;
+ }
+
+ @VisibleForTesting
+ public int getSocketTimeoutMs() {
+ return _socketTimeoutMs;
+ }
+
+
@Override
protected void doInit(PinotConfiguration config) {
if (_httpClient == null) {
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpsSegmentFetcher.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpsSegmentFetcher.java
index 71528595d8..e6b08532a7 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpsSegmentFetcher.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/fetcher/HttpsSegmentFetcher.java
@@ -71,5 +71,6 @@ public class HttpsSegmentFetcher extends HttpSegmentFetcher {
SSLContext sslContext = new
ClientSSLContextGenerator(sslConfig).generate();
_httpClient = new
FileUploadDownloadClient(HttpClientConfig.newBuilder(config).build(),
sslContext);
+ super.doInit(config);
}
}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcherTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcherTest.java
index 1a567901b9..c83036a633 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcherTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/fetcher/HttpSegmentFetcherTest.java
@@ -21,14 +21,17 @@ package org.apache.pinot.common.utils.fetcher;
import java.io.File;
import java.net.URI;
import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Supplier;
import org.apache.commons.io.FileUtils;
import org.apache.pinot.common.utils.FileUploadDownloadClient;
import org.apache.pinot.spi.env.PinotConfiguration;
import org.apache.pinot.spi.utils.retry.AttemptsExceededException;
+import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import static org.apache.pinot.common.utils.fetcher.HttpSegmentFetcher.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -46,12 +49,23 @@ public class HttpSegmentFetcherTest {
_fetcherConfig.setProperty(BaseSegmentFetcher.RETRY_COUNT_CONFIG_KEY, 3);
_fetcherConfig.setProperty(BaseSegmentFetcher.RETRY_WAIT_MS_CONFIG_KEY,
10);
_fetcherConfig.setProperty(BaseSegmentFetcher.RETRY_DELAY_SCALE_FACTOR_CONFIG_KEY,
1.1);
+ _fetcherConfig.setProperty(CONNECTION_REQUEST_TIMEOUT_CONFIG_KEY, 1000);
+ _fetcherConfig.setProperty(SOCKET_TIMEOUT_CONFIG_KEY, 1000);
}
private HttpSegmentFetcher getSegmentFetcher(FileUploadDownloadClient
client) {
- HttpSegmentFetcher segmentFetcher = new HttpSegmentFetcher();
- segmentFetcher.setHttpClient(client);
+ HttpSegmentFetcher segmentFetcher;
+ if (ThreadLocalRandom.current().nextBoolean()) {
+ segmentFetcher = new HttpsSegmentFetcher();
+ } else {
+ segmentFetcher = new HttpSegmentFetcher();
+ }
+
segmentFetcher.init(_fetcherConfig);
+ segmentFetcher.setHttpClient(client);
+
+ Assert.assertEquals(segmentFetcher.getSocketTimeoutMs(), 1000);
+ Assert.assertEquals(segmentFetcher.getConnectionRequestTimeoutMs(), 1000);
return segmentFetcher;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
