This is an automated email from the ASF dual-hosted git repository.
kharekartik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3062bd0e54 Allow configuring helix timeouts for EV dropped in Instance
manager (#10510)
3062bd0e54 is described below
commit 3062bd0e54723375157994f1921e526008f3748c
Author: Kartik Khare <[email protected]>
AuthorDate: Sat Apr 1 19:41:26 2023 +0530
Allow configuring helix timeouts for EV dropped in Instance manager (#10510)
* Allow configuring helix timeouts for data manager
* Refactor: rename method and cache configs
---------
Co-authored-by: Kartik Khare
<[email protected]>
---
.../starter/helix/HelixInstanceDataManager.java | 12 +++++++-----
.../starter/helix/HelixInstanceDataManagerConfig.java | 19 +++++++++++++++++++
.../config/instance/InstanceDataManagerConfig.java | 4 ++++
3 files changed, 30 insertions(+), 5 deletions(-)
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
index aabe2dd715..6806d16c10 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManager.java
@@ -80,9 +80,6 @@ import org.slf4j.LoggerFactory;
@ThreadSafe
public class HelixInstanceDataManager implements InstanceDataManager {
private static final Logger LOGGER =
LoggerFactory.getLogger(HelixInstanceDataManager.class);
- // TODO: Make this configurable
- private static final long EXTERNAL_VIEW_DROPPED_MAX_WAIT_MS = 20 * 60_000L;
// 20 minutes
- private static final long EXTERNAL_VIEW_CHECK_INTERVAL_MS = 1_000L; // 1
second
private final ConcurrentHashMap<String, TableDataManager>
_tableDataManagerMap = new ConcurrentHashMap<>();
@@ -93,6 +90,8 @@ public class HelixInstanceDataManager implements
InstanceDataManager {
private ZkHelixPropertyStore<ZNRecord> _propertyStore;
private SegmentUploader _segmentUploader;
private Supplier<Boolean> _isServerReadyToServeQueries = () -> false;
+ private long _externalViewDroppedMaxWaitMs;
+ private long _externalViewDroppedCheckInternalMs;
// Fixed size LRU cache for storing last N errors on the instance.
// Key is TableNameWithType-SegmentName pair.
@@ -116,6 +115,9 @@ public class HelixInstanceDataManager implements
InstanceDataManager {
_segmentUploader = new
PinotFSSegmentUploader(_instanceDataManagerConfig.getSegmentStoreUri(),
PinotFSSegmentUploader.DEFAULT_SEGMENT_UPLOAD_TIMEOUT_MILLIS);
+ _externalViewDroppedMaxWaitMs =
_instanceDataManagerConfig.getExternalViewDroppedMaxWaitMs();
+ _externalViewDroppedCheckInternalMs =
_instanceDataManagerConfig.getExternalViewDroppedCheckIntervalMs();
+
File instanceDataDir = new
File(_instanceDataManagerConfig.getInstanceDataDir());
initInstanceDataDir(instanceDataDir);
@@ -232,7 +234,7 @@ public class HelixInstanceDataManager implements
InstanceDataManager {
public void deleteTable(String tableNameWithType)
throws Exception {
// Wait externalview to converge
- long endTimeMs = System.currentTimeMillis() +
EXTERNAL_VIEW_DROPPED_MAX_WAIT_MS;
+ long endTimeMs = System.currentTimeMillis() +
_externalViewDroppedMaxWaitMs;
do {
ExternalView externalView = _helixManager.getHelixDataAccessor()
.getProperty(_helixManager.getHelixDataAccessor().keyBuilder().externalView(tableNameWithType));
@@ -249,7 +251,7 @@ public class HelixInstanceDataManager implements
InstanceDataManager {
});
return;
}
- Thread.sleep(EXTERNAL_VIEW_CHECK_INTERVAL_MS);
+ Thread.sleep(_externalViewDroppedCheckInternalMs);
} while (System.currentTimeMillis() < endTimeMs);
throw new TimeoutException(
"Timeout while waiting for ExternalView to converge for the table to
delete: " + tableNameWithType);
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
index 1cc1e07609..8f31fe1b0b 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/starter/helix/HelixInstanceDataManagerConfig.java
@@ -128,10 +128,17 @@ public class HelixInstanceDataManagerConfig implements
InstanceDataManagerConfig
private static final String DELETED_SEGMENTS_CACHE_TTL_MINUTES =
"table.deleted.segments.cache.ttl.minutes";
private static final String PEER_DOWNLOAD_SCHEME = "peer.download.scheme";
+ // Check if the external view is dropped for a table, and if so, wait for
the external view to
+ // be updated for a maximum of this time.
+ private static final String EXTERNAL_VIEW_DROPPED_MAX_WAIT_MS =
"external.view.dropped.max.wait.ms";
+ private static final String EXTERNAL_VIEW_DROPPED_CHECK_INTERVAL_MS =
"external.view.dropped.check.interval.ms";
+
private final static String[] REQUIRED_KEYS = {INSTANCE_ID};
private static final long DEFAULT_ERROR_CACHE_SIZE = 100L;
private static final int DEFAULT_DELETED_SEGMENTS_CACHE_SIZE = 10_000;
private static final int DEFAULT_DELETED_SEGMENTS_CACHE_TTL_MINUTES = 2;
+ public static final long DEFAULT_EXTERNAL_VIEW_DROPPED_MAX_WAIT_MS = 20 *
60_000L;
+ public static final long DEFAULT_EXTERNAL_VIEW_DROPPED_CHECK_INTERVAL_MS =
1_000L;
private final PinotConfiguration _instanceDataManagerConfiguration;
@@ -283,6 +290,18 @@ public class HelixInstanceDataManagerConfig implements
InstanceDataManagerConfig
return _instanceDataManagerConfiguration.getProperty(PEER_DOWNLOAD_SCHEME);
}
+ @Override
+ public long getExternalViewDroppedMaxWaitMs() {
+ return
_instanceDataManagerConfiguration.getProperty(EXTERNAL_VIEW_DROPPED_MAX_WAIT_MS,
+ DEFAULT_EXTERNAL_VIEW_DROPPED_MAX_WAIT_MS);
+ }
+
+ @Override
+ public long getExternalViewDroppedCheckIntervalMs() {
+ return
_instanceDataManagerConfiguration.getProperty(EXTERNAL_VIEW_DROPPED_CHECK_INTERVAL_MS,
+ DEFAULT_EXTERNAL_VIEW_DROPPED_CHECK_INTERVAL_MS);
+ }
+
@Override
public String toString() {
String configString = "";
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
index 9069ee7a57..4c380cb5b2 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/instance/InstanceDataManagerConfig.java
@@ -68,4 +68,8 @@ public interface InstanceDataManagerConfig {
int getDeletedSegmentsCacheTtlMinutes();
String getSegmentPeerDownloadScheme();
+
+ long getExternalViewDroppedMaxWaitMs();
+
+ long getExternalViewDroppedCheckIntervalMs();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
