This is an automated email from the ASF dual-hosted git repository.
nagarwal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 218d4a6 [HUDI-1135] Make timeline server timeout settings
configurable.
218d4a6 is described below
commit 218d4a6836ce86216d289fa5fbb389871ad7ba0f
Author: Prashant Wason <pwa...@uber.com>
AuthorDate: Mon Aug 24 14:10:48 2020 -0700
[HUDI-1135] Make timeline server timeout settings configurable.
---
.../hudi/common/table/view/FileSystemViewManager.java | 5 +++--
.../common/table/view/FileSystemViewStorageConfig.java | 15 ++++++++++++++-
.../table/view/RemoteHoodieTableFileSystemView.java | 8 +++++++-
3 files changed, 24 insertions(+), 4 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index c4ab712..d310181 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -169,9 +169,10 @@ public class FileSystemViewManager {
private static RemoteHoodieTableFileSystemView
createRemoteFileSystemView(SerializableConfiguration conf,
FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
LOG.info("Creating remote view for basePath " + metaClient.getBasePath() +
". Server="
- + viewConf.getRemoteViewServerHost() + ":" +
viewConf.getRemoteViewServerPort());
+ + viewConf.getRemoteViewServerHost() + ":" +
viewConf.getRemoteViewServerPort() + ", Timeout="
+ + viewConf.getRemoteTimelineClientTimeoutSecs());
return new
RemoteHoodieTableFileSystemView(viewConf.getRemoteViewServerHost(),
viewConf.getRemoteViewServerPort(),
- metaClient);
+ metaClient, viewConf.getRemoteTimelineClientTimeoutSecs());
}
/**
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
index 434f873..603f88f 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java
@@ -44,6 +44,8 @@ public class FileSystemViewStorageConfig extends
DefaultHoodieConfig {
public static final String FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION =
"hoodie.filesystem.view.spillable.bootstrap.base.file.mem.fraction";
private static final String ROCKSDB_BASE_PATH_PROP =
"hoodie.filesystem.view.rocksdb.base.path";
+ public static final String FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS =
+ "hoodie.filesystem.view.remote.timeout.secs";
public static final FileSystemViewStorageType DEFAULT_VIEW_STORAGE_TYPE =
FileSystemViewStorageType.MEMORY;
public static final FileSystemViewStorageType
DEFAULT_SECONDARY_VIEW_STORAGE_TYPE = FileSystemViewStorageType.MEMORY;
@@ -52,7 +54,7 @@ public class FileSystemViewStorageConfig extends
DefaultHoodieConfig {
public static final String DEFAULT_FILESYSTEM_VIEW_INCREMENTAL_SYNC_MODE =
"false";
public static final String DEFUALT_REMOTE_VIEW_SERVER_HOST = "localhost";
public static final Integer DEFAULT_REMOTE_VIEW_SERVER_PORT = 26754;
-
+ public static final Integer DEFAULT_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS = 5
* 60; // 5 min
public static final String DEFAULT_VIEW_SPILLABLE_DIR = "/tmp/view_map/";
private static final Double DEFAULT_MEM_FRACTION_FOR_PENDING_COMPACTION =
0.01;
private static final Double DEFAULT_MEM_FRACTION_FOR_EXTERNAL_DATA_FILE =
0.05;
@@ -91,6 +93,10 @@ public class FileSystemViewStorageConfig extends
DefaultHoodieConfig {
return Integer.parseInt(props.getProperty(FILESYSTEM_VIEW_REMOTE_PORT));
}
+ public Integer getRemoteTimelineClientTimeoutSecs() {
+ return
Integer.parseInt(props.getProperty(FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS));
+ }
+
public long getMaxMemoryForFileGroupMap() {
long totalMemory =
Long.parseLong(props.getProperty(FILESYSTEM_VIEW_SPILLABLE_MEM));
return totalMemory - getMaxMemoryForPendingCompaction() -
getMaxMemoryForBootstrapBaseFile();
@@ -175,6 +181,11 @@ public class FileSystemViewStorageConfig extends
DefaultHoodieConfig {
return this;
}
+ public Builder withRemoteTimelineClientTimeoutSecs(Long
timelineClientTimeoutSecs) {
+ props.setProperty(FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS,
timelineClientTimeoutSecs.toString());
+ return this;
+ }
+
public Builder withMemFractionForPendingCompaction(Double
memFractionForPendingCompaction) {
props.setProperty(FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION,
memFractionForPendingCompaction.toString());
return this;
@@ -216,6 +227,8 @@ public class FileSystemViewStorageConfig extends
DefaultHoodieConfig {
DEFAULT_VIEW_SPILLABLE_DIR);
setDefaultOnCondition(props,
!props.containsKey(FILESYSTEM_VIEW_SPILLABLE_MEM),
FILESYSTEM_VIEW_SPILLABLE_MEM,
DEFAULT_MAX_MEMORY_FOR_VIEW.toString());
+ setDefaultOnCondition(props,
!props.containsKey(FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS),
+ FILESTYSTEM_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS,
DEFAULT_REMOTE_TIMELINE_CLIENT_TIMEOUT_SECS.toString());
setDefaultOnCondition(props,
!props.containsKey(FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION),
FILESYSTEM_VIEW_PENDING_COMPACTION_MEM_FRACTION,
DEFAULT_MEM_FRACTION_FOR_PENDING_COMPACTION.toString());
setDefaultOnCondition(props,
!props.containsKey(FILESYSTEM_VIEW_BOOTSTRAP_BASE_FILE_FRACTION),
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
index c99cc98..a0fe60c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java
@@ -115,6 +115,7 @@ public class RemoteHoodieTableFileSystemView implements
SyncableFileSystemView,
private final HoodieTableMetaClient metaClient;
private final HoodieTimeline timeline;
private final ObjectMapper mapper;
+ private final int timeoutSecs;
private boolean closed = false;
@@ -123,12 +124,17 @@ public class RemoteHoodieTableFileSystemView implements
SyncableFileSystemView,
}
public RemoteHoodieTableFileSystemView(String server, int port,
HoodieTableMetaClient metaClient) {
+ this(server, port, metaClient, 300);
+ }
+
+ public RemoteHoodieTableFileSystemView(String server, int port,
HoodieTableMetaClient metaClient, int timeoutSecs) {
this.basePath = metaClient.getBasePath();
this.serverHost = server;
this.serverPort = port;
this.mapper = new ObjectMapper();
this.metaClient = metaClient;
this.timeline =
metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
+ this.timeoutSecs = timeoutSecs;
}
private <T> T executeRequest(String requestPath, Map<String, String>
queryParameters, TypeReference reference,
@@ -147,7 +153,7 @@ public class RemoteHoodieTableFileSystemView implements
SyncableFileSystemView,
String url = builder.toString();
LOG.info("Sending request : (" + url + ")");
Response response;
- int timeout = 1000 * 300; // 5 min timeout
+ int timeout = this.timeoutSecs * 1000; // msec
switch (method) {
case GET:
response =
Request.Get(url).connectTimeout(timeout).socketTimeout(timeout).execute();
