This is an automated email from the ASF dual-hosted git repository. vinoth pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push: new fdae388 [HUDI-1203] add port configuration for EmbeddedTimelineService (#2142) fdae388 is described below commit fdae388626b8d97acc01191aa0e7075c36a41132 Author: lw0090 <lw309637...@gmail.com> AuthorDate: Tue Oct 6 02:36:54 2020 +0800 [HUDI-1203] add port configuration for EmbeddedTimelineService (#2142) --- .../apache/hudi/client/AbstractHoodieClient.java | 2 +- .../client/embedded/EmbeddedTimelineService.java | 6 ++-- .../org/apache/hudi/config/HoodieWriteConfig.java | 11 +++++++ .../hudi/timeline/service/TimelineService.java | 37 +++++++++++++++++++--- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java index 9628e41..7a59ebc 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java @@ -103,7 +103,7 @@ public abstract class AbstractHoodieClient implements Serializable, AutoCloseabl LOG.info("Starting Timeline service !!"); Option<String> hostAddr = context.getProperty(EngineProperty.EMBEDDED_SERVER_HOST); timelineServer = Option.of(new EmbeddedTimelineService(context, hostAddr.orElse(null), - config.getClientSpecifiedViewStorageConfig())); + config.getEmbeddedTimelineServerPort(), config.getClientSpecifiedViewStorageConfig())); try { timelineServer.get().startServer(); // Allow executor to find this newly instantiated timeline service diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java index ca0c713..738119c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java @@ -39,17 +39,19 @@ public class EmbeddedTimelineService { private static final Logger LOG = LogManager.getLogger(EmbeddedTimelineService.class); private int serverPort; + private int preferredPort; private String hostAddr; private final SerializableConfiguration hadoopConf; private final FileSystemViewStorageConfig config; private transient FileSystemViewManager viewManager; private transient TimelineService server; - public EmbeddedTimelineService(HoodieEngineContext context, String embeddedTimelineServiceHostAddr, FileSystemViewStorageConfig config) { + public EmbeddedTimelineService(HoodieEngineContext context, String embeddedTimelineServiceHostAddr, int embeddedTimelineServerPort, FileSystemViewStorageConfig config) { setHostAddr(embeddedTimelineServiceHostAddr); this.config = config; this.hadoopConf = context.getHadoopConf(); this.viewManager = createViewManager(); + this.preferredPort = embeddedTimelineServerPort; } private FileSystemViewManager createViewManager() { @@ -66,7 +68,7 @@ public class EmbeddedTimelineService { } public void startServer() throws IOException { - server = new TimelineService(0, viewManager, hadoopConf.newCopy()); + server = new TimelineService(preferredPort, viewManager, hadoopConf.newCopy()); serverPort = server.startService(); LOG.info("Started embedded timeline server at " + hostAddr + ":" + serverPort); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 60ca3c2..bb65600 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -99,6 +99,8 @@ public class HoodieWriteConfig extends DefaultHoodieConfig { public static final String EMBEDDED_TIMELINE_SERVER_ENABLED = "hoodie.embed.timeline.server"; public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_ENABLED = "true"; + public static final String EMBEDDED_TIMELINE_SERVER_PORT = "hoodie.embed.timeline.server.port"; + public static final String DEFAULT_EMBEDDED_TIMELINE_SERVER_PORT = "0"; public static final String FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP = "hoodie.fail.on.timeline.archiving"; public static final String DEFAULT_FAIL_ON_TIMELINE_ARCHIVING_ENABLED = "true"; @@ -255,6 +257,10 @@ public class HoodieWriteConfig extends DefaultHoodieConfig { return Boolean.parseBoolean(props.getProperty(EMBEDDED_TIMELINE_SERVER_ENABLED)); } + public int getEmbeddedTimelineServerPort() { + return Integer.parseInt(props.getProperty(EMBEDDED_TIMELINE_SERVER_PORT, DEFAULT_EMBEDDED_TIMELINE_SERVER_PORT)); + } + public boolean isFailOnTimelineArchivingEnabled() { return Boolean.parseBoolean(props.getProperty(FAIL_ON_TIMELINE_ARCHIVING_ENABLED_PROP)); } @@ -953,6 +959,11 @@ public class HoodieWriteConfig extends DefaultHoodieConfig { return this; } + public Builder withEmbeddedTimelineServerPort(int port) { + props.setProperty(EMBEDDED_TIMELINE_SERVER_PORT, String.valueOf(port)); + return this; + } + public Builder withBulkInsertSortMode(String mode) { props.setProperty(BULKINSERT_SORT_MODE, mode); return this; diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java index f38909f..969f824 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java @@ -41,6 +41,7 @@ import java.io.Serializable; public class TimelineService { private static final Logger LOG = LogManager.getLogger(TimelineService.class); + private static final int START_SERVICE_MAX_RETRIES = 16; private int serverPort; private Configuration conf; @@ -98,16 +99,42 @@ public class TimelineService { public Boolean help = false; } + private int startServiceOnPort(int port) throws IOException { + if (!(port == 0 || (1024 <= port && port < 65536))) { + throw new IllegalArgumentException(String.format("startPort should be between 1024 and 65535 (inclusive), " + + "or 0 for a random free port. but now is %s.", port)); + } + for (int attempt = 0; attempt < START_SERVICE_MAX_RETRIES; attempt++) { + // Returns port to try when trying to bind a service. Handles wrapping and skipping privileged ports. + int tryPort = port == 0 ? port : (port + attempt - 1024) % (65536 - 1024) + 1024; + try { + app.start(tryPort); + return app.port(); + } catch (Exception e) { + if (e.getMessage() != null && e.getMessage().contains("Failed to bind to")) { + if (tryPort == 0) { + LOG.warn("Timeline server could not bind on a random free port."); + } else { + LOG.warn(String.format("Timeline server could not bind on port %d. " + + "Attempting port %d + 1.",tryPort, tryPort)); + } + } else { + LOG.warn(String.format("Timeline server start failed on port %d. Attempting port %d + 1.",tryPort, tryPort), e); + } + } + } + throw new IOException(String.format("Timeline server start failed on port %d, after retry %d times", port, START_SERVICE_MAX_RETRIES)); + } + public int startService() throws IOException { app = Javalin.create(); FileSystemViewHandler router = new FileSystemViewHandler(app, conf, fsViewsManager); app.get("/", ctx -> ctx.result("Hello World")); router.register(); - app.start(serverPort); - // If port = 0, a dynamic port is assigned. Store it. - serverPort = app.port(); - LOG.info("Starting Timeline server on port :" + serverPort); - return serverPort; + int realServerPort = startServiceOnPort(serverPort); + LOG.info("Starting Timeline server on port :" + realServerPort); + this.serverPort = realServerPort; + return realServerPort; } public void run() throws IOException {