hadoop git commit: YARN-7827. Stop and Delete Yarn Service from RM UI fails with HTTP ERROR 404. Contributed by Sunil G
Repository: hadoop Updated Branches: refs/heads/trunk 1bc03ddf9 -> ddec08d7c YARN-7827. Stop and Delete Yarn Service from RM UI fails with HTTP ERROR 404. Contributed by Sunil G Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddec08d7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddec08d7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddec08d7 Branch: refs/heads/trunk Commit: ddec08d7ccc8e43492fca2784203bd8af5e968cc Parents: 1bc03dd Author: Jian HeAuthored: Thu Feb 8 21:32:02 2018 -0800 Committer: Jian He Committed: Thu Feb 8 21:32:40 2018 -0800 -- .../src/main/webapp/app/adapters/yarn-servicedef.js | 9 ++--- .../src/main/webapp/app/components/deploy-service.js| 12 +--- .../src/main/webapp/app/controllers/yarn-app.js | 4 ++-- .../src/main/webapp/app/controllers/yarn-app/info.js| 4 ++-- .../main/webapp/app/controllers/yarn-deploy-service.js | 12 ++-- .../webapp/app/templates/components/deploy-service.hbs | 10 ++ .../src/main/webapp/app/templates/yarn-app.hbs | 4 ++-- .../src/main/webapp/app/templates/yarn-app/info.hbs | 4 ++-- .../src/main/webapp/app/utils/info-seeder.js| 3 ++- .../src/main/webapp/config/default-config.js| 2 +- 10 files changed, 42 insertions(+), 22 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddec08d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js index 3fb4a81..03685fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js @@ -24,21 +24,24 @@ export default RESTAbstractAdapter.extend({ restNameSpace: "dashService", serverName: "DASH", - deployService(request) { + deployService(request, user) { var url = this.buildURL(); +url += "/?user.name=" + user; return this.ajax(url, "POST", {data: request}); }, - stopService(serviceName) { + stopService(serviceName, user) { var url = this.buildURL(); url += "/" + serviceName; +url += "/?user.name=" + user; var data = {"state": "STOPPED", "name": serviceName}; return this.ajax(url, "PUT", {data: data}); }, - deleteService(serviceName) { + deleteService(serviceName, user) { var url = this.buildURL(); url += "/" + serviceName; +url += "/?user.name=" + user; return this.ajax(url, "DELETE", {data: {}}); } }); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddec08d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js index 90e10e5..36895d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js @@ -27,6 +27,7 @@ export default Ember.Component.extend({ customServiceDef: '', serviceResp: null, isLoading: false, + userName: '', actions: { showSaveTemplateModal() { @@ -36,11 +37,11 @@ export default Ember.Component.extend({ deployService() { this.set('serviceResp', null); if (this.get('isStandardViewType')) { -this.sendAction("deployServiceDef", this.get('serviceDef')); +this.sendAction("deployServiceDef", this.get('serviceDef'), this.get('userName')); } else { try { var parsed = JSON.parse(this.get('customServiceDef')); - this.sendAction("deployServiceJson", parsed); + this.sendAction("deployServiceJson", parsed, this.get('userName')); } catch (err) { this.set('serviceResp', {type: 'error', message: 'Invalid JSON: ' + err.message}); throw err; @@ -148,16 +149,21 @@ export default Ember.Component.extend({ isValidTemplateName: Ember.computed.notEmpty('savedTemplateName'), + isUserNameGiven: Ember.computed.empty('userName'), + isValidServiceDef: Ember.computed('serviceDef.name', 'serviceDef.queue', 'serviceDef.serviceComponents.[]', function () { return
hadoop git commit: YARN-5428. Allow for specifying the docker client configuration directory. Contributed by Shane Kumpf
Repository: hadoop Updated Branches: refs/heads/trunk 996796f10 -> eb2449d53 YARN-5428. Allow for specifying the docker client configuration directory. Contributed by Shane Kumpf Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb2449d5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb2449d5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb2449d5 Branch: refs/heads/trunk Commit: eb2449d5398e9ac869bc088e10d838a7f13deac0 Parents: 996796f Author: Jian HeAuthored: Wed Feb 7 10:59:38 2018 -0800 Committer: Jian He Committed: Thu Feb 8 11:35:30 2018 -0800 -- .../applications/distributedshell/Client.java | 38 +++- .../DockerCredentialTokenIdentifier.java| 159 .../yarn/util/DockerClientConfigHandler.java| 183 +++ .../src/main/proto/yarn_security_token.proto| 5 + ...apache.hadoop.security.token.TokenIdentifier | 1 + .../security/TestDockerClientConfigHandler.java | 129 + .../runtime/DockerLinuxContainerRuntime.java| 39 .../linux/runtime/docker/DockerCommand.java | 16 ++ .../runtime/TestDockerContainerRuntime.java | 109 +++ .../runtime/docker/TestDockerRunCommand.java| 8 + .../src/site/markdown/DockerContainers.md | 13 +- 11 files changed, 690 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb2449d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 2aafa94..0aef83f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -87,6 +87,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.DockerClientConfigHandler; import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; @@ -225,6 +226,9 @@ public class Client { private String flowVersion = null; private long flowRunId = 0L; + // Docker client configuration + private String dockerClientConfig = null; + // Command line options private Options opts; @@ -368,6 +372,10 @@ public class Client { "If container could retry, it specifies max retires"); opts.addOption("container_retry_interval", true, "Interval between each retry, unit is milliseconds"); +opts.addOption("docker_client_config", true, +"The docker client configuration path. The scheme should be supplied" ++ " (i.e. file:// or hdfs://)." ++ " Only used when the Docker runtime is enabled and requested."); opts.addOption("placement_spec", true, "Placement specification. Please note, if this option is specified," + " The \"num_containers\" option will be ignored. All requested" @@ -585,6 +593,9 @@ public class Client { "Flow run is not a valid long value", e); } } +if (cliParser.hasOption("docker_client_config")) { + dockerClientConfig = cliParser.getOptionValue("docker_client_config"); +} return true; } @@ -884,9 +895,10 @@ public class Client { // amContainer.setServiceData(serviceData); // Setup security tokens +Credentials rmCredentials = null; if (UserGroupInformation.isSecurityEnabled()) { // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce - Credentials credentials = new Credentials(); + rmCredentials = new Credentials(); String tokenRenewer = YarnClientUtils.getRmPrincipal(conf); if (tokenRenewer == null || tokenRenewer.length() == 0) { throw new IOException( @@ -895,16 +907,32 @@ public class Client { //
hadoop git commit: YARN-7889. Missing kerberos token when check for RM REST API availability. Contributed by Eric Yang
Repository: hadoop Updated Branches: refs/heads/trunk a196ee936 -> 87429affc YARN-7889. Missing kerberos token when check for RM REST API availability. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87429aff Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87429aff Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87429aff Branch: refs/heads/trunk Commit: 87429affcea2de61791c1b048b6db8c8275de051 Parents: a196ee9 Author: Jian HeAuthored: Mon Feb 5 22:55:56 2018 -0800 Committer: Jian He Committed: Mon Feb 5 22:56:25 2018 -0800 -- .../yarn/service/client/ApiServiceClient.java | 41 ++-- 1 file changed, 29 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/87429aff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index cb91946..49702e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -87,21 +87,38 @@ public class ApiServiceClient extends AppAdminClient { rmAddress = conf .get("yarn.resourcemanager.webapp.https.address"); } - +boolean useKerberos = UserGroupInformation.isSecurityEnabled(); List rmServers = RMHAUtils .getRMHAWebappAddresses(new YarnConfiguration(conf)); for (String host : rmServers) { - StringBuilder sb = new StringBuilder(); - sb.append(scheme); - sb.append(host); - sb.append(path); - Client client = Client.create(); - WebResource webResource = client - .resource(sb.toString()); - String test = webResource.get(String.class); - if (test.contains("hadoop_version")) { -rmAddress = host; -break; + try { +Client client = Client.create(); +StringBuilder sb = new StringBuilder(); +sb.append(scheme); +sb.append(host); +sb.append(path); +if (!useKerberos) { + try { +String username = UserGroupInformation.getCurrentUser().getShortUserName(); +sb.append("?user.name="); +sb.append(username); + } catch (IOException e) { +LOG.debug("Fail to resolve username: {}", e); + } +} +WebResource webResource = client +.resource(sb.toString()); +if (useKerberos) { + AuthenticatedURL.Token token = new AuthenticatedURL.Token(); + webResource.header("WWW-Authenticate", token); +} +ClientResponse test = webResource.get(ClientResponse.class); +if (test.getStatus() == 200) { + rmAddress = host; + break; +} + } catch (Exception e) { +LOG.debug("Fail to connect to: "+host, e); } } return scheme+rmAddress; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7831. YARN Service CLI should use hadoop.http.authentication.type to determine authentication method. Contributed by Eric Yang
Repository: hadoop Updated Branches: refs/heads/trunk 51cb6c538 -> b6e50fad5 YARN-7831. YARN Service CLI should use hadoop.http.authentication.type to determine authentication method. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6e50fad Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6e50fad Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6e50fad Branch: refs/heads/trunk Commit: b6e50fad53f26e2b718a85ec0678e3161decc691 Parents: 51cb6c5 Author: Jian HeAuthored: Fri Feb 2 14:37:58 2018 -0800 Committer: Jian He Committed: Fri Feb 2 14:38:33 2018 -0800 -- .../org/apache/hadoop/yarn/service/client/ApiServiceClient.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e50fad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 8c2edb5..cb91946 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -123,7 +123,8 @@ public class ApiServiceClient extends AppAdminClient { api.append("/"); api.append(appName); } -if (!UserGroupInformation.isSecurityEnabled()) { +Configuration conf = getConfig(); +if (conf.get("hadoop.http.authentication.type").equalsIgnoreCase("simple")) { api.append("?user.name=" + UrlEncoded .encodeString(System.getProperty("user.name"))); } @@ -147,7 +148,7 @@ public class ApiServiceClient extends AppAdminClient { client.setChunkedEncodingSize(null); Builder builder = client .resource(getApiUrl(appName)).type(MediaType.APPLICATION_JSON); -if (conf.get("hadoop.security.authentication").equals("kerberos")) { +if (conf.get("hadoop.http.authentication.type").equals("kerberos")) { AuthenticatedURL.Token token = new AuthenticatedURL.Token(); builder.header("WWW-Authenticate", token); } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7868. Provide improved error message when YARN service is disabled. Contributed by Eric Yang
Repository: hadoop Updated Branches: refs/heads/trunk 6e5ba9366 -> f9a4d4cf2 YARN-7868. Provide improved error message when YARN service is disabled. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9a4d4cf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9a4d4cf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9a4d4cf Branch: refs/heads/trunk Commit: f9a4d4cf237d64ccb52ab1792372c89e9d36f41d Parents: 6e5ba93 Author: Jian HeAuthored: Fri Feb 2 11:10:18 2018 -0800 Committer: Jian He Committed: Fri Feb 2 11:10:47 2018 -0800 -- .../org/apache/hadoop/yarn/service/client/ApiServiceClient.java | 4 1 file changed, 4 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9a4d4cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java index 34e62b6..8c2edb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java @@ -171,6 +171,10 @@ public class ApiServiceClient extends AppAdminClient { LOG.error("Authentication required"); return EXIT_EXCEPTION_THROWN; } +if (response.getStatus() == 503) { + LOG.error("YARN Service is unavailable or disabled."); + return EXIT_EXCEPTION_THROWN; +} try { ServiceStatus ss = response.getEntity(ServiceStatus.class); output = ss.getDiagnostics(); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S
Repository: hadoop Updated Branches: refs/heads/branch-2 312c57169 -> 0497e0933 YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0497e093 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0497e093 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0497e093 Branch: refs/heads/branch-2 Commit: 0497e09339415ffd6b6f84beeff7e1809647662c Parents: 312c571 Author: Jian HeAuthored: Sat Jan 27 22:18:51 2018 -0800 Committer: Jian He Committed: Sun Jan 28 13:24:49 2018 -0800 -- .../hadoop/yarn/server/nodemanager/NodeManager.java | 11 --- .../nodemanager/timelineservice/NMTimelinePublisher.java | 1 + .../timelineservice/storage/HBaseTimelineWriterImpl.java | 5 + 3 files changed, 10 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0497e093/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index c23c071..431a894 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -440,18 +440,15 @@ public class NodeManager extends CompositeService addService(nodeStatusUpdater); ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater); -super.serviceInit(conf); -// TODO add local dirs to del - } - - @Override - protected void serviceStart() throws Exception { +// Do secure login before calling init for added services. try { doSecureLogin(); } catch (IOException e) { throw new YarnRuntimeException("Failed NodeManager login", e); } -super.serviceStart(); + +super.serviceInit(conf); +// TODO add local dirs to del } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/0497e093/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index 2124c1a..52895c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -99,6 +99,7 @@ public class NMTimelinePublisher extends CompositeService { this.nmLoginUGI = UserGroupInformation.isSecurityEnabled() ? UserGroupInformation.getLoginUser() : UserGroupInformation.getCurrentUser(); +LOG.info("Initialized NMTimelinePublisher UGI to " + nmLoginUGI); super.serviceInit(conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0497e093/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java index 9e9134c..f938185 100644 ---
hadoop git commit: YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S
Repository: hadoop Updated Branches: refs/heads/branch-3.0 ec4f5f0f0 -> 331677936 YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33167793 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33167793 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33167793 Branch: refs/heads/branch-3.0 Commit: 33167793641857a1be3032ff5b8288f124bf92f7 Parents: ec4f5f0 Author: Jian HeAuthored: Sat Jan 27 22:18:51 2018 -0800 Committer: Jian He Committed: Sat Jan 27 22:20:49 2018 -0800 -- .../hadoop/yarn/server/nodemanager/NodeManager.java | 11 --- .../nodemanager/timelineservice/NMTimelinePublisher.java | 1 + .../timelineservice/storage/HBaseTimelineWriterImpl.java | 5 + 3 files changed, 10 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/33167793/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index a97b3f2..fc48450 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -437,18 +437,15 @@ public class NodeManager extends CompositeService addService(nodeStatusUpdater); ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater); -super.serviceInit(conf); -// TODO add local dirs to del - } - - @Override - protected void serviceStart() throws Exception { +// Do secure login before calling init for added services. try { doSecureLogin(); } catch (IOException e) { throw new YarnRuntimeException("Failed NodeManager login", e); } -super.serviceStart(); + +super.serviceInit(conf); +// TODO add local dirs to del } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/33167793/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index b8192ca..2ce3c5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -100,6 +100,7 @@ public class NMTimelinePublisher extends CompositeService { this.nmLoginUGI = UserGroupInformation.isSecurityEnabled() ? UserGroupInformation.getLoginUser() : UserGroupInformation.getCurrentUser(); +LOG.info("Initialized NMTimelinePublisher UGI to " + nmLoginUGI); super.serviceInit(conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/33167793/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java index 9e9134c..f938185 100644 ---
hadoop git commit: YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S
Repository: hadoop Updated Branches: refs/heads/trunk 649ef7ac3 -> 443523f9c YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/443523f9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/443523f9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/443523f9 Branch: refs/heads/trunk Commit: 443523f9c0fcc4ba2503791090b1879c6031759b Parents: 649ef7a Author: Jian HeAuthored: Sat Jan 27 22:18:51 2018 -0800 Committer: Jian He Committed: Sat Jan 27 22:18:51 2018 -0800 -- .../hadoop/yarn/server/nodemanager/NodeManager.java | 11 --- .../nodemanager/timelineservice/NMTimelinePublisher.java | 1 + .../timelineservice/storage/HBaseTimelineWriterImpl.java | 5 + 3 files changed, 10 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 5cacd20..42b7b5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -454,18 +454,15 @@ public class NodeManager extends CompositeService ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater); nmStore.setNodeStatusUpdater(nodeStatusUpdater); -super.serviceInit(conf); -// TODO add local dirs to del - } - - @Override - protected void serviceStart() throws Exception { +// Do secure login before calling init for added services. try { doSecureLogin(); } catch (IOException e) { throw new YarnRuntimeException("Failed NodeManager login", e); } -super.serviceStart(); + +super.serviceInit(conf); +// TODO add local dirs to del } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java index b8192ca..2ce3c5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java @@ -100,6 +100,7 @@ public class NMTimelinePublisher extends CompositeService { this.nmLoginUGI = UserGroupInformation.isSecurityEnabled() ? UserGroupInformation.getLoginUser() : UserGroupInformation.getCurrentUser(); +LOG.info("Initialized NMTimelinePublisher UGI to " + nmLoginUGI); super.serviceInit(conf); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java index 9e9134c..f938185 100644 ---
hadoop git commit: YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan
Repository: hadoop Updated Branches: refs/heads/branch-2.9 258fb117d -> cbaf6d242 YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbaf6d24 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbaf6d24 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbaf6d24 Branch: refs/heads/branch-2.9 Commit: cbaf6d242d5b8c878cfd23f4dd4c438a40e967b4 Parents: 258fb11 Author: Jian HeAuthored: Wed Jan 24 11:16:54 2018 -0800 Committer: Jian He Committed: Wed Jan 24 16:18:36 2018 -0800 -- .../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbaf6d24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index c3ddc54..27742e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer { } sb.setLength(sb.length() - 1); params.put(AmIpFilter.PROXY_URI_BASES, sb.toString()); -container.addFilter(FILTER_NAME, FILTER_CLASS, params); // Handle RM HA urls // Include yarn-site.xml in the classpath @@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer { params.put(RM_HA_URLS, StringUtils.join(",", urls)); } } + +container.addFilter(FILTER_NAME, FILTER_CLASS, params); } private Collection getRmIds(Configuration conf) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan
Repository: hadoop Updated Branches: refs/heads/branch-3.0 ecfdd01be -> 7598f7332 YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7598f733 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7598f733 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7598f733 Branch: refs/heads/branch-3.0 Commit: 7598f7332042f5c2733e191567f1b1420f1e390a Parents: ecfdd01 Author: Jian HeAuthored: Wed Jan 24 11:16:54 2018 -0800 Committer: Jian He Committed: Wed Jan 24 11:20:23 2018 -0800 -- .../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7598f733/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index c3ddc54..27742e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer { } sb.setLength(sb.length() - 1); params.put(AmIpFilter.PROXY_URI_BASES, sb.toString()); -container.addFilter(FILTER_NAME, FILTER_CLASS, params); // Handle RM HA urls // Include yarn-site.xml in the classpath @@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer { params.put(RM_HA_URLS, StringUtils.join(",", urls)); } } + +container.addFilter(FILTER_NAME, FILTER_CLASS, params); } private Collection getRmIds(Configuration conf) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan
Repository: hadoop Updated Branches: refs/heads/branch-2 ce43e93f9 -> c611707e2 YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c611707e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c611707e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c611707e Branch: refs/heads/branch-2 Commit: c611707e28c7094ff0c8a9558db1c7d04bbc8b6d Parents: ce43e93 Author: Jian HeAuthored: Wed Jan 24 11:16:54 2018 -0800 Committer: Jian He Committed: Wed Jan 24 11:19:46 2018 -0800 -- .../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c611707e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index c3ddc54..27742e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer { } sb.setLength(sb.length() - 1); params.put(AmIpFilter.PROXY_URI_BASES, sb.toString()); -container.addFilter(FILTER_NAME, FILTER_CLASS, params); // Handle RM HA urls // Include yarn-site.xml in the classpath @@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer { params.put(RM_HA_URLS, StringUtils.join(",", urls)); } } + +container.addFilter(FILTER_NAME, FILTER_CLASS, params); } private Collection getRmIds(Configuration conf) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan
Repository: hadoop Updated Branches: refs/heads/trunk 97607617c -> 55c32776b YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55c32776 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55c32776 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55c32776 Branch: refs/heads/trunk Commit: 55c32776b1f130c6b81ef7aff5fc1562180f4601 Parents: 9760761 Author: Jian HeAuthored: Wed Jan 24 11:16:54 2018 -0800 Committer: Jian He Committed: Wed Jan 24 11:17:12 2018 -0800 -- .../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c32776/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java index c3ddc54..27742e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java @@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer { } sb.setLength(sb.length() - 1); params.put(AmIpFilter.PROXY_URI_BASES, sb.toString()); -container.addFilter(FILTER_NAME, FILTER_CLASS, params); // Handle RM HA urls // Include yarn-site.xml in the classpath @@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer { params.put(RM_HA_URLS, StringUtils.join(",", urls)); } } + +container.addFilter(FILTER_NAME, FILTER_CLASS, params); } private Collection getRmIds(Configuration conf) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang
Repository: hadoop Updated Branches: refs/heads/branch-2 ab482ebe7 -> a80c2dbab YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a80c2dba Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a80c2dba Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a80c2dba Branch: refs/heads/branch-2 Commit: a80c2dbab5ddfe71aa66f529629fa9964c7689b6 Parents: ab482eb Author: Jian HeAuthored: Tue Jan 23 14:03:53 2018 -0800 Committer: Jian He Committed: Tue Jan 23 14:12:00 2018 -0800 -- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a80c2dba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 905f4da..a975a5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1667,7 +1667,7 @@ Property to enable docker user remapping yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed -false +true - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang
Repository: hadoop Updated Branches: refs/heads/branch-2.9 93220f90e -> 6da08837d YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6da08837 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6da08837 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6da08837 Branch: refs/heads/branch-2.9 Commit: 6da08837da9bb0f409626b41c9583d6d15da9a5b Parents: 93220f9 Author: Jian HeAuthored: Tue Jan 23 14:03:53 2018 -0800 Committer: Jian He Committed: Tue Jan 23 14:07:41 2018 -0800 -- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da08837/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 71dd72a..ce8c0f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1659,7 +1659,7 @@ Property to enable docker user remapping yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed -false +true - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang
Repository: hadoop Updated Branches: refs/heads/branch-3.0 1de071257 -> 92f789a13 YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92f789a1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92f789a1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92f789a1 Branch: refs/heads/branch-3.0 Commit: 92f789a131d8ae5ee46302f3c88554e7314f516f Parents: 1de0712 Author: Jian HeAuthored: Tue Jan 23 14:03:53 2018 -0800 Committer: Jian He Committed: Tue Jan 23 14:06:48 2018 -0800 -- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/92f789a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 4e26be6..f3a9b09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1656,7 +1656,7 @@ Property to enable docker user remapping yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed -false +true - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang
Repository: hadoop Updated Branches: refs/heads/trunk a72cdcc47 -> 39b999aba YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39b999ab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39b999ab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39b999ab Branch: refs/heads/trunk Commit: 39b999aba2c1a1fc1df17bdf952834ba516f3b2a Parents: a72cdcc Author: Jian HeAuthored: Tue Jan 23 14:03:53 2018 -0800 Committer: Jian He Committed: Tue Jan 23 14:03:53 2018 -0800 -- .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/39b999ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a9938c3..45aa635 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1688,7 +1688,7 @@ Property to enable docker user remapping yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed -false +true - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7766. Introduce a new config property for YARN Service dependency tarball location. Contributed by Gour Saha
Repository: hadoop Updated Branches: refs/heads/trunk f63d13f10 -> a72cdcc47 YARN-7766. Introduce a new config property for YARN Service dependency tarball location. Contributed by Gour Saha Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a72cdcc4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a72cdcc4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a72cdcc4 Branch: refs/heads/trunk Commit: a72cdcc47a1b3a4fcadbc9e12bd1bae2bbf04d83 Parents: f63d13f Author: Jian HeAuthored: Tue Jan 23 10:53:27 2018 -0800 Committer: Jian He Committed: Tue Jan 23 10:53:27 2018 -0800 -- .../yarn/service/client/ServiceClient.java | 35 +++- .../yarn/service/conf/YarnServiceConf.java | 6 .../tarball/TarballProviderService.java | 4 +-- .../yarn/service/utils/CoreFileSystem.java | 32 -- .../hadoop/yarn/client/api/AppAdminClient.java | 12 --- .../hadoop/yarn/client/cli/ApplicationCLI.java | 13 +--- .../hadoop/yarn/client/cli/TestYarnCLI.java | 5 ++- 7 files changed, 68 insertions(+), 39 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index c224089..b8034b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.client.api.AppAdminClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.client.cli.ApplicationCLI; import org.apache.hadoop.yarn.client.util.YarnClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -718,14 +719,20 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, libPath, "lib", false); Path dependencyLibTarGzip = fs.getDependencyTarGzip(); if (fs.isFile(dependencyLibTarGzip)) { - LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/" - + dependencyLibTarGzip); + LOG.info("Loading lib tar from " + dependencyLibTarGzip); fs.submitTarGzipAndUpdate(localResources); } else { + if (dependencyLibTarGzip != null) { +LOG.warn("Property {} has a value {}, but is not a valid file", +YarnServiceConf.DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip); + } String[] libs = ServiceUtils.getLibDirs(); - LOG.info("Uploading all dependency jars to HDFS. For faster submission of" + - " apps, pre-upload dependency jars to HDFS " - + "using command: yarn app -enableFastLaunch"); + LOG.info("Uploading all dependency jars to HDFS. For faster submission of" + + " apps, set config property {} to the dependency tarball location." + + " Dependency tarball can be uploaded to any HDFS path directly" + + " or by using command: yarn app -{} []", + YarnServiceConf.DEPENDENCY_TARBALL_PATH, + ApplicationCLI.ENABLE_FAST_LAUNCH); for (String libDirProp : libs) { ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib", libDirProp); @@ -988,16 +995,23 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, return this.yarnClient; } - public int enableFastLaunch() throws IOException, YarnException { -return actionDependency(true); + public int enableFastLaunch(String destinationFolder) + throws IOException, YarnException { +return actionDependency(destinationFolder, true); } - public int actionDependency(boolean overwrite) + public int actionDependency(String destinationFolder, boolean overwrite) throws IOException, YarnException { String currentUser =
hadoop git commit: YARN-7671. Improve Diagonstic message for stop yarn native service. Contributed by Chandni Singh
Repository: hadoop Updated Branches: refs/heads/trunk 3d65dbe03 -> c05b84703 YARN-7671. Improve Diagonstic message for stop yarn native service. Contributed by Chandni Singh Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c05b8470 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c05b8470 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c05b8470 Branch: refs/heads/trunk Commit: c05b84703b7754b6c2fbcec39f22b5d7802563ec Parents: 3d65dbe Author: Jian HeAuthored: Fri Jan 12 14:04:15 2018 -0800 Committer: Jian He Committed: Fri Jan 12 14:04:15 2018 -0800 -- .../org/apache/hadoop/yarn/service/ClientAMService.java | 6 +- .../org/apache/hadoop/yarn/service/ServiceScheduler.java | 11 --- 2 files changed, 9 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b8470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java index 94dd8d6..fb73f15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -117,7 +118,10 @@ public class ClientAMService extends AbstractService @Override public StopResponseProto stop(StopRequestProto requestProto) throws IOException, YarnException { -LOG.info("Stop the service."); +LOG.info("Stop the service by {}", UserGroupInformation.getCurrentUser()); +context.scheduler.getDiagnostics() +.append("Stopped by user " + UserGroupInformation.getCurrentUser()); + // Stop the service in 2 seconds delay to make sure this rpc call is completed. // shutdown hook will be executed which will stop AM gracefully. Thread thread = new Thread() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b8470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 0a4ea07..eb4783f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -262,13 +262,10 @@ public class ServiceScheduler extends CompositeService { serviceTimelinePublisher .serviceAttemptUnregistered(context, diagnostics.toString()); } -String msg = diagnostics.toString() -+ "Navigate to the failed component for more details."; -amRMClient -.unregisterApplicationMaster(FinalApplicationStatus.ENDED, msg, ""); -LOG.info("Service " + app.getName() -+ " unregistered with RM, with attemptId = " + context.attemptId -+ ", diagnostics = " + diagnostics); +amRMClient.unregisterApplicationMaster(FinalApplicationStatus.ENDED, +diagnostics.toString(), ""); +LOG.info("Service {} unregistered with RM, with attemptId = {} " + +", diagnostics = {} ", app.getName(), context.attemptId, diagnostics); super.serviceStop();
hadoop git commit: Revert "YARN-7540. Route YARN service CLI function through YARN Service API. (Contributed by Eric Yang)"
Repository: hadoop Updated Branches: refs/heads/trunk a81144daa -> 836e3c45e Revert "YARN-7540. Route YARN service CLI function through YARN Service API. (Contributed by Eric Yang)" This reverts commit 438c1d333ebc0a3071bb556532ed959a4bd1e6d6. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/836e3c45 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/836e3c45 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/836e3c45 Branch: refs/heads/trunk Commit: 836e3c45e8232fc4c8c795c0f93d2f3d7353f392 Parents: a81144d Author: Jian HeAuthored: Fri Jan 5 18:25:00 2018 -0800 Committer: Jian He Committed: Fri Jan 5 18:25:00 2018 -0800 -- .../hadoop-yarn-services-api/pom.xml| 9 - .../yarn/service/client/ApiServiceClient.java | 430 --- .../hadoop/yarn/service/webapp/ApiServer.java | 34 +- .../yarn/service/webapp/package-info.java | 28 -- .../service/client/TestApiServiceClient.java| 259 --- .../src/test/resources/example-app.json | 15 - .../src/test/resources/log4j.properties | 19 - .../service/api/records/ReadinessCheck.java | 2 - .../yarn/service/api/records/Resource.java | 1 + .../yarn/service/api/records/ServiceState.java | 2 +- .../hadoop/yarn/service/utils/JsonSerDeser.java | 3 - .../client/TestBuildExternalComponents.java | 2 +- .../yarn/service/client/TestServiceCLI.java | 14 +- .../hadoop/yarn/client/api/AppAdminClient.java | 6 - 14 files changed, 10 insertions(+), 814 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/836e3c45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index bae62c6..ddea2a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -65,15 +65,6 @@ - -org.apache.rat -apache-rat-plugin - - -**/*.json - - - http://git-wip-us.apache.org/repos/asf/hadoop/blob/836e3c45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java deleted file mode 100644 index f4133a5..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java +++ /dev/null @@ -1,430 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.service.client; - -import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser; - -import java.io.File; -import java.io.IOException; -import java.text.MessageFormat; -import java.util.List; -import java.util.Map; - -import javax.ws.rs.core.MediaType; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import
hadoop git commit: YARN-7565. Yarn service pre-maturely releases the container after AM restart. Contributed by Chandni Singh
Repository: hadoop Updated Branches: refs/heads/trunk 06f0eb2dc -> 3ebe6a781 YARN-7565. Yarn service pre-maturely releases the container after AM restart. Contributed by Chandni Singh Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ebe6a78 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ebe6a78 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ebe6a78 Branch: refs/heads/trunk Commit: 3ebe6a7819292ce6bd557e36137531b59890c845 Parents: 06f0eb2 Author: Jian HeAuthored: Tue Dec 12 13:35:56 2017 -0800 Committer: Jian He Committed: Tue Dec 12 13:35:56 2017 -0800 -- .../hadoop/yarn/service/ServiceScheduler.java | 72 +- .../yarn/service/component/Component.java | 8 ++ .../component/instance/ComponentInstance.java | 10 +- .../yarn/service/conf/YarnServiceConf.java | 10 ++ .../hadoop/yarn/service/MockServiceAM.java | 144 --- .../hadoop/yarn/service/TestServiceAM.java | 98 - .../yarn/client/api/async/AMRMClientAsync.java | 10 ++ .../api/async/impl/AMRMClientAsyncImpl.java | 5 + .../types/yarn/YarnRegistryAttributes.java | 1 + .../markdown/yarn-service/Configurations.md | 1 + 10 files changed, 330 insertions(+), 29 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index bea31cf..2697050 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventT import org.apache.hadoop.yarn.service.component.Component; import org.apache.hadoop.yarn.service.component.ComponentEvent; import org.apache.hadoop.yarn.service.component.ComponentEventType; +import org.apache.hadoop.yarn.service.conf.YarnServiceConf; import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; import org.apache.hadoop.yarn.service.provider.ProviderUtils; @@ -137,6 +138,9 @@ public class ServiceScheduler extends CompositeService { private YarnRegistryViewForProviders yarnRegistryOperations; private ServiceContext context; private ContainerLaunchService containerLaunchService; + private final Map unRecoveredInstances = + new ConcurrentHashMap<>(); + private long containerRecoveryTimeout; public ServiceScheduler(ServiceContext context) { super(context.service.getName()); @@ -212,6 +216,9 @@ public class ServiceScheduler extends CompositeService { createConfigFileCache(context.fs.getFileSystem()); createAllComponents(); +containerRecoveryTimeout = getConfig().getInt( +YarnServiceConf.CONTAINER_RECOVERY_TIMEOUT_MS, +YarnServiceConf.DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS); } protected YarnRegistryViewForProviders createYarnRegistryOperations( @@ -320,7 +327,7 @@ public class ServiceScheduler extends CompositeService { } for (Container container : containersFromPrevAttempt) { LOG.info("Handling {} from previous attempt", container.getId()); - ServiceRecord record = existingRecords.get(RegistryPathUtils + ServiceRecord record = existingRecords.remove(RegistryPathUtils .encodeYarnID(container.getId().toString())); if (record != null) { Component comp = componentsById.get(container.getAllocationRequestId()); @@ -337,6 +344,40 @@ public class ServiceScheduler extends CompositeService { amRMClient.releaseAssignedContainer(container.getId()); } } + +existingRecords.forEach((encodedContainerId, record) -> { + String componentName = record.get(YarnRegistryAttributes.YARN_COMPONENT); + if (componentName != null) { +Component component = componentsByName.get(componentName); +ComponentInstance compInstance =
hadoop git commit: YARN-6168. Restarted RM may not inform AM about all existing containers. Contributed by Chandni Singh
Repository: hadoop Updated Branches: refs/heads/trunk 3cd75845d -> fedabcad4 YARN-6168. Restarted RM may not inform AM about all existing containers. Contributed by Chandni Singh Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fedabcad Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fedabcad Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fedabcad Branch: refs/heads/trunk Commit: fedabcad42067ac7dd24de40fab6be2d3485a540 Parents: 3cd7584 Author: Jian HeAuthored: Mon Nov 27 09:55:08 2017 -0800 Committer: Jian He Committed: Mon Nov 27 10:19:58 2017 -0800 -- .../api/protocolrecords/AllocateResponse.java | 54 +++ .../src/main/proto/yarn_service_protos.proto| 1 + .../impl/pb/AllocateResponsePBImpl.java | 37 + .../resourcemanager/DefaultAMSProcessor.java| 3 + .../scheduler/AbstractYarnScheduler.java| 4 +- .../resourcemanager/scheduler/Allocation.java | 13 +- .../scheduler/SchedulerApplicationAttempt.java | 48 ++ .../scheduler/common/fica/FiCaSchedulerApp.java | 5 +- .../scheduler/fair/FairScheduler.java | 3 +- .../applicationsmanager/TestAMRestart.java | 149 +++ 10 files changed, 310 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fedabcad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java index 9b254ae..98346ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java @@ -372,6 +372,44 @@ public abstract class AllocateResponse { public void setUpdateErrors(List updateErrors) { } + /** + * Get the list of running containers as viewed by + * ResourceManager from previous application attempts which + * have not been reported to the Application Master yet. + * + * These containers were recovered by the RM after the application master + * had already registered. This may happen after RM restart when some NMs get + * delayed in connecting to the RM and reporting the active containers. + * Since they were not reported in the registration + * response, they are reported in the response to the AM heartbeat. + * + * @return the list of running containers as viewed by + * ResourceManager from previous application attempts. + */ + @Public + @Unstable + public abstract List getContainersFromPreviousAttempts(); + + /** + * Set the list of running containers as viewed by + * ResourceManager from previous application attempts which have + * not been reported to the Application Master yet. + * + * These containers were recovered by the RM after the application master + * had already registered. This may happen after RM restart when some NMs get + * delayed in connecting to the RM and reporting the active containers. + * Since they were not reported in the registration + * response, they are reported in the response to the AM heartbeat. + * + * @param containersFromPreviousAttempt + * the list of running containers as viewed by + * ResourceManager from previous application attempts. + */ + @Private + @Unstable + public abstract void setContainersFromPreviousAttempts( + List containersFromPreviousAttempt); + @Private @Unstable public static AllocateResponseBuilder newBuilder() { @@ -590,6 +628,22 @@ public abstract class AllocateResponse { } /** + * Set the containersFromPreviousAttempt of the response. + * @see AllocateResponse#setContainersFromPreviousAttempts(List) + * @param containersFromPreviousAttempt + * containersFromPreviousAttempt of the response + * @return {@link AllocateResponseBuilder} + */ +@Private +@Unstable +public AllocateResponseBuilder containersFromPreviousAttempt( +List containersFromPreviousAttempt) { + allocateResponse.setContainersFromPreviousAttempts( + containersFromPreviousAttempt); + return this; +} + +/** * Return generated {@link AllocateResponse} object. * @return {@link AllocateResponse} */
hadoop git commit: YARN-7503. Configurable heap size / JVM opts in service AM. Contributed by Jonathan Hung
Repository: hadoop Updated Branches: refs/heads/trunk 28d0fcbef -> 6bf2c3019 YARN-7503. Configurable heap size / JVM opts in service AM. Contributed by Jonathan Hung Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bf2c301 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bf2c301 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bf2c301 Branch: refs/heads/trunk Commit: 6bf2c301924a3acae5a7510b8473f6292a5a471b Parents: 28d0fcb Author: Jian HeAuthored: Thu Nov 16 10:53:55 2017 -0800 Committer: Jian He Committed: Thu Nov 16 10:53:55 2017 -0800 -- .../apache/hadoop/yarn/service/client/ServiceClient.java | 10 +- .../apache/hadoop/yarn/service/conf/YarnServiceConf.java | 5 + .../service/containerlaunch/JavaCommandLineBuilder.java | 11 +-- 3 files changed, 15 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf2c301/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index af43f8a..d1b6026 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -559,7 +559,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, Map env = addAMEnv(); // create AM CLI -String cmdStr = buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j); +String cmdStr = buildCommandLine(app, conf, appRootDir, hasAMLog4j); submissionContext.setResource(Resource.newInstance(YarnServiceConf .getLong(YarnServiceConf.AM_RESOURCE_MEM, YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM, app.getConfiguration(), @@ -624,12 +624,12 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, LOG.debug(builder.toString()); } - private String buildCommandLine(String serviceName, Configuration conf, + private String buildCommandLine(Service app, Configuration conf, Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException { JavaCommandLineBuilder CLI = new JavaCommandLineBuilder(); CLI.forceIPv4().headless(); -//TODO CLI.setJVMHeap -//TODO CLI.addJVMOPTS +CLI.setJVMOpts(YarnServiceConf.get(YarnServiceConf.JVM_OPTS, null, +app.getConfiguration(), conf)); if (hasSliderAMLog4j) { CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME); CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR); @@ -637,7 +637,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, CLI.add(ServiceMaster.class.getCanonicalName()); //TODO debugAM CLI.add(Arguments.ARG_DEBUG) CLI.add("-" + ServiceMaster.YARNFILE_OPTION, new Path(appRootDir, -serviceName + ".json")); +app.getName() + ".json")); // pass the registry binding CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT, RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf2c301/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java index a7bd58d..684d980 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java +++
hadoop git commit: YARN-7488. Make ServiceClient.getAppId method public to return ApplicationId for a service name. Contributed by Gour Saha
Repository: hadoop Updated Branches: refs/heads/trunk 8b1257416 -> 4f40cd314 YARN-7488. Make ServiceClient.getAppId method public to return ApplicationId for a service name. Contributed by Gour Saha Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f40cd31 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f40cd31 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f40cd31 Branch: refs/heads/trunk Commit: 4f40cd314ab14f735a465fb9dff2dc1bf118e703 Parents: 8b12574 Author: Jian HeAuthored: Mon Nov 13 18:55:12 2017 -0800 Committer: Jian He Committed: Mon Nov 13 18:57:56 2017 -0800 -- .../java/org/apache/hadoop/yarn/service/client/ServiceClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f40cd31/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 11cd30d..af43f8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -943,7 +943,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes, UserGroupInformation.getCurrentUser(), rpc, address); } - private synchronized ApplicationId getAppId(String serviceName) + public synchronized ApplicationId getAppId(String serviceName) throws IOException, YarnException { if (cachedAppIds.containsKey(serviceName)) { return cachedAppIds.get(serviceName); - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7466. ResourceRequest has a different default for allocationRequestId than Container. Contributed by Chandni Singh
Repository: hadoop Updated Branches: refs/heads/trunk e14f03dfb -> 5323b0048 YARN-7466. ResourceRequest has a different default for allocationRequestId than Container. Contributed by Chandni Singh Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5323b004 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5323b004 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5323b004 Branch: refs/heads/trunk Commit: 5323b0048b743771276ba860b10c27b23a70bf9e Parents: e14f03d Author: Jian HeAuthored: Mon Nov 13 15:37:39 2017 -0800 Committer: Jian He Committed: Mon Nov 13 15:37:39 2017 -0800 -- .../hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5323b004/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index e69c07b..7769c48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -388,7 +388,7 @@ message ResourceRequestProto { optional bool relax_locality = 5 [default = true]; optional string node_label_expression = 6; optional ExecutionTypeRequestProto execution_type_request = 7; - optional int64 allocation_request_id = 8 [default = 0]; + optional int64 allocation_request_id = 8 [default = -1]; optional ProfileCapabilityProto profile = 9; } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[43/60] [abbrv] hadoop git commit: Revert "YARN-6626. Embed REST API service into RM. Contributed by Eric Yang"
Revert "YARN-6626. Embed REST API service into RM. Contributed by Eric Yang" This reverts commit 63d1084e9781e0fee876916190b69f6242dd00e4. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/673c0db4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/673c0db4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/673c0db4 Branch: refs/heads/trunk Commit: 673c0db43cfa4a0e9e7a21c5ea31660ecf61b8c3 Parents: 9e677fa Author: Jian HeAuthored: Wed Oct 4 23:39:27 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:17 2017 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 2 -- .../hadoop/yarn/service/webapp/ApiServer.java | 7 .../hadoop-yarn-services-core/pom.xml | 6 .../yarn/service/api/records/Artifact.java | 8 - .../yarn/service/api/records/ConfigFile.java| 8 - .../service/api/records/ReadinessCheck.java | 8 - .../src/main/resources/yarn-default.xml | 8 - .../server/resourcemanager/webapp/RMWebApp.java | 19 --- .../site/markdown/yarn-service/QuickStart.md| 34 +++- 9 files changed, 11 insertions(+), 89 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 4799137..640e86e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -336,8 +336,6 @@ public class YarnConfiguration extends Configuration { public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn." + "webapp.ui2.war-file-path"; - public static final String YARN_API_SERVICES_ENABLE = "yarn." - + "webapp.api-service.enable"; public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 89b020d..e8286ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -17,7 +17,6 @@ package org.apache.hadoop.yarn.service.webapp; -import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.VersionInfo; @@ -58,12 +57,6 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; @Singleton @Path(CONTEXT_ROOT) public class ApiServer { - - @Inject - public ApiServer(Configuration conf) { -super(); - } - private static final Logger LOG = LoggerFactory.getLogger(ApiServer.class); private static Configuration YARN_CONFIG = new YarnConfiguration(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 205a64d..fb07edc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -245,6 +245,12 @@ + org.apache.hadoop + hadoop-yarn-server-resourcemanager + test + + +
[59/60] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He
YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6c4bd74 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6c4bd74 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6c4bd74 Branch: refs/heads/trunk Commit: a6c4bd74b636d0715ffb4c1aca9ef4e49201aa02 Parents: a288b0a Author: Billie RinaldiAuthored: Fri Nov 3 11:25:57 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:20 2017 -0800 -- .../definition/YARN-Services-Examples.md| 8 +- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 6 +- .../examples/httpd-no-dns/httpd-no-dns.json | 6 +- .../examples/httpd/httpd.json | 4 +- .../hadoop/yarn/service/ServiceMaster.java | 6 + .../hadoop/yarn/service/ServiceMetrics.java | 4 - .../hadoop/yarn/service/ServiceScheduler.java | 15 -- .../yarn/service/api/ServiceApiConstants.java | 4 +- .../yarn/service/api/records/ConfigFile.java| 32 ++-- .../yarn/service/api/records/Configuration.java | 4 +- .../service/api/records/ReadinessCheck.java | 24 +-- .../yarn/service/client/ServiceClient.java | 61 --- .../yarn/service/component/Component.java | 29 ++-- .../component/instance/ComponentInstance.java | 37 +++-- .../yarn/service/conf/YarnServiceConf.java | 14 +- .../yarn/service/conf/YarnServiceConstants.java | 2 + .../service/monitor/probe/MonitorUtils.java | 4 +- .../provider/AbstractClientProvider.java| 17 +- .../yarn/service/provider/ProviderUtils.java| 12 +- .../timelineservice/ServiceTimelineEvent.java | 4 +- .../ServiceTimelinePublisher.java | 48 +++--- .../yarn/service/conf/TestAppJsonResolve.java | 8 +- .../TestServiceTimelinePublisher.java | 2 +- .../service/conf/examples/app-override.json | 6 +- .../markdown/yarn-service/Configurations.md | 165 +++ .../src/site/markdown/yarn-service/Examples.md | 4 +- .../src/site/markdown/yarn-service/Overview.md | 3 +- .../markdown/yarn-service/YarnServiceAPI.md | 14 +- .../app/controllers/yarn-app/components.js | 2 +- .../app/templates/components/metrics-table.hbs | 2 +- .../templates/yarn-component-instance/info.hbs | 4 +- .../templates/yarn-component-instances/info.hbs | 2 +- 32 files changed, 384 insertions(+), 169 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md index 3cd3d48..00b21dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md @@ -176,14 +176,14 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 { "type": "XML", "dest_file": "/etc/hadoop/conf/core-site.xml", -"props": { +"properties": { "fs.defaultFS": "${CLUSTER_FS_URI}" } }, { "type": "XML", "dest_file": "/etc/hbase/conf/hbase-site.xml", -"props": { +"properties": { "hbase.cluster.distributed": "true", "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}", "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase", @@ -216,14 +216,14 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 { "type": "XML", "dest_file": "/etc/hadoop/conf/core-site.xml", -"props": { +"properties": { "fs.defaultFS": "${CLUSTER_FS_URI}" } }, { "type": "XML", "dest_file": "/etc/hbase/conf/hbase-site.xml", -"props": { +"properties": { "hbase.cluster.distributed": "true", "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}", "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
[49/60] [abbrv] hadoop git commit: YARN-7393. RegistryDNS doesn't work in tcp channel. Contributed by Eric Yang
YARN-7393. RegistryDNS doesn't work in tcp channel. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ec9e72c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ec9e72c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ec9e72c Branch: refs/heads/trunk Commit: 8ec9e72cc37957d8cf9af5a7b78086441ae64b00 Parents: d2775fb Author: Billie RinaldiAuthored: Fri Oct 27 11:44:55 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:19 2017 -0800 -- .../hadoop/registry/server/dns/RegistryDNS.java | 20 +++- 1 file changed, 15 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ec9e72c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index 9097c46..d7ebece 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -189,8 +189,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, LOG.info("Opening TCP and UDP channels on {} port {}", addr, port); addNIOUDP(addr, port); -//TODO Fix dns lookup over TCP -//addNIOTCP(addr, port); +addNIOTCP(addr, port); } /** @@ -768,7 +767,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, buf.get(in, 0, messageLength); Message query; - byte[] response = null; + byte[] response; try { query = new Message(in); LOG.info("received TCP query {}", query.getQuestion()); @@ -781,10 +780,19 @@ public class RegistryDNS extends AbstractService implements DNSOperations, } ByteBuffer out = ByteBuffer.allocate(response.length + 2); - out.putShort(0, (short) (response.length & 0x)); + out.clear(); + byte[] data = new byte[2]; + + data[1] = (byte)(response.length & 0xFF); + data[0] = (byte)((response.length >> 8) & 0xFF); + out.put(data); out.put(response); + out.flip(); + + while(out.hasRemaining()) { +ch.write(out); + } - ch.write(out); } catch (IOException e) { throw NetUtils.wrapException(ch.socket().getInetAddress().getHostName(), ch.socket().getPort(), @@ -836,6 +844,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations, } }); +} else { + Thread.sleep(500); } } } catch (IOException e) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[52/60] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java deleted file mode 100644 index 65f0472..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.client.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; -import org.apache.hadoop.yarn.service.exceptions.UsageException; - -import java.io.File; - -import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE; -import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY; - -/** - * Resolve registry entries - * - * --path {path} - * --out {destfile} - * --verbose - * --list - */ -@Parameters(commandNames = {ACTION_RESOLVE}, -commandDescription = DESCRIBE_ACTION_REGISTRY) -public class ActionResolveArgs extends AbstractActionArgs { - - public static final String USAGE = - "Usage: " + SliderActions.ACTION_RESOLVE - + " " - + ARG_PATH + " " - + "[" + ARG_LIST + "] " - + "[" + ARG_OUTPUT + " ] " - + "[" + ARG_DESTDIR + " ] " - ; - public ActionResolveArgs() { - } - - @Override - public String getActionName() { -return ACTION_RESOLVE; - } - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - @Override - public int getMinParams() { -return 0; - } - - @Parameter(names = {ARG_LIST}, - description = "list services") - public boolean list; - - @Parameter(names = {ARG_PATH}, - description = "resolve a path") - public String path; - - @Parameter(names = {ARG_DESTDIR}, - description = "destination directory for operations") - public File destdir; - - @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, - description = "dest file") - public File out; - - @Override - public String toString() { -final StringBuilder sb = -new StringBuilder(ACTION_RESOLVE).append(" "); -sb.append(ARG_PATH).append(" ").append(path).append(" "); -if (list) { - sb.append(ARG_LIST).append(" "); -} -if (destdir != null) { - sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" "); -} -if (out != null) { - sb.append(ARG_OUTPUT).append(" ").append(out).append(" "); -} -return sb.toString(); - } - - @Override - public void validate() throws BadCommandArgumentsException, UsageException { -super.validate(); -if (StringUtils.isEmpty(path)) { - throw new BadCommandArgumentsException("Missing mandatory argument " - + ARG_PATH); -} -if (list && out != null) { - throw new BadCommandArgumentsException("Argument " - + ARG_OUTPUT + - " not supported for " + ARG_LIST); -} -if (out != null && destdir != null) { - throw new BadCommandArgumentsException( - ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together" - ); -} - } - - public String getPath() { -return path; - } - - public void setPath(String path) { -this.path = path; - } - - public boolean isList() { -return list; -
[54/60] [abbrv] hadoop git commit: YARN-7326. Add recursion support and configure RegistryDNS to lookup upstream. Contributed by Eric Yang
YARN-7326. Add recursion support and configure RegistryDNS to lookup upstream. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/501be9b4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/501be9b4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/501be9b4 Branch: refs/heads/trunk Commit: 501be9b4be7d549b23cc501b26fdfdce0a7aa911 Parents: 68acd88 Author: Billie RinaldiAuthored: Wed Oct 25 07:37:03 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:19 2017 -0800 -- .../hadoop/registry/server/dns/RegistryDNS.java | 148 ++- .../registry/server/dns/TestRegistryDNS.java| 8 +- 2 files changed, 148 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/501be9b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index d7a415d..37e8429 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -37,8 +37,10 @@ import org.xbill.DNS.DNSKEYRecord; import org.xbill.DNS.DNSSEC; import org.xbill.DNS.DSRecord; import org.xbill.DNS.ExtendedFlags; +import org.xbill.DNS.ExtendedResolver; import org.xbill.DNS.Flags; import org.xbill.DNS.Header; +import org.xbill.DNS.Lookup; import org.xbill.DNS.Message; import org.xbill.DNS.NSRecord; import org.xbill.DNS.Name; @@ -49,9 +51,12 @@ import org.xbill.DNS.RRSIGRecord; import org.xbill.DNS.RRset; import org.xbill.DNS.Rcode; import org.xbill.DNS.Record; +import org.xbill.DNS.Resolver; +import org.xbill.DNS.ResolverConfig; import org.xbill.DNS.SOARecord; import org.xbill.DNS.Section; import org.xbill.DNS.SetResponse; +import org.xbill.DNS.SimpleResolver; import org.xbill.DNS.TSIG; import org.xbill.DNS.TSIGRecord; import org.xbill.DNS.TextParseException; @@ -66,8 +71,11 @@ import java.io.IOException; import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.NetworkInterface; import java.net.Socket; import java.net.SocketAddress; +import java.net.SocketException; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.channels.DatagramChannel; import java.nio.channels.ServerSocketChannel; @@ -78,10 +86,13 @@ import java.security.PrivateKey; import java.security.spec.InvalidKeySpecException; import java.security.spec.RSAPrivateKeySpec; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.Date; +import java.util.Enumeration; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.Callable; @@ -134,6 +145,16 @@ public class RegistryDNS extends AbstractService implements DNSOperations, private boolean channelsInitialized = false; /** + * Lock to update resolver only once per request. + */ + private final Object resolverUpdateLock = new Object(); + + /** + * Whether resolver update has been requested. + */ + private boolean resolverUpdateRequested = true; + + /** * Construct the service. * * @param name service name @@ -172,6 +193,79 @@ public class RegistryDNS extends AbstractService implements DNSOperations, } /** + * Initialize registryDNS to use /etc/resolv.conf values + * as default resolvers. + */ + private void updateDNSServer(Configuration conf) { +synchronized (resolverUpdateLock) { + if (!resolverUpdateRequested) { +return; + } + int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT); + resolverUpdateRequested = false; + List list = new ArrayList(); + try { +// If resolv.conf contains the server's own IP address, +// and RegistryDNS handles the lookup. Local IP address +// must be filter out from default resolvers to prevent +// self recursive loop. +if (port != 53) { + // When registryDNS is not running on default port, + // registryDNS can utilize local DNS server as upstream lookup. + throw new SocketException("Bypass filtering local DNS server."); +} +Enumeration net = +
[33/60] [abbrv] hadoop git commit: YARN-7161. Fix special chars in DNS documentation. Contributed by Gour Saha
YARN-7161. Fix special chars in DNS documentation. Contributed by Gour Saha Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9ca13e3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9ca13e3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9ca13e3 Branch: refs/heads/trunk Commit: b9ca13e34bc8a14787e94e9b70ae0aa249f00954 Parents: eed9e3e Author: Jian HeAuthored: Thu Sep 7 18:17:02 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:15 2017 -0800 -- .../native-services/NativeServicesDiscovery.md | 87 ++-- 1 file changed, 45 insertions(+), 42 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9ca13e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md index fa54a09..a927118 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md @@ -13,39 +13,42 @@ --> # YARN DNS Server + + + ## Introduction The YARN DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions: -1. **Exposing existing service discovery information via DNS**â - Information provided in +1. **Exposing existing service-discovery information via DNS** - Information provided in the current YARN service registryâs records will be converted into DNS entries, thus allowing users to discover information about YARN applications using standard DNS client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port number for services). -2. **Enabling Container to IP mappingsâÂ** - Enables discovery of the IPs of containers via +2. **Enabling Container to IP mappings** - Enables discovery of the IPs of containers via standard DNS lookups. Given the availability of the records via DNS, container -nameÂbased communication will be facilitated (e.g. âcurl +name-based communication will be facilitated (e.g. âcurl http://myContainer.myDomain.com/endpointâ). ## Service Properties The existing YARN Service Registry is leveraged as the source of information for the DNS Service. -The following core functions are supported by the DNS Server: +The following core functions are supported by the DNS-Server: -###Functional properties +### Functional properties -1. Supports creation of DNS records for endÂpoints of the deployed YARN applications +1. Supports creation of DNS records for end-points of the deployed YARN applications 2. Record names remain unchanged during restart of containers and/or applications 3. Supports reverse lookups (name based on IP). 4. Supports security using the standards defined by The Domain Name System Security Extensions (DNSSEC) 5. Highly available -6. Scalable Â- The service provides the responsiveness (e.g. low latency) required to +6. Scalable - The service provides the responsiveness (e.g. low-latency) required to respond to DNS queries (timeouts yield attempts to invoke other configured name servers). -###Deployment properties +### Deployment properties 1. Supports integration with existing DNS assets (e.g. a corporate DNS server) by acting as a DNS server for a Hadoop cluster zone/domain. The server is not intended to act as a @@ -55,59 +58,59 @@ DNS standards. The default port for DNS protocols is in a restricted, administra range (53), so the port is configurable for deployments in which the service may not be managed via an administrative account. -##DNS Record Name Structure +## DNS Record Name Structure The DNS names of generated records are composed from the following elements (labels). Note that these elements must be compatible with DNS conventions (see âPreferred Name Syntaxâ in RFC 1035): -* **domain** -â the name of the cluster DNS domain. This name is provided as a +* **domain** - the name of the cluster DNS domain. This name is provided as a configuration property. In addition, it is this name that is configured at a parent DNS server as the zone name for the defined yDNS zone (the zone for which the parent DNS server will forward requests to yDNS). E.g. yarncluster.com -* **userÂname**â - the name of the application deployer. This name is the
[36/60] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He
YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/541b64c7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/541b64c7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/541b64c7 Branch: refs/heads/trunk Commit: 541b64c7985db425c5ce8bd5f71a2d5980809708 Parents: b9ca13e Author: Gour SahaAuthored: Thu Sep 7 23:25:31 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:15 2017 -0800 -- hadoop-project/pom.xml | 4 +- hadoop-project/src/site/site.xml| 10 +- hadoop-yarn-project/hadoop-yarn/bin/yarn| 7 +- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 11 +- .../hadoop/yarn/service/ServiceMaster.java | 10 +- .../hadoop/yarn/service/client/ServiceCLI.java | 5 +- .../yarn/service/client/ServiceClient.java | 20 +- .../client/params/AbstractActionArgs.java | 33 +- .../AbstractClusterBuildingActionArgs.java | 22 +- .../service/client/params/ActionBuildArgs.java | 5 + .../service/client/params/ActionCreateArgs.java | 5 + .../client/params/ActionDependencyArgs.java | 4 +- .../service/client/params/ActionExamples.java | 26 + .../service/client/params/ActionUpdateArgs.java | 7 +- .../yarn/service/client/params/Arguments.java | 9 +- .../yarn/service/client/params/ClientArgs.java | 10 - .../yarn/service/client/params/CommonArgs.java | 28 +- .../service/client/params/ServiceAMArgs.java| 50 ++ .../client/params/ServiceAMCreateAction.java| 37 ++ .../service/client/params/SliderAMArgs.java | 57 -- .../client/params/SliderAMCreateAction.java | 73 --- .../service/client/params/SliderActions.java| 9 +- .../ServiceTimelinePublisher.java | 5 +- .../client/TestBuildExternalComponents.java | 6 +- .../yarn/service/client/TestServiceCLI.java | 30 +- .../hadoop/yarn/service/conf/examples/app.json | 2 +- .../registry/client/api/RegistryConstants.java | 2 +- .../src/site/markdown/YarnCommands.md | 56 +- .../native-services/NativeServicesAPI.md| 606 --- .../native-services/NativeServicesDiscovery.md | 144 - .../native-services/NativeServicesIntro.md | 107 .../src/site/markdown/yarn-service/Concepts.md | 77 +++ .../src/site/markdown/yarn-service/Overview.md | 58 ++ .../site/markdown/yarn-service/QuickStart.md| 218 +++ .../markdown/yarn-service/ServiceDiscovery.md | 150 + .../markdown/yarn-service/YarnServiceAPI.md | 592 ++ 36 files changed, 1363 insertions(+), 1132 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3cb9568..d18c9a8 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -143,10 +143,10 @@ ${project.version} - + 1.5.4 1.16 -1.30 +1.48 http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-project/src/site/site.xml -- diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index ad893a2..57cff9a 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -155,10 +155,12 @@ - - - - + + + + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/bin/yarn -- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 1db585e..05d892b 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -31,7 +31,7 @@ function hadoop_usage hadoop_add_option "--hosts filename" "list of hosts to use in worker mode" hadoop_add_option "--workers" "turn on worker mode" - hadoop_add_subcommand "apiserver" "run yarn-native-service rest server" + hadoop_add_subcommand "apiserver" daemon "run the api-server for deploying/managing services on YARN" hadoop_add_subcommand "application" client "prints application(s) report/kill application" hadoop_add_subcommand "applicationattempt" client "prints applicationattempt(s) report" hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries" @@ -50,10 +50,9 @@ function hadoop_usage hadoop_add_subcommand "router" daemon "run the Router daemon"
[46/60] [abbrv] hadoop git commit: YARN-7202. Add UT for api-server. Contributed by Eric Yang
YARN-7202. Add UT for api-server. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57144a1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57144a1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57144a1 Branch: refs/heads/trunk Commit: b57144a1726e0b02abbdc3e8f37769d3f7c039c8 Parents: b8a7ef1 Author: Jian HeAuthored: Thu Oct 12 10:57:35 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:18 2017 -0800 -- .../hadoop-yarn-services-api/pom.xml| 34 +- .../hadoop/yarn/service/webapp/ApiServer.java | 21 +- .../yarn/service/webapp/ApiServerWebApp.java| 2 +- .../hadoop/yarn/service/ServiceClientTest.java | 107 ++ .../hadoop/yarn/service/TestApiServer.java | 366 +++ .../hadoop/yarn/service/ServiceTestUtils.java | 171 + .../yarn/service/TestYarnNativeServices.java| 162 +--- 7 files changed, 690 insertions(+), 173 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 74d9681..ddea2a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -41,7 +41,6 @@ - org.apache.maven.plugins maven-jar-plugin @@ -66,17 +65,6 @@ - - -org.apache.maven.plugins -maven-surefire-plugin - - -${java.home} - - - - @@ -84,6 +72,7 @@ + org.apache.hadoop hadoop-yarn-services-core @@ -116,5 +105,26 @@ javax.ws.rs jsr311-api + + org.mockito + mockito-all + test + + + + + + + + org.apache.hadoop + hadoop-common + test-jar + + + junit + junit + test + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 4fe12e3..5773069 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -165,12 +165,12 @@ public class ApiServer { } else { LOG.info("Successfully stopped service {}", appName); } - return Response.status(Status.NO_CONTENT).build(); + return Response.status(Status.OK).build(); } catch (ApplicationNotFoundException e) { ServiceStatus serviceStatus = new ServiceStatus(); serviceStatus.setDiagnostics( - "Service " + appName + " not found " + e.getMessage()); - return Response.status(Status.NOT_FOUND).entity(serviceStatus) + "Service " + appName + " is not found in YARN: " + e.getMessage()); + return Response.status(Status.BAD_REQUEST).entity(serviceStatus) .build(); } catch (Exception e) { ServiceStatus serviceStatus = new ServiceStatus(); @@ -245,7 +245,8 @@ public class ApiServer { // flex a single component app if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil .hasComponent(updateServiceData)) { - Component defaultComp = ServiceApiUtil.createDefaultComponent(updateServiceData); + Component defaultComp = ServiceApiUtil + .createDefaultComponent(updateServiceData); return updateComponent(updateServiceData.getName(), defaultComp.getName(), defaultComp); } @@ -291,4 +292,16 @@ public class ApiServer { .entity(status).build(); } } + + /** + * Used by negative test case. + * + * @param mockServerClient - A mocked version of ServiceClient + */ +
[40/60] [abbrv] hadoop git commit: YARN-6626. Embed REST API service into RM. Contributed by Eric Yang
YARN-6626. Embed REST API service into RM. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7230215 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7230215 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7230215 Branch: refs/heads/trunk Commit: c72302157907b44b71bc15008aabd9cf3d806f12 Parents: 673c0db Author: Jian HeAuthored: Thu Sep 28 16:29:22 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:17 2017 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 2 ++ .../hadoop/yarn/service/webapp/ApiServer.java | 11 +++ .../hadoop-yarn-services-core/pom.xml | 6 .../yarn/service/api/records/Artifact.java | 8 + .../yarn/service/api/records/ConfigFile.java| 8 + .../service/api/records/ReadinessCheck.java | 8 + .../src/main/resources/yarn-default.xml | 8 + .../server/resourcemanager/webapp/RMWebApp.java | 19 +++ .../site/markdown/yarn-service/QuickStart.md| 34 +--- 9 files changed, 93 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 640e86e..4799137 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration { public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn." + "webapp.ui2.war-file-path"; + public static final String YARN_API_SERVICES_ENABLE = "yarn." + + "webapp.api-service.enable"; public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index e8286ef..4fe12e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -17,6 +17,7 @@ package org.apache.hadoop.yarn.service.webapp; +import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.VersionInfo; @@ -57,6 +58,16 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; @Singleton @Path(CONTEXT_ROOT) public class ApiServer { + + public ApiServer() { +super(); + } + + @Inject + public ApiServer(Configuration conf) { +super(); + } + private static final Logger LOG = LoggerFactory.getLogger(ApiServer.class); private static Configuration YARN_CONFIG = new YarnConfiguration(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index fb07edc..205a64d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -245,12 +245,6 @@ - org.apache.hadoop - hadoop-yarn-server-resourcemanager - test - - - org.apache.curator
[42/60] [abbrv] hadoop git commit: YARN-6626. Embed REST API service into RM. Contributed by Eric Yang
YARN-6626. Embed REST API service into RM. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e677fa0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e677fa0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e677fa0 Branch: refs/heads/trunk Commit: 9e677fa05c6a822cf034bffd25bbee0bad84084b Parents: 8851209 Author: Jian HeAuthored: Thu Sep 28 16:29:22 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:17 2017 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 2 ++ .../hadoop/yarn/service/webapp/ApiServer.java | 7 .../hadoop-yarn-services-core/pom.xml | 6 .../yarn/service/api/records/Artifact.java | 8 + .../yarn/service/api/records/ConfigFile.java| 8 + .../service/api/records/ReadinessCheck.java | 8 + .../src/main/resources/yarn-default.xml | 8 + .../server/resourcemanager/webapp/RMWebApp.java | 19 +++ .../site/markdown/yarn-service/QuickStart.md| 34 +--- 9 files changed, 89 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 640e86e..4799137 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration { public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn." + "webapp.ui2.war-file-path"; + public static final String YARN_API_SERVICES_ENABLE = "yarn." + + "webapp.api-service.enable"; public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index e8286ef..89b020d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -17,6 +17,7 @@ package org.apache.hadoop.yarn.service.webapp; +import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.VersionInfo; @@ -57,6 +58,12 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; @Singleton @Path(CONTEXT_ROOT) public class ApiServer { + + @Inject + public ApiServer(Configuration conf) { +super(); + } + private static final Logger LOG = LoggerFactory.getLogger(ApiServer.class); private static Configuration YARN_CONFIG = new YarnConfiguration(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index fb07edc..205a64d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -245,12 +245,6 @@ - org.apache.hadoop - hadoop-yarn-server-resourcemanager - test - - - org.apache.curator curator-test test
[57/60] [abbrv] hadoop git commit: YARN-7331. Change in few metrics in new YARN UI related to native-services. Contributed by Sunil G
YARN-7331. Change in few metrics in new YARN UI related to native-services. Contributed by Sunil G Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a288b0ac Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a288b0ac Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a288b0ac Branch: refs/heads/trunk Commit: a288b0accf8e0df0375cd5ca3aafdbfb142aa8c3 Parents: ba7ed7b Author: Jian HeAuthored: Mon Oct 30 13:16:19 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:20 2017 -0800 -- .../main/webapp/app/adapters/yarn-servicedef.js | 2 +- .../app/controllers/yarn-app/components.js | 8 ++--- .../yarn-component-instances/info.js| 2 +- .../app/serializers/yarn-component-instance.js | 2 +- .../app/templates/components/metrics-table.hbs | 37 +++- .../templates/yarn-component-instance/info.hbs | 6 +--- .../src/main/webapp/config/configs.env | 7 .../src/main/webapp/config/default-config.js| 3 +- 8 files changed, 12 insertions(+), 55 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js index dc5dbfd..3fb4a81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js @@ -20,7 +20,7 @@ import Ember from 'ember'; import RESTAbstractAdapter from './restabstract'; export default RESTAbstractAdapter.extend({ - address: "dashWebAddress", + address: "rmWebAddress", restNameSpace: "dashService", serverName: "DASH", http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js index 947cb98..b4bac0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js @@ -30,7 +30,7 @@ export default Ember.Controller.extend({ cols.push({ id: 'name', - headerTitle: 'Component Group', + headerTitle: 'Component', contentPath: 'name', cellComponentName: 'em-table-linked-cell', getCellContent: function(row) { @@ -49,13 +49,9 @@ export default Ember.Controller.extend({ contentPath: 'memory' }, { id: 'instances', - headerTitle: '# Components', + headerTitle: 'Number Of Instances', contentPath: 'instances', observePath: true -}, { - id: 'createdDate', - headerTitle: 'Created Time', - contentPath: 'createdDate' }); return ColumnDef.make(cols); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js index a676b34..0d1a4bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js @@ -31,7 +31,7 @@ export default Ember.Controller.extend({ cols.push({ id: 'instanceName', - headerTitle: 'Component Name', + headerTitle: 'Component Instance', contentPath: 'instanceName', cellComponentName: 'em-table-linked-cell', getCellContent: function(row) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js -- diff --git
[35/60] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md deleted file mode 100644 index f56139a..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md +++ /dev/null @@ -1,606 +0,0 @@ - - -# YARN Simplified API layer for services - -## Overview -Bringing a new service on YARN today is not a simple experience. The APIs of -existing frameworks are either too low level (native YARN), require writing -new code (for frameworks with programmatic APIs) or writing a complex spec -(for declarative frameworks). In addition to building critical building blocks -inside YARN (as part of other efforts at -[YARN-4692](https://issues.apache.org/jira/browse/YARN-4692)), there is a need for -simplifying the user facing story for building services. Experience of projects -like Apache Slider running real-life services like HBase, Storm, Accumulo, -Solr etc, gives us some very good insights on how simplified APIs for services -should look like. - -To this end, we should look at a new simple-services API layer backed by REST -interfaces. This API can be used to create and manage the lifecycle of YARN -services. Services here can range from simple single-component service to -complex multi-component assemblies needing orchestration. -[YARN-4793](https://issues.apache.org/jira/browse/YARN-4793) tracks this -effort. - -This document spotlights on this specification. In most of the cases, the -application owner will not be forced to make any changes to their applications. -This is primarily true if the application is packaged with containerization -technologies like docker. Irrespective of how complex the application is, -there will be hooks provided at appropriate layers to allow pluggable and -customizable application behavior. - - -### Version information -Version: 1.0.0 - -### License information -License: Apache 2.0 -License URL: http://www.apache.org/licenses/LICENSE-2.0.html - -### URI scheme -Host: host.mycompany.com - -BasePath: /ws/v1/ - -Schemes: HTTP - -### Consumes - -* application/json - - -### Produces - -* application/json - - -## Paths -### Create a service -``` -POST /services -``` - - Description - -Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state. - - Parameters -|Type|Name|Description|Required|Schema|Default| -||||||| -|BodyParameter|Service|Service request object|true|Service|| - - - Responses -|HTTP Code|Description|Schema| -|||| -|202|The request to create a service is accepted|No Content| -|400|Invalid service definition provided in the request body|No Content| -|500|Failed to create a service|No Content| -|default|Unexpected error|ServiceStatus| - - -### (TBD) List of services running in the cluster. -``` -GET /services -``` - - Description - -Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. - - Responses -|HTTP Code|Description|Schema| -|||| -|200|An array of services|Service array| -|default|Unexpected error|ServiceStatus| - - -### Get current version of the API server. -``` -GET /services/version -``` - - Description - -Get current version of the API server. - - Responses -|HTTP Code|Description|Schema| -|||| -|200|Successful request|No Content| - - -### Update a service or upgrade the binary version of the components of a running service -``` -PUT /services/{service_name} -``` - - Description - -Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD). - - Parameters -|Type|Name|Description|Required|Schema|Default| -||||||| -|PathParameter|service_name|Service name|true|string|| -|BodyParameter|Service|The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED)
[51/60] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index f22d487..df4b1df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -20,88 +20,71 @@ package org.apache.hadoop.yarn.service.client; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.client.cli.ApplicationCLI; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.ClientAMProtocol; import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.client.params.ClientArgs; +import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.conf.ExampleAppJson; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS; -import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; -import static org.mockito.Mockito.*; public class TestServiceCLI { + private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI + .class); - protected Configuration conf = new YarnConfiguration(); + private Configuration conf = new YarnConfiguration(); private File basedir; - private ServiceCLI cli; private SliderFileSystem fs; + private String basedirProp; - private void buildApp(String appDef) throws Throwable { -String[] args = -{ "build", ARG_FILE, ExampleAppJson.resourceName(appDef)}; -ClientArgs clientArgs = new ClientArgs(args); -clientArgs.parse(); -cli.exec(clientArgs); + private void runCLI(String[] args) throws Exception { +LOG.info("running CLI: yarn {}", Arrays.asList(args)); +ApplicationCLI cli = new ApplicationCLI(); +cli.setSysOutPrintStream(System.out); +cli.setSysErrPrintStream(System.err); +int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args)); +cli.stop(); + } + + private void buildApp(String serviceName, String appDef) throws Throwable { +String[] args = {"app", "-D", basedirProp, "-save", serviceName, +ExampleAppJson.resourceName(appDef)}; +runCLI(args); + } + + private void buildApp(String serviceName, String appDef, String lifetime, + String queue) throws Throwable { +String[] args = {"app", "-D", basedirProp, "-save", serviceName, +ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime, +"-changeQueue", queue}; +runCLI(args); } @Before public void setup() throws Throwable { basedir = new File("target", "apps"); +basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath(); conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); -conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0); -conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1); -conf.setInt( -CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); -conf.setInt(CommonConfigurationKeysPublic. -IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0); fs = new SliderFileSystem(conf); if (basedir.exists()) { FileUtils.deleteDirectory(basedir); } else { basedir.mkdirs(); } - -// create a CLI and skip
[47/60] [abbrv] hadoop git commit: YARN-7323. Data structure update in service REST API. Contributed by Jian He
YARN-7323. Data structure update in service REST API. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68acd88d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68acd88d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68acd88d Branch: refs/heads/trunk Commit: 68acd88dcbfe03a0134c60b5398dfaa31ad2b786 Parents: 3c30b1a Author: Billie RinaldiAuthored: Tue Oct 17 11:50:16 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:18 2017 -0800 -- .../hadoop/yarn/service/webapp/ApiServer.java | 9 --- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 67 - .../hadoop/yarn/service/ServiceScheduler.java | 1 + .../yarn/service/api/records/Component.java | 22 +- .../service/api/records/ComponentState.java | 30 .../yarn/service/api/records/Container.java | 15 ++-- .../yarn/service/api/records/Service.java | 76 .../yarn/service/api/records/ServiceState.java | 2 +- .../yarn/service/client/ServiceClient.java | 45 .../yarn/service/component/Component.java | 12 .../component/instance/ComponentInstance.java | 48 + .../yarn/service/conf/RestApiConstants.java | 4 -- .../yarn/service/utils/ServiceApiUtil.java | 53 ++ .../hadoop/yarn/service/ServiceTestUtils.java | 4 +- .../hadoop/yarn/service/TestServiceApiUtil.java | 74 ++- .../yarn/service/TestYarnNativeServices.java| 4 +- .../service/conf/examples/app-override.json | 8 ++- .../hadoop/yarn/service/conf/examples/app.json | 8 ++- .../yarn/service/conf/examples/external0.json | 15 ++-- .../markdown/yarn-service/YarnServiceAPI.md | 22 +++--- 20 files changed, 232 insertions(+), 287 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 5773069..1bb6c93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -242,15 +242,6 @@ public class ApiServer { return updateLifetime(appName, updateServiceData); } -// flex a single component app -if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil -.hasComponent(updateServiceData)) { - Component defaultComp = ServiceApiUtil - .createDefaultComponent(updateServiceData); - return updateComponent(updateServiceData.getName(), defaultComp.getName(), - defaultComp); -} - // If nothing happens consider it a no-op return Response.status(Status.NO_CONTENT).build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index b9b5b3a..cc76259 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -19,28 +19,17 @@ swagger: '2.0' info: title: "YARN Simplified API layer for services" description: | -Bringing a new service on YARN today is not a simple experience. The APIs of -existing frameworks are either too low level (native YARN), require writing -new code (for frameworks with programmatic APIs) or writing a complex spec -(for declarative frameworks). In addition to building critical building
[32/60] [abbrv] hadoop git commit: Rebased onto trunk - fix conflicts
Rebased onto trunk - fix conflicts Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/307d55b3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/307d55b3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/307d55b3 Branch: refs/heads/trunk Commit: 307d55b3e1ec9261c73e63286887c0b90a0c961e Parents: 541b64c Author: Jian HeAuthored: Mon Sep 11 17:34:30 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:15 2017 -0800 -- .../org/apache/hadoop/yarn/service/ServiceScheduler.java | 11 +++ 1 file changed, 11 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/307d55b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index fb2fd16..7b809b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -574,6 +574,11 @@ public class ServiceScheduler extends CompositeService { } +@Override public void onContainerResourceUpdated(ContainerId containerId, +Resource resource) { + +} + @Override public void onGetContainerStatusError(ContainerId containerId, Throwable t) { @@ -586,6 +591,12 @@ public class ServiceScheduler extends CompositeService { } @Override +public void onUpdateContainerResourceError(ContainerId containerId, +Throwable t) { + +} + +@Override public void onStopContainerError(ContainerId containerId, Throwable t) { } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[38/60] [abbrv] hadoop git commit: YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He
YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce74e643 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce74e643 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce74e643 Branch: refs/heads/trunk Commit: ce74e64363abc64561263be70a923ab3e67f043f Parents: 37c9b73 Author: Billie RinaldiAuthored: Thu Sep 21 10:18:42 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:16 2017 -0800 -- .../hadoop/yarn/service/ServiceScheduler.java | 6 +- .../hadoop/yarn/service/client/ServiceClient.java | 3 +++ .../params/AbstractClusterBuildingActionArgs.java | 2 +- .../yarn/service/client/params/Arguments.java | 1 + .../component/instance/ComponentInstance.java | 10 +- .../service/provider/AbstractProviderService.java | 18 +++--- .../client/binding/RegistryTypeUtils.java | 3 ++- .../dns/ApplicationServiceRecordProcessor.java| 15 ++- .../server/dns/BaseServiceRecordProcessor.java| 8 .../registry/server/dns/TestRegistryDNS.java | 12 .../runtime/DockerLinuxContainerRuntime.java | 4 ++-- .../src/site/markdown/YarnCommands.md | 1 + 12 files changed, 53 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 7b809b9..ec5f3ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService { attemptId.getApplicationId().toString()); serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.APPLICATION); -serviceRecord.description = "Yarn Service Master"; - -serviceRecord.addExternalEndpoint(RegistryTypeUtils -.ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc", -context.clientAMService.getBindAddress())); +serviceRecord.description = "YarnServiceMaster"; // set any provided attributes setUserProvidedServiceRecordAttributes(service.getConfiguration(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 6890aef..a3a9fd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService if (!StringUtils.isEmpty(args.getServiceName())) { service.setName(args.getServiceName()); } +if (!StringUtils.isEmpty(args.queue)) { + service.setQueue(args.queue); +} return service; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java -- diff --git
[20/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java new file mode 100644 index 000..cb7131e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java @@ -0,0 +1,494 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.component; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; +import org.apache.hadoop.yarn.service.ContainerFailureTracker; +import org.apache.hadoop.yarn.service.ServiceContext; +import org.apache.hadoop.yarn.service.ServiceScheduler; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent; +import org.apache.hadoop.yarn.service.ServiceMetrics; +import org.apache.hadoop.yarn.state.InvalidStateTransitionException; +import org.apache.hadoop.yarn.state.MultipleArcTransition; +import org.apache.hadoop.yarn.state.SingleArcTransition; +import org.apache.hadoop.yarn.state.StateMachine; +import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.Apps; +import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils; +import org.apache.hadoop.yarn.service.monitor.probe.Probe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*; +import static org.apache.hadoop.yarn.service.component.ComponentEventType.*; +import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START; +import static org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP; +import static org.apache.hadoop.yarn.service.component.ComponentState.*; +import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD; + +public class Component implements EventHandler { + private static final Logger LOG = LoggerFactory.getLogger(Component.class); + + private org.apache.hadoop.yarn.service.api.records.Component componentSpec; + private long allocateId; + private Priority priority; + private ServiceMetrics componentMetrics; + private ServiceScheduler scheduler; + private ServiceContext context; + private AMRMClientAsync amrmClient; + private AtomicLong instanceIdCounter = new AtomicLong(); + private MapcompInstances = + new ConcurrentHashMap<>(); + // component instances to be assigned with a container + private List pendingInstances = new LinkedList<>(); + private ContainerFailureTracker
[55/60] [abbrv] hadoop git commit: YARN-7423. Improve service client loggings. Contributed by Jian He
YARN-7423. Improve service client loggings. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a127f7b0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a127f7b0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a127f7b0 Branch: refs/heads/trunk Commit: a127f7b0fbde3942ca4329d5463bb8fe421daf7b Parents: a6c4bd7 Author: Billie RinaldiAuthored: Sat Nov 4 12:27:42 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:20 2017 -0800 -- .../hadoop/yarn/service/ServiceMaster.java | 4 +- .../yarn/service/api/records/Configuration.java | 8 +- .../yarn/service/client/ServiceClient.java | 50 +- .../yarn/service/component/Component.java | 10 +- .../component/instance/ComponentInstance.java | 4 +- .../containerlaunch/AbstractLauncher.java | 6 +- .../containerlaunch/ClasspathConstructor.java | 4 +- .../containerlaunch/CommandLineBuilder.java | 4 +- .../containerlaunch/JavaCommandLineBuilder.java | 4 +- .../yarn/service/monitor/probe/HttpProbe.java | 4 +- .../yarn/service/monitor/probe/PortProbe.java | 4 +- .../provider/AbstractClientProvider.java| 8 +- .../provider/AbstractProviderService.java | 4 +- .../yarn/service/provider/ProviderUtils.java| 10 +- .../registry/YarnRegistryViewForProviders.java | 6 +- .../yarn/service/utils/CoreFileSystem.java | 2 +- .../hadoop/yarn/service/utils/PortScanner.java | 4 +- .../yarn/service/utils/ServiceApiUtil.java | 2 +- .../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++ .../hadoop/yarn/service/utils/SliderUtils.java | 564 --- .../yarn/service/utils/ZookeeperUtils.java | 4 +- .../service/conf/TestValidateServiceNames.java | 1 - 22 files changed, 614 insertions(+), 638 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java index 2d86a5a..b0b4f06 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java @@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager; import org.apache.hadoop.yarn.service.monitor.ServiceMonitor; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.service.utils.SliderUtils; +import org.apache.hadoop.yarn.service.utils.ServiceUtils; import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,7 +112,7 @@ public class ServiceMaster extends CompositeService { } protected ContainerId getAMContainerId() throws BadClusterStateException { -return ContainerId.fromString(SliderUtils.mandatoryEnvVariable( +return ContainerId.fromString(ServiceUtils.mandatoryEnvVariable( ApplicationConstants.Environment.CONTAINER_ID.name())); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java index dc6dff8..859c1ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java +++
[07/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java deleted file mode 100644 index 7baa284..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.containerlaunch; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.service.utils.SliderUtils; - -import java.util.ArrayList; -import java.util.List; - -/** - * Build a single command line to include in the container commands; - * Special support for JVM command buildup. - */ -public class CommandLineBuilder { - protected final List argumentList = new ArrayList<>(20); - - /** - * Add an entry to the command list - * @param args arguments -these will be converted strings - */ - public void add(Object... args) { -for (Object arg : args) { - argumentList.add(arg.toString()); -} - } - - // Get the number of arguments - public int size() { -return argumentList.size(); - } - - /** - * Append the output and error files to the tail of the command - * @param stdout out - * @param stderr error. Set this to null to append into stdout - */ - public void addOutAndErrFiles(String stdout, String stderr) { -Preconditions.checkNotNull(stdout, "Null output file"); -Preconditions.checkState(!stdout.isEmpty(), "output filename invalid"); -// write out the path output -argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + - stdout); -if (stderr != null) { - argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + - stderr); -} else { - argumentList.add("2>&1"); -} - } - - /** - * This just returns the command line - * @see #build() - * @return the command line - */ - @Override - public String toString() { -return build(); - } - - /** - * Build the command line - * @return the command line - */ - public String build() { -return SliderUtils.join(argumentList, " "); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java deleted file mode 100644 index fcbb69b..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you
[45/60] [abbrv] hadoop git commit: YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi
YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92734800 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92734800 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92734800 Branch: refs/heads/trunk Commit: 92734800844b93bb19f3c0fca6be88d2801f7b1b Parents: b57144a Author: Jian HeAuthored: Fri Oct 13 10:49:38 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:18 2017 -0800 -- hadoop-yarn-project/hadoop-yarn/bin/yarn| 6 ++ .../hadoop-yarn/conf/yarn-env.sh| 12 +++ .../hadoop-yarn/hadoop-yarn-registry/pom.xml| 73 +- .../dns/PrivilegedRegistryDNSStarter.java | 80 .../hadoop/registry/server/dns/RegistryDNS.java | 45 +++ .../registry/server/dns/RegistryDNSServer.java | 57 +- .../src/site/markdown/YarnCommands.md | 8 ++ .../src/site/markdown/registry/index.md | 1 + .../markdown/registry/registry-configuration.md | 4 +- .../site/markdown/yarn-service/RegistryDNS.md | 40 +- .../markdown/yarn-service/ServiceDiscovery.md | 21 - 11 files changed, 282 insertions(+), 65 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/bin/yarn -- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 05d892b..727384c 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -45,6 +45,7 @@ function hadoop_usage hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker" hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server" hadoop_add_subcommand "queue" client "prints queue information" + hadoop_add_subcommand "registrydns" daemon "run the registry DNS server" hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager" hadoop_add_subcommand "rmadmin" admin "admin tools" hadoop_add_subcommand "router" daemon "run the Router daemon" @@ -143,6 +144,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" queue) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI ;; +registrydns) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter' + HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer' +;; resourcemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh -- diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh index 90a87bf..4bd1d3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh @@ -160,3 +160,15 @@ # See ResourceManager for some examples # #export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')" + +### +# Registry DNS specific parameters +### +# For privileged registry DNS, user to run as after dropping privileges +# This will replace the hadoop.id.str Java property in secure mode. +# export YARN_REGISTRYDNS_SECURE_USER=yarn + +# Supplemental options for privileged registry DNS +# By default, Hadoop uses jsvc which needs to know to launch a +# server jvm. +# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server" http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml index e083312..4e805cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml @@ -34,6 +34,21 @@ + org.slf4j + slf4j-api + + + + org.apache.hadoop + hadoop-auth + + + + org.apache.hadoop + hadoop-annotations + + + org.apache.hadoop hadoop-yarn-api @@ -70,14 +85,68 @@ + org.apache.zookeeper + zookeeper + + + + org.apache.curator + curator-client + + +
[29/60] [abbrv] hadoop git commit: Rebase onto trunk - fix some issues. Contributed by Rohith Sharma K S
Rebase onto trunk - fix some issues. Contributed by Rohith Sharma K S Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eed9e3e8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eed9e3e8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eed9e3e8 Branch: refs/heads/trunk Commit: eed9e3e8f4724ca42e0140754b8fe598b6472747 Parents: e660e52 Author: Jian HeAuthored: Mon Sep 4 22:07:24 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:14 2017 -0800 -- .../hadoop-yarn-services-api/pom.xml| 2 +- .../hadoop-yarn-services-core/pom.xml | 2 +- .../ServiceTimelinePublisher.java | 32 .../TestServiceTimelinePublisher.java | 2 -- .../hadoop-yarn-services/pom.xml| 2 +- .../src/main/webapp/app/routes/yarn-services.js | 2 +- 6 files changed, 23 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 1077ccd..74d9681 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -19,7 +19,7 @@ org.apache.hadoop hadoop-yarn-applications -3.0.0-beta1-SNAPSHOT +3.1.0-SNAPSHOT hadoop-yarn-services-api Apache Hadoop YARN Services API http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index d9b7adb..fb07edc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -19,7 +19,7 @@ org.apache.hadoop hadoop-yarn-services -3.0.0-beta1-SNAPSHOT +3.1.0-SNAPSHOT hadoop-yarn-services-core jar http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java index 5e65ad9..9e7d8e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java @@ -66,6 +66,7 @@ public class ServiceTimelinePublisher extends CompositeService { protected void serviceInit(org.apache.hadoop.conf.Configuration configuration) throws Exception { addService(timelineClient); +super.serviceInit(configuration); } @@ -95,8 +96,8 @@ public class ServiceTimelinePublisher extends CompositeService { // create info keys Map entityInfos = new HashMap (); entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName()); -entityInfos.put(ServiceTimelineMetricsConstants.STATE, -service.getState().toString()); +//entityInfos.put(ServiceTimelineMetricsConstants.STATE, +//service.getState().toString()); entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME, currentTimeMillis); entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS, @@ -232,18 +233,23 @@ public class
[12/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java deleted file mode 100644 index 8c968dc..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ /dev/null @@ -1,655 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.UpdatedContainer; -import org.apache.hadoop.yarn.client.api.AMRMClient; -import org.apache.hadoop.yarn.client.api.TimelineV2Client; -import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; -import org.apache.hadoop.yarn.client.api.async.NMClientAsync; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.AsyncDispatcher; -import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants; -import org.apache.hadoop.yarn.service.api.records.Application; -import org.apache.hadoop.yarn.service.api.records.ConfigFile; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstance; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent; -import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType; -import org.apache.hadoop.yarn.service.component.Component; -import org.apache.hadoop.yarn.service.component.ComponentEvent; -import org.apache.hadoop.yarn.service.component.ComponentEventType; -import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; -import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; -import org.apache.hadoop.yarn.service.metrics.ServiceMetrics; -import org.apache.hadoop.yarn.service.provider.ProviderUtils; -import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; -import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink; -import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; -import
[22/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java new file mode 100644 index 000..c3a2752 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -0,0 +1,892 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client; + +import org.apache.commons.lang.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryNTimes; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.util.VersionInfo; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; +import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; +import org.apache.hadoop.yarn.service.ClientAMProtocol; +import org.apache.hadoop.yarn.service.ServiceMaster; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs; +import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs; +import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs; +import org.apache.hadoop.yarn.service.client.params.Arguments; +import
[53/60] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi
YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba7ed7b6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba7ed7b6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba7ed7b6 Branch: refs/heads/trunk Commit: ba7ed7b66a48c61086ea837f39ff0f22ffe31eca Parents: 8ec9e72 Author: Jian HeAuthored: Sat Oct 28 23:04:19 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:19 2017 -0800 -- NOTICE.txt | 4 - hadoop-project/pom.xml | 6 - hadoop-yarn-project/hadoop-yarn/bin/yarn| 27 +- .../hadoop-yarn/conf/yarn-env.sh| 18 +- .../hadoop-yarn-services-core/pom.xml | 5 - .../hadoop/yarn/service/ServiceMaster.java | 21 +- .../hadoop/yarn/service/client/ServiceCLI.java | 113 --- .../yarn/service/client/ServiceClient.java | 187 +-- .../client/params/AbstractActionArgs.java | 139 .../client/params/AbstractArgsDelegate.java | 28 -- .../AbstractClusterBuildingActionArgs.java | 46 --- .../service/client/params/ActionBuildArgs.java | 39 --- .../service/client/params/ActionClientArgs.java | 71 - .../service/client/params/ActionCreateArgs.java | 46 --- .../client/params/ActionDependencyArgs.java | 65 .../client/params/ActionDestroyArgs.java| 37 --- .../service/client/params/ActionExistsArgs.java | 49 --- .../service/client/params/ActionFlexArgs.java | 50 --- .../service/client/params/ActionFreezeArgs.java | 56 .../service/client/params/ActionHelpArgs.java | 44 --- .../service/client/params/ActionKeytabArgs.java | 76 - .../service/client/params/ActionListArgs.java | 76 - .../client/params/ActionRegistryArgs.java | 218 - .../client/params/ActionResolveArgs.java| 153 - .../client/params/ActionResourceArgs.java | 70 .../service/client/params/ActionStatusArgs.java | 51 --- .../service/client/params/ActionThawArgs.java | 67 .../service/client/params/ActionTokensArgs.java | 78 - .../service/client/params/ActionUpdateArgs.java | 37 --- .../yarn/service/client/params/ArgOps.java | 156 - .../yarn/service/client/params/Arguments.java | 103 -- .../yarn/service/client/params/ClientArgs.java | 193 --- .../yarn/service/client/params/CommonArgs.java | 245 -- .../client/params/ComponentArgsDelegate.java| 52 --- .../client/params/DontSplitArguments.java | 34 -- .../client/params/LaunchArgsAccessor.java | 30 -- .../client/params/LaunchArgsDelegate.java | 51 --- .../client/params/OptionArgsDelegate.java | 66 .../client/params/PathArgumentConverter.java| 34 -- .../service/client/params/ServiceAMArgs.java| 50 --- .../client/params/ServiceAMCreateAction.java| 37 --- .../service/client/params/SliderActions.java| 79 - .../service/client/params/WaitArgsDelegate.java | 42 --- .../service/client/params/WaitTimeAccessor.java | 24 -- .../hadoop/yarn/service/utils/SliderUtils.java | 6 +- .../client/TestBuildExternalComponents.java | 19 +- .../yarn/service/client/TestServiceCLI.java | 125 +++- .../hadoop/yarn/client/api/AppAdminClient.java | 222 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 318 --- .../hadoop/yarn/client/cli/TestYarnCLI.java | 158 ++--- .../src/site/markdown/YarnCommands.md | 86 ++--- .../src/site/markdown/yarn-service/Examples.md | 12 +- .../site/markdown/yarn-service/QuickStart.md| 46 ++- .../markdown/yarn-service/YarnServiceAPI.md | 20 +- 54 files changed, 840 insertions(+), 3245 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/NOTICE.txt -- diff --git a/NOTICE.txt b/NOTICE.txt index f3af2f7..bc7a26f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -582,10 +582,6 @@ Ehcache 3.3.1, which has the following notices: * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. -JCommander (https://github.com/cbeust/jcommander), -which has the following notices: - * Copyright 2010 Cedric Beust ced...@beust.com - The binary distribution of this product bundles binaries of snakeyaml (https://bitbucket.org/asomov/snakeyaml), which has the following notices: http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index d18c9a8..f99e413
[50/60] [abbrv] hadoop git commit: YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He
YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2775fbc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2775fbc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2775fbc Branch: refs/heads/trunk Commit: d2775fbccaa187729d86459542210434ce4c3b8b Parents: 501be9b Author: Billie RinaldiAuthored: Wed Oct 25 13:01:01 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:19 2017 -0800 -- .../org/apache/hadoop/registry/server/dns/RegistryDNS.java | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2775fbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index 37e8429..9097c46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -189,7 +189,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations, LOG.info("Opening TCP and UDP channels on {} port {}", addr, port); addNIOUDP(addr, port); -addNIOTCP(addr, port); +//TODO Fix dns lookup over TCP +//addNIOTCP(addr, port); } /** @@ -770,6 +771,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, byte[] response = null; try { query = new Message(in); +LOG.info("received TCP query {}", query.getQuestion()); response = generateReply(query, ch.socket()); if (response == null) { return; @@ -947,7 +949,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, input.flip(); input.get(in); query = new Message(in); - LOG.info("{}: received query {}", remoteAddress, + LOG.info("{}: received UDP query {}", remoteAddress, query.getQuestion()); response = generateReply(query, null); if (response == null) { @@ -960,7 +962,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, output.put(response); output.flip(); -LOG.info("{}: sending response", remoteAddress); +LOG.debug("{}: sending response", remoteAddress); channel.send(output, remoteAddress); } } catch (Exception e) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[27/60] [abbrv] hadoop git commit: YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang
YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59fe98cd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59fe98cd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59fe98cd Branch: refs/heads/trunk Commit: 59fe98cd50b34eb75c5f522aa6f3a53bd650b15e Parents: fecf22b Author: Jian HeAuthored: Tue Aug 29 20:37:38 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:13 2017 -0800 -- .../java/org/apache/hadoop/yarn/service/api/records/Resource.java | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/59fe98cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java index cec9de9..dfdf92a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty; import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -103,6 +104,7 @@ public class Resource extends BaseResource implements Cloneable { this.memory = memory; } + @JsonIgnore public long getMemoryMB() { if (this.memory == null) { return 0; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[02/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java deleted file mode 100644 index 1fa07ce..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import com.google.common.net.HostAndPort; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.service.exceptions.BadConfigException; - -import java.util.ArrayList; -import java.util.List; - -public class ZookeeperUtils { - public static final int DEFAULT_PORT = 2181; - - public static String buildConnectionString(String zkHosts, int port) { -String zkPort = Integer.toString(port); -//parse the hosts -String[] hostlist = zkHosts.split(",", 0); -String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false); -return quorum; - } - - /** - * Take a quorum list and split it to (trimmed) pairs - * @param hostPortQuorumList list of form h1:port, h2:port2,... - * @return a possibly empty list of values between commas. They may not be - * valid hostname:port pairs - */ - public static List splitToPairs(String hostPortQuorumList) { -// split an address hot -String[] strings = StringUtils.getStrings(hostPortQuorumList); -int len = 0; -if (strings != null) { - len = strings.length; -} -List tuples = new ArrayList(len); -if (strings != null) { - for (String s : strings) { -tuples.add(s.trim()); - } -} -return tuples; - } - - /** - * Split a quorum list into a list of hostnames and ports - * @param hostPortQuorumList split to a list of hosts and ports - * @return a list of values - */ - public static List splitToHostsAndPorts(String hostPortQuorumList) { -// split an address hot -String[] strings = StringUtils.getStrings(hostPortQuorumList); -int len = 0; -if (strings != null) { - len = strings.length; -} -List list = new ArrayList(len); -if (strings != null) { - for (String s : strings) { - list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT)); - } -} -return list; - } - - /** - * Build up to a hosts only list - * @param hostAndPorts - * @return a list of the hosts only - */ - public static String buildHostsOnlyList(List hostAndPorts) { -StringBuilder sb = new StringBuilder(); -for (HostAndPort hostAndPort : hostAndPorts) { - sb.append(hostAndPort.getHostText()).append(","); -} -if (sb.length() > 0) { - sb.delete(sb.length() - 1, sb.length()); -} -return sb.toString(); - } - - public static String buildQuorumEntry(HostAndPort hostAndPort, -int defaultPort) { -String s = hostAndPort.toString(); -if (hostAndPort.hasPort()) { - return s; -} else { - return s + ":" + defaultPort; -} - } - - /** - * Build a quorum list, injecting a ":defaultPort" ref if needed on - * any entry without one - * @param hostAndPorts - * @param defaultPort - * @return - */ - public static String buildQuorum(List hostAndPorts, int defaultPort) { -List entries = new ArrayList(hostAndPorts.size()); -for (HostAndPort hostAndPort : hostAndPorts) { - entries.add(buildQuorumEntry(hostAndPort, defaultPort)); -} -return SliderUtils.join(entries, ",", false); - } - - public static String convertToHostsOnlyList(String quorum) throws - BadConfigException { -List
[37/60] [abbrv] hadoop git commit: YARN-7186. Add examples in yarn-service. Contributed by Jian He
YARN-7186. Add examples in yarn-service. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37c9b732 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37c9b732 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37c9b732 Branch: refs/heads/trunk Commit: 37c9b7327d188ccad7fd36b7466a65f68ad0c899 Parents: 307d55b Author: Billie RinaldiAuthored: Thu Sep 14 15:43:00 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:16 2017 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml | 7 .../examples/sleeper/sleeper.json | 15 +++ .../yarn/service/client/ServiceClient.java | 26 ++-- .../client/params/AbstractActionArgs.java | 2 +- .../AbstractClusterBuildingActionArgs.java | 2 +- .../service/client/params/ActionBuildArgs.java | 7 +++- .../service/client/params/ActionCreateArgs.java | 12 +- .../service/client/params/ActionExamples.java | 26 .../yarn/service/client/params/Arguments.java | 4 +- .../yarn/service/client/params/ClientArgs.java | 44 .../yarn/service/client/params/CommonArgs.java | 15 +-- .../service/client/params/SliderActions.java| 2 - .../src/site/markdown/YarnCommands.md | 34 --- .../site/markdown/yarn-service/QuickStart.md| 9 +++- 14 files changed, 91 insertions(+), 114 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml -- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 8b3d292..83633ac 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -98,6 +98,13 @@ etc/hadoop + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples + /share/hadoop/${hadoop.component}/yarn-service-examples + +**/* + + + hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target /share/hadoop/${hadoop.component}/sources http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json new file mode 100644 index 000..89ce527 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json @@ -0,0 +1,15 @@ +{ + "name": "sleeper-service", + "components" : + [ +{ + "name": "sleeper", + "number_of_containers": 2, + "launch_command": "sleep 90", + "resource": { +"cpus": 1, +"memory": "256" + } +} + ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index bfddc44..6890aef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs; +import
[08/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java deleted file mode 100644 index 982a114..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java +++ /dev/null @@ -1,493 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.compinstance; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.registry.client.binding.RegistryPathUtils; -import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; -import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; -import org.apache.hadoop.util.ExitUtil; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.client.api.NMClient; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.service.ServiceScheduler; -import org.apache.hadoop.yarn.service.api.records.ContainerState; -import org.apache.hadoop.yarn.service.component.Component; -import org.apache.hadoop.yarn.state.InvalidStateTransitionException; -import org.apache.hadoop.yarn.state.SingleArcTransition; -import org.apache.hadoop.yarn.state.StateMachine; -import org.apache.hadoop.yarn.state.StateMachineFactory; -import org.apache.hadoop.yarn.util.BoundedAppender; -import org.apache.hadoop.yarn.service.utils.SliderUtils; -import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; -import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus; -import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.text.MessageFormat; -import java.util.Date; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; - -import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER; -import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE; -import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.*; -import static org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.*; - -public class ComponentInstance implements EventHandler, -Comparable { - private static final Logger LOG = - LoggerFactory.getLogger(ComponentInstance.class); - - private StateMachinestateMachine; - private Component component; - private final ReadLock readLock; - private final WriteLock writeLock; - - private ComponentInstanceId compInstanceId = null; - private Path compInstanceDir; - private Container container; - private YarnRegistryViewForProviders yarnRegistryOperations; - private FileSystem fs; - private boolean timelineServiceEnabled = false; - private
[39/60] [abbrv] hadoop git commit: YARN-7201. Added an apache httpd example YARN service. Contributed by Billie Rinaldi
YARN-7201. Added an apache httpd example YARN service. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c70e5df1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c70e5df1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c70e5df1 Branch: refs/heads/trunk Commit: c70e5df100ba800f7170f44f63d388ae056cb74d Parents: ce74e64 Author: Jian HeAuthored: Mon Sep 25 16:36:43 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:16 2017 -0800 -- .../examples/httpd-no-dns/httpd-no-dns.json | 62 .../httpd-no-dns/httpd-proxy-no-dns.conf| 24 +++ .../examples/httpd/httpd-proxy.conf | 24 +++ .../examples/httpd/httpd.json | 55 +++ .../yarn/service/api/ServiceApiConstants.java | 5 + .../yarn/service/component/Component.java | 31 .../yarn/service/provider/ProviderUtils.java| 1 + .../hadoop/registry/server/dns/RegistryDNS.java | 4 + .../src/site/markdown/yarn-service/Examples.md | 159 +++ .../src/site/markdown/yarn-service/Overview.md | 3 +- 10 files changed, 367 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json new file mode 100644 index 000..6b35538 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json @@ -0,0 +1,62 @@ +{ + "name": "httpd-service-no-dns", + "lifetime": "3600", + "components": [ +{ + "name": "httpd", + "number_of_containers": 2, + "artifact": { +"id": "centos/httpd-24-centos7:latest", +"type": "DOCKER" + }, + "launch_command": "/usr/bin/run-httpd", + "resource": { +"cpus": 1, +"memory": "1024" + }, + "readiness_check": { +"type": "HTTP", +"props": { + "url": "http://${THIS_HOST}:8080; +} + }, + "configuration": { +"files": [ + { +"type": "ENV", +"dest_file": "/var/www/html/index.html", +"props": { + "content": "TitleHello from ${COMPONENT_INSTANCE_NAME}!" +} + } +] + } +}, +{ + "name": "httpd-proxy", + "number_of_containers": 1, + "dependencies": [ "httpd" ], + "artifact": { +"id": "centos/httpd-24-centos7:latest", +"type": "DOCKER" + }, + "launch_command": "/usr/bin/run-httpd", + "resource": { +"cpus": 1, +"memory": "1024" + }, + "configuration": { +"files": [ + { +"type": "TEMPLATE", +"dest_file": "/etc/httpd/conf.d/httpd-proxy.conf", +"src_file": "httpd-proxy-no-dns.conf" + } +] + } +} + ], + "quicklinks": { +"Apache HTTP Server": "http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf new file mode 100644 index 000..9894e64 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +#
[19/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java new file mode 100644 index 000..7baa284 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.containerlaunch; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.service.utils.SliderUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Build a single command line to include in the container commands; + * Special support for JVM command buildup. + */ +public class CommandLineBuilder { + protected final List argumentList = new ArrayList<>(20); + + /** + * Add an entry to the command list + * @param args arguments -these will be converted strings + */ + public void add(Object... args) { +for (Object arg : args) { + argumentList.add(arg.toString()); +} + } + + // Get the number of arguments + public int size() { +return argumentList.size(); + } + + /** + * Append the output and error files to the tail of the command + * @param stdout out + * @param stderr error. Set this to null to append into stdout + */ + public void addOutAndErrFiles(String stdout, String stderr) { +Preconditions.checkNotNull(stdout, "Null output file"); +Preconditions.checkState(!stdout.isEmpty(), "output filename invalid"); +// write out the path output +argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + + stdout); +if (stderr != null) { + argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + + stderr); +} else { + argumentList.add("2>&1"); +} + } + + /** + * This just returns the command line + * @see #build() + * @return the command line + */ + @Override + public String toString() { +return build(); + } + + /** + * Build the command line + * @return the command line + */ + public String build() { +return SliderUtils.join(argumentList, " "); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java new file mode 100644 index 000..0e51a62 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0
[16/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java new file mode 100644 index 000..7b22e3e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.JsonParseException; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.PropertyNamingStrategy; +import org.codehaus.jackson.map.SerializationConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Support for marshalling objects to and from JSON. + * This class is NOT thread safe; it constructs an object mapper + * as an instance field. + * @param + */ +public class JsonSerDeser { + + private static final Logger log = LoggerFactory.getLogger(JsonSerDeser.class); + private static final String UTF_8 = "UTF-8"; + + private final Class classType; + private final ObjectMapper mapper; + + /** + * Create an instance bound to a specific type + * @param classType class type + */ + public JsonSerDeser(Class classType) { +this.classType = classType; +this.mapper = new ObjectMapper(); +mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public JsonSerDeser(Class classType, PropertyNamingStrategy namingStrategy) { +this(classType); +mapper.setPropertyNamingStrategy(namingStrategy); + } + + /** + * Convert from JSON + * @param json input + * @return the parsed JSON + * @throws IOException IO + * @throws JsonMappingException failure to map from the JSON to this class + */ + public T fromJson(String json) +throws IOException, JsonParseException, JsonMappingException { +try { + return mapper.readValue(json, classType); +} catch (IOException e) { + log.error("Exception while parsing json : " + e + "\n" + json, e); + throw e; +} + } + + /** + * Convert from a JSON file + * @param jsonFile input file + * @return the parsed JSON + * @throws IOException IO problems + * @throws JsonMappingException failure to map from the JSON to this class + */ + public T fromFile(File jsonFile) +throws IOException, JsonParseException, JsonMappingException { +File absoluteFile = jsonFile.getAbsoluteFile(); +try { + return mapper.readValue(absoluteFile, classType); +} catch (IOException e) { + log.error("Exception while parsing json file {}", absoluteFile, e); + throw e; +} + } + + /** + * Convert from a JSON file + * @param resource input file + * @return the parsed JSON + * @throws IOException IO problems + * @throws JsonMappingException failure to map from the JSON to this class + */ + public T fromResource(String resource) +throws IOException, JsonParseException, JsonMappingException { +
[13/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java new file mode 100644 index 000..b742553 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.timelineservice; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier; +import org.apache.hadoop.yarn.client.api.TimelineV2Client; +import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.ServiceContext; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.api.records.Artifact; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Container; +import org.apache.hadoop.yarn.service.api.records.ContainerState; +import org.apache.hadoop.yarn.service.api.records.PlacementPolicy; +import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test class for ServiceTimelinePublisher. + */ +public class TestServiceTimelinePublisher { + private TimelineV2Client timelineClient; + private Configuration config; + private ServiceTimelinePublisher serviceTimelinePublisher; + private static String SERVICE_NAME = "HBASE"; + private static String SERVICEID = "application_1490093646524_0005"; + private static String ARTIFACTID = "ARTIFACTID"; + private static String COMPONENT_NAME = "DEFAULT"; + private static String CONTAINER_ID = + "container_e02_1490093646524_0005_01_01"; + private static String CONTAINER_IP = + "localhost"; + private static String CONTAINER_HOSTNAME = + "cnl124-localhost.site"; + private static String CONTAINER_BAREHOST = + "localhost.com"; + + @Before + public void setUp() throws Exception { +config = new Configuration(); +config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); +config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); +timelineClient = +new DummyTimelineClient(ApplicationId.fromString(SERVICEID)); +serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient); +timelineClient.init(config); +serviceTimelinePublisher.init(config); +timelineClient.start(); +serviceTimelinePublisher.start(); + } + + @After + public void tearDown() throws Exception { +if
[34/60] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md new file mode 100644 index 000..9022268 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -0,0 +1,592 @@ + + +# YARN Service API + +Bringing a new service on YARN today is not a simple experience. The APIs of existing +frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) +or writing a complex spec (for declarative frameworks). + +This simplified REST API can be used to create and manage the lifecycle of YARN services. +In most cases, the application owner will not be forced to make any changes to their applications. +This is primarily true if the application is packaged with containerization technologies like Docker. + +This document describes the API specifications (aka. YarnFile) for deploying/managing +containerized services on YARN. The same JSON spec can be used for both REST API +and CLI to manage the services. + + +### Version information +Version: 1.0.0 + +### License information +License: Apache 2.0 +License URL: http://www.apache.org/licenses/LICENSE-2.0.html + +### URI scheme +Host: host.mycompany.com + +Port: 9191(default) + +Schemes: HTTP + +### Consumes + +* application/json + + +### Produces + +* application/json + + +## Paths +### Create a service +``` +POST /ws/v1/services +``` + + Description + +Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the service creation request. There is no guarantee that the service will actually reach a RUNNING state. Resource availability and several other factors determines if the service will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the service and determine its state. + + Parameters +|Type|Name|Description|Required|Schema|Default| +||||||| +|BodyParameter|Service|Service request object|true|Service|| + + + Responses +|HTTP Code|Description|Schema| +|||| +|202|The request to create a service is accepted|No Content| +|400|Invalid service definition provided in the request body|No Content| +|500|Failed to create a service|No Content| +|default|Unexpected error|ServiceStatus| + + +### (TBD) List of services running in the cluster. +``` +GET /ws/v1/services +``` + + Description + +Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. + + Responses +|HTTP Code|Description|Schema| +|||| +|200|An array of services|Service array| +|default|Unexpected error|ServiceStatus| + + +### Get current version of the API server. +``` +GET /ws/v1/services/version +``` + + Description + +Get current version of the API server. + + Responses +|HTTP Code|Description|Schema| +|||| +|200|Successful request|No Content| + + +### Update a service or upgrade the binary version of the components of a running service +``` +PUT /ws/v1/services/{service_name} +``` + + Description + +Update the runtime properties of a service. Currently the following operations are supported - update lifetime, stop/start a service. The PUT operation is also used to orchestrate an upgrade of the service containers to a newer version of their artifacts (TBD). + + Parameters +|Type|Name|Description|Required|Schema|Default| +||||||| +|PathParameter|service_name|Service name|true|string|| +|BodyParameter|Service|The updated service definition. It can contain the updated lifetime of a service or the desired state (STOPPED/STARTED) of a service to initiate a start/stop operation against the specified service|true|Service|| + + + Responses +|HTTP Code|Description|Schema| +|||| +|204|Update or upgrade was successful|No Content| +|404|Service does not exist|No Content| +|default|Unexpected error|ServiceStatus| + + +### Destroy a service +``` +DELETE /ws/v1/services/{service_name} +``` + + Description + +Destroy a service and release all resources. This API might have to return JSON data providing location of logs (TBD), etc. + + Parameters +|Type|Name|Description|Required|Schema|Default| +||||||| +|PathParameter|service_name|Service name|true|string|| + + + Responses +|HTTP Code|Description|Schema| +||||
[56/60] [abbrv] hadoop git commit: YARN-7371. Added allocateRequestId in NMContainerStatus for recovery. Contributed by Chandni Singh
YARN-7371. Added allocateRequestId in NMContainerStatus for recovery. Contributed by Chandni Singh Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a55d0738 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a55d0738 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a55d0738 Branch: refs/heads/trunk Commit: a55d0738f1eba225c45578bbdabd0e5a6d6f5392 Parents: a127f7b Author: Jian HeAuthored: Sat Nov 4 23:15:21 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:20 2017 -0800 -- .../hadoop/yarn/service/ServiceTestUtils.java | 10 +- .../yarn/service/TestYarnNativeServices.java| 105 --- .../yarn/security/ContainerTokenIdentifier.java | 20 +++- .../src/main/proto/yarn_security_token.proto| 1 + .../api/protocolrecords/NMContainerStatus.java | 18 +++- .../impl/pb/NMContainerStatusPBImpl.java| 15 ++- .../OpportunisticContainerAllocator.java| 2 +- .../yarn_server_common_service_protos.proto | 1 + .../containermanager/ContainerManagerImpl.java | 5 +- .../container/ContainerImpl.java| 8 +- .../scheduler/AbstractYarnScheduler.java| 1 + .../scheduler/SchedulerApplicationAttempt.java | 3 +- .../security/RMContainerTokenSecretManager.java | 36 ++- .../server/resourcemanager/TestRMRestart.java | 2 +- .../TestResourceTrackerService.java | 6 +- .../capacity/TestContainerAllocation.java | 5 +- 16 files changed, 177 insertions(+), 61 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java index 19a5177..cf32880 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java @@ -131,12 +131,16 @@ public class ServiceTestUtils { return fs; } + protected MiniYARNCluster getYarnCluster() { +return yarnCluster; + } + protected void setupInternal(int numNodeManager) throws Exception { LOG.info("Starting up YARN cluster"); -//Logger rootLogger = LogManager.getRootLogger(); -//rootLogger.setLevel(Level.DEBUG); -setConf(new YarnConfiguration()); +if (conf == null) { + setConf(new YarnConfiguration()); +} conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); // reduce the teardown waiting time conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 64fcf57..f98d90a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -18,12 +18,13 @@ package org.apache.hadoop.yarn.service; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import
[58/60] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs index 17d38cb..0f0156a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs @@ -20,7 +20,7 @@ -{{type}} Metrics: Detailed Information +{{type}} Metrics: http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs index 5a94575..ef517d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs @@ -20,7 +20,7 @@ {{#if model.container}} - Component Information + Component Instance Information @@ -30,7 +30,7 @@ {{check-availability model.container.instanceName}} -Component Group +Component {{check-availability model.container.component}} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs index 0b642b0..fb4dba2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs @@ -18,7 +18,7 @@ -Active Components: {{model.componentName}} +Component: {{model.componentName}} {{em-table columns=tableColumns rows=model.instances}} - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[44/60] [abbrv] hadoop git commit: YARN-6744. Recover component information on YARN native services AM restart. Contributed by Billie Rinaldi
YARN-6744. Recover component information on YARN native services AM restart. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8a7ef1b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8a7ef1b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8a7ef1b Branch: refs/heads/trunk Commit: b8a7ef1b64392094562e7782e0fd092934724ad2 Parents: c723021 Author: Jian HeAuthored: Wed Oct 11 21:05:06 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:18 2017 -0800 -- .../hadoop-yarn-services-core/pom.xml | 5 ++ .../hadoop/yarn/service/ServiceScheduler.java | 94 ++-- .../yarn/service/component/Component.java | 76 +--- .../service/component/ComponentEventType.java | 1 + .../component/instance/ComponentInstance.java | 27 +- .../yarn/service/provider/ProviderUtils.java| 15 ++-- .../registry/YarnRegistryViewForProviders.java | 52 --- 7 files changed, 211 insertions(+), 59 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index 205a64d..851f73b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -159,6 +159,11 @@ org.apache.hadoop + hadoop-yarn-server-common + + + + org.apache.hadoop hadoop-common http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index ec5f3ed..f3824df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.binding.RegistryUtils; import org.apache.hadoop.registry.client.types.ServiceRecord; import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; @@ -237,11 +237,6 @@ public class ServiceScheduler extends CompositeService { serviceTimelinePublisher .serviceAttemptUnregistered(context, diagnostics.toString()); } -// Cleanup each component instance. no need to release containers as -// they will be automatically released by RM -for (ComponentInstance instance : liveInstances.values()) { - instance.cleanupRegistryAndCompHdfsDir(); -} String msg = diagnostics.toString() + "Navigate to the failed component for more details."; amRMClient @@ -266,11 +261,67 @@ public class ServiceScheduler extends CompositeService { } registerServiceInstance(context.attemptId, app); -//TODO handle containers recover - } +// recover components based on containers sent from RM +recoverComponents(response); - private void recover() { +for (Component component : componentsById.values()) { + // Trigger initial evaluation of components + if (component.areDependenciesReady()) { +LOG.info("Triggering initial evaluation of component {}", +component.getName()); +ComponentEvent event = new
[23/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java new file mode 100644 index 000..c64b1b5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.api.records; + +import io.swagger.annotations.ApiModelProperty; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Unstable +@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") +public class Error { + + private Integer code = null; + private String message = null; + private String fields = null; + + /** + **/ + public Error code(Integer code) { +this.code = code; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("code") + public Integer getCode() { +return code; + } + + public void setCode(Integer code) { +this.code = code; + } + + /** + **/ + public Error message(String message) { +this.message = message; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("message") + public String getMessage() { +return message; + } + + public void setMessage(String message) { +this.message = message; + } + + /** + **/ + public Error fields(String fields) { +this.fields = fields; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("fields") + public String getFields() { +return fields; + } + + public void setFields(String fields) { +this.fields = fields; + } + + @Override + public boolean equals(java.lang.Object o) { +if (this == o) { + return true; +} +if (o == null || getClass() != o.getClass()) { + return false; +} +Error error = (Error) o; +return Objects.equals(this.code, error.code) +&& Objects.equals(this.message, error.message) +&& Objects.equals(this.fields, error.fields); + } + + @Override + public int hashCode() { +return Objects.hash(code, message, fields); + } + + @Override + public String toString() { +StringBuilder sb = new StringBuilder(); +sb.append("class Error {\n"); + +sb.append("code: ").append(toIndentedString(code)).append("\n"); +sb.append("message: ").append(toIndentedString(message)).append("\n"); +sb.append("fields: ").append(toIndentedString(fields)).append("\n"); +sb.append("}"); +return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { +if (o == null) { + return "null"; +} +return o.toString().replace("\n", "\n"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java -- diff --git
[03/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java deleted file mode 100644 index 405f690..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; -import org.codehaus.jackson.map.annotate.JsonSerialize; - -import java.io.IOException; - -/** - * Serialized form of an application report which can be persisted - * and then parsed. It can not be converted back into a - * real YARN application report - * - * Useful for testing - */ - -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) - -public class SerializedApplicationReport { - - public String applicationId; - public String applicationAttemptId; - public String name; - public String applicationType; - public String user; - public String queue; - public String host; - public Integer rpcPort; - public String state; - public String diagnostics; - public String url; - /** - * This value is non-null only when a report is generated from a submission context. - * The YARN {@link ApplicationReport} structure does not propagate this value - * from the RM. - */ - public Long submitTime; - public Long startTime; - public Long finishTime; - public String finalStatus; - public String origTrackingUrl; - public Float progress; - - public SerializedApplicationReport() { - } - - public SerializedApplicationReport(ApplicationReport report) { -this.applicationId = report.getApplicationId().toString(); -ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId(); -this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A"; -this.name = report.getName(); -this.applicationType = report.getApplicationType(); -this.user = report.getUser(); -this.queue = report.getQueue(); -this.host = report.getHost(); -this.rpcPort = report.getRpcPort(); -this.state = report.getYarnApplicationState().toString(); -this.diagnostics = report.getDiagnostics(); -this.startTime = report.getStartTime(); -this.finishTime = report.getFinishTime(); -FinalApplicationStatus appStatus = report.getFinalApplicationStatus(); -this.finalStatus = appStatus == null ? "" : appStatus.toString(); -this.progress = report.getProgress(); -this.url = report.getTrackingUrl(); -this.origTrackingUrl= report.getOriginalTrackingUrl(); - } - - @Override - public String toString() { -try { - return ApplicationReportSerDeser.toString(this); -} catch (IOException e) { - return super.toString(); -} - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java -- diff --git
[11/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java deleted file mode 100644 index e10305a..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.api.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import java.util.Locale; - -@InterfaceAudience.Public -@InterfaceStability.Unstable -public enum ConfigFormat { - - JSON("json"), - PROPERTIES("properties"), - XML("xml"), - HADOOP_XML("hadoop_xml"), - ENV("env"), - TEMPLATE("template"), - YAML("yaml"), - ; - ConfigFormat(String suffix) { -this.suffix = suffix; - } - - private final String suffix; - - public String getSuffix() { -return suffix; - } - - - @Override - public String toString() { -return suffix; - } - - /** - * Get a matching format or null - * @param type - * @return the format - */ - public static ConfigFormat resolve(String type) { -for (ConfigFormat format: values()) { - if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) { -return format; - } -} -return null; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java deleted file mode 100644 index 0ac508b..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.api.records; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import
[31/60] [abbrv] hadoop git commit: YARN-7073. Yarn native services rest API documentation. Contributed by Gour Saha
YARN-7073. Yarn native services rest API documentation. Contributed by Gour Saha Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dd4490a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dd4490a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dd4490a Branch: refs/heads/trunk Commit: 7dd4490a2bd2b682a5f1a8605e6f6c053037 Parents: bd96c4c Author: Jian HeAuthored: Thu Aug 31 17:17:22 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:14 2017 -0800 -- hadoop-project/src/site/site.xml| 1 + .../hadoop/yarn/service/webapp/ApiServer.java | 47 +- .../definition/YARN-Services-Examples.md| 245 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 128 ++-- .../native-services/NativeServicesAPI.md| 606 +++ 5 files changed, 973 insertions(+), 54 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-project/src/site/site.xml -- diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml index 8cb57c8..ad893a2 100644 --- a/hadoop-project/src/site/site.xml +++ b/hadoop-project/src/site/site.xml @@ -157,6 +157,7 @@ + http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index f55e3f1..e8286ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -24,12 +24,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.api.records.ServiceStatus; import org.apache.hadoop.yarn.service.client.ServiceClient; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.utils.SliderUtils; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +48,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED; import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; /** @@ -76,11 +76,11 @@ public class ApiServer { @GET @Path(VERSION) @Consumes({ MediaType.APPLICATION_JSON }) - @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN }) + @Produces({ MediaType.APPLICATION_JSON }) public Response getVersion() { String version = VersionInfo.getBuildVersion(); LOG.info(version); -return Response.ok(version).build(); +return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build(); } @POST @@ -94,11 +94,11 @@ public class ApiServer { ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service); LOG.info("Successfully created service " + service.getName() + " applicationId = " + applicationId); - serviceStatus.setState(ServiceState.ACCEPTED); + serviceStatus.setState(ACCEPTED); serviceStatus.setUri( CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service .getName()); - return Response.status(Status.CREATED).entity(serviceStatus).build(); + return Response.status(Status.ACCEPTED).entity(serviceStatus).build(); } catch (IllegalArgumentException e) { serviceStatus.setDiagnostics(e.getMessage()); return Response.status(Status.BAD_REQUEST).entity(serviceStatus) @@ -182,16 +182,16 @@ public class ApiServer { + ": Invalid number of containers specified " + component .getNumberOfContainers()).build(); } +ServiceStatus status = new
[25/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index 88f74ef..17f8c95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -22,7 +22,7 @@ info: Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like. -To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component apps to complex multi-component assemblies needing orchestration. +To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component service to complex multi-component assemblies needing orchestration. We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management. @@ -41,177 +41,177 @@ schemes: - http - https # will be prefixed to all paths -basePath: /services/v1/ +basePath: /ws/v1/ consumes: - application/json produces: - application/json paths: - /applications: + /services: get: - summary: List of applications/services running in the cluster - description: Get a list of all currently running applications (response includes a minimal projection of the application info). For more details do a GET on a specific application name. + summary: List of services running in the cluster + description: Get a list of all currently running services (response includes a minimal projection of the service info). For more details do a GET on a specific service name. responses: 200: - description: An array of applications + description: An array of services schema: type: array items: - $ref: '#/definitions/Application' + $ref: '#/definitions/Service' default: description: Unexpected error schema: -$ref: '#/definitions/ApplicationStatus' +$ref: '#/definitions/ServiceStatus' post: - summary: Create an application/service - description: Create an application. The request JSON is an Application object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the application creation request. There is no guarantee that the application will actually reach a RUNNING state. Resource availability and several other factors determines if the application will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the application and determine its state. + summary: Create a service + description: Create a service. The request JSON is a service object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only
[26/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40ab068e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40ab068e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40ab068e Branch: refs/heads/trunk Commit: 40ab068eabe50875449d601471619a4e82fff86d Parents: 2ba38ce Author: Billie RinaldiAuthored: Mon Aug 28 09:59:55 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:13 2017 -0800 -- .../resources/assemblies/hadoop-yarn-dist.xml |8 +- .../assemblies/hadoop-yarn-services-dist.xml|2 +- hadoop-project/pom.xml |2 +- hadoop-yarn-project/hadoop-yarn/bin/yarn| 14 +- .../hadoop-yarn/conf/yarn-env.sh|2 +- .../hadoop-yarn-services-api/pom.xml|2 +- .../hadoop/yarn/service/webapp/ApiServer.java | 274 +++ .../yarn/service/webapp/ApiServerWebApp.java| 159 ++ .../service/webapp/ApplicationApiService.java | 275 --- .../service/webapp/ApplicationApiWebApp.java| 123 -- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 185 +- .../src/main/scripts/run_rest_service.sh| 28 - .../dev-support/findbugs-exclude.xml| 48 + .../conf/yarnservice-log4j.properties | 62 + .../hadoop-yarn-services-core/pom.xml | 408 + .../hadoop/yarn/service/ClientAMProtocol.java | 40 + .../hadoop/yarn/service/ClientAMService.java| 132 ++ .../yarn/service/ContainerFailureTracker.java | 89 + .../hadoop/yarn/service/ServiceContext.java | 41 + .../hadoop/yarn/service/ServiceMaster.java | 156 ++ .../hadoop/yarn/service/ServiceMetrics.java | 98 + .../hadoop/yarn/service/ServiceScheduler.java | 654 +++ .../yarn/service/api/ServiceApiConstants.java | 69 + .../yarn/service/api/records/Artifact.java | 160 ++ .../yarn/service/api/records/BaseResource.java | 52 + .../yarn/service/api/records/Component.java | 412 + .../yarn/service/api/records/ConfigFile.java| 225 +++ .../yarn/service/api/records/ConfigFormat.java | 67 + .../yarn/service/api/records/Configuration.java | 225 +++ .../yarn/service/api/records/Container.java | 297 +++ .../service/api/records/ContainerState.java | 30 + .../hadoop/yarn/service/api/records/Error.java | 129 ++ .../service/api/records/PlacementPolicy.java| 102 ++ .../service/api/records/ReadinessCheck.java | 175 ++ .../yarn/service/api/records/Resource.java | 159 ++ .../yarn/service/api/records/Service.java | 466 + .../yarn/service/api/records/ServiceState.java | 33 + .../yarn/service/api/records/ServiceStatus.java | 148 ++ .../yarn/service/client/ClientAMProxy.java | 57 + .../hadoop/yarn/service/client/ServiceCLI.java | 112 ++ .../yarn/service/client/ServiceClient.java | 892 + .../client/params/AbstractActionArgs.java | 158 ++ .../client/params/AbstractArgsDelegate.java | 28 + .../AbstractClusterBuildingActionArgs.java | 58 + .../service/client/params/ActionBuildArgs.java | 31 + .../service/client/params/ActionClientArgs.java | 71 + .../service/client/params/ActionCreateArgs.java | 33 + .../client/params/ActionDependencyArgs.java | 65 + .../client/params/ActionDestroyArgs.java| 37 + .../service/client/params/ActionExistsArgs.java | 49 + .../service/client/params/ActionFlexArgs.java | 50 + .../service/client/params/ActionFreezeArgs.java | 56 + .../service/client/params/ActionHelpArgs.java | 44 + .../service/client/params/ActionKDiagArgs.java | 76 + .../service/client/params/ActionKeytabArgs.java | 76 + .../service/client/params/ActionListArgs.java | 76 + .../client/params/ActionRegistryArgs.java | 218 +++ .../client/params/ActionResolveArgs.java| 153 ++ .../client/params/ActionResourceArgs.java | 70 + .../service/client/params/ActionStatusArgs.java | 51 + .../service/client/params/ActionThawArgs.java | 67 + .../service/client/params/ActionTokensArgs.java | 78 + .../service/client/params/ActionUpdateArgs.java | 32 + .../yarn/service/client/params/ArgOps.java | 156 ++ .../yarn/service/client/params/Arguments.java | 103 ++ .../yarn/service/client/params/ClientArgs.java | 252 +++ .../yarn/service/client/params/CommonArgs.java | 282 +++ .../client/params/ComponentArgsDelegate.java| 52 + .../client/params/DontSplitArguments.java | 34 + .../client/params/LaunchArgsAccessor.java | 30 + .../client/params/LaunchArgsDelegate.java | 51 + .../client/params/OptionArgsDelegate.java | 66 + .../client/params/PathArgumentConverter.java| 34 +
[41/60] [abbrv] hadoop git commit: YARN-7191. Improve yarn-service documentation. Contributed by Jian He
YARN-7191. Improve yarn-service documentation. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88512097 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88512097 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88512097 Branch: refs/heads/trunk Commit: 8851209788a4e110e899dd28f628657a21981db5 Parents: c70e5df Author: Billie RinaldiAuthored: Wed Sep 27 15:08:33 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:17 2017 -0800 -- .../src/site/markdown/yarn-service/Concepts.md | 47 +--- .../src/site/markdown/yarn-service/Overview.md | 3 +- .../site/markdown/yarn-service/QuickStart.md| 34 +-- .../site/markdown/yarn-service/RegistryDNS.md | 166 + .../markdown/yarn-service/ServiceDiscovery.md | 235 --- 5 files changed, 286 insertions(+), 199 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md index 7b62c36..e567d03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md @@ -22,6 +22,8 @@ It also does all the heavy lifting work such as resolving the service definition failed containers, monitoring components' healthiness and readiness, ensuring dependency start order across components, flexing up/down components, upgrading components etc. The end goal of the framework is to make sure the service is up and running as the state that user desired. +In addition, it leverages a lot of features in YARN core to accomplish scheduling constraints, such as +affinity and anti-affinity scheduling, log aggregation for services, automatically restart a container if it fails, and do in-place upgrade of a container. ### A Restful API-Server for deploying/managing services on YARN A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users @@ -34,44 +36,11 @@ support HA, distribute the load etc. ### Service Discovery A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup. -The DNS server essentially exposes the information in YARN service registry by translating them into DNS records such as A record and SRV record. -Clients can discover the IPs of containers via standard DNS lookup. -The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult -to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [DNS doc](ServiceDiscovery.md) -for more details. - -### Scheduling - -A host of scheduling features are being developed to support long running services. - -* Affinity and anti-affinity scheduling across containers ([YARN-6592](https://issues.apache.org/jira/browse/YARN-6592)). -* Container resizing ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197)) -* Special handling of container preemption/reservation for services - -### Container auto-restarts - -[YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails. -The service REST API provides users a way to enable NodeManager to automatically restart the container if it fails. -The advantage is that it avoids the entire cycle of releasing the failed containers, re-asking new containers, re-do resource localizations and so on, which -greatly minimizes container downtime. +The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.md). And the DNS server essentially exposes the +information in YARN service registry by translating them into DNS records such as A record and SRV record. +Clients can then discover the IPs of containers via standard DNS lookup. -### Container in-place upgrade - -[YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) aims to support upgrading containers in-place, that is, without losing the container allocations. -It opens up a few APIs in NodeManager to allow ApplicationMasters to upgrade their containers via a simple API call. -Under the hood, NodeManager does below steps: -* Downloading the new resources such as jars, docker container
[60/60] [abbrv] hadoop git commit: Merge branch 'yarn-native-services' into trunk
Merge branch 'yarn-native-services' into trunk Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbc632d9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbc632d9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbc632d9 Branch: refs/heads/trunk Commit: cbc632d9abf08c56a7fc02be51b2718af30bad28 Parents: dcd99c4 a55d073 Author: Jian HeAuthored: Mon Nov 6 14:02:19 2017 -0800 Committer: Jian He Committed: Mon Nov 6 14:02:19 2017 -0800 -- LICENSE.txt |1 + NOTICE.txt | 10 + .../resources/assemblies/hadoop-yarn-dist.xml | 25 + hadoop-project/pom.xml | 44 +- hadoop-project/src/site/site.xml| 10 +- hadoop-yarn-project/hadoop-yarn/bin/yarn| 18 +- .../hadoop-yarn/conf/yarn-env.sh| 18 + .../dev-support/findbugs-exclude.xml| 16 +- .../hadoop/yarn/conf/YarnConfiguration.java |2 + .../dev-support/findbugs-exclude.xml| 20 + .../hadoop-yarn-services-api/pom.xml| 130 ++ .../hadoop/yarn/service/webapp/ApiServer.java | 298 +++ .../yarn/service/webapp/ApiServerWebApp.java| 161 ++ .../definition/YARN-Services-Examples.md| 245 +++ ...RN-Simplified-V1-API-Layer-For-Services.yaml | 471 + .../src/main/resources/log4j-server.properties | 76 + .../src/main/resources/webapps/api-server/app | 16 + .../src/main/webapp/WEB-INF/web.xml | 36 + .../hadoop/yarn/service/ServiceClientTest.java | 107 ++ .../hadoop/yarn/service/TestApiServer.java | 366 .../dev-support/findbugs-exclude.xml| 48 + .../conf/yarnservice-log4j.properties | 62 + .../examples/httpd-no-dns/httpd-no-dns.json | 62 + .../httpd-no-dns/httpd-proxy-no-dns.conf| 24 + .../examples/httpd/httpd-proxy.conf | 24 + .../examples/httpd/httpd.json | 55 + .../examples/sleeper/sleeper.json | 15 + .../hadoop-yarn-services-core/pom.xml | 255 +++ .../hadoop/yarn/service/ClientAMProtocol.java | 40 + .../hadoop/yarn/service/ClientAMService.java| 132 ++ .../yarn/service/ContainerFailureTracker.java | 89 + .../hadoop/yarn/service/ServiceContext.java | 41 + .../hadoop/yarn/service/ServiceMaster.java | 169 ++ .../hadoop/yarn/service/ServiceMetrics.java | 94 + .../hadoop/yarn/service/ServiceScheduler.java | 691 +++ .../yarn/service/api/ServiceApiConstants.java | 74 + .../yarn/service/api/records/Artifact.java | 168 ++ .../yarn/service/api/records/BaseResource.java | 52 + .../yarn/service/api/records/Component.java | 430 + .../service/api/records/ComponentState.java | 30 + .../yarn/service/api/records/ConfigFile.java| 233 +++ .../yarn/service/api/records/ConfigFormat.java | 67 + .../yarn/service/api/records/Configuration.java | 225 +++ .../yarn/service/api/records/Container.java | 298 +++ .../service/api/records/ContainerState.java | 30 + .../hadoop/yarn/service/api/records/Error.java | 129 ++ .../service/api/records/PlacementPolicy.java| 102 + .../service/api/records/ReadinessCheck.java | 183 ++ .../yarn/service/api/records/Resource.java | 161 ++ .../yarn/service/api/records/Service.java | 390 .../yarn/service/api/records/ServiceState.java | 33 + .../yarn/service/api/records/ServiceStatus.java | 148 ++ .../yarn/service/client/ClientAMProxy.java | 57 + .../yarn/service/client/ServiceClient.java | 960 ++ .../yarn/service/component/Component.java | 584 ++ .../yarn/service/component/ComponentEvent.java | 83 + .../service/component/ComponentEventType.java | 27 + .../yarn/service/component/ComponentState.java | 25 + .../component/instance/ComponentInstance.java | 549 ++ .../instance/ComponentInstanceEvent.java| 58 + .../instance/ComponentInstanceEventType.java| 26 + .../component/instance/ComponentInstanceId.java | 91 + .../instance/ComponentInstanceState.java| 26 + .../yarn/service/conf/RestApiConstants.java | 39 + .../yarn/service/conf/SliderExitCodes.java | 88 + .../yarn/service/conf/YarnServiceConf.java | 113 ++ .../yarn/service/conf/YarnServiceConstants.java | 92 + .../containerlaunch/AbstractLauncher.java | 271 +++ .../containerlaunch/ClasspathConstructor.java | 172 ++ .../containerlaunch/CommandLineBuilder.java | 86 + .../containerlaunch/ContainerLaunchService.java | 101 + .../containerlaunch/CredentialUtils.java| 319 .../containerlaunch/JavaCommandLineBuilder.java | 181 ++ .../exceptions/BadClusterStateException.java|
[48/60] [abbrv] hadoop git commit: Rebase to trunk
Rebase to trunk Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c30b1a9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c30b1a9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c30b1a9 Branch: refs/heads/trunk Commit: 3c30b1a97d4d93a5714a91db90749c2a581bfdc4 Parents: 9273480 Author: Jian HeAuthored: Fri Oct 13 11:15:02 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:18 2017 -0800 -- hadoop-yarn-project/hadoop-yarn/bin/yarn | 1 - 1 file changed, 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c30b1a9/hadoop-yarn-project/hadoop-yarn/bin/yarn -- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 727384c..cc269d8 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -51,7 +51,6 @@ function hadoop_usage hadoop_add_subcommand "router" daemon "run the Router daemon" hadoop_add_subcommand "schedulerconf" client "Updates scheduler configuration" hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools" - hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" hadoop_add_subcommand "service" client "deploy/manage a service on YARN" hadoop_add_subcommand "sharedcachemanager" daemon "run the SharedCacheManager daemon" hadoop_add_subcommand "timelinereader" client "run the timeline reader server" - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[05/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java deleted file mode 100644 index b3fdef9..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.rest; - -import com.google.common.base.Preconditions; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.api.json.JSONConfiguration; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; - -/** - * Class to bond to a Jersey client, for UGI integration and SPNEGO. - * - * Usage: create an instance, then when creating a Jersey Client - * pass in to the constructor the handler provided by {@link #getHandler()} - * - * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey docs - */ -public class UgiJerseyBinding implements -HttpURLConnectionFactory { - private static final Logger log = - LoggerFactory.getLogger(UgiJerseyBinding.class); - - private final UrlConnectionOperations operations; - private final URLConnectionClientHandler handler; - - /** - * Construct an instance - * @param operations operations instance - */ - @SuppressWarnings("ThisEscapedInObjectConstruction") - public UgiJerseyBinding(UrlConnectionOperations operations) { -Preconditions.checkArgument(operations != null, "Null operations"); -this.operations = operations; -handler = new URLConnectionClientHandler(this); - } - - /** - * Create an instance off the configuration. The SPNEGO policy - * is derived from the current UGI settings. - * @param conf config - */ - public UgiJerseyBinding(Configuration conf) { -this(new UrlConnectionOperations(conf)); - } - - /** - * Get a URL connection. - * @param url URL to connect to - * @return the connection - * @throws IOException any problem. {@link AuthenticationException} - * errors are wrapped - */ - @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { -try { - // open a connection handling status codes and so redirections - // but as it opens a connection, it's less useful than you think. - - return operations.openConnection(url); -} catch (AuthenticationException e) { - throw new IOException(e); -} - } - - public UrlConnectionOperations getOperations() { -return operations; - } - - public URLConnectionClientHandler getHandler() { -return handler; - } - - /** - * Get the SPNEGO flag (as found in the operations instance - * @return the spnego policy - */ - public boolean isUseSpnego() { -return operations.isUseSpnego(); - } - - - /** - * Uprate error codes 400 and up into faults; - * - * see {@link ExceptionConverter#convertJerseyException(String, String, UniformInterfaceException)} - */ - public static IOException uprateFaults(HttpVerb verb, String url, -
[09/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java deleted file mode 100644 index c2866cf..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.client.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; -import org.apache.hadoop.yarn.service.exceptions.UsageException; -import org.apache.hadoop.yarn.service.api.records.ConfigFormat; - -import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY; -import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY; -import java.io.File; - -/** - * Registry actions - * - * --instance {app name}, if a / is in it, refers underneath? - * --dest {destfile} - * --list : list instances of slider service - * --listfiles - */ -@Parameters(commandNames = {ACTION_REGISTRY}, -commandDescription = DESCRIBE_ACTION_REGISTRY) - -public class ActionRegistryArgs extends AbstractActionArgs { - - public static final String USAGE = - "Usage: " + SliderActions.ACTION_REGISTRY - + " (" - + Arguments.ARG_LIST + "|" - + Arguments.ARG_LISTCONF + "|" - + Arguments.ARG_LISTEXP + "|" - + Arguments.ARG_LISTFILES + "|" - + Arguments.ARG_GETCONF + "|" - + Arguments.ARG_GETEXP + "> " - + Arguments.ARG_NAME + " " - + " )" - + "[" + Arguments.ARG_VERBOSE + "] " - + "[" + Arguments.ARG_USER + "] " - + "[" + Arguments.ARG_OUTPUT + " ] " - + "[" + Arguments.ARG_SERVICETYPE + " ] " - + "[" + Arguments.ARG_FORMAT + "] " - + System.getProperty("line.separator") - + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json" - ; - public ActionRegistryArgs() { - } - - public ActionRegistryArgs(String name) { -this.name = name; - } - - @Override - public String getActionName() { -return ACTION_REGISTRY; - } - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - @Override - public int getMinParams() { -return 0; - } - - @Parameter(names = {ARG_LIST}, - description = "list services") - public boolean list; - - @Parameter(names = {ARG_LISTCONF}, - description = "list configurations") - public boolean listConf; - - @Parameter(names = {ARG_GETCONF}, - description = "get configuration") - public String getConf; - - @Parameter(names = {ARG_LISTEXP}, - description = "list exports") - public boolean listExports; - - @Parameter(names = {ARG_GETEXP}, - description = "get export") - public String getExport; - - @Parameter(names = {ARG_LISTFILES}, - description = "list files") - public String listFiles; - - @Parameter(names = {ARG_GETFILES}, - description = "get files") - public String getFiles; - - //--format - @Parameter(names = ARG_FORMAT, - description = "Format for a response: ") - public String format = ConfigFormat.XML.toString() ; - - @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST}, - description = "Output destination") - public File out; - - @Parameter(names
[06/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java deleted file mode 100644 index 83c9961..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.provider; - -import org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory; -import org.apache.hadoop.yarn.service.api.records.Artifact; -import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory; -import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Base class for factories. - */ -public abstract class ProviderFactory { - protected static final Logger LOG = - LoggerFactory.getLogger(ProviderFactory.class); - - protected ProviderFactory() {} - - public abstract AbstractClientProvider createClientProvider(); - - public abstract ProviderService createServerProvider(); - - public static synchronized ProviderService getProviderService(Artifact - artifact) { -return createSliderProviderFactory(artifact).createServerProvider(); - } - - public static synchronized AbstractClientProvider getClientProvider(Artifact - artifact) { -return createSliderProviderFactory(artifact).createClientProvider(); - } - - /** - * Create a provider for a specific application - * @param artifact artifact - * @return provider factory - */ - public static synchronized ProviderFactory createSliderProviderFactory( - Artifact artifact) { -if (artifact == null || artifact.getType() == null) { - LOG.debug("Loading service provider type default"); - return DefaultProviderFactory.getInstance(); -} -LOG.debug("Loading service provider type {}", artifact.getType()); -switch (artifact.getType()) { - // TODO add handling for custom types? - // TODO handle application - case DOCKER: -return DockerProviderFactory.getInstance(); - case TARBALL: -return TarballProviderFactory.getInstance(); - default: -throw new IllegalArgumentException(String.format("Resolution error, " + -"%s should not be passed to createSliderProviderFactory", -artifact.getType())); -} - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java deleted file mode 100644 index 9ef0176..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership.
[10/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java deleted file mode 100644 index 1049698..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ /dev/null @@ -1,872 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.client; - -import org.apache.commons.lang.StringUtils; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryNTimes; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.registry.client.api.RegistryConstants; -import org.apache.hadoop.registry.client.api.RegistryOperations; -import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; -import org.apache.hadoop.registry.client.binding.RegistryUtils; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.service.CompositeService; -import org.apache.hadoop.util.VersionInfo; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ApplicationTimeout; -import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.ipc.YarnRPC; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto; -import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto; -import org.apache.hadoop.yarn.service.ClientAMProtocol; -import org.apache.hadoop.yarn.service.ServiceMaster; -import org.apache.hadoop.yarn.service.api.records.Application; -import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs; -import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs; -import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs; -import org.apache.hadoop.yarn.service.client.params.Arguments; -import org.apache.hadoop.yarn.service.client.params.ClientArgs; -import org.apache.hadoop.yarn.service.client.params.CommonArgs; -import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
[14/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java new file mode 100644 index 000..be36335 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java @@ -0,0 +1,530 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.service; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; +import org.apache.hadoop.yarn.service.api.records.Artifact; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.utils.JsonSerDeser; +import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; +import org.apache.hadoop.yarn.service.utils.SliderFileSystem; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME; +import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME; +import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*; +import static org.easymock.EasyMock.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Test for ServiceApiUtil helper methods. + */ +public class TestServiceApiUtil { + private static final Logger LOG = LoggerFactory + .getLogger(TestServiceApiUtil.class); + private static final String EXCEPTION_PREFIX = "Should have thrown " + + "exception: "; + private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " + + "exception: "; + + private static final String LEN_64_STR = + "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01"; + + private static final YarnConfiguration CONF_DEFAULT_DNS = new + YarnConfiguration(); + private static final YarnConfiguration CONF_DNS_ENABLED = new + YarnConfiguration(); + + @BeforeClass + public static void init() { +CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true); + } + + @Test(timeout = 9) + public void testResourceValidation() throws Exception { +assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR +.length()); + +SliderFileSystem sfs = initMock(null); + +Service app = new Service(); + +// no name +try { + ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); + Assert.fail(EXCEPTION_PREFIX + "service with no name"); +} catch (IllegalArgumentException e) { + assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage()); +} + +// bad format name +String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR}; +for (String badName : badNames) { + app.setName(badName); + try { +ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED); +Assert.fail(EXCEPTION_PREFIX + "service with bad name " + badName); + } catch (IllegalArgumentException e) { +
[21/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java new file mode 100644 index 000..3e53418 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.client.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.UsageException; +import org.apache.hadoop.yarn.service.api.records.ConfigFormat; + +import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY; +import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY; +import java.io.File; + +/** + * Registry actions + * + * --instance {app name}, if a / is in it, refers underneath? + * --dest {destfile} + * --list : list instances of slider service + * --listfiles + */ +@Parameters(commandNames = {ACTION_REGISTRY}, +commandDescription = DESCRIBE_ACTION_REGISTRY) + +public class ActionRegistryArgs extends AbstractActionArgs { + + public static final String USAGE = + "Usage: " + SliderActions.ACTION_REGISTRY + + " (" + + Arguments.ARG_LIST + "|" + + Arguments.ARG_LISTCONF + "|" + + Arguments.ARG_LISTEXP + "|" + + Arguments.ARG_LISTFILES + "|" + + Arguments.ARG_GETCONF + "|" + + Arguments.ARG_GETEXP + "> " + + Arguments.ARG_NAME + " " + + " )" + + "[" + Arguments.ARG_VERBOSE + "] " + + "[" + Arguments.ARG_USER + "] " + + "[" + Arguments.ARG_OUTPUT + " ] " + + "[" + Arguments.ARG_SERVICETYPE + " ] " + + "[" + Arguments.ARG_FORMAT + "] " + + System.getProperty("line.separator") + + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json" + ; + public ActionRegistryArgs() { + } + + public ActionRegistryArgs(String name) { +this.name = name; + } + + @Override + public String getActionName() { +return ACTION_REGISTRY; + } + + /** + * Get the min #of params expected + * @return the min number of params in the {@link #parameters} field + */ + @Override + public int getMinParams() { +return 0; + } + + @Parameter(names = {ARG_LIST}, + description = "list services") + public boolean list; + + @Parameter(names = {ARG_LISTCONF}, + description = "list configurations") + public boolean listConf; + + @Parameter(names = {ARG_GETCONF}, + description = "get configuration") + public String getConf; + + @Parameter(names = {ARG_LISTEXP}, + description = "list exports") + public boolean listExports; + + @Parameter(names = {ARG_GETEXP}, + description = "get export") + public String getExport; + + @Parameter(names = {ARG_LISTFILES}, + description = "list files") + public String listFiles; + + @Parameter(names = {ARG_GETFILES}, + description = "get files") + public String getFiles; + + //--format + @Parameter(names = ARG_FORMAT, + description = "Format for a response: ") + public String format = ConfigFormat.XML.toString() ; + + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST}, + description = "Output destination") + public File out; + +
[01/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
Repository: hadoop Updated Branches: refs/heads/trunk dcd99c4b9 -> cbc632d9a http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java deleted file mode 100644 index 8310530..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.conf; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.api.records.Application; -import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; -import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.util.Arrays; -import java.util.Collection; - -import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER; -import static org.easymock.EasyMock.*; - -/** - * Test loading example resources. - */ -@RunWith(value = Parameterized.class) -public class TestLoadExampleAppJson extends Assert { - private String resource; - - public TestLoadExampleAppJson(String resource) { -this.resource = resource; - } - - @Parameterized.Parameters - public static Collectionfilenames() { -String[][] stringArray = new String[ExampleAppJson -.ALL_EXAMPLE_RESOURCES.size()][1]; -int i = 0; -for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) { - stringArray[i++][0] = s; -} -return Arrays.asList(stringArray); - } - - @Test - public void testLoadResource() throws Throwable { -try { - Application application = JSON_SER_DESER.fromResource(resource); - - SliderFileSystem sfs = createNiceMock(SliderFileSystem.class); - FileSystem mockFs = createNiceMock(FileSystem.class); - expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes(); - expect(sfs.buildClusterDirPath(anyObject())).andReturn( - new Path("cluster_dir_path")).anyTimes(); - replay(sfs, mockFs); - - ServiceApiUtil.validateAndResolveApplication(application, sfs, - new YarnConfiguration()); -} catch (Exception e) { - throw new Exception("exception loading " + resource + ":" + e.toString()); -} - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java deleted file mode 100644 index 98c78d3..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional
[30/60] [abbrv] hadoop git commit: YARN-7126. Create introductory site documentation for YARN native services. Contributed by Gour Saha
YARN-7126. Create introductory site documentation for YARN native services. Contributed by Gour Saha Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e660e52a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e660e52a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e660e52a Branch: refs/heads/trunk Commit: e660e52a048eae39f6b57f3c085ebbd34c04dd17 Parents: 7dd4490 Author: Jian HeAuthored: Fri Sep 1 16:19:31 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:14 2017 -0800 -- LICENSE.txt | 1 + .../native-services/NativeServicesIntro.md | 96 +++- 2 files changed, 96 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e660e52a/LICENSE.txt -- diff --git a/LICENSE.txt b/LICENSE.txt index 67472b6..b0cef03 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1777,6 +1777,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The binary distribution of this product bundles these dependencies under the following license: FindBugs-jsr305 3.0.0 +dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. (2-clause BSD) Redistribution and use in source and binary forms, with or without http://git-wip-us.apache.org/repos/asf/hadoop/blob/e660e52a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md index 89fefe9..e6a4e91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md @@ -10,4 +10,98 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. See accompanying LICENSE file. ---> \ No newline at end of file +--> + +# Introduction: YARN Native Services + +## Overview +YARN Native Services provides first class framework support and APIs to host long running services natively in YARN. In addition to launching services, the new APIs support performing lifecycle management operations, such as flex service components up/down, manage lifetime, upgrade the service to a newer version, and stop/restart/delete the service. + +The native services capabilities are built on the existing low-level resource management API provided by YARN that can support any type of application. Other application frameworks like Hadoop MapReduce already expose higher level APIs that users can leverage to run applications on top of YARN. With the advent of containerization technologies like Docker, providing first class support and APIs for long running services at the framework level made sense. + +Relying on a framework has the advantage of exposing a simpler usage model to the user by enabling service configuration and launch through specification (without writing new code), as well as hiding complex low-level details including state management and fault-tolerance etc. Users/operators of existing services typically like to avoid modifying an existing service to be aware of YARN. With first class support capable of running a single Docker image as well as complex assemblies comprised of multiple Docker images, there is no need for service owners to be aware of YARN. Developers of new services do not have to worry about YARN internals and only need to focus on containerization of their service(s). + +## First class support for services +In order to natively provide first class support for long running services, several new features and improvements have been made at the framework level. + +### Incorporate Apache Slider into Apache YARN +Apache Slider, which existed as a separate incubator project has been merged into YARN to kick start the first class support. Apache Slider is a universal Application Master (AM) which had several key features built in - fault tolerance of service containers and AM, work-preserving AM restarts, service logs management, service management like flex up/down, stop/start, and rolling upgrade to newer service versions, etc. Of course lot more work has been done on top of what Apache Slider brought in, details
[17/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java new file mode 100644 index 000..add2475 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.registry; + +import com.google.common.base.Preconditions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.registry.client.api.BindFlags; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; + +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; +import org.apache.hadoop.yarn.service.utils.SliderUtils; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join; + +/** + * Registry view for providers. This tracks where the service + * is registered, offers access to the record and other things. + */ +public class YarnRegistryViewForProviders { + private static final Log LOG = + LogFactory.getLog(YarnRegistryViewForProviders.class); + + private final RegistryOperations registryOperations; + private final String user; + private final String sliderServiceClass; + private final String instanceName; + /** + * Record used where the service registered itself. + * Null until the service is registered + */ + private ServiceRecord selfRegistration; + + /** + * Path where record was registered + * Null until the service is registered + */ + private String selfRegistrationPath; + + public YarnRegistryViewForProviders(RegistryOperations registryOperations, + String user, + String sliderServiceClass, + String instanceName, + ApplicationAttemptId applicationAttemptId) { +Preconditions.checkArgument(registryOperations != null, +"null registry operations"); +Preconditions.checkArgument(user != null, "null user"); +Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass), +"unset service class"); +Preconditions.checkArgument(SliderUtils.isSet(instanceName), +"instanceName"); +Preconditions.checkArgument(applicationAttemptId != null, +"null applicationAttemptId"); +this.registryOperations = registryOperations; +this.user = user; +this.sliderServiceClass = sliderServiceClass; +this.instanceName = instanceName; + } + + public String getUser() { +return user; + } + + + private void setSelfRegistration(ServiceRecord selfRegistration) { +this.selfRegistration = selfRegistration; + } + + /** + * Get the path to where the service has registered itself. + * Null until the service is registered + * @return the service registration path. + */ + public String getSelfRegistrationPath() { +return selfRegistrationPath; + } + + /** + * Get the absolute path to where the service has registered itself. + * This includes the base registry path + * Null until the service is registered + * @return the service
[15/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java new file mode 100644 index 000..6e6f4dd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java @@ -0,0 +1,1654 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.utils; + +import com.google.common.base.Preconditions; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.GlobFilter; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.client.params.Arguments; +import org.apache.hadoop.yarn.service.client.params.SliderActions; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor; +import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException; +import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; +import org.apache.hadoop.yarn.service.exceptions.BadConfigException; +import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.zookeeper.server.util.KerberosUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.Serializable; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.URL; +import java.net.URLDecoder; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; +import java.util.zip.GZIPOutputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** + * These are slider-specific Util methods + */ +public final class SliderUtils { + + private static final Logger log = LoggerFactory.getLogger(SliderUtils.class); + + /**
[18/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java new file mode 100644 index 000..684f655 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.monitor.probe; + +import org.apache.hadoop.yarn.service.api.records.ReadinessCheck; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Formatter; +import java.util.Locale; + +/** + * Various utils to work with the monitor + */ +public final class MonitorUtils { + protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils + .class); + + private MonitorUtils() { + } + + public static String toPlural(int val) { +return val != 1 ? "s" : ""; + } + + /** + * Convert milliseconds to human time -the exact format is unspecified + * @param milliseconds a time in milliseconds + * @return a time that is converted to human intervals + */ + public static String millisToHumanTime(long milliseconds) { +StringBuilder sb = new StringBuilder(); +// Send all output to the Appendable object sb +Formatter formatter = new Formatter(sb, Locale.US); + +long s = Math.abs(milliseconds / 1000); +long m = Math.abs(milliseconds % 1000); +if (milliseconds > 0) { + formatter.format("%d.%03ds", s, m); +} else if (milliseconds == 0) { + formatter.format("0"); +} else { + formatter.format("-%d.%03ds", s, m); +} +return sb.toString(); + } + + public static Probe getProbe(ReadinessCheck readinessCheck) { +if (readinessCheck == null) { + return null; +} +if (readinessCheck.getType() == null) { + return null; +} +try { + switch (readinessCheck.getType()) { + case HTTP: +return HttpProbe.create(readinessCheck.getProps()); + case PORT: +return PortProbe.create(readinessCheck.getProps()); + default: +return null; + } +} catch (Throwable t) { + throw new IllegalArgumentException("Error creating readiness check " + + t); +} + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java new file mode 100644 index 000..aba5859 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You
[28/60] [abbrv] hadoop git commit: Rebase onto latest trunk. minor conflicts
Rebase onto latest trunk. minor conflicts Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd96c4c2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd96c4c2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd96c4c2 Branch: refs/heads/trunk Commit: bd96c4c235c15c1df6ba37544f8e15004ad1d09d Parents: 59fe98c Author: Jian HeAuthored: Wed Aug 30 22:48:35 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:13 2017 -0800 -- .../java/org/apache/hadoop/yarn/util/Apps.java | 2 ++ .../rmapp/attempt/RMAppAttemptImpl.java | 35 ++-- 2 files changed, 4 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java index 5efbd2e..685c6d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java @@ -150,6 +150,8 @@ public class Apps { } // Check if should black list the node based on container exit status + @Private + @Unstable public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { switch (exitStatus) { case ContainerExitStatus.PREEMPTED: http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 4997bc6..d3fbcb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -107,6 +107,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.BoundedAppender; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -1539,38 +1540,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { -switch (exitStatus) { -case ContainerExitStatus.PREEMPTED: -case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER: -case ContainerExitStatus.KILLED_BY_APPMASTER: -case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION: -case ContainerExitStatus.ABORTED: - // Neither the app's fault nor the system's fault. This happens by design, - // so no need for skipping nodes - return false; -case ContainerExitStatus.DISKS_FAILED: - // This container is marked with this exit-status means that the node is - // already marked as unhealthy given that most of the disks failed. So, no - // need for any explicit skipping of nodes. - return false; -case ContainerExitStatus.KILLED_EXCEEDED_VMEM: -case ContainerExitStatus.KILLED_EXCEEDED_PMEM: - // No point in skipping the node as it's not the system's fault - return false; -case ContainerExitStatus.SUCCESS: - return false; -case ContainerExitStatus.INVALID: - // Ideally, this shouldn't be considered for skipping a node. But in - // reality, it seems like there are cases where we are not setting - // exit-code correctly and so it's better to be conservative. See - // YARN-4284. - return true; -default: - return true; -} - } - private static final class UnmanagedAMAttemptSavedTransition extends
[04/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java deleted file mode 100644 index fe8cce8..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.utils; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.service.exceptions.BadConfigException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.StringWriter; -import java.net.URL; -import java.util.Map; - -/** - * Methods to aid in config, both in the Configuration class and - * with other parts of setting up Slider-initated processes. - * - * Some of the methods take an argument of a map iterable for their sources; this allows - * the same method - */ -public class ConfigHelper { - private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); - - /** - * Set an entire map full of values - * - * @param config config to patch - * @param map map of data - * @param origin origin data - */ - public static void addConfigMap(Configuration config, - Mapmap, - String origin) throws BadConfigException { -addConfigMap(config, map.entrySet(), origin); - } - - /** - * Set an entire map full of values - * - * @param config config to patch - * @param map map of data - * @param origin origin data - */ - public static void addConfigMap(Configuration config, - Iterable > map, - String origin) throws BadConfigException { -for (Map.Entry mapEntry : map) { - String key = mapEntry.getKey(); - String value = mapEntry.getValue(); - if (value == null) { -throw new BadConfigException("Null value for property " + key); - } - config.set(key, value, origin); -} - } - - /** - * Convert to an XML string - * @param conf configuration - * @return conf - * @throws IOException - */ - public static String toXml(Configuration conf) throws IOException { -StringWriter writer = new StringWriter(); -conf.writeXml(writer); -return writer.toString(); - } - - - /** - * Register a resource as a default resource. - * Do not attempt to use this unless you understand that the - * order in which default resources are loaded affects the outcome, - * and that subclasses of Configuration often register new default - * resources - * @param resource the resource name - * @return the URL or null - */ - public static URL registerDefaultResource(String resource) { -URL resURL = getResourceUrl(resource); -if (resURL != null) { - Configuration.addDefaultResource(resource); -} -return resURL; - } - - /** - * Load a configuration from a resource on this classpath. - * If the resource is not found, an empty configuration is returned - * @param resource the resource name - * @return the loaded configuration. - */ - public static Configuration loadFromResource(String resource) { -Configuration conf = new Configuration(false); -URL resURL = getResourceUrl(resource); -if (resURL != null) { - log.debug("loaded resources from {}", resURL); - conf.addResource(resource); -} else{ - log.debug("failed to find {} on
[24/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java new file mode 100644 index 000..fb2fd16 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -0,0 +1,654 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.api.RegistryOperationsFactory; +import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.CompositeService; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.UpdatedContainer; +import org.apache.hadoop.yarn.client.api.AMRMClient; +import org.apache.hadoop.yarn.client.api.TimelineV2Client; +import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; +import org.apache.hadoop.yarn.client.api.async.NMClientAsync; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.service.api.ServiceApiConstants; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ConfigFile; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType; +import org.apache.hadoop.yarn.service.component.Component; +import org.apache.hadoop.yarn.service.component.ComponentEvent; +import org.apache.hadoop.yarn.service.component.ComponentEventType; +import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; +import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; +import org.apache.hadoop.yarn.service.provider.ProviderUtils; +import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders; +import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink; +import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher; +import
[27/58] [abbrv] hadoop git commit: Rebase onto latest trunk. minor conflicts
Rebase onto latest trunk. minor conflicts Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd96c4c2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd96c4c2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd96c4c2 Branch: refs/heads/yarn-native-services Commit: bd96c4c235c15c1df6ba37544f8e15004ad1d09d Parents: 59fe98c Author: Jian HeAuthored: Wed Aug 30 22:48:35 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:13 2017 -0800 -- .../java/org/apache/hadoop/yarn/util/Apps.java | 2 ++ .../rmapp/attempt/RMAppAttemptImpl.java | 35 ++-- 2 files changed, 4 insertions(+), 33 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java index 5efbd2e..685c6d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java @@ -150,6 +150,8 @@ public class Apps { } // Check if should black list the node based on container exit status + @Private + @Unstable public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { switch (exitStatus) { case ContainerExitStatus.PREEMPTED: http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index 4997bc6..d3fbcb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -107,6 +107,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.BoundedAppender; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -1539,38 +1540,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { } } - private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) { -switch (exitStatus) { -case ContainerExitStatus.PREEMPTED: -case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER: -case ContainerExitStatus.KILLED_BY_APPMASTER: -case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION: -case ContainerExitStatus.ABORTED: - // Neither the app's fault nor the system's fault. This happens by design, - // so no need for skipping nodes - return false; -case ContainerExitStatus.DISKS_FAILED: - // This container is marked with this exit-status means that the node is - // already marked as unhealthy given that most of the disks failed. So, no - // need for any explicit skipping of nodes. - return false; -case ContainerExitStatus.KILLED_EXCEEDED_VMEM: -case ContainerExitStatus.KILLED_EXCEEDED_PMEM: - // No point in skipping the node as it's not the system's fault - return false; -case ContainerExitStatus.SUCCESS: - return false; -case ContainerExitStatus.INVALID: - // Ideally, this shouldn't be considered for skipping a node. But in - // reality, it seems like there are cases where we are not setting - // exit-code correctly and so it's better to be conservative. See - // YARN-4284. - return true; -default: - return true; -} - } - private static final class UnmanagedAMAttemptSavedTransition
[23/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java new file mode 100644 index 000..c64b1b5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.api.records; + +import io.swagger.annotations.ApiModelProperty; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Unstable +@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00") +public class Error { + + private Integer code = null; + private String message = null; + private String fields = null; + + /** + **/ + public Error code(Integer code) { +this.code = code; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("code") + public Integer getCode() { +return code; + } + + public void setCode(Integer code) { +this.code = code; + } + + /** + **/ + public Error message(String message) { +this.message = message; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("message") + public String getMessage() { +return message; + } + + public void setMessage(String message) { +this.message = message; + } + + /** + **/ + public Error fields(String fields) { +this.fields = fields; +return this; + } + + @ApiModelProperty(example = "null", value = "") + @JsonProperty("fields") + public String getFields() { +return fields; + } + + public void setFields(String fields) { +this.fields = fields; + } + + @Override + public boolean equals(java.lang.Object o) { +if (this == o) { + return true; +} +if (o == null || getClass() != o.getClass()) { + return false; +} +Error error = (Error) o; +return Objects.equals(this.code, error.code) +&& Objects.equals(this.message, error.message) +&& Objects.equals(this.fields, error.fields); + } + + @Override + public int hashCode() { +return Objects.hash(code, message, fields); + } + + @Override + public String toString() { +StringBuilder sb = new StringBuilder(); +sb.append("class Error {\n"); + +sb.append("code: ").append(toIndentedString(code)).append("\n"); +sb.append("message: ").append(toIndentedString(message)).append("\n"); +sb.append("fields: ").append(toIndentedString(fields)).append("\n"); +sb.append("}"); +return sb.toString(); + } + + /** + * Convert the given object to string with each line indented by 4 spaces + * (except the first line). + */ + private String toIndentedString(java.lang.Object o) { +if (o == null) { + return "null"; +} +return o.toString().replace("\n", "\n"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java -- diff --git
[46/58] [abbrv] hadoop git commit: YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi
YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92734800 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92734800 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92734800 Branch: refs/heads/yarn-native-services Commit: 92734800844b93bb19f3c0fca6be88d2801f7b1b Parents: b57144a Author: Jian HeAuthored: Fri Oct 13 10:49:38 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:18 2017 -0800 -- hadoop-yarn-project/hadoop-yarn/bin/yarn| 6 ++ .../hadoop-yarn/conf/yarn-env.sh| 12 +++ .../hadoop-yarn/hadoop-yarn-registry/pom.xml| 73 +- .../dns/PrivilegedRegistryDNSStarter.java | 80 .../hadoop/registry/server/dns/RegistryDNS.java | 45 +++ .../registry/server/dns/RegistryDNSServer.java | 57 +- .../src/site/markdown/YarnCommands.md | 8 ++ .../src/site/markdown/registry/index.md | 1 + .../markdown/registry/registry-configuration.md | 4 +- .../site/markdown/yarn-service/RegistryDNS.md | 40 +- .../markdown/yarn-service/ServiceDiscovery.md | 21 - 11 files changed, 282 insertions(+), 65 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/bin/yarn -- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 05d892b..727384c 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -45,6 +45,7 @@ function hadoop_usage hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker" hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server" hadoop_add_subcommand "queue" client "prints queue information" + hadoop_add_subcommand "registrydns" daemon "run the registry DNS server" hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager" hadoop_add_subcommand "rmadmin" admin "admin tools" hadoop_add_subcommand "router" daemon "run the Router daemon" @@ -143,6 +144,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}" queue) HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI ;; +registrydns) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter' + HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer' +;; resourcemanager) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager' http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh -- diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh index 90a87bf..4bd1d3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh @@ -160,3 +160,15 @@ # See ResourceManager for some examples # #export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')" + +### +# Registry DNS specific parameters +### +# For privileged registry DNS, user to run as after dropping privileges +# This will replace the hadoop.id.str Java property in secure mode. +# export YARN_REGISTRYDNS_SECURE_USER=yarn + +# Supplemental options for privileged registry DNS +# By default, Hadoop uses jsvc which needs to know to launch a +# server jvm. +# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server" http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml index e083312..4e805cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml @@ -34,6 +34,21 @@ + org.slf4j + slf4j-api + + + + org.apache.hadoop + hadoop-auth + + + + org.apache.hadoop + hadoop-annotations + + + org.apache.hadoop hadoop-yarn-api @@ -70,14 +85,68 @@ + org.apache.zookeeper + zookeeper + + + + org.apache.curator +
[41/58] [abbrv] hadoop git commit: YARN-6626. Embed REST API service into RM. Contributed by Eric Yang
YARN-6626. Embed REST API service into RM. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e677fa0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e677fa0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e677fa0 Branch: refs/heads/yarn-native-services Commit: 9e677fa05c6a822cf034bffd25bbee0bad84084b Parents: 8851209 Author: Jian HeAuthored: Thu Sep 28 16:29:22 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:17 2017 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 2 ++ .../hadoop/yarn/service/webapp/ApiServer.java | 7 .../hadoop-yarn-services-core/pom.xml | 6 .../yarn/service/api/records/Artifact.java | 8 + .../yarn/service/api/records/ConfigFile.java| 8 + .../service/api/records/ReadinessCheck.java | 8 + .../src/main/resources/yarn-default.xml | 8 + .../server/resourcemanager/webapp/RMWebApp.java | 19 +++ .../site/markdown/yarn-service/QuickStart.md| 34 +--- 9 files changed, 89 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 640e86e..4799137 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration { public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn." + "webapp.ui2.war-file-path"; + public static final String YARN_API_SERVICES_ENABLE = "yarn." + + "webapp.api-service.enable"; public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index e8286ef..89b020d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -17,6 +17,7 @@ package org.apache.hadoop.yarn.service.webapp; +import com.google.inject.Inject; import com.google.inject.Singleton; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.VersionInfo; @@ -57,6 +58,12 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*; @Singleton @Path(CONTEXT_ROOT) public class ApiServer { + + @Inject + public ApiServer(Configuration conf) { +super(); + } + private static final Logger LOG = LoggerFactory.getLogger(ApiServer.class); private static Configuration YARN_CONFIG = new YarnConfiguration(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml index fb07edc..205a64d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml @@ -245,12 +245,6 @@ - org.apache.hadoop - hadoop-yarn-server-resourcemanager - test - - - org.apache.curator curator-test test
[43/58] [abbrv] hadoop git commit: YARN-7202. Add UT for api-server. Contributed by Eric Yang
YARN-7202. Add UT for api-server. Contributed by Eric Yang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57144a1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57144a1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57144a1 Branch: refs/heads/yarn-native-services Commit: b57144a1726e0b02abbdc3e8f37769d3f7c039c8 Parents: b8a7ef1 Author: Jian HeAuthored: Thu Oct 12 10:57:35 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:18 2017 -0800 -- .../hadoop-yarn-services-api/pom.xml| 34 +- .../hadoop/yarn/service/webapp/ApiServer.java | 21 +- .../yarn/service/webapp/ApiServerWebApp.java| 2 +- .../hadoop/yarn/service/ServiceClientTest.java | 107 ++ .../hadoop/yarn/service/TestApiServer.java | 366 +++ .../hadoop/yarn/service/ServiceTestUtils.java | 171 + .../yarn/service/TestYarnNativeServices.java| 162 +--- 7 files changed, 690 insertions(+), 173 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml index 74d9681..ddea2a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml @@ -41,7 +41,6 @@ - org.apache.maven.plugins maven-jar-plugin @@ -66,17 +65,6 @@ - - -org.apache.maven.plugins -maven-surefire-plugin - - -${java.home} - - - - @@ -84,6 +72,7 @@ + org.apache.hadoop hadoop-yarn-services-core @@ -116,5 +105,26 @@ javax.ws.rs jsr311-api + + org.mockito + mockito-all + test + + + + + + + + org.apache.hadoop + hadoop-common + test-jar + + + junit + junit + test + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 4fe12e3..5773069 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -165,12 +165,12 @@ public class ApiServer { } else { LOG.info("Successfully stopped service {}", appName); } - return Response.status(Status.NO_CONTENT).build(); + return Response.status(Status.OK).build(); } catch (ApplicationNotFoundException e) { ServiceStatus serviceStatus = new ServiceStatus(); serviceStatus.setDiagnostics( - "Service " + appName + " not found " + e.getMessage()); - return Response.status(Status.NOT_FOUND).entity(serviceStatus) + "Service " + appName + " is not found in YARN: " + e.getMessage()); + return Response.status(Status.BAD_REQUEST).entity(serviceStatus) .build(); } catch (Exception e) { ServiceStatus serviceStatus = new ServiceStatus(); @@ -245,7 +245,8 @@ public class ApiServer { // flex a single component app if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil .hasComponent(updateServiceData)) { - Component defaultComp = ServiceApiUtil.createDefaultComponent(updateServiceData); + Component defaultComp = ServiceApiUtil + .createDefaultComponent(updateServiceData); return updateComponent(updateServiceData.getName(), defaultComp.getName(), defaultComp); } @@ -291,4 +292,16 @@ public class ApiServer { .entity(status).build(); } } + + /** + * Used by negative test case. + * + * @param mockServerClient - A mocked version of
[44/58] [abbrv] hadoop git commit: YARN-7323. Data structure update in service REST API. Contributed by Jian He
YARN-7323. Data structure update in service REST API. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68acd88d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68acd88d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68acd88d Branch: refs/heads/yarn-native-services Commit: 68acd88dcbfe03a0134c60b5398dfaa31ad2b786 Parents: 3c30b1a Author: Billie RinaldiAuthored: Tue Oct 17 11:50:16 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:18 2017 -0800 -- .../hadoop/yarn/service/webapp/ApiServer.java | 9 --- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 67 - .../hadoop/yarn/service/ServiceScheduler.java | 1 + .../yarn/service/api/records/Component.java | 22 +- .../service/api/records/ComponentState.java | 30 .../yarn/service/api/records/Container.java | 15 ++-- .../yarn/service/api/records/Service.java | 76 .../yarn/service/api/records/ServiceState.java | 2 +- .../yarn/service/client/ServiceClient.java | 45 .../yarn/service/component/Component.java | 12 .../component/instance/ComponentInstance.java | 48 + .../yarn/service/conf/RestApiConstants.java | 4 -- .../yarn/service/utils/ServiceApiUtil.java | 53 ++ .../hadoop/yarn/service/ServiceTestUtils.java | 4 +- .../hadoop/yarn/service/TestServiceApiUtil.java | 74 ++- .../yarn/service/TestYarnNativeServices.java| 4 +- .../service/conf/examples/app-override.json | 8 ++- .../hadoop/yarn/service/conf/examples/app.json | 8 ++- .../yarn/service/conf/examples/external0.json | 15 ++-- .../markdown/yarn-service/YarnServiceAPI.md | 22 +++--- 20 files changed, 232 insertions(+), 287 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java index 5773069..1bb6c93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java @@ -242,15 +242,6 @@ public class ApiServer { return updateLifetime(appName, updateServiceData); } -// flex a single component app -if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil -.hasComponent(updateServiceData)) { - Component defaultComp = ServiceApiUtil - .createDefaultComponent(updateServiceData); - return updateComponent(updateServiceData.getName(), defaultComp.getName(), - defaultComp); -} - // If nothing happens consider it a no-op return Response.status(Status.NO_CONTENT).build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index b9b5b3a..cc76259 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -19,28 +19,17 @@ swagger: '2.0' info: title: "YARN Simplified API layer for services" description: | -Bringing a new service on YARN today is not a simple experience. The APIs of -existing frameworks are either too low level (native YARN), require writing -new code (for frameworks with programmatic APIs) or writing a complex spec -(for declarative frameworks). In addition to building
[56/58] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He
YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6c4bd74 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6c4bd74 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6c4bd74 Branch: refs/heads/yarn-native-services Commit: a6c4bd74b636d0715ffb4c1aca9ef4e49201aa02 Parents: a288b0a Author: Billie RinaldiAuthored: Fri Nov 3 11:25:57 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:20 2017 -0800 -- .../definition/YARN-Services-Examples.md| 8 +- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 6 +- .../examples/httpd-no-dns/httpd-no-dns.json | 6 +- .../examples/httpd/httpd.json | 4 +- .../hadoop/yarn/service/ServiceMaster.java | 6 + .../hadoop/yarn/service/ServiceMetrics.java | 4 - .../hadoop/yarn/service/ServiceScheduler.java | 15 -- .../yarn/service/api/ServiceApiConstants.java | 4 +- .../yarn/service/api/records/ConfigFile.java| 32 ++-- .../yarn/service/api/records/Configuration.java | 4 +- .../service/api/records/ReadinessCheck.java | 24 +-- .../yarn/service/client/ServiceClient.java | 61 --- .../yarn/service/component/Component.java | 29 ++-- .../component/instance/ComponentInstance.java | 37 +++-- .../yarn/service/conf/YarnServiceConf.java | 14 +- .../yarn/service/conf/YarnServiceConstants.java | 2 + .../service/monitor/probe/MonitorUtils.java | 4 +- .../provider/AbstractClientProvider.java| 17 +- .../yarn/service/provider/ProviderUtils.java| 12 +- .../timelineservice/ServiceTimelineEvent.java | 4 +- .../ServiceTimelinePublisher.java | 48 +++--- .../yarn/service/conf/TestAppJsonResolve.java | 8 +- .../TestServiceTimelinePublisher.java | 2 +- .../service/conf/examples/app-override.json | 6 +- .../markdown/yarn-service/Configurations.md | 165 +++ .../src/site/markdown/yarn-service/Examples.md | 4 +- .../src/site/markdown/yarn-service/Overview.md | 3 +- .../markdown/yarn-service/YarnServiceAPI.md | 14 +- .../app/controllers/yarn-app/components.js | 2 +- .../app/templates/components/metrics-table.hbs | 2 +- .../templates/yarn-component-instance/info.hbs | 4 +- .../templates/yarn-component-instances/info.hbs | 2 +- 32 files changed, 384 insertions(+), 169 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md index 3cd3d48..00b21dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md @@ -176,14 +176,14 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 { "type": "XML", "dest_file": "/etc/hadoop/conf/core-site.xml", -"props": { +"properties": { "fs.defaultFS": "${CLUSTER_FS_URI}" } }, { "type": "XML", "dest_file": "/etc/hbase/conf/hbase-site.xml", -"props": { +"properties": { "hbase.cluster.distributed": "true", "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}", "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase", @@ -216,14 +216,14 @@ POST URL - http://localhost:9191:/ws/v1/services/hbase-app-1 { "type": "XML", "dest_file": "/etc/hadoop/conf/core-site.xml", -"props": { +"properties": { "fs.defaultFS": "${CLUSTER_FS_URI}" } }, { "type": "XML", "dest_file": "/etc/hbase/conf/hbase-site.xml", -"props": { +"properties": { "hbase.cluster.distributed": "true", "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}", "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
[09/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java deleted file mode 100644 index c2866cf..000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.service.client.params; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import org.apache.hadoop.yarn.service.conf.YarnServiceConstants; -import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException; -import org.apache.hadoop.yarn.service.exceptions.UsageException; -import org.apache.hadoop.yarn.service.api.records.ConfigFormat; - -import static org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY; -import static org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY; -import java.io.File; - -/** - * Registry actions - * - * --instance {app name}, if a / is in it, refers underneath? - * --dest {destfile} - * --list : list instances of slider service - * --listfiles - */ -@Parameters(commandNames = {ACTION_REGISTRY}, -commandDescription = DESCRIBE_ACTION_REGISTRY) - -public class ActionRegistryArgs extends AbstractActionArgs { - - public static final String USAGE = - "Usage: " + SliderActions.ACTION_REGISTRY - + " (" - + Arguments.ARG_LIST + "|" - + Arguments.ARG_LISTCONF + "|" - + Arguments.ARG_LISTEXP + "|" - + Arguments.ARG_LISTFILES + "|" - + Arguments.ARG_GETCONF + "|" - + Arguments.ARG_GETEXP + "> " - + Arguments.ARG_NAME + " " - + " )" - + "[" + Arguments.ARG_VERBOSE + "] " - + "[" + Arguments.ARG_USER + "] " - + "[" + Arguments.ARG_OUTPUT + " ] " - + "[" + Arguments.ARG_SERVICETYPE + " ] " - + "[" + Arguments.ARG_FORMAT + "] " - + System.getProperty("line.separator") - + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json" - ; - public ActionRegistryArgs() { - } - - public ActionRegistryArgs(String name) { -this.name = name; - } - - @Override - public String getActionName() { -return ACTION_REGISTRY; - } - - /** - * Get the min #of params expected - * @return the min number of params in the {@link #parameters} field - */ - @Override - public int getMinParams() { -return 0; - } - - @Parameter(names = {ARG_LIST}, - description = "list services") - public boolean list; - - @Parameter(names = {ARG_LISTCONF}, - description = "list configurations") - public boolean listConf; - - @Parameter(names = {ARG_GETCONF}, - description = "get configuration") - public String getConf; - - @Parameter(names = {ARG_LISTEXP}, - description = "list exports") - public boolean listExports; - - @Parameter(names = {ARG_GETEXP}, - description = "get export") - public String getExport; - - @Parameter(names = {ARG_LISTFILES}, - description = "list files") - public String listFiles; - - @Parameter(names = {ARG_GETFILES}, - description = "get files") - public String getFiles; - - //--format - @Parameter(names = ARG_FORMAT, - description = "Format for a response: ") - public String format = ConfigFormat.XML.toString() ; - - @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST}, - description = "Output destination") - public File out; - - @Parameter(names
[40/58] [abbrv] hadoop git commit: YARN-7191. Improve yarn-service documentation. Contributed by Jian He
YARN-7191. Improve yarn-service documentation. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88512097 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88512097 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88512097 Branch: refs/heads/yarn-native-services Commit: 8851209788a4e110e899dd28f628657a21981db5 Parents: c70e5df Author: Billie RinaldiAuthored: Wed Sep 27 15:08:33 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:17 2017 -0800 -- .../src/site/markdown/yarn-service/Concepts.md | 47 +--- .../src/site/markdown/yarn-service/Overview.md | 3 +- .../site/markdown/yarn-service/QuickStart.md| 34 +-- .../site/markdown/yarn-service/RegistryDNS.md | 166 + .../markdown/yarn-service/ServiceDiscovery.md | 235 --- 5 files changed, 286 insertions(+), 199 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md index 7b62c36..e567d03 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md @@ -22,6 +22,8 @@ It also does all the heavy lifting work such as resolving the service definition failed containers, monitoring components' healthiness and readiness, ensuring dependency start order across components, flexing up/down components, upgrading components etc. The end goal of the framework is to make sure the service is up and running as the state that user desired. +In addition, it leverages a lot of features in YARN core to accomplish scheduling constraints, such as +affinity and anti-affinity scheduling, log aggregation for services, automatically restart a container if it fails, and do in-place upgrade of a container. ### A Restful API-Server for deploying/managing services on YARN A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users @@ -34,44 +36,11 @@ support HA, distribute the load etc. ### Service Discovery A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup. -The DNS server essentially exposes the information in YARN service registry by translating them into DNS records such as A record and SRV record. -Clients can discover the IPs of containers via standard DNS lookup. -The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult -to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [DNS doc](ServiceDiscovery.md) -for more details. - -### Scheduling - -A host of scheduling features are being developed to support long running services. - -* Affinity and anti-affinity scheduling across containers ([YARN-6592](https://issues.apache.org/jira/browse/YARN-6592)). -* Container resizing ([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197)) -* Special handling of container preemption/reservation for services - -### Container auto-restarts - -[YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a retry-policy to let NM re-launch a service container when it fails. -The service REST API provides users a way to enable NodeManager to automatically restart the container if it fails. -The advantage is that it avoids the entire cycle of releasing the failed containers, re-asking new containers, re-do resource localizations and so on, which -greatly minimizes container downtime. +The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.md). And the DNS server essentially exposes the +information in YARN service registry by translating them into DNS records such as A record and SRV record. +Clients can then discover the IPs of containers via standard DNS lookup. -### Container in-place upgrade - -[YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) aims to support upgrading containers in-place, that is, without losing the container allocations. -It opens up a few APIs in NodeManager to allow ApplicationMasters to upgrade their containers via a simple API call. -Under the hood, NodeManager does below steps: -* Downloading the new resources such as jars,
[19/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java new file mode 100644 index 000..7baa284 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.containerlaunch; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.service.utils.SliderUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Build a single command line to include in the container commands; + * Special support for JVM command buildup. + */ +public class CommandLineBuilder { + protected final List argumentList = new ArrayList<>(20); + + /** + * Add an entry to the command list + * @param args arguments -these will be converted strings + */ + public void add(Object... args) { +for (Object arg : args) { + argumentList.add(arg.toString()); +} + } + + // Get the number of arguments + public int size() { +return argumentList.size(); + } + + /** + * Append the output and error files to the tail of the command + * @param stdout out + * @param stderr error. Set this to null to append into stdout + */ + public void addOutAndErrFiles(String stdout, String stderr) { +Preconditions.checkNotNull(stdout, "Null output file"); +Preconditions.checkState(!stdout.isEmpty(), "output filename invalid"); +// write out the path output +argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + + stdout); +if (stderr != null) { + argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + + stderr); +} else { + argumentList.add("2>&1"); +} + } + + /** + * This just returns the command line + * @see #build() + * @return the command line + */ + @Override + public String toString() { +return build(); + } + + /** + * Build the command line + * @return the command line + */ + public String build() { +return SliderUtils.join(argumentList, " "); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java new file mode 100644 index 000..0e51a62 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0
[49/58] [abbrv] hadoop git commit: YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He
YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2775fbc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2775fbc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2775fbc Branch: refs/heads/yarn-native-services Commit: d2775fbccaa187729d86459542210434ce4c3b8b Parents: 501be9b Author: Billie RinaldiAuthored: Wed Oct 25 13:01:01 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:19 2017 -0800 -- .../org/apache/hadoop/registry/server/dns/RegistryDNS.java | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2775fbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java index 37e8429..9097c46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java @@ -189,7 +189,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations, LOG.info("Opening TCP and UDP channels on {} port {}", addr, port); addNIOUDP(addr, port); -addNIOTCP(addr, port); +//TODO Fix dns lookup over TCP +//addNIOTCP(addr, port); } /** @@ -770,6 +771,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, byte[] response = null; try { query = new Message(in); +LOG.info("received TCP query {}", query.getQuestion()); response = generateReply(query, ch.socket()); if (response == null) { return; @@ -947,7 +949,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, input.flip(); input.get(in); query = new Message(in); - LOG.info("{}: received query {}", remoteAddress, + LOG.info("{}: received UDP query {}", remoteAddress, query.getQuestion()); response = generateReply(query, null); if (response == null) { @@ -960,7 +962,7 @@ public class RegistryDNS extends AbstractService implements DNSOperations, output.put(response); output.flip(); -LOG.info("{}: sending response", remoteAddress); +LOG.debug("{}: sending response", remoteAddress); channel.send(output, remoteAddress); } } catch (Exception e) { - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[13/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java new file mode 100644 index 000..b742553 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.timelineservice; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier; +import org.apache.hadoop.yarn.client.api.TimelineV2Client; +import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.service.ServiceContext; +import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.api.records.Artifact; +import org.apache.hadoop.yarn.service.api.records.Component; +import org.apache.hadoop.yarn.service.api.records.Container; +import org.apache.hadoop.yarn.service.api.records.ContainerState; +import org.apache.hadoop.yarn.service.api.records.PlacementPolicy; +import org.apache.hadoop.yarn.service.api.records.Resource; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Test class for ServiceTimelinePublisher. + */ +public class TestServiceTimelinePublisher { + private TimelineV2Client timelineClient; + private Configuration config; + private ServiceTimelinePublisher serviceTimelinePublisher; + private static String SERVICE_NAME = "HBASE"; + private static String SERVICEID = "application_1490093646524_0005"; + private static String ARTIFACTID = "ARTIFACTID"; + private static String COMPONENT_NAME = "DEFAULT"; + private static String CONTAINER_ID = + "container_e02_1490093646524_0005_01_01"; + private static String CONTAINER_IP = + "localhost"; + private static String CONTAINER_HOSTNAME = + "cnl124-localhost.site"; + private static String CONTAINER_BAREHOST = + "localhost.com"; + + @Before + public void setUp() throws Exception { +config = new Configuration(); +config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); +config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); +timelineClient = +new DummyTimelineClient(ApplicationId.fromString(SERVICEID)); +serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient); +timelineClient.init(config); +serviceTimelinePublisher.init(config); +timelineClient.start(); +serviceTimelinePublisher.start(); + } + + @After + public void tearDown() throws Exception { +if
[17/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java new file mode 100644 index 000..add2475 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.service.registry; + +import com.google.common.base.Preconditions; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.registry.client.api.BindFlags; +import org.apache.hadoop.registry.client.api.RegistryOperations; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; + +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId; +import org.apache.hadoop.yarn.service.utils.SliderUtils; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join; + +/** + * Registry view for providers. This tracks where the service + * is registered, offers access to the record and other things. + */ +public class YarnRegistryViewForProviders { + private static final Log LOG = + LogFactory.getLog(YarnRegistryViewForProviders.class); + + private final RegistryOperations registryOperations; + private final String user; + private final String sliderServiceClass; + private final String instanceName; + /** + * Record used where the service registered itself. + * Null until the service is registered + */ + private ServiceRecord selfRegistration; + + /** + * Path where record was registered + * Null until the service is registered + */ + private String selfRegistrationPath; + + public YarnRegistryViewForProviders(RegistryOperations registryOperations, + String user, + String sliderServiceClass, + String instanceName, + ApplicationAttemptId applicationAttemptId) { +Preconditions.checkArgument(registryOperations != null, +"null registry operations"); +Preconditions.checkArgument(user != null, "null user"); +Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass), +"unset service class"); +Preconditions.checkArgument(SliderUtils.isSet(instanceName), +"instanceName"); +Preconditions.checkArgument(applicationAttemptId != null, +"null applicationAttemptId"); +this.registryOperations = registryOperations; +this.user = user; +this.sliderServiceClass = sliderServiceClass; +this.instanceName = instanceName; + } + + public String getUser() { +return user; + } + + + private void setSelfRegistration(ServiceRecord selfRegistration) { +this.selfRegistration = selfRegistration; + } + + /** + * Get the path to where the service has registered itself. + * Null until the service is registered + * @return the service registration path. + */ + public String getSelfRegistrationPath() { +return selfRegistrationPath; + } + + /** + * Get the absolute path to where the service has registered itself. + * This includes the base registry path + * Null until the service is registered + * @return the service
[52/58] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi
YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba7ed7b6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba7ed7b6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba7ed7b6 Branch: refs/heads/yarn-native-services Commit: ba7ed7b66a48c61086ea837f39ff0f22ffe31eca Parents: 8ec9e72 Author: Jian HeAuthored: Sat Oct 28 23:04:19 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:19 2017 -0800 -- NOTICE.txt | 4 - hadoop-project/pom.xml | 6 - hadoop-yarn-project/hadoop-yarn/bin/yarn| 27 +- .../hadoop-yarn/conf/yarn-env.sh| 18 +- .../hadoop-yarn-services-core/pom.xml | 5 - .../hadoop/yarn/service/ServiceMaster.java | 21 +- .../hadoop/yarn/service/client/ServiceCLI.java | 113 --- .../yarn/service/client/ServiceClient.java | 187 +-- .../client/params/AbstractActionArgs.java | 139 .../client/params/AbstractArgsDelegate.java | 28 -- .../AbstractClusterBuildingActionArgs.java | 46 --- .../service/client/params/ActionBuildArgs.java | 39 --- .../service/client/params/ActionClientArgs.java | 71 - .../service/client/params/ActionCreateArgs.java | 46 --- .../client/params/ActionDependencyArgs.java | 65 .../client/params/ActionDestroyArgs.java| 37 --- .../service/client/params/ActionExistsArgs.java | 49 --- .../service/client/params/ActionFlexArgs.java | 50 --- .../service/client/params/ActionFreezeArgs.java | 56 .../service/client/params/ActionHelpArgs.java | 44 --- .../service/client/params/ActionKeytabArgs.java | 76 - .../service/client/params/ActionListArgs.java | 76 - .../client/params/ActionRegistryArgs.java | 218 - .../client/params/ActionResolveArgs.java| 153 - .../client/params/ActionResourceArgs.java | 70 .../service/client/params/ActionStatusArgs.java | 51 --- .../service/client/params/ActionThawArgs.java | 67 .../service/client/params/ActionTokensArgs.java | 78 - .../service/client/params/ActionUpdateArgs.java | 37 --- .../yarn/service/client/params/ArgOps.java | 156 - .../yarn/service/client/params/Arguments.java | 103 -- .../yarn/service/client/params/ClientArgs.java | 193 --- .../yarn/service/client/params/CommonArgs.java | 245 -- .../client/params/ComponentArgsDelegate.java| 52 --- .../client/params/DontSplitArguments.java | 34 -- .../client/params/LaunchArgsAccessor.java | 30 -- .../client/params/LaunchArgsDelegate.java | 51 --- .../client/params/OptionArgsDelegate.java | 66 .../client/params/PathArgumentConverter.java| 34 -- .../service/client/params/ServiceAMArgs.java| 50 --- .../client/params/ServiceAMCreateAction.java| 37 --- .../service/client/params/SliderActions.java| 79 - .../service/client/params/WaitArgsDelegate.java | 42 --- .../service/client/params/WaitTimeAccessor.java | 24 -- .../hadoop/yarn/service/utils/SliderUtils.java | 6 +- .../client/TestBuildExternalComponents.java | 19 +- .../yarn/service/client/TestServiceCLI.java | 125 +++- .../hadoop/yarn/client/api/AppAdminClient.java | 222 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 318 --- .../hadoop/yarn/client/cli/TestYarnCLI.java | 158 ++--- .../src/site/markdown/YarnCommands.md | 86 ++--- .../src/site/markdown/yarn-service/Examples.md | 12 +- .../site/markdown/yarn-service/QuickStart.md| 46 ++- .../markdown/yarn-service/YarnServiceAPI.md | 20 +- 54 files changed, 840 insertions(+), 3245 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/NOTICE.txt -- diff --git a/NOTICE.txt b/NOTICE.txt index f3af2f7..bc7a26f 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -582,10 +582,6 @@ Ehcache 3.3.1, which has the following notices: * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. -JCommander (https://github.com/cbeust/jcommander), -which has the following notices: - * Copyright 2010 Cedric Beust ced...@beust.com - The binary distribution of this product bundles binaries of snakeyaml (https://bitbucket.org/asomov/snakeyaml), which has the following notices: http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-project/pom.xml -- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index
[50/58] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java index f22d487..df4b1df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java @@ -20,88 +20,71 @@ package org.apache.hadoop.yarn.service.client; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hadoop.yarn.client.cli.ApplicationCLI; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.service.ClientAMProtocol; import org.apache.hadoop.yarn.service.api.records.Component; -import org.apache.hadoop.yarn.service.client.params.ClientArgs; +import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.conf.ExampleAppJson; import org.apache.hadoop.yarn.service.utils.ServiceApiUtil; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; -import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.List; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS; -import static org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS; -import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE; import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH; -import static org.mockito.Mockito.*; public class TestServiceCLI { + private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI + .class); - protected Configuration conf = new YarnConfiguration(); + private Configuration conf = new YarnConfiguration(); private File basedir; - private ServiceCLI cli; private SliderFileSystem fs; + private String basedirProp; - private void buildApp(String appDef) throws Throwable { -String[] args = -{ "build", ARG_FILE, ExampleAppJson.resourceName(appDef)}; -ClientArgs clientArgs = new ClientArgs(args); -clientArgs.parse(); -cli.exec(clientArgs); + private void runCLI(String[] args) throws Exception { +LOG.info("running CLI: yarn {}", Arrays.asList(args)); +ApplicationCLI cli = new ApplicationCLI(); +cli.setSysOutPrintStream(System.out); +cli.setSysErrPrintStream(System.err); +int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args)); +cli.stop(); + } + + private void buildApp(String serviceName, String appDef) throws Throwable { +String[] args = {"app", "-D", basedirProp, "-save", serviceName, +ExampleAppJson.resourceName(appDef)}; +runCLI(args); + } + + private void buildApp(String serviceName, String appDef, String lifetime, + String queue) throws Throwable { +String[] args = {"app", "-D", basedirProp, "-save", serviceName, +ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime, +"-changeQueue", queue}; +runCLI(args); } @Before public void setup() throws Throwable { basedir = new File("target", "apps"); +basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath(); conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath()); -conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0); -conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1); -conf.setInt( -CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0); -conf.setInt(CommonConfigurationKeysPublic. -IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0); fs = new SliderFileSystem(conf); if (basedir.exists()) { FileUtils.deleteDirectory(basedir); } else { basedir.mkdirs(); } - -// create a CLI and skip
[36/58] [abbrv] hadoop git commit: YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He
YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce74e643 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce74e643 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce74e643 Branch: refs/heads/yarn-native-services Commit: ce74e64363abc64561263be70a923ab3e67f043f Parents: 37c9b73 Author: Billie RinaldiAuthored: Thu Sep 21 10:18:42 2017 -0700 Committer: Jian He Committed: Mon Nov 6 13:30:16 2017 -0800 -- .../hadoop/yarn/service/ServiceScheduler.java | 6 +- .../hadoop/yarn/service/client/ServiceClient.java | 3 +++ .../params/AbstractClusterBuildingActionArgs.java | 2 +- .../yarn/service/client/params/Arguments.java | 1 + .../component/instance/ComponentInstance.java | 10 +- .../service/provider/AbstractProviderService.java | 18 +++--- .../client/binding/RegistryTypeUtils.java | 3 ++- .../dns/ApplicationServiceRecordProcessor.java| 15 ++- .../server/dns/BaseServiceRecordProcessor.java| 8 .../registry/server/dns/TestRegistryDNS.java | 12 .../runtime/DockerLinuxContainerRuntime.java | 4 ++-- .../src/site/markdown/YarnCommands.md | 1 + 12 files changed, 53 insertions(+), 30 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index 7b809b9..ec5f3ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService { attemptId.getApplicationId().toString()); serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.APPLICATION); -serviceRecord.description = "Yarn Service Master"; - -serviceRecord.addExternalEndpoint(RegistryTypeUtils -.ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc", -context.clientAMService.getBindAddress())); +serviceRecord.description = "YarnServiceMaster"; // set any provided attributes setUserProvidedServiceRecordAttributes(service.getConfiguration(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java index 6890aef..a3a9fd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java @@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService if (!StringUtils.isEmpty(args.getServiceName())) { service.setName(args.getServiceName()); } +if (!StringUtils.isEmpty(args.queue)) { + service.setQueue(args.queue); +} return service; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java -- diff --git