hadoop git commit: YARN-7118. AHS REST API can return NullPointerException. (Billie Rinaldi via wangda)

[wangda]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 e7199cb49 -> 179d8dc83


YARN-7118. AHS REST API can return NullPointerException. (Billie Rinaldi via 
wangda)

Change-Id: I16ca93385ab80e3680d4d173b8d1b79929ea7174


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/179d8dc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/179d8dc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/179d8dc8

Branch: refs/heads/branch-2
Commit: 179d8dc83c1a26adff178c32f1e286b40b1b6a88
Parents: e7199cb
Author: Wangda Tan 
Authored: Mon Sep 25 21:41:00 2017 -0700
Committer: Wangda Tan 
Committed: Mon Sep 25 21:41:00 2017 -0700

--
 ...pplicationHistoryManagerOnTimelineStore.java | 16 +---
 .../webapp/TestAHSWebServices.java  | 27 +---
 .../hadoop/yarn/server/webapp/WebServices.java  |  3 ++-
 3 files changed, 37 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/179d8dc8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 4c3705f..99bbe32 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -146,7 +146,11 @@ public class TestApplicationHistoryManagerOnTimelineStore {
   } else if (i == 3) {
 entities.addEntity(createApplicationTimelineEntity(
 appId, false, false, false, false, YarnApplicationState.FINISHED,
-true));
+true, false));
+  } else if (i == SCALE + 1) {
+entities.addEntity(createApplicationTimelineEntity(
+appId, false, false, false, false, YarnApplicationState.FINISHED,
+false, true));
   } else {
 entities.addEntity(createApplicationTimelineEntity(
 appId, false, false, false, false, YarnApplicationState.FINISHED));
@@ -497,13 +501,14 @@ public class TestApplicationHistoryManagerOnTimelineStore 
{
   boolean wrongAppId, boolean enableUpdateEvent,
   YarnApplicationState state) {
 return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId,
-wrongAppId, enableUpdateEvent, state, false);
+wrongAppId, enableUpdateEvent, state, false, false);
   }
 
   private static TimelineEntity createApplicationTimelineEntity(
   ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
   boolean wrongAppId, boolean enableUpdateEvent,
-  YarnApplicationState state, boolean missingPreemptMetrics) {
+  YarnApplicationState state, boolean missingPreemptMetrics,
+  boolean missingQueue) {
 TimelineEntity entity = new TimelineEntity();
 entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
 if (wrongAppId) {
@@ -519,7 +524,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
 entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
 "test app type");
 entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1");
-entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test 
queue");
+if (!missingQueue) {
+  entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
+  "test queue");
+}
 entityInfo.put(
 ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, 
"false");
 entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/179d8dc8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
--
diff --git 

hadoop git commit: YARN-7251. Misc changes to YARN-5734

[jhung]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-5734 e8b0e3b9c -> 9cac5d353


YARN-7251. Misc changes to YARN-5734


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9cac5d35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9cac5d35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9cac5d35

Branch: refs/heads/YARN-5734
Commit: 9cac5d35393e491525a82379ffc423a57fb428b2
Parents: e8b0e3b
Author: Jonathan Hung 
Authored: Mon Sep 25 17:52:40 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Sep 25 17:52:40 2017 -0700

--
 hadoop-yarn-project/hadoop-yarn/bin/yarn|  3 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd|  3 +-
 .../hadoop/yarn/conf/YarnConfiguration.java | 28 +
 .../hadoop/yarn/client/cli/SchedConfCLI.java| 42 ++--
 .../yarn/client/cli/TestSchedConfCLI.java   | 20 +-
 .../src/main/resources/yarn-default.xml |  4 +-
 .../resourcemanager/webapp/RMWebServices.java   |  2 +-
 .../TestRMWebServicesConfigurationMutation.java | 24 +--
 .../src/site/markdown/CapacityScheduler.md  |  2 +-
 .../src/site/markdown/ResourceManagerRest.md| 10 ++---
 .../src/site/markdown/YarnCommands.md   | 10 ++---
 11 files changed, 97 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9cac5d35/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 331fcfe..a9b152d 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,6 +47,7 @@ function hadoop_usage
   hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
   hadoop_add_subcommand "rmadmin" admin "admin tools"
   hadoop_add_subcommand "router" daemon "run the Router daemon"
+  hadoop_add_subcommand "schedulerconf" client "Updates scheduler 
configuration"
   hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
   hadoop_add_subcommand "sharedcachemanager" admin "run the SharedCacheManager 
daemon"
   hadoop_add_subcommand "timelinereader" client "run the timeline reader 
server"
@@ -142,7 +143,7 @@ function yarncmd_case
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.router.Router'
 ;;
-schedconf)
+schedulerconf)
 HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.SchedConfCLI'
 ;;
 scmadmin)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9cac5d35/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 7ec9848..fed3d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -295,7 +295,7 @@ goto :eof
   set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
   goto :eof
 
-:schedconf
+:schedulerconf
   set CLASS=org.apache.hadoop.yarn.client.cli.SchedConfCLI
   set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
   goto :eof
@@ -345,6 +345,7 @@ goto :eof
   @echo   node prints node report(s)
   @echo   queueprints queue information
   @echo   logs dump container logs
+  @echo   schedulerconfupdates scheduler configuration
   @echo   classpathprints the class path needed to get the
   @echoHadoop jar and the required libraries
   @echo   daemonlogget/set the log level for each daemon

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9cac5d35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e3ce3ecc..5af44ac 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -674,33 +674,61 @@ public class YarnConfiguration extends Configuration {
   public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
   "org.apache.hadoop.yarn.LocalConfigurationProvider";
 
+  @Private
+  @Unstable
   public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
   YARN_PREFIX 

hadoop git commit: YARN-7238. Documentation for API based scheduler configuration management

[jhung]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-5734 166be597b -> e8b0e3b9c


YARN-7238. Documentation for API based scheduler configuration management


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8b0e3b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8b0e3b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8b0e3b9

Branch: refs/heads/YARN-5734
Commit: e8b0e3b9cf4b86cc19c22efbd61701a9ac5a7747
Parents: 166be59
Author: Jonathan Hung 
Authored: Mon Sep 25 17:47:32 2017 -0700
Committer: Jonathan Hung 
Committed: Mon Sep 25 17:47:32 2017 -0700

--
 .../src/site/markdown/CapacityScheduler.md  |  24 ++-
 .../src/site/markdown/ResourceManagerRest.md| 186 +++
 .../src/site/markdown/YarnCommands.md   |  13 ++
 3 files changed, 221 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b0e3b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 6bb8489..d70f891 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -295,9 +295,29 @@ The `ReservationSystem` is integrated with the 
`CapacityScheduler` queue hierach
 Changing Queue Configuration
 
 
-Changing queue properties and adding new queues is very simple. You need to 
edit **conf/capacity-scheduler.xml** and run *yarn rmadmin -refreshQueues*.
+Changing queue/scheduler properties and adding/removing queues can be done in 
two ways, via file or via API.
+
+### Changing queue configuration via file
+
+  To edit by file, you need to edit **conf/capacity-scheduler.xml** and run 
*yarn rmadmin -refreshQueues*.
 
 $ vi $HADOOP_CONF_DIR/capacity-scheduler.xml
 $ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues
 
-**Note:** Queues cannot be *deleted*, only addition of new queues is supported 
- the updated queue configuration should be a valid one i.e. queue-capacity at 
each *level* should be equal to 100%.
+### Changing queue configuration via API
+
+  Editing by API uses a backing store for the scheduler configuration. To 
enable this, the following parameters can be configured in yarn-site.xml.
+
+  | Property | Description |
+  |: |: |
+  | `yarn.scheduler.capacity.configuration.provider` | The type of 
configuration provider to use for capacity scheduler. To enable changing queue 
configuration via API, this should be set to *store*. Default value is *file*, 
which disables the API and reverts back to changing queue configuration via 
file. |
+  | `yarn.scheduler.configuration.store.class` | The type of backing store to 
use. Default value is *memory*, which stores the scheduler configuration in 
memory (and does not persist configuration changes across restarts). Other 
values are *leveldb* (using a leveldb-based implementation), and *zk* (using a 
zookeeper-based implementation). |
+  | `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy 
can be configured to restrict which users can modify which queues. Default 
value is 
*org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*,
 which only allows YARN admins to make any configuration modifications. Another 
value is 
*org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*,
 which only allows queue modifications if the caller is an admin of the queue. |
+  | `yarn.scheduler.configuration.store.max-logs` | Configuration changes are 
audit logged in the backing store, if using leveldb or zookeeper. This 
configuration controls the maximum number of audit logs to store, dropping the 
oldest logs when exceeded. Default is 1000. |
+  | `yarn.scheduler.configuration.leveldb-store.path` | The storage path of 
the configuration store when using leveldb. Default value is 
*${hadoop.tmp.dir}/yarn/system/confstore*. |
+  | `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | 
The interval for compacting the configuration store in seconds, when using 
leveldb. Default value is 86400, or one day. |
+  | `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root 
node path for configuration store related information, when using zookeeper. 
Default value is */confstore*. |
+
+  **Note:** When enabling backing store for scheduler configuration, *yarn 

hadoop git commit: HADOOP-14890 Move up to AWS SDK 1.11.199. Contributed by Steve Loughran.

[fabbri]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 60b966cb4 -> f3dbcd6a8


HADOOP-14890 Move up to AWS SDK 1.11.199. Contributed by Steve Loughran.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3dbcd6a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3dbcd6a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3dbcd6a

Branch: refs/heads/branch-3.0
Commit: f3dbcd6a890e7f96d8a383bf596b23e55ff143d4
Parents: 60b966c
Author: Aaron Fabbri 
Authored: Mon Sep 25 17:11:12 2017 -0700
Committer: Aaron Fabbri 
Committed: Mon Sep 25 17:19:46 2017 -0700

--
 hadoop-project/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3dbcd6a/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a106c17..db4937c 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -135,7 +135,7 @@
 1.0-beta-1
 1.0-alpha-8
 900
-1.11.134
+1.11.199
 2.3.4
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14890 Move up to AWS SDK 1.11.199. Contributed by Steve Loughran.

[fabbri]

Repository: hadoop
Updated Branches:
  refs/heads/trunk df800f6cf -> a2b31e355


HADOOP-14890 Move up to AWS SDK 1.11.199. Contributed by Steve Loughran.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2b31e35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2b31e35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2b31e35

Branch: refs/heads/trunk
Commit: a2b31e355a73c37dada15a18a90a690314be7fd3
Parents: df800f6
Author: Aaron Fabbri 
Authored: Mon Sep 25 17:11:12 2017 -0700
Committer: Aaron Fabbri 
Committed: Mon Sep 25 17:12:01 2017 -0700

--
 hadoop-project/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2b31e35/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7b342c4..915bd4e 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -135,7 +135,7 @@
 1.0-beta-1
 1.0-alpha-8
 900
-1.11.134
+1.11.199
 2.3.4
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: Block Storage: volume creation times out while creating 3TB volume because of too many containers. Contributed by Mukul Kumar Singh.

[cliang]

Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 e01245495 -> 087c69ba2


Block Storage: volume creation times out while creating 3TB volume because of 
too many containers. Contributed by Mukul Kumar Singh.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/087c69ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/087c69ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/087c69ba

Branch: refs/heads/HDFS-7240
Commit: 087c69ba2434813044089f733581607d172efbd1
Parents: e012454
Author: Chen Liang 
Authored: Mon Sep 25 16:50:55 2017 -0700
Committer: Chen Liang 
Committed: Mon Sep 25 16:50:55 2017 -0700

--
 .../apache/hadoop/cblock/CBlockConfigKeys.java  |  15 ++
 .../cblock/client/CBlockVolumeClient.java   |  28 ++-
 .../cblock/jscsiHelper/BlockWriterTask.java |   6 +-
 .../cache/impl/AsyncBlockWriter.java|   8 +-
 .../hadoop/cblock/meta/VolumeDescriptor.java|   8 +-
 .../hadoop/cblock/storage/StorageManager.java   | 188 ++-
 .../src/main/resources/ozone-default.xml|  16 ++
 .../cblock/util/ContainerLookUpService.java |   6 +-
 .../hadoop/cblock/util/MockStorageClient.java   |  19 +-
 9 files changed, 212 insertions(+), 82 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/087c69ba/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
index d77091f..87f40b3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
@@ -172,6 +172,21 @@ public final class CBlockConfigKeys {
   public static final int DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT =
   64 * 1024;
 
+  /**
+   * Cblock CLI configs.
+   */
+  public static final String DFS_CBLOCK_MANAGER_POOL_SIZE =
+  "dfs.cblock.manager.pool.size";
+  public static final int DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT = 16;
+
+  /**
+   * currently the largest supported volume is about 8TB, which might take
+   * > 20 seconds to finish creating containers. thus set timeout to 30 sec.
+   */
+  public static final String DFS_CBLOCK_RPC_TIMEOUT_SECONDS =
+  "dfs.cblock.rpc.timeout.seconds";
+  public static final int DFS_CBLOCK_RPC_TIMEOUT_SECONDS_DEFAULT = 300;
+
   private CBlockConfigKeys() {
 
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/087c69ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
index 90a16ce..11965a3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.cblock.client;
 
+import org.apache.hadoop.cblock.CBlockConfigKeys;
 import org.apache.hadoop.cblock.meta.VolumeInfo;
 import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
 import org.apache.hadoop.io.retry.RetryPolicies;
@@ -36,32 +37,25 @@ import java.util.concurrent.TimeUnit;
  */
 public class CBlockVolumeClient {
   private final CBlockServiceProtocolClientSideTranslatorPB cblockClient;
-  private final OzoneConfiguration conf;
 
   public CBlockVolumeClient(OzoneConfiguration conf) throws IOException {
-this.conf = conf;
-long version = RPC.getProtocolVersion(CBlockServiceProtocolPB.class);
-InetSocketAddress address = OzoneClientUtils.getCblockServiceRpcAddr(conf);
-// currently the largest supported volume is about 8TB, which might take
-// > 20 seconds to finish creating containers. thus set timeout to 30 sec.
-cblockClient = new CBlockServiceProtocolClientSideTranslatorPB(
-RPC.getProtocolProxy(CBlockServiceProtocolPB.class, version,
-address, UserGroupInformation.getCurrentUser(), conf,
-NetUtils.getDefaultSocketFactory(conf), 3, RetryPolicies
-.retryUpToMaximumCountWithFixedSleep(300, 1, TimeUnit
-.SECONDS)).getProxy());
+this(conf, null);
   }
 
   public 

[54/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
deleted file mode 100644
index c2866cf..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- * 
- * --instance {app name}, if  a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles 
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
-commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
-  public static final String USAGE =
-  "Usage: " + SliderActions.ACTION_REGISTRY
-  + " ("
-  + Arguments.ARG_LIST + "|"
-  + Arguments.ARG_LISTCONF + "|"
-  + Arguments.ARG_LISTEXP + "|"
-  + Arguments.ARG_LISTFILES + "|"
-  + Arguments.ARG_GETCONF + "|"
-  + Arguments.ARG_GETEXP + "> "
-  + Arguments.ARG_NAME + "  "
-  + " )"
-  + "[" + Arguments.ARG_VERBOSE + "] "
-  + "[" + Arguments.ARG_USER + "] "
-  + "[" + Arguments.ARG_OUTPUT + "  ] "
-  + "[" + Arguments.ARG_SERVICETYPE + "  ] "
-  + "[" + Arguments.ARG_FORMAT + " ] "
-  + System.getProperty("line.separator")
-  + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
-  ;
-  public ActionRegistryArgs() {
-  }
-
-  public ActionRegistryArgs(String name) {
-this.name = name;
-  }
-
-  @Override
-  public String getActionName() {
-return ACTION_REGISTRY;
-  }
-
-  /**
-   * Get the min #of params expected
-   * @return the min number of params in the {@link #parameters} field
-   */
-  @Override
-  public int getMinParams() {
-return 0;
-  }
-  
-  @Parameter(names = {ARG_LIST}, 
-  description = "list services")
-  public boolean list;
-
-  @Parameter(names = {ARG_LISTCONF}, 
-  description = "list configurations")
-  public boolean listConf;
-
-  @Parameter(names = {ARG_GETCONF},
-  description = "get configuration")
-  public String getConf;
-
-  @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
-  public boolean listExports;
-
-  @Parameter(names = {ARG_GETEXP},
- description = "get export")
-  public String getExport;
-
-  @Parameter(names = {ARG_LISTFILES},
-  description = "list files")
-  public String listFiles;
-
-  @Parameter(names = {ARG_GETFILES},
-  description = "get files")
-  public String getFiles;
-
-  //--format 
-  @Parameter(names = ARG_FORMAT,
-  description = "Format for a response: ")
-  public String format = ConfigFormat.XML.toString() ;
-
-  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
-  description = "Output destination")
-  public File out;
-
-  @Parameter(names 

[60/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 000..6e6f4dd
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+  private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+  /**

[51/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
deleted file mode 100644
index 83c9961..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import 
org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base class for factories.
- */
-public abstract class ProviderFactory {
-  protected static final Logger LOG =
-  LoggerFactory.getLogger(ProviderFactory.class);
-
-  protected ProviderFactory() {}
-
-  public abstract AbstractClientProvider createClientProvider();
-
-  public abstract ProviderService createServerProvider();
-
-  public static synchronized ProviderService getProviderService(Artifact
-  artifact) {
-return createSliderProviderFactory(artifact).createServerProvider();
-  }
-
-  public static synchronized AbstractClientProvider getClientProvider(Artifact
-  artifact) {
-return createSliderProviderFactory(artifact).createClientProvider();
-  }
-
-  /**
-   * Create a provider for a specific application
-   * @param artifact artifact
-   * @return provider factory
-   */
-  public static synchronized ProviderFactory createSliderProviderFactory(
-  Artifact artifact) {
-if (artifact == null || artifact.getType() == null) {
-  LOG.debug("Loading service provider type default");
-  return DefaultProviderFactory.getInstance();
-}
-LOG.debug("Loading service provider type {}", artifact.getType());
-switch (artifact.getType()) {
-  // TODO add handling for custom types?
-  // TODO handle application
-  case DOCKER:
-return DockerProviderFactory.getInstance();
-  case TARBALL:
-return TarballProviderFactory.getInstance();
-  default:
-throw new IllegalArgumentException(String.format("Resolution error, " +
-"%s should not be passed to createSliderProviderFactory",
-artifact.getType()));
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
deleted file mode 100644
index 9ef0176..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  

[75/86] [abbrv] hadoop git commit: Rebase onto trunk - fix some issues. Contributed by Rohith Sharma K S

[jianhe]

Rebase onto trunk - fix some issues. Contributed by Rohith Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7164b079
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7164b079
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7164b079

Branch: refs/heads/yarn-native-services
Commit: 7164b079b0f77eab6cd1fccf9fa95ee2f24688c0
Parents: 54d2e6b
Author: Jian He 
Authored: Mon Sep 4 22:07:24 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:24 2017 -0700

--
 .../hadoop-yarn-services-api/pom.xml|  2 +-
 .../hadoop-yarn-services-core/pom.xml   |  2 +-
 .../ServiceTimelinePublisher.java   | 32 
 .../TestServiceTimelinePublisher.java   |  2 --
 .../hadoop-yarn-services/pom.xml|  2 +-
 .../src/main/webapp/app/routes/yarn-services.js |  2 +-
 6 files changed, 23 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 1077ccd..74d9681 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -19,7 +19,7 @@
   
 org.apache.hadoop
 hadoop-yarn-applications
-3.0.0-beta1-SNAPSHOT
+3.1.0-SNAPSHOT
   
   hadoop-yarn-services-api
   Apache Hadoop YARN Services API

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index d9b7adb..fb07edc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -19,7 +19,7 @@
   
 org.apache.hadoop
 hadoop-yarn-services
-3.0.0-beta1-SNAPSHOT
+3.1.0-SNAPSHOT
   
   hadoop-yarn-services-core
   jar

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7164b079/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index 5e65ad9..9e7d8e8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -66,6 +66,7 @@ public class ServiceTimelinePublisher extends 
CompositeService {
   protected void serviceInit(org.apache.hadoop.conf.Configuration 
configuration)
   throws Exception {
 addService(timelineClient);
+super.serviceInit(configuration);
   }
 
 
@@ -95,8 +96,8 @@ public class ServiceTimelinePublisher extends 
CompositeService {
 // create info keys
 Map entityInfos = new HashMap();
 entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName());
-entityInfos.put(ServiceTimelineMetricsConstants.STATE,
-service.getState().toString());
+//entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+//service.getState().toString());
 entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
 currentTimeMillis);
 entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
@@ -232,18 +233,23 @@ public class 

[37/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
new file mode 100644
index 000..b851fb7
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Base class of all probes.
+ */
+public abstract class Probe implements MonitorKeys {
+
+  protected final Configuration conf;
+  private String name;
+
+  /**
+   * Create a probe of a specific name
+   *
+   * @param name probe name
+   * @param conf configuration being stored.
+   */
+  public Probe(String name, Configuration conf) {
+this.name = name;
+this.conf = conf;
+  }
+
+
+  protected void setName(String name) {
+this.name = name;
+  }
+
+  public String getName() {
+return name;
+  }
+
+
+  @Override
+  public String toString() {
+return getName();
+  }
+
+  public static String getProperty(Map props, String name,
+  String defaultValue) throws IOException {
+String value = props.get(name);
+if (StringUtils.isEmpty(value)) {
+  if (defaultValue == null) {
+throw new IOException(name + " not specified");
+  }
+  return defaultValue;
+}
+return value;
+  }
+
+  public static int getPropertyInt(Map props, String name,
+  Integer defaultValue) throws IOException {
+String value = props.get(name);
+if (StringUtils.isEmpty(value)) {
+  if (defaultValue == null) {
+throw new IOException(name + " not specified");
+  }
+  return defaultValue;
+}
+return Integer.parseInt(value);
+  }
+
+  /**
+   * perform any prelaunch initialization
+   */
+  public void init() throws IOException {
+
+  }
+
+  /**
+   * Ping the endpoint. All exceptions must be caught and included in the
+   * (failure) status.
+   *
+   * @param instance instance to ping
+   * @return the status
+   */
+  public abstract ProbeStatus ping(ComponentInstance instance);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
new file mode 100644
index 000..7cd761c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache 

[64/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
new file mode 100644
index 000..7baa284
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Build a single command line to include in the container commands;
+ * Special support for JVM command buildup.
+ */
+public class CommandLineBuilder {
+  protected final List argumentList = new ArrayList<>(20);
+
+  /**
+   * Add an entry to the command list
+   * @param args arguments -these will be converted strings
+   */
+  public void add(Object... args) {
+for (Object arg : args) {
+  argumentList.add(arg.toString());
+}
+  }
+
+  // Get the number of arguments
+  public int size() {
+return argumentList.size();
+  }
+  
+  /**
+   * Append the output and error files to the tail of the command
+   * @param stdout out
+   * @param stderr error. Set this to null to append into stdout
+   */
+  public void addOutAndErrFiles(String stdout, String stderr) {
+Preconditions.checkNotNull(stdout, "Null output file");
+Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
+// write out the path output
+argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stdout);
+if (stderr != null) {
+  argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" 
+
+   stderr);
+} else {
+  argumentList.add("2>&1");
+}
+  }
+
+  /**
+   * This just returns the command line
+   * @see #build()
+   * @return the command line
+   */
+  @Override
+  public String toString() {
+return build();
+  }
+
+  /**
+   * Build the command line
+   * @return the command line
+   */
+  public String build() {
+return SliderUtils.join(argumentList, " ");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
new file mode 100644
index 000..0e51a62
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 

[84/86] [abbrv] hadoop git commit: Rebased onto trunk - fix conflicts

[jianhe]

Rebased onto trunk - fix conflicts


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b69afb7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b69afb7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b69afb7a

Branch: refs/heads/yarn-native-services
Commit: b69afb7af79a60abb6985326f10b93fc275a7c76
Parents: 803eb06
Author: Jian He 
Authored: Mon Sep 11 17:34:30 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:25 2017 -0700

--
 .../org/apache/hadoop/yarn/service/ServiceScheduler.java | 11 +++
 1 file changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b69afb7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index fb2fd16..7b809b9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -574,6 +574,11 @@ public class ServiceScheduler extends CompositeService {
 
 }
 
+@Override public void onContainerResourceUpdated(ContainerId containerId,
+Resource resource) {
+
+}
+
 @Override public void onGetContainerStatusError(ContainerId containerId,
 Throwable t) {
 
@@ -586,6 +591,12 @@ public class ServiceScheduler extends CompositeService {
 }
 
 @Override
+public void onUpdateContainerResourceError(ContainerId containerId,
+Throwable t) {
+
+}
+
+@Override
 public void onStopContainerError(ContainerId containerId, Throwable t) {
 
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[62/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
new file mode 100644
index 000..add2475
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
+
+/**
+ * Registry view for providers. This tracks where the service
+ * is registered, offers access to the record and other things.
+ */
+public class YarnRegistryViewForProviders {
+  private static final Log LOG =
+  LogFactory.getLog(YarnRegistryViewForProviders.class);
+
+  private final RegistryOperations registryOperations;
+  private final String user;
+  private final String sliderServiceClass;
+  private final String instanceName;
+  /**
+   * Record used where the service registered itself.
+   * Null until the service is registered
+   */
+  private ServiceRecord selfRegistration;
+
+  /**
+   * Path where record was registered
+   * Null until the service is registered
+   */
+  private String selfRegistrationPath;
+
+  public YarnRegistryViewForProviders(RegistryOperations registryOperations,
+  String user,
+  String sliderServiceClass,
+  String instanceName,
+  ApplicationAttemptId applicationAttemptId) {
+Preconditions.checkArgument(registryOperations != null,
+"null registry operations");
+Preconditions.checkArgument(user != null, "null user");
+Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+"unset service class");
+Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+"instanceName");
+Preconditions.checkArgument(applicationAttemptId != null,
+"null applicationAttemptId");
+this.registryOperations = registryOperations;
+this.user = user;
+this.sliderServiceClass = sliderServiceClass;
+this.instanceName = instanceName;
+  }
+
+  public String getUser() {
+return user;
+  }
+
+
+  private void setSelfRegistration(ServiceRecord selfRegistration) {
+this.selfRegistration = selfRegistration;
+  }
+
+  /**
+   * Get the path to where the service has registered itself.
+   * Null until the service is registered
+   * @return the service registration path.
+   */
+  public String getSelfRegistrationPath() {
+return selfRegistrationPath;
+  }
+
+  /**
+   * Get the absolute path to where the service has registered itself.
+   * This includes the base registry path
+   * Null until the service is registered
+   * @return the service 

[74/86] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi

[jianhe]

YARN-7113. Clean up packaging and dependencies for yarn-native-services. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bfd8815
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bfd8815
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bfd8815

Branch: refs/heads/yarn-native-services
Commit: 7bfd881503987a48a36db522ed6ce13ef2a1ccae
Parents: 4f8fe17
Author: Jian He 
Authored: Tue Aug 29 11:09:00 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:23 2017 -0700

--
 NOTICE.txt  |   14 +
 .../resources/assemblies/hadoop-yarn-dist.xml   |8 -
 .../assemblies/hadoop-yarn-services-api.xml |   36 -
 .../assemblies/hadoop-yarn-services-dist.xml|   30 -
 hadoop-project/pom.xml  |   19 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   37 +-
 .../hadoop-yarn-services-api/pom.xml|  104 +-
 .../yarn/service/webapp/ApiServerWebApp.java|4 +-
 .../src/main/resources/webapps/api-server/app   |   16 +
 .../resources/webapps/services-rest-api/app |   16 -
 .../hadoop-yarn-services-core/pom.xml   |  213 +---
 .../service/client/params/ActionKDiagArgs.java  |   76 --
 .../yarn/service/client/params/ClientArgs.java  |5 -
 .../registry/YarnRegistryViewForProviders.java  |8 +-
 .../yarn/service/utils/KerberosDiags.java   |  680 ---
 .../hadoop/yarn/service/utils/SliderUtils.java  | 1088 --
 .../hadoop/yarn/service/ServiceTestUtils.java   |   28 +
 .../hadoop/yarn/service/TestServiceApiUtil.java |   38 +-
 .../yarn/service/TestYarnNativeServices.java|   10 +-
 .../yarn/service/client/TestServiceCLI.java |1 -
 .../yarn/service/conf/TestAppJsonResolve.java   |   30 +-
 .../service/conf/TestLoadExampleAppJson.java|   11 +-
 .../providers/TestAbstractClientProvider.java   |   10 +-
 hadoop-yarn-project/pom.xml |4 +
 24 files changed, 175 insertions(+), 2311 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index 0718909..f3af2f7 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -581,3 +581,17 @@ The binary distribution of this product bundles binaries of
 Ehcache 3.3.1,
 which has the following notices:
  * Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
+
+JCommander (https://github.com/cbeust/jcommander),
+which has the following notices:
+ * Copyright 2010 Cedric Beust ced...@beust.com
+
+The binary distribution of this product bundles binaries of
+snakeyaml (https://bitbucket.org/asomov/snakeyaml),
+which has the following notices:
+ * Copyright (c) 2008, http://www.snakeyaml.org
+
+The binary distribution of this product bundles binaries of
+swagger-annotations (https://github.com/swagger-api/swagger-core),
+which has the following notices:
+ * Copyright 2016 SmartBear Software

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 8aeeabd..8b3d292 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -98,10 +98,6 @@
   etc/hadoop
 
 
-  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version}
-  
/share/hadoop/${hadoop.component}/lib/services
-
-
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target
   
/share/hadoop/${hadoop.component}/sources
   
@@ -109,10 +105,6 @@
   
 
 
-  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}
-  
/share/hadoop/${hadoop.component}/lib/services-api
-
-
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target
   
/share/hadoop/${hadoop.component}/sources
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
deleted file mode 100644
index 589f724..000
--- 

[61/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
new file mode 100644
index 000..7b22e3e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Support for marshalling objects to and from JSON.
+ * This class is NOT thread safe; it constructs an object mapper
+ * as an instance field.
+ * @param 
+ */
+public class JsonSerDeser {
+
+  private static final Logger log = 
LoggerFactory.getLogger(JsonSerDeser.class);
+  private static final String UTF_8 = "UTF-8";
+
+  private final Class classType;
+  private final ObjectMapper mapper;
+
+  /**
+   * Create an instance bound to a specific type
+   * @param classType class type
+   */
+  public JsonSerDeser(Class classType) {
+this.classType = classType;
+this.mapper = new ObjectMapper();
+mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, 
false);
+  }
+
+  public JsonSerDeser(Class classType, PropertyNamingStrategy 
namingStrategy) {
+this(classType);
+mapper.setPropertyNamingStrategy(namingStrategy);
+  }
+
+  /**
+   * Convert from JSON
+   * @param json input
+   * @return the parsed JSON
+   * @throws IOException IO
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+  public T fromJson(String json)
+throws IOException, JsonParseException, JsonMappingException {
+try {
+  return mapper.readValue(json, classType);
+} catch (IOException e) {
+  log.error("Exception while parsing json : " + e + "\n" + json, e);
+  throw e;
+}
+  }
+
+  /**
+   * Convert from a JSON file
+   * @param jsonFile input file
+   * @return the parsed JSON
+   * @throws IOException IO problems
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+  public T fromFile(File jsonFile)
+throws IOException, JsonParseException, JsonMappingException {
+File absoluteFile = jsonFile.getAbsoluteFile();
+try {
+  return mapper.readValue(absoluteFile, classType);
+} catch (IOException e) {
+  log.error("Exception while parsing json file {}", absoluteFile, e);
+  throw e;
+}
+  }
+
+  /**
+   * Convert from a JSON file
+   * @param resource input file
+   * @return the parsed JSON
+   * @throws IOException IO problems
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+ public T fromResource(String resource)
+throws IOException, JsonParseException, JsonMappingException {
+

[77/86] [abbrv] hadoop git commit: YARN-7126. Create introductory site documentation for YARN native services. Contributed by Gour Saha

[jianhe]

YARN-7126. Create introductory site documentation for YARN native services. 
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54d2e6be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54d2e6be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54d2e6be

Branch: refs/heads/yarn-native-services
Commit: 54d2e6be4a3ce5bc8eb4bfc77fd277835a11cea0
Parents: 5fec5b7
Author: Jian He 
Authored: Fri Sep 1 16:19:31 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:24 2017 -0700

--
 LICENSE.txt |  1 +
 .../native-services/NativeServicesIntro.md  | 96 +++-
 2 files changed, 96 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d2e6be/LICENSE.txt
--
diff --git a/LICENSE.txt b/LICENSE.txt
index 3f50521..46ee108 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1776,6 +1776,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 
DAMAGE.
 The binary distribution of this product bundles these dependencies under the
 following license:
 FindBugs-jsr305 3.0.0
+dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved.
 

 (2-clause BSD)
 Redistribution and use in source and binary forms, with or without

http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d2e6be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
index 89fefe9..e6a4e91 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
@@ -10,4 +10,98 @@
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
--->
\ No newline at end of file
+-->
+
+# Introduction: YARN Native Services
+
+## Overview
+YARN Native Services provides first class framework support and APIs to host 
long running services natively in YARN. In addition to launching services, the 
new APIs support performing lifecycle management operations, such as flex 
service components up/down, manage lifetime, upgrade the service to a newer 
version, and stop/restart/delete the service.
+
+The native services capabilities are built on the existing low-level resource 
management API provided by YARN that can support any type of application. Other 
application frameworks like Hadoop MapReduce already expose higher level APIs 
that users can leverage to run applications on top of YARN. With the advent of 
containerization technologies like Docker, providing first class support and 
APIs for long running services at the framework level made sense.
+
+Relying on a framework has the advantage of exposing a simpler usage model to 
the user by enabling service configuration and launch through specification 
(without writing new code), as well as hiding complex low-level details 
including state management and fault-tolerance etc. Users/operators of existing 
services typically like to avoid modifying an existing service to be aware of 
YARN. With first class support capable of running a single Docker image as well 
as complex assemblies comprised of multiple Docker images, there is no need for 
service owners to be aware of YARN. Developers of new services do not have to 
worry about YARN internals and only need to focus on containerization of their 
service(s).
+
+## First class support for services
+In order to natively provide first class support for long running services, 
several new features and improvements have been made at the framework level.
+
+### Incorporate Apache Slider into Apache YARN
+Apache Slider, which existed as a separate incubator project has been merged 
into YARN to kick start the first class support. Apache Slider is a universal 
Application Master (AM) which had several key features built in - fault 
tolerance of service containers and AM, work-preserving AM restarts, service 
logs management, service management like flex up/down, stop/start, and rolling 
upgrade to newer service versions, etc. Of course lot more work has been done 
on top of what Apache Slider 

[56/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
deleted file mode 100644
index e10305a..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import java.util.Locale;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public enum ConfigFormat {
-
-  JSON("json"),
-  PROPERTIES("properties"),
-  XML("xml"),
-  HADOOP_XML("hadoop_xml"),
-  ENV("env"),
-  TEMPLATE("template"),
-  YAML("yaml"),
-  ;
-  ConfigFormat(String suffix) {
-this.suffix = suffix;
-  }
-
-  private final String suffix;
-
-  public String getSuffix() {
-return suffix;
-  }
-
-
-  @Override
-  public String toString() {
-return suffix;
-  }
-
-  /**
-   * Get a matching format or null
-   * @param type
-   * @return the format
-   */
-  public static ConfigFormat resolve(String type) {
-for (ConfigFormat format: values()) {
-  if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
-return format;
-  }
-}
-return null;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
deleted file mode 100644
index 0ac508b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import 

[73/86] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bfd8815/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
index 6e6f4dd..7e53d18 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -24,35 +24,17 @@ import 
org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.service.client.params.Arguments;
 import org.apache.hadoop.yarn.service.client.params.SliderActions;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
 import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
 import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.zookeeper.server.util.KerberosUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,32 +45,19 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.io.Serializable;
-import java.net.InetSocketAddress;
 import java.net.ServerSocket;
-import java.net.Socket;
 import java.net.URL;
 import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 
 /**
  * These are slider-specific Util methods
@@ -97,41 +66,6 @@ public final class SliderUtils {
 
   private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
 
-  /**
-   * Atomic bool to track whether or not process security has already been
-   * turned on (prevents re-entrancy)
-   */
-  private static final AtomicBoolean processSecurityAlreadyInitialized =
-  new AtomicBoolean(false);
-  public static final String JAVA_SECURITY_KRB5_REALM =
-  "java.security.krb5.realm";
-  public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
-  /**
-   * Winutils
-   */
-  public static final String WINUTILS = "WINUTILS.EXE";
-  /**
-   * name of openssl program
-   */
-  public static final String OPENSSL = "openssl";
-
-  /**
-   * name of python program
-   */
-  public static final String PYTHON = "python";
-
-  /**
-   * type of docker standalone service
-   */
-  public static final String DOCKER = "docker";
-  /**
-   * type of docker on yarn service
-   */
-  public static final String DOCKER_YARN = "yarn_docker";
-

[52/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
deleted file mode 100644
index 7baa284..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Build a single command line to include in the container commands;
- * Special support for JVM command buildup.
- */
-public class CommandLineBuilder {
-  protected final List argumentList = new ArrayList<>(20);
-
-  /**
-   * Add an entry to the command list
-   * @param args arguments -these will be converted strings
-   */
-  public void add(Object... args) {
-for (Object arg : args) {
-  argumentList.add(arg.toString());
-}
-  }
-
-  // Get the number of arguments
-  public int size() {
-return argumentList.size();
-  }
-  
-  /**
-   * Append the output and error files to the tail of the command
-   * @param stdout out
-   * @param stderr error. Set this to null to append into stdout
-   */
-  public void addOutAndErrFiles(String stdout, String stderr) {
-Preconditions.checkNotNull(stdout, "Null output file");
-Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
-// write out the path output
-argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stdout);
-if (stderr != null) {
-  argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" 
+
-   stderr);
-} else {
-  argumentList.add("2>&1");
-}
-  }
-
-  /**
-   * This just returns the command line
-   * @see #build()
-   * @return the command line
-   */
-  @Override
-  public String toString() {
-return build();
-  }
-
-  /**
-   * Build the command line
-   * @return the command line
-   */
-  public String build() {
-return SliderUtils.join(argumentList, " ");
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
deleted file mode 100644
index fcbb69b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you 

[81/86] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
new file mode 100644
index 000..9022268
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -0,0 +1,592 @@
+
+
+# YARN Service API 
+
+Bringing a new service on YARN today is not a simple experience. The APIs of 
existing 
+frameworks are either too low level (native YARN), require writing new code 
(for frameworks with programmatic APIs)
+or writing a complex spec (for declarative frameworks).
+
+This simplified REST API can be used to create and manage the lifecycle of 
YARN services. 
+In most cases, the application owner will not be forced to make any changes to 
their applications. 
+This is primarily true if the application is packaged with containerization 
technologies like Docker.
+
+This document describes the API specifications (aka. YarnFile) for 
deploying/managing
+containerized services on YARN. The same JSON spec can be used for both REST 
API
+and CLI to manage the services. 
+
+
+### Version information
+Version: 1.0.0
+
+### License information
+License: Apache 2.0
+License URL: http://www.apache.org/licenses/LICENSE-2.0.html
+
+### URI scheme
+Host: host.mycompany.com
+
+Port: 9191(default)
+
+Schemes: HTTP
+
+### Consumes
+
+* application/json
+
+
+### Produces
+
+* application/json
+
+
+## Paths
+### Create a service
+```
+POST /ws/v1/services
+```
+
+ Description
+
+Create a service. The request JSON is a service object with details required 
for creation. If the request is successful it returns 202 Accepted. A success 
of this API only confirms success in submission of the service creation 
request. There is no guarantee that the service will actually reach a RUNNING 
state. Resource availability and several other factors determines if the 
service will be deployed in the cluster. It is expected that clients would 
subsequently call the GET API to get details of the service and determine its 
state.
+
+ Parameters
+|Type|Name|Description|Required|Schema|Default|
+|||||||
+|BodyParameter|Service|Service request object|true|Service||
+
+
+ Responses
+|HTTP Code|Description|Schema|
+||||
+|202|The request to create a service is accepted|No Content|
+|400|Invalid service definition provided in the request body|No Content|
+|500|Failed to create a service|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### (TBD) List of services running in the cluster.
+```
+GET /ws/v1/services
+```
+
+ Description
+
+Get a list of all currently running services (response includes a minimal 
projection of the service info). For more details do a GET on a specific 
service name.
+
+ Responses
+|HTTP Code|Description|Schema|
+||||
+|200|An array of services|Service array|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get current version of the API server.
+```
+GET /ws/v1/services/version
+```
+
+ Description
+
+Get current version of the API server.
+
+ Responses
+|HTTP Code|Description|Schema|
+||||
+|200|Successful request|No Content|
+
+
+### Update a service or upgrade the binary version of the components of a 
running service
+```
+PUT /ws/v1/services/{service_name}
+```
+
+ Description
+
+Update the runtime properties of a service. Currently the following operations 
are supported - update lifetime, stop/start a service. The PUT operation is 
also used to orchestrate an upgrade of the service containers to a newer 
version of their artifacts (TBD).
+
+ Parameters
+|Type|Name|Description|Required|Schema|Default|
+|||||||
+|PathParameter|service_name|Service name|true|string||
+|BodyParameter|Service|The updated service definition. It can contain the 
updated lifetime of a service or the desired state (STOPPED/STARTED) of a 
service to initiate a start/stop operation against the specified 
service|true|Service||
+
+
+ Responses
+|HTTP Code|Description|Schema|
+||||
+|204|Update or upgrade was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Destroy a service
+```
+DELETE /ws/v1/services/{service_name}
+```
+
+ Description
+
+Destroy a service and release all resources. This API might have to return 
JSON data providing location of logs (TBD), etc.
+
+ Parameters
+|Type|Name|Description|Required|Schema|Default|
+|||||||
+|PathParameter|service_name|Service name|true|string||
+
+
+ Responses
+|HTTP Code|Description|Schema|
+||||

[63/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
new file mode 100644
index 000..684f655
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Formatter;
+import java.util.Locale;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+  protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
+  .class);
+
+  private MonitorUtils() {
+  }
+
+  public static String toPlural(int val) {
+return val != 1 ? "s" : "";
+  }
+
+  /**
+   * Convert milliseconds to human time -the exact format is unspecified
+   * @param milliseconds a time in milliseconds
+   * @return a time that is converted to human intervals
+   */
+  public static String millisToHumanTime(long milliseconds) {
+StringBuilder sb = new StringBuilder();
+// Send all output to the Appendable object sb
+Formatter formatter = new Formatter(sb, Locale.US);
+
+long s = Math.abs(milliseconds / 1000);
+long m = Math.abs(milliseconds % 1000);
+if (milliseconds > 0) {
+  formatter.format("%d.%03ds", s, m);
+} else if (milliseconds == 0) {
+  formatter.format("0");
+} else {
+  formatter.format("-%d.%03ds", s, m);
+}
+return sb.toString();
+  }
+
+  public static Probe getProbe(ReadinessCheck readinessCheck) {
+if (readinessCheck == null) {
+  return null;
+}
+if (readinessCheck.getType() == null) {
+  return null;
+}
+try {
+  switch (readinessCheck.getType()) {
+  case HTTP:
+return HttpProbe.create(readinessCheck.getProps());
+  case PORT:
+return PortProbe.create(readinessCheck.getProps());
+  default:
+return null;
+  }
+} catch (Throwable t) {
+  throw new IllegalArgumentException("Error creating readiness check " +
+  t);
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
new file mode 100644
index 000..aba5859
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You 

[83/86] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He

[jianhe]

YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/803eb069
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/803eb069
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/803eb069

Branch: refs/heads/yarn-native-services
Commit: 803eb069d3bea41bac62915c4ad1b58f1221a663
Parents: e8af82c
Author: Gour Saha 
Authored: Thu Sep 7 23:25:31 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:25 2017 -0700

--
 hadoop-project/pom.xml  |   4 +-
 hadoop-project/src/site/site.xml|  10 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   4 +-
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |  11 +-
 .../hadoop/yarn/service/ServiceMaster.java  |  10 +-
 .../hadoop/yarn/service/client/ServiceCLI.java  |   5 +-
 .../yarn/service/client/ServiceClient.java  |  20 +-
 .../client/params/AbstractActionArgs.java   |  33 +-
 .../AbstractClusterBuildingActionArgs.java  |  22 +-
 .../service/client/params/ActionBuildArgs.java  |   5 +
 .../service/client/params/ActionCreateArgs.java |   5 +
 .../client/params/ActionDependencyArgs.java |   4 +-
 .../service/client/params/ActionExamples.java   |  26 +
 .../service/client/params/ActionUpdateArgs.java |   7 +-
 .../yarn/service/client/params/Arguments.java   |   9 +-
 .../yarn/service/client/params/ClientArgs.java  |  10 -
 .../yarn/service/client/params/CommonArgs.java  |  28 +-
 .../service/client/params/ServiceAMArgs.java|  50 ++
 .../client/params/ServiceAMCreateAction.java|  37 ++
 .../service/client/params/SliderAMArgs.java |  57 --
 .../client/params/SliderAMCreateAction.java |  73 ---
 .../service/client/params/SliderActions.java|   9 +-
 .../ServiceTimelinePublisher.java   |   5 +-
 .../client/TestBuildExternalComponents.java |   6 +-
 .../yarn/service/client/TestServiceCLI.java |  30 +-
 .../hadoop/yarn/service/conf/examples/app.json  |   2 +-
 .../registry/client/api/RegistryConstants.java  |   2 +-
 .../src/site/markdown/YarnCommands.md   |  56 +-
 .../native-services/NativeServicesAPI.md| 606 ---
 .../native-services/NativeServicesDiscovery.md  | 144 -
 .../native-services/NativeServicesIntro.md  | 107 
 .../src/site/markdown/yarn-service/Concepts.md  |  77 +++
 .../src/site/markdown/yarn-service/Overview.md  |  58 ++
 .../site/markdown/yarn-service/QuickStart.md| 218 +++
 .../markdown/yarn-service/ServiceDiscovery.md   | 150 +
 .../markdown/yarn-service/YarnServiceAPI.md | 592 ++
 36 files changed, 1362 insertions(+), 1130 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 1697744..93475a5 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -141,10 +141,10 @@
 
 ${project.version}
-
+
 1.5.4
 1.16
-1.30
+1.48
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 755bc61..4b4b13f 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -152,10 +152,12 @@
   
 
 
-
-  
-  
-  
+
+  
+  
+  
+  
+  
 
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 43bcb6b..022287a 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -31,7 +31,7 @@ function hadoop_usage
   hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
   hadoop_add_option "--workers" "turn on worker mode"
 
-  hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
+  hadoop_add_subcommand "apiserver" daemon "run the api-server for 
deploying/managing services on YARN"
   hadoop_add_subcommand "application" client "prints application(s) 
report/kill application"
   hadoop_add_subcommand "applicationattempt" client "prints 
applicationattempt(s) report"
   hadoop_add_subcommand "classpath" client "prints the class path needed to 
get the hadoop jar and the required libraries"
@@ -49,7 +49,7 @@ function hadoop_usage
   hadoop_add_subcommand "rmadmin" admin "admin 

[86/86] [abbrv] hadoop git commit: YARN-7201. Added an apache httpd example YARN service. Contributed by Billie Rinaldi

[jianhe]

YARN-7201. Added an apache httpd example YARN service. Contributed by Billie 
Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f7a50d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f7a50d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f7a50d8

Branch: refs/heads/yarn-native-services
Commit: 3f7a50d8d50787331ae5806a946d6d1d52b5ec55
Parents: 360b2d2
Author: Jian He 
Authored: Mon Sep 25 16:36:43 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:27 2017 -0700

--
 .../examples/httpd-no-dns/httpd-no-dns.json |  62 
 .../httpd-no-dns/httpd-proxy-no-dns.conf|  24 +++
 .../examples/httpd/httpd-proxy.conf |  24 +++
 .../examples/httpd/httpd.json   |  55 +++
 .../yarn/service/api/ServiceApiConstants.java   |   5 +
 .../yarn/service/component/Component.java   |  31 
 .../yarn/service/provider/ProviderUtils.java|   1 +
 .../hadoop/registry/server/dns/RegistryDNS.java |   4 +
 .../src/site/markdown/yarn-service/Examples.md  | 159 +++
 .../src/site/markdown/yarn-service/Overview.md  |   3 +-
 10 files changed, 367 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
new file mode 100644
index 000..6b35538
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
@@ -0,0 +1,62 @@
+{
+  "name": "httpd-service-no-dns",
+  "lifetime": "3600",
+  "components": [
+{
+  "name": "httpd",
+  "number_of_containers": 2,
+  "artifact": {
+"id": "centos/httpd-24-centos7:latest",
+"type": "DOCKER"
+  },
+  "launch_command": "/usr/bin/run-httpd",
+  "resource": {
+"cpus": 1,
+"memory": "1024"
+  },
+  "readiness_check": {
+"type": "HTTP",
+"props": {
+  "url": "http://${THIS_HOST}:8080;
+}
+  },
+  "configuration": {
+"files": [
+  {
+"type": "ENV",
+"dest_file": "/var/www/html/index.html",
+"props": {
+  "content": 
"TitleHello from 
${COMPONENT_INSTANCE_NAME}!"
+}
+  }
+]
+  }
+},
+{
+  "name": "httpd-proxy",
+  "number_of_containers": 1,
+  "dependencies": [ "httpd" ],
+  "artifact": {
+"id": "centos/httpd-24-centos7:latest",
+"type": "DOCKER"
+  },
+  "launch_command": "/usr/bin/run-httpd",
+  "resource": {
+"cpus": 1,
+"memory": "1024"
+  },
+  "configuration": {
+"files": [
+  {
+"type": "TEMPLATE",
+"dest_file": "/etc/httpd/conf.d/httpd-proxy.conf",
+"src_file": "httpd-proxy-no-dns.conf"
+  }
+]
+  }
+}
+  ],
+  "quicklinks": {
+"Apache HTTP Server": 
"http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f7a50d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
new file mode 100644
index 000..9894e64
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at

[70/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 88f74ef..17f8c95 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -22,7 +22,7 @@ info:
 Bringing a new service on YARN today is not a simple experience. The APIs 
of existing frameworks are either too low level (native YARN), require writing 
new code (for frameworks with programmatic APIs) or writing a complex spec (for 
declarative frameworks). In addition to building critical building blocks 
inside YARN (as part of other efforts at 
link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a 
need for simplifying the user facing story for building services. Experience of 
projects like Apache Slider running real-life services like HBase, Storm, 
Accumulo, Solr etc, gives us some very good insights on how simplified APIs for 
services should look like.
 
 
-To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component apps to 
complex multi-component assemblies needing orchestration.
+To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component service to 
complex multi-component assemblies needing orchestration.
 
 
 We should also look at making this a unified REST based entry point for 
other important features like resource-profile management 
(link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), 
package-definitions' lifecycle-management and service-discovery 
(link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]).
 We also need to flesh out its relation to our present much lower level REST 
APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN 
for application-submission and management.
@@ -41,177 +41,177 @@ schemes:
   - http
   - https
 # will be prefixed to all paths
-basePath: /services/v1/
+basePath: /ws/v1/
 consumes:
   - application/json
 produces:
   - application/json
 paths:
-  /applications:
+  /services:
 get:
-  summary: List of applications/services running in the cluster
-  description: Get a list of all currently running applications (response 
includes a minimal projection of the application info). For more details do a 
GET on a specific application name.
+  summary: List of services running in the cluster
+  description: Get a list of all currently running services (response 
includes a minimal projection of the service info). For more details do a GET 
on a specific service name.
   responses:
 200:
-  description: An array of applications
+  description: An array of services
   schema:
 type: array
 items:
-  $ref: '#/definitions/Application'
+  $ref: '#/definitions/Service'
 default:
   description: Unexpected error
   schema:
-$ref: '#/definitions/ApplicationStatus'
+$ref: '#/definitions/ServiceStatus'
 post:
-  summary: Create an application/service
-  description: Create an application. The request JSON is an Application 
object with details required for creation. If the request is successful it 
returns 202 Accepted. A success of this API only confirms success in submission 
of the application creation request. There is no guarantee that the application 
will actually reach a RUNNING state. Resource availability and several other 
factors determines if the application will be deployed in the cluster. It is 
expected that clients would subsequently call the GET API to get details of the 
application and determine its state.
+  summary: Create a service
+  description: Create a service. The request JSON is a service object with 
details required for creation. If the request is successful it returns 202 
Accepted. A success of this API only 

[80/86] [abbrv] hadoop git commit: YARN-7186. Add examples in yarn-service. Contributed by Jian He

[jianhe]

YARN-7186. Add examples in yarn-service. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c329bba2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c329bba2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c329bba2

Branch: refs/heads/yarn-native-services
Commit: c329bba24cd2d55db476347aee451eacf257
Parents: b69afb7
Author: Billie Rinaldi 
Authored: Thu Sep 14 15:43:00 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:25 2017 -0700

--
 .../resources/assemblies/hadoop-yarn-dist.xml   |  7 
 .../examples/sleeper/sleeper.json   | 15 +++
 .../yarn/service/client/ServiceClient.java  | 26 ++--
 .../client/params/AbstractActionArgs.java   |  2 +-
 .../AbstractClusterBuildingActionArgs.java  |  2 +-
 .../service/client/params/ActionBuildArgs.java  |  7 +++-
 .../service/client/params/ActionCreateArgs.java | 12 +-
 .../service/client/params/ActionExamples.java   | 26 
 .../yarn/service/client/params/Arguments.java   |  4 +-
 .../yarn/service/client/params/ClientArgs.java  | 44 
 .../yarn/service/client/params/CommonArgs.java  | 15 +--
 .../service/client/params/SliderActions.java|  2 -
 .../src/site/markdown/YarnCommands.md   | 34 ---
 .../site/markdown/yarn-service/QuickStart.md|  9 +++-
 14 files changed, 91 insertions(+), 114 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 8b3d292..83633ac 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -98,6 +98,13 @@
   etc/hadoop
 
 
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples
+  
/share/hadoop/${hadoop.component}/yarn-service-examples
+  
+**/*
+  
+
+
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target
   
/share/hadoop/${hadoop.component}/sources
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
new file mode 100644
index 000..89ce527
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
@@ -0,0 +1,15 @@
+{
+  "name": "sleeper-service",
+  "components" :
+  [
+{
+  "name": "sleeper",
+  "number_of_containers": 2,
+  "launch_command": "sleep 90",
+  "resource": {
+"cpus": 1,
+"memory": "256"
+  }
+}
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c329bba2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index bfddc44..6890aef 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.ServiceState;
 import 

[71/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

YARN-7091. Rename application to service in yarn-native-services. Contributed 
by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f8fe178
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f8fe178
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f8fe178

Branch: refs/heads/yarn-native-services
Commit: 4f8fe178e6faa567391d26358680e52fc0232bab
Parents: 538c63e
Author: Billie Rinaldi 
Authored: Mon Aug 28 09:59:55 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:22 2017 -0700

--
 .../resources/assemblies/hadoop-yarn-dist.xml   |8 +-
 .../assemblies/hadoop-yarn-services-dist.xml|2 +-
 hadoop-project/pom.xml  |2 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   14 +-
 .../hadoop-yarn/conf/yarn-env.sh|2 +-
 .../hadoop-yarn-services-api/pom.xml|2 +-
 .../hadoop/yarn/service/webapp/ApiServer.java   |  274 +++
 .../yarn/service/webapp/ApiServerWebApp.java|  159 ++
 .../service/webapp/ApplicationApiService.java   |  275 ---
 .../service/webapp/ApplicationApiWebApp.java|  123 --
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |  185 +-
 .../src/main/scripts/run_rest_service.sh|   28 -
 .../dev-support/findbugs-exclude.xml|   48 +
 .../conf/yarnservice-log4j.properties   |   62 +
 .../hadoop-yarn-services-core/pom.xml   |  408 +
 .../hadoop/yarn/service/ClientAMProtocol.java   |   40 +
 .../hadoop/yarn/service/ClientAMService.java|  132 ++
 .../yarn/service/ContainerFailureTracker.java   |   89 +
 .../hadoop/yarn/service/ServiceContext.java |   41 +
 .../hadoop/yarn/service/ServiceMaster.java  |  156 ++
 .../hadoop/yarn/service/ServiceMetrics.java |   98 +
 .../hadoop/yarn/service/ServiceScheduler.java   |  654 +++
 .../yarn/service/api/ServiceApiConstants.java   |   69 +
 .../yarn/service/api/records/Artifact.java  |  160 ++
 .../yarn/service/api/records/BaseResource.java  |   52 +
 .../yarn/service/api/records/Component.java |  412 +
 .../yarn/service/api/records/ConfigFile.java|  225 +++
 .../yarn/service/api/records/ConfigFormat.java  |   67 +
 .../yarn/service/api/records/Configuration.java |  225 +++
 .../yarn/service/api/records/Container.java |  297 +++
 .../service/api/records/ContainerState.java |   30 +
 .../hadoop/yarn/service/api/records/Error.java  |  129 ++
 .../service/api/records/PlacementPolicy.java|  102 ++
 .../service/api/records/ReadinessCheck.java |  175 ++
 .../yarn/service/api/records/Resource.java  |  159 ++
 .../yarn/service/api/records/Service.java   |  466 +
 .../yarn/service/api/records/ServiceState.java  |   33 +
 .../yarn/service/api/records/ServiceStatus.java |  148 ++
 .../yarn/service/client/ClientAMProxy.java  |   57 +
 .../hadoop/yarn/service/client/ServiceCLI.java  |  112 ++
 .../yarn/service/client/ServiceClient.java  |  892 +
 .../client/params/AbstractActionArgs.java   |  158 ++
 .../client/params/AbstractArgsDelegate.java |   28 +
 .../AbstractClusterBuildingActionArgs.java  |   58 +
 .../service/client/params/ActionBuildArgs.java  |   31 +
 .../service/client/params/ActionClientArgs.java |   71 +
 .../service/client/params/ActionCreateArgs.java |   33 +
 .../client/params/ActionDependencyArgs.java |   65 +
 .../client/params/ActionDestroyArgs.java|   37 +
 .../service/client/params/ActionExistsArgs.java |   49 +
 .../service/client/params/ActionFlexArgs.java   |   50 +
 .../service/client/params/ActionFreezeArgs.java |   56 +
 .../service/client/params/ActionHelpArgs.java   |   44 +
 .../service/client/params/ActionKDiagArgs.java  |   76 +
 .../service/client/params/ActionKeytabArgs.java |   76 +
 .../service/client/params/ActionListArgs.java   |   76 +
 .../client/params/ActionRegistryArgs.java   |  218 +++
 .../client/params/ActionResolveArgs.java|  153 ++
 .../client/params/ActionResourceArgs.java   |   70 +
 .../service/client/params/ActionStatusArgs.java |   51 +
 .../service/client/params/ActionThawArgs.java   |   67 +
 .../service/client/params/ActionTokensArgs.java |   78 +
 .../service/client/params/ActionUpdateArgs.java |   32 +
 .../yarn/service/client/params/ArgOps.java  |  156 ++
 .../yarn/service/client/params/Arguments.java   |  103 ++
 .../yarn/service/client/params/ClientArgs.java  |  252 +++
 .../yarn/service/client/params/CommonArgs.java  |  282 +++
 .../client/params/ComponentArgsDelegate.java|   52 +
 .../client/params/DontSplitArguments.java   |   34 +
 .../client/params/LaunchArgsAccessor.java   |   30 +
 .../client/params/LaunchArgsDelegate.java   |   51 +
 .../client/params/OptionArgsDelegate.java   |   66 +
 .../client/params/PathArgumentConverter.java|   34 +
 

[82/86] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/803eb069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
deleted file mode 100644
index f56139a..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
+++ /dev/null
@@ -1,606 +0,0 @@
-
-
-# YARN Simplified API layer for services
-
-## Overview
-Bringing a new service on YARN today is not a simple experience. The APIs of
-existing frameworks are either too low level (native YARN), require writing
-new code (for frameworks with programmatic APIs) or writing a complex spec
-(for declarative frameworks). In addition to building critical building blocks
-inside YARN (as part of other efforts at
-[YARN-4692](https://issues.apache.org/jira/browse/YARN-4692)), there is a need 
for
-simplifying the user facing story for building services. Experience of projects
-like Apache Slider running real-life services like HBase, Storm, Accumulo,
-Solr etc, gives us some very good insights on how simplified APIs for services
-should look like.
-
-To this end, we should look at a new simple-services API layer backed by REST
-interfaces. This API can be used to create and manage the lifecycle of YARN
-services. Services here can range from simple single-component service to
-complex multi-component assemblies needing orchestration.
-[YARN-4793](https://issues.apache.org/jira/browse/YARN-4793) tracks this
-effort.
-
-This document spotlights on this specification. In most of the cases, the
-application owner will not be forced to make any changes to their applications.
-This is primarily true if the application is packaged with containerization
-technologies like docker. Irrespective of how complex the application is,
-there will be hooks provided at appropriate layers to allow pluggable and
-customizable application behavior.
-
-
-### Version information
-Version: 1.0.0
-
-### License information
-License: Apache 2.0
-License URL: http://www.apache.org/licenses/LICENSE-2.0.html
-
-### URI scheme
-Host: host.mycompany.com
-
-BasePath: /ws/v1/
-
-Schemes: HTTP
-
-### Consumes
-
-* application/json
-
-
-### Produces
-
-* application/json
-
-
-## Paths
-### Create a service
-```
-POST /services
-```
-
- Description
-
-Create a service. The request JSON is a service object with details required 
for creation. If the request is successful it returns 202 Accepted. A success 
of this API only confirms success in submission of the service creation 
request. There is no guarantee that the service will actually reach a RUNNING 
state. Resource availability and several other factors determines if the 
service will be deployed in the cluster. It is expected that clients would 
subsequently call the GET API to get details of the service and determine its 
state.
-
- Parameters
-|Type|Name|Description|Required|Schema|Default|
-|||||||
-|BodyParameter|Service|Service request object|true|Service||
-
-
- Responses
-|HTTP Code|Description|Schema|
-||||
-|202|The request to create a service is accepted|No Content|
-|400|Invalid service definition provided in the request body|No Content|
-|500|Failed to create a service|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### (TBD) List of services running in the cluster.
-```
-GET /services
-```
-
- Description
-
-Get a list of all currently running services (response includes a minimal 
projection of the service info). For more details do a GET on a specific 
service name.
-
- Responses
-|HTTP Code|Description|Schema|
-||||
-|200|An array of services|Service array|
-|default|Unexpected error|ServiceStatus|
-
-
-### Get current version of the API server.
-```
-GET /services/version
-```
-
- Description
-
-Get current version of the API server.
-
- Responses
-|HTTP Code|Description|Schema|
-||||
-|200|Successful request|No Content|
-
-
-### Update a service or upgrade the binary version of the components of a 
running service
-```
-PUT /services/{service_name}
-```
-
- Description
-
-Update the runtime properties of a service. Currently the following operations 
are supported - update lifetime, stop/start a service. The PUT operation is 
also used to orchestrate an upgrade of the service containers to a newer 
version of their artifacts (TBD).
-
- Parameters
-|Type|Name|Description|Required|Schema|Default|
-|||||||
-|PathParameter|service_name|Service name|true|string||
-|BodyParameter|Service|The updated service definition. It can contain the 
updated lifetime of a service or the desired state (STOPPED/STARTED) 

[78/86] [abbrv] hadoop git commit: YARN-7073. Yarn native services rest API documentation. Contributed by Gour Saha

[jianhe]

YARN-7073. Yarn native services rest API documentation. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fec5b7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fec5b7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fec5b7a

Branch: refs/heads/yarn-native-services
Commit: 5fec5b7a2cec4ae2b002ec7d54f4639f19c15648
Parents: 3819081
Author: Jian He 
Authored: Thu Aug 31 17:17:22 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:24 2017 -0700

--
 hadoop-project/src/site/site.xml|   1 +
 .../hadoop/yarn/service/webapp/ApiServer.java   |  47 +-
 .../definition/YARN-Services-Examples.md| 245 
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 128 ++--
 .../native-services/NativeServicesAPI.md| 606 +++
 5 files changed, 973 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fec5b7a/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index e6e9d86..755bc61 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -154,6 +154,7 @@
 
 
   
+  
   
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fec5b7a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index f55e3f1..e8286ef 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -24,12 +24,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.api.records.ServiceState;
 import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
 import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,6 +48,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 
+import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED;
 import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
 
 /**
@@ -76,11 +76,11 @@ public class ApiServer {
   @GET
   @Path(VERSION)
   @Consumes({ MediaType.APPLICATION_JSON })
-  @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+  @Produces({ MediaType.APPLICATION_JSON })
   public Response getVersion() {
 String version = VersionInfo.getBuildVersion();
 LOG.info(version);
-return Response.ok(version).build();
+return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build();
   }
 
   @POST
@@ -94,11 +94,11 @@ public class ApiServer {
   ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service);
   LOG.info("Successfully created service " + service.getName()
   + " applicationId = " + applicationId);
-  serviceStatus.setState(ServiceState.ACCEPTED);
+  serviceStatus.setState(ACCEPTED);
   serviceStatus.setUri(
   CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
   .getName());
-  return Response.status(Status.CREATED).entity(serviceStatus).build();
+  return Response.status(Status.ACCEPTED).entity(serviceStatus).build();
 } catch (IllegalArgumentException e) {
   serviceStatus.setDiagnostics(e.getMessage());
   return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
@@ -182,16 +182,16 @@ public class ApiServer {
   + ": Invalid number of containers specified " + component
   .getNumberOfContainers()).build();
 }
+ServiceStatus 

[85/86] [abbrv] hadoop git commit: YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He

[jianhe]

YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/360b2d26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/360b2d26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/360b2d26

Branch: refs/heads/yarn-native-services
Commit: 360b2d263cd1eee10b9df991841031188fbce374
Parents: c329bba
Author: Billie Rinaldi 
Authored: Thu Sep 21 10:18:42 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:26 2017 -0700

--
 .../hadoop/yarn/service/ServiceScheduler.java |  6 +-
 .../hadoop/yarn/service/client/ServiceClient.java |  3 +++
 .../params/AbstractClusterBuildingActionArgs.java |  2 +-
 .../yarn/service/client/params/Arguments.java |  1 +
 .../component/instance/ComponentInstance.java | 10 +-
 .../service/provider/AbstractProviderService.java | 18 +++---
 .../client/binding/RegistryTypeUtils.java |  3 ++-
 .../dns/ApplicationServiceRecordProcessor.java| 15 ++-
 .../server/dns/BaseServiceRecordProcessor.java|  8 
 .../registry/server/dns/TestRegistryDNS.java  | 12 
 .../runtime/DockerLinuxContainerRuntime.java  |  4 ++--
 .../src/site/markdown/YarnCommands.md |  1 +
 12 files changed, 53 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 7b809b9..ec5f3ed 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService {
 attemptId.getApplicationId().toString());
 serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
 PersistencePolicies.APPLICATION);
-serviceRecord.description = "Yarn Service Master";
-
-serviceRecord.addExternalEndpoint(RegistryTypeUtils
-.ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
-context.clientAMService.getBindAddress()));
+serviceRecord.description = "YarnServiceMaster";
 
 // set any provided attributes
 setUserProvidedServiceRecordAttributes(service.getConfiguration(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 6890aef..a3a9fd0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService
 if (!StringUtils.isEmpty(args.getServiceName())) {
   service.setName(args.getServiceName());
 }
+if (!StringUtils.isEmpty(args.queue)) {
+  service.setQueue(args.queue);
+}
 return service;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/360b2d26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
--
diff --git 

[59/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
new file mode 100644
index 000..be36335
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
+import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
+import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for ServiceApiUtil helper methods.
+ */
+public class TestServiceApiUtil {
+  private static final Logger LOG = LoggerFactory
+  .getLogger(TestServiceApiUtil.class);
+  private static final String EXCEPTION_PREFIX = "Should have thrown " +
+  "exception: ";
+  private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+  "exception: ";
+
+  private static final String LEN_64_STR =
+  "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
+
+  private static final YarnConfiguration CONF_DEFAULT_DNS = new
+  YarnConfiguration();
+  private static final YarnConfiguration CONF_DNS_ENABLED = new
+  YarnConfiguration();
+
+  @BeforeClass
+  public static void init() {
+CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
+  }
+
+  @Test(timeout = 9)
+  public void testResourceValidation() throws Exception {
+assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
+.length());
+
+SliderFileSystem sfs = initMock(null);
+
+Service app = new Service();
+
+// no name
+try {
+  ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+  Assert.fail(EXCEPTION_PREFIX + "service with no name");
+} catch (IllegalArgumentException e) {
+  assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+}
+
+// bad format name
+String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
+for (String badName : badNames) {
+  app.setName(badName);
+  try {
+ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+Assert.fail(EXCEPTION_PREFIX + "service with bad name " + badName);
+  } catch (IllegalArgumentException e) {
+

[76/86] [abbrv] hadoop git commit: Rebase onto latest trunk. minor conflicts

[jianhe]

Rebase onto latest trunk. minor conflicts


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38190819
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38190819
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38190819

Branch: refs/heads/yarn-native-services
Commit: 3819081963d1f28f7a18c5eb00f7b9fe9c6558bf
Parents: d5fad4e
Author: Jian He 
Authored: Wed Aug 30 22:48:35 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:24 2017 -0700

--
 .../java/org/apache/hadoop/yarn/util/Apps.java  |  2 ++
 .../rmapp/attempt/RMAppAttemptImpl.java | 35 ++--
 2 files changed, 4 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/38190819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 75b4633..e88d2b2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -150,6 +150,8 @@ public class Apps {
   }
 
   // Check if should black list the node based on container exit status
+  @Private
+  @Unstable
   public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
 switch (exitStatus) {
 case ContainerExitStatus.PREEMPTED:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38190819/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index fa29e00..a0aa284 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -109,6 +109,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.BoundedAppender;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
@@ -1556,38 +1557,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, 
Recoverable {
 }
   }
 
-  private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
-switch (exitStatus) {
-case ContainerExitStatus.PREEMPTED:
-case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER:
-case ContainerExitStatus.KILLED_BY_APPMASTER:
-case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION:
-case ContainerExitStatus.ABORTED:
-  // Neither the app's fault nor the system's fault. This happens by 
design,
-  // so no need for skipping nodes
-  return false;
-case ContainerExitStatus.DISKS_FAILED:
-  // This container is marked with this exit-status means that the node is
-  // already marked as unhealthy given that most of the disks failed. So, 
no
-  // need for any explicit skipping of nodes.
-  return false;
-case ContainerExitStatus.KILLED_EXCEEDED_VMEM:
-case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
-  // No point in skipping the node as it's not the system's fault
-  return false;
-case ContainerExitStatus.SUCCESS:
-  return false;
-case ContainerExitStatus.INVALID:
-  // Ideally, this shouldn't be considered for skipping a node. But in
-  // reality, it seems like there are cases where we are not setting
-  // exit-code correctly and so it's better to be conservative. See
-  // YARN-4284.
-  return true;
-default:
-  return true;
-}
-  }
-
   private static final class UnmanagedAMAttemptSavedTransition

[57/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
deleted file mode 100644
index 8c968dc..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ /dev/null
@@ -1,655 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.UpdatedContainer;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.component.ComponentEventType;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import 

[16/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
deleted file mode 100644
index 3e8a3c3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.Container;
-
-/**
- * Static assignment structure
- */
-public class ContainerAssignment {
-
-  /**
-   * Container that has been allocated
-   */
-  public final Container container;
-
-  /**
-   * Role to assign to it
-   */
-  public final RoleStatus role;
-
-  /**
-   * Placement outcome: was this from history or not
-   */
-  public final ContainerAllocationOutcome placement;
-
-  public ContainerAssignment(Container container,
-  RoleStatus role,
-  ContainerAllocationOutcome placement) {
-this.container = container;
-this.role = role;
-this.placement = placement;
-  }
-
-  @Override
-  public String toString() {
-final StringBuilder sb = new StringBuilder("ContainerAssignment{");
-sb.append("container=").append(container);
-sb.append(", role=").append(role);
-sb.append(", placement=").append(placement);
-sb.append('}');
-return sb.toString();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
deleted file mode 100644
index 6df4bf4..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-
-/**
- * Container outcomes we care about; slightly simplified from
- * {@link ContainerExitStatus} -and hopefully able to handle
- * any new exit codes.
- */
-public enum ContainerOutcome {
-  Completed,
-  Failed,
-  Failed_limits_exceeded,
-  Disk_failure,
-  Preempted;
-
-  /**
-   * Build a container outcome from 

[58/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
new file mode 100644
index 000..b742553
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class for ServiceTimelinePublisher.
+ */
+public class TestServiceTimelinePublisher {
+  private TimelineV2Client timelineClient;
+  private Configuration config;
+  private ServiceTimelinePublisher serviceTimelinePublisher;
+  private static String SERVICE_NAME = "HBASE";
+  private static String SERVICEID = "application_1490093646524_0005";
+  private static String ARTIFACTID = "ARTIFACTID";
+  private static String COMPONENT_NAME = "DEFAULT";
+  private static String CONTAINER_ID =
+  "container_e02_1490093646524_0005_01_01";
+  private static String CONTAINER_IP =
+  "localhost";
+  private static String CONTAINER_HOSTNAME =
+  "cnl124-localhost.site";
+  private static String CONTAINER_BAREHOST =
+  "localhost.com";
+
+  @Before
+  public void setUp() throws Exception {
+config = new Configuration();
+config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+timelineClient =
+new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
+serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+timelineClient.init(config);
+serviceTimelinePublisher.init(config);
+timelineClient.start();
+serviceTimelinePublisher.start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+if 

[43/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
deleted file mode 100644
index 333859e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
+++ /dev/null
@@ -1,68 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-# This is the log4j configuration for Slider Application Master
-
-# Log rotation based on size (100KB) with a max of 10 backup files
-log4j.rootLogger=INFO, amlog
-log4j.threshhold=ALL
-log4j.appender.amlog=org.apache.log4j.RollingFileAppender
-log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
-log4j.appender.amlog.File=${LOG_DIR}/slider.log
-log4j.appender.amlog.MaxFileSize=1MB
-log4j.appender.amlog.MaxBackupIndex=10
-
-# log layout skips stack-trace creation operations by avoiding line numbers 
and method
-log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - 
%m%n
-
-# debug edition is much more expensive
-#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} 
(%F:%M(%L)) - %m%n
-
-# configure stderr
-# set the conversion pattern of stderr
-# Print the date in ISO 8601 format
-log4j.appender.stderr=org.apache.log4j.ConsoleAppender
-log4j.appender.stderr.Target=System.err
-log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - 
%m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
-#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
-
-# for debugging Slider
-#log4j.logger.org.apache.slider=DEBUG
-
-# uncomment to debug service lifecycle issues
-#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-
-# uncomment for YARN operations
-#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-
-# uncomment this to debug security problems
-#log4j.logger.org.apache.hadoop.security=DEBUG
-
-#crank back on some noise
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.hadoop.hdfs=WARN
-log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
-
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.curator.framework.state=ERROR
-log4j.logger.org.apache.curator.framework.imps=WARN

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
new file mode 100644
index 000..58c8e27
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in 

[24/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
deleted file mode 100644
index 1964459..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-public class PersistKeys {
-
-  public static final String SCHEMA =
-"http://example.org/specification/v2.0.0;;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
deleted file mode 100644
index ac8fca5..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-
-/**
- * Miscellaneous methods to assist slider registry work
- * 
- */
-public class SliderRegistryUtils {
-
-
-  /**
-   * Get the registry path for an instance under the user's home node
-   * @param instanceName application instance
-   * @return a path to the registry location for this application instance.
-   */
-  public static String registryPathForInstance(String instanceName) {
-return RegistryUtils.servicePath(
-RegistryUtils.currentUser(), SliderKeys.APP_TYPE, instanceName
-);
-  }
-
-  /**
-   * Process a path expanding it if needed.
-   * Validation is delegated to later as the core registry will need
-   * to do that anyway
-   * @param path path
-   * @return a path maybe with some expansion
-   */
-  public static String resolvePath(String path) {
-Preconditions.checkArgument(path!=null, "null path");
-Preconditions.checkArgument(!path.isEmpty(), "empty path");
-String newpath = path;
-if (path.startsWith("~/")) {
-  // add user expansion
-  newpath = RegistryUtils.homePathForCurrentUser() + path.substring(1);
-} else if (path.equals("~")) {
-  newpath = 

[53/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
deleted file mode 100644
index 982a114..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.ServiceScheduler;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.Date;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-
-import static 
org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
-import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
-import static 
org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.*;
-import static 
org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.*;
-
-public class ComponentInstance implements EventHandler,
-Comparable {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(ComponentInstance.class);
-
-  private  StateMachine stateMachine;
-  private Component component;
-  private final ReadLock readLock;
-  private final WriteLock writeLock;
-
-  private ComponentInstanceId compInstanceId = null;
-  private Path compInstanceDir;
-  private Container container;
-  private YarnRegistryViewForProviders yarnRegistryOperations;
-  private FileSystem fs;
-  private boolean timelineServiceEnabled = false;
-  private 

[35/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 000..415392a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1699 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+  private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+  /**
+   * Atomic 

[65/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
new file mode 100644
index 000..cb7131e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -0,0 +1,494 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.ContainerFailureTracker;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.ServiceMetrics;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.Probe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+import static 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START;
+import static 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
+import static org.apache.hadoop.yarn.service.component.ComponentState.*;
+import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
+
+public class Component implements EventHandler {
+  private static final Logger LOG = LoggerFactory.getLogger(Component.class);
+
+  private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
+  private long allocateId;
+  private Priority priority;
+  private ServiceMetrics componentMetrics;
+  private ServiceScheduler scheduler;
+  private ServiceContext context;
+  private AMRMClientAsync amrmClient;
+  private AtomicLong instanceIdCounter = new AtomicLong();
+  private Map compInstances =
+  new ConcurrentHashMap<>();
+  // component instances to be assigned with a container
+  private List pendingInstances = new LinkedList<>();
+  private ContainerFailureTracker 

[67/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
new file mode 100644
index 000..c3a2752
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -0,0 +1,892 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
+import 
org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.ServiceMaster;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import 

[23/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
deleted file mode 100644
index bf71861..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.restclient;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Class to bond to a Jersey client, for UGI integration and SPNEGO.
- * 
- *   Usage: create an instance, then when creating a Jersey Client
- *   pass in to the constructor the handler provided by {@link #getHandler()}
- *
- * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey
 docs
- */
-public class UgiJerseyBinding implements
-HttpURLConnectionFactory {
-  private static final Logger log =
-  LoggerFactory.getLogger(UgiJerseyBinding.class);
-
-  private final UrlConnectionOperations operations;
-  private final URLConnectionClientHandler handler;
-
-  /**
-   * Construct an instance
-   * @param operations operations instance
-   */
-  @SuppressWarnings("ThisEscapedInObjectConstruction")
-  public UgiJerseyBinding(UrlConnectionOperations operations) {
-Preconditions.checkArgument(operations != null, "Null operations");
-this.operations = operations;
-handler = new URLConnectionClientHandler(this);
-  }
-
-  /**
-   * Create an instance off the configuration. The SPNEGO policy
-   * is derived from the current UGI settings.
-   * @param conf config
-   */
-  public UgiJerseyBinding(Configuration conf) {
-this(new UrlConnectionOperations(conf));
-  }
-
-  /**
-   * Get a URL connection. 
-   * @param url URL to connect to
-   * @return the connection
-   * @throws IOException any problem. {@link AuthenticationException} 
-   * errors are wrapped
-   */
-  @Override
-  public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
-try {
-  // open a connection handling status codes and so redirections
-  // but as it opens a connection, it's less useful than you think.
-
-  return operations.openConnection(url);
-} catch (AuthenticationException e) {
-  throw new IOException(e);
-}
-  }
-
-  public UrlConnectionOperations getOperations() {
-return operations;
-  }
-
-  public URLConnectionClientHandler getHandler() {
-return handler;
-  }
-  
-  /**
-   * Get the SPNEGO flag (as found in the operations instance
-   * @return the spnego policy
-   */
-  public boolean isUseSpnego() {
-return operations.isUseSpnego();
-  }
-
-
-  /**
-   * Uprate error codes 400 and up into faults; 
-   * 
-   * see {@link ExceptionConverter#convertJerseyException(String, String, 
UniformInterfaceException)}
-   */
-  public static 

[47/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
deleted file mode 100644
index 1fa07ce..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ZookeeperUtils {
-  public static final int DEFAULT_PORT = 2181;
-
-  public static String buildConnectionString(String zkHosts, int port) {
-String zkPort = Integer.toString(port);
-//parse the hosts
-String[] hostlist = zkHosts.split(",", 0);
-String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
-return quorum;
-  }
-
-  /**
-   * Take a quorum list and split it to (trimmed) pairs
-   * @param hostPortQuorumList list of form h1:port, h2:port2,...
-   * @return a possibly empty list of values between commas. They may not be
-   * valid hostname:port pairs
-   */
-  public static List splitToPairs(String hostPortQuorumList) {
-// split an address hot
-String[] strings = StringUtils.getStrings(hostPortQuorumList);
-int len = 0;
-if (strings != null) {
-  len = strings.length;
-}
-List tuples = new ArrayList(len);
-if (strings != null) {
-  for (String s : strings) {
-tuples.add(s.trim());
-  }
-}
-return tuples;
-  }
-
-  /**
-   * Split a quorum list into a list of hostnames and ports
-   * @param hostPortQuorumList split to a list of hosts and ports
-   * @return a list of values
-   */
-  public static List splitToHostsAndPorts(String 
hostPortQuorumList) {
-// split an address hot
-String[] strings = StringUtils.getStrings(hostPortQuorumList);
-int len = 0;
-if (strings != null) {
-  len = strings.length;
-}
-List list = new ArrayList(len);
-if (strings != null) {
-  for (String s : strings) {
-
list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
-  }
-}
-return list;
-  }
-
-  /**
-   * Build up to a hosts only list
-   * @param hostAndPorts
-   * @return a list of the hosts only
-   */
-  public static String buildHostsOnlyList(List hostAndPorts) {
-StringBuilder sb = new StringBuilder();
-for (HostAndPort hostAndPort : hostAndPorts) {
-  sb.append(hostAndPort.getHostText()).append(",");
-}
-if (sb.length() > 0) {
-  sb.delete(sb.length() - 1, sb.length());
-}
-return sb.toString();
-  }
-
-  public static String buildQuorumEntry(HostAndPort hostAndPort,
-int defaultPort) {
-String s = hostAndPort.toString();
-if (hostAndPort.hasPort()) {
-  return s;
-} else {
-  return s + ":" + defaultPort;
-}
-  }
-
-  /**
-   * Build a quorum list, injecting a ":defaultPort" ref if needed on
-   * any entry without one
-   * @param hostAndPorts
-   * @param defaultPort
-   * @return
-   */
-  public static String buildQuorum(List hostAndPorts, int 
defaultPort) {
-List entries = new ArrayList(hostAndPorts.size());
-for (HostAndPort hostAndPort : hostAndPorts) {
-  entries.add(buildQuorumEntry(hostAndPort, defaultPort));
-}
-return SliderUtils.join(entries, ",", false);
-  }
-  
-  public static String convertToHostsOnlyList(String quorum) throws
-  BadConfigException {
-List 

[11/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
deleted file mode 100644
index 9c653f3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * An extended composite service which stops itself if any child service
- * fails, or when all its children have successfully stopped without failure.
- *
- * Lifecycle
- * 
- *   If any child exits with a failure: this service stops, propagating
- *   the exception.
- *   When all child services has stopped, this service stops itself
- * 
- *
- */
-public class WorkflowCompositeService extends CompositeService
-implements ServiceParent, ServiceStateChangeListener {
-
-  private static final Logger LOG =
-LoggerFactory.getLogger(WorkflowCompositeService.class);
-
-  /**
-   * Deadlock-avoiding overridden config for slider services; see SLIDER-1052
-   */
-  private volatile Configuration configuration;
-
-  /**
-   * Construct an instance
-   * @param name name of this service instance
-   */
-  public WorkflowCompositeService(String name) {
-super(name);
-  }
-
-  @Override
-  public Configuration getConfig() {
-return configuration;
-  }
-
-  @Override
-  protected void setConfig(Configuration conf) {
-super.setConfig(conf);
-configuration = conf;
-  }
-
-  /**
-   * Construct an instance with the default name.
-   */
-  public WorkflowCompositeService() {
-this("WorkflowCompositeService");
-  }
-
-  /**
-   * Varargs constructor
-   * @param name name of this service instance
-   * @param children children
-   */
-  public WorkflowCompositeService(String name, Service... children) {
-this(name);
-for (Service child : children) {
-  addService(child);
-}
-  }
-
-  /**
-   * Construct with a list of children
-   * @param name name of this service instance
-   * @param children children to add
-   */
-  public WorkflowCompositeService(String name, List children) {
-this(name);
-for (Service child : children) {
-  addService(child);
-}
-  }
-
-  /**
-   * Add a service, and register it
-   * @param service the {@link Service} to be added.
-   * Important: do not add a service to a parent during your own 
serviceInit/start,
-   * in Hadoop 2.2; you will trigger a ConcurrentModificationException.
-   */
-  @Override
-  public synchronized void addService(Service service) {
-Preconditions.checkArgument(service != null, "null service argument");
-service.registerServiceListener(this);
-super.addService(service);
-  }
-
-  /**
-   * When this service is started, any service stopping with a failure
-   * exception is converted immediately into a failure of this service, 
-   * storing the failure and stopping ourselves.
-   * @param child the service that has changed.
-   */
-  @Override
-  public void stateChanged(Service child) {
-//if that child stopped while we are running:
-if (isInState(STATE.STARTED) && child.isInState(STATE.STOPPED)) {
-  // a child service 

[19/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
deleted file mode 100644
index 864a1cf..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.JmxReporter;
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.MetricSet;
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.Slf4jReporter;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.slider.server.services.workflow.ClosingService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * YARN service which hooks up Codahale metrics to 
- * JMX, and, if enabled Ganglia and/or an SLF4J log.
- */
-public class MetricsBindingService extends CompositeService
-implements MetricsKeys {
-  protected static final Logger log =
-  LoggerFactory.getLogger(MetricsBindingService.class);
-  private final MetricRegistry metrics;
-
-  private String reportingDetails = "not started";
-
-
-  public MetricsBindingService(String name,
-  MetricRegistry metrics) {
-super(name);
-Preconditions.checkArgument(metrics != null, "Null metrics");
-this.metrics = metrics;
-  }
-
-  /**
-   * Instantiate...create a metric registry in the process
-   * @param name service name
-   */
-  public MetricsBindingService(String name) {
-this(name, new MetricRegistry());
-  }
-
-  /**
-   * Accessor for the metrics instance
-   * @return the metrics
-   */
-  public MetricRegistry getMetrics() {
-return metrics;
-  }
-
-  @Override
-  protected void serviceStart() throws Exception {
-super.serviceStart();
-
-StringBuilder summary = new StringBuilder();
-Configuration conf = getConfig();
-
-summary.append("Reporting to JMX");
-// always start the JMX binding
-JmxReporter jmxReporter;
-jmxReporter = JmxReporter.forRegistry(metrics).build();
-jmxReporter.start();
-addService(new ClosingService<>(jmxReporter));
-
-
-// Ganglia
-if (conf.getBoolean(METRICS_GANGLIA_ENABLED, false)) {
-  log.warn("Ganglia integration is not implemented");
-/*
-  // This is all disabled due to transitive dependencies on an LGPL library
-  com.codahale.metrics.ganglia.GangliaReporter gangliaReporter;
-  String host = conf.getTrimmed(METRICS_GANGLIA_HOST, "");
-  int port = conf.getInt(METRICS_GANGLIA_PORT, DEFAULT_GANGLIA_PORT);
-  int interval = conf.getInt(METRICS_GANGLIA_REPORT_INTERVAL, 60);
-  int ttl = 1;
-  info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode
-  mcast = 
info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode.getModeForAddress(host);
-  boolean ganglia31 = conf.getBoolean(METRICS_GANGLIA_VERSION_31, true);
-
-  final info.ganglia.gmetric4j.gmetric.GMetric ganglia =
-  new info.ganglia.gmetric4j.gmetric.GMetric(
-  host,
-  port,
-  mcast,
-  ttl,
-  ganglia31);
-  gangliaReporter = 
com.codahale.metrics.ganglia.GangliaReporter.forRegistry(metrics)
-

[06/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
deleted file mode 100644
index 8686479..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
+++ /dev/null
@@ -1,430 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static 
org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority;
-import static 
org.apache.slider.server.appmaster.state.ContainerPriority.extractRole;
-
-/**
- * Test app state RM operations.
- */
-public class TestMockAppStateRMOperations extends BaseMockAppStateTest
-implements MockRoles {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(BaseMockAppStateTest.class);
-
-  @Override
-  public String getTestName() {
-return "TestMockAppStateRMOperations";
-  }
-
-  //@Test
-  public void testPriorityOnly() throws Throwable {
-assertEquals(5, extractRole(buildPriority(5, false)));
-  }
-
-  //@Test
-  public void testPriorityRoundTrip() throws Throwable {
-assertEquals(5, extractRole(buildPriority(5, false)));
-  }
-
-  //@Test
-  public void testPriorityRoundTripWithRequest() throws Throwable {
-int priority = buildPriority(5, false);
-assertEquals(5, extractRole(priority));
-  }
-
-  //@Test
-  public void testMockAddOp() throws Throwable {
-getRole0Status().setDesired(1);
-List ops = appState.reviewRequestAndReleaseNodes();
-assertListLength(ops, 1);
-ContainerRequestOperation operation = 
(ContainerRequestOperation)ops.get(0);
-int priority = operation.getRequest().getPriority().getPriority();
-assertEquals(extractRole(priority), getRole0Status().getKey());
-MockRMOperationHandler handler = new MockRMOperationHandler();
-handler.execute(ops);
-
-AbstractRMOperation op = handler.getFirstOp();
-assertTrue(op instanceof ContainerRequestOperation);
-  }
-
-  /**
-   * Test of a flex up and down op which verifies that outstanding
-   * requests are cancelled first.
-   * 
-   *   request 5 nodes, assert 5 request made
-   *   allocate 1 of them
-   *   flex cluster size to 3
-   *   assert this generates 2 cancel requests
-   * 
-   */
-  //@Test
-  public void testRequestThenCancelOps() 

[34/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
deleted file mode 100644
index 2f71004..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * These are the keys valid in resource options
- *
- /*
-
- Container failure window.
-
- The window is calculated in minutes as as (days * 24 *60 + hours* 24 + 
minutes)
-
- Every interval of this period after the AM is started/restarted becomes
- the time period in which the CONTAINER_FAILURE_THRESHOLD value is calculated.
- 
- After the window limit is reached, the failure counts are reset. This
- is not a sliding window/moving average policy, simply a rule such as
- "every six hours the failure count is reset"
-
-
- 
- ===
- 
-
- */
-public interface ResourceKeys {
-
-
-  /**
-   * #of instances of a component: {@value}
-   *
-  */
-  String COMPONENT_INSTANCES = "yarn.component.instances";
-
-  /**
-   * Whether to use unique names for each instance of a component: {@value}
-   */
-  String UNIQUE_NAMES = "component.unique.names";
-
-  /**
-   *  Amount of memory to ask YARN for in MB.
-   *  Important: this may be a hard limit on the
-   *  amount of RAM that the service can use
-   *  {@value}
-   */
-  String YARN_MEMORY = "yarn.memory";
-  
-  /** {@value} */
-  int DEF_YARN_MEMORY = 256;
-  
-  /**
-   * Number of cores/virtual cores to ask YARN for
-   *  {@value}
-   */
-  String YARN_CORES = "yarn.vcores";
-
-  /**
-   * If normalization is set to false, then if the resource (memory and/or
-   * vcore) requested by a role is higher than YARN limits, then the resource
-   * request is not normalized. If this causes failures at the YARN level then
-   * applications are expecting that to happen. Default value is true.
-   */
-  String YARN_RESOURCE_NORMALIZATION_ENABLED =
-  "yarn.resource.normalization.enabled";
-
-  /**
-   * Number of disks per instance to ask YARN for
-   *  {@value}
-   */
-  String YARN_DISKS = "yarn.disks.count-per-instance";
-
-  /**
-   * Disk size per disk to ask YARN for
-   *  {@value}
-   */
-  String YARN_DISK_SIZE = "yarn.disk.size";
-
-  /** {@value} */
-  int DEF_YARN_CORES = 1;
-
-
-  /**
-   * Label expression that this container must satisfy
-   *  {@value}
-   */
-  String YARN_LABEL_EXPRESSION = "yarn.label.expression";
-
-  /** default label expression: */
-  String DEF_YARN_LABEL_EXPRESSION = null;
-
-
-  /**
-   * Constant to indicate that the requirements of a YARN resource limit
-   * (cores, memory, ...) should be set to the maximum allowed by
-   * the queue into which the YARN container requests are placed.
-   */
-  String YARN_RESOURCE_MAX = "max";
-  
-  /**
-   * Mandatory property for all roles
-   * 1. this must be defined.
-   * 2. this must be >= 1
-   * 3. this must not match any other role priority in the cluster.
-   */
-  String COMPONENT_PRIORITY = "yarn.role.priority";
-  
-  /**
-   * placement policy
-   */
-  String COMPONENT_PLACEMENT_POLICY = "yarn.component.placement.policy";
-
-  /**
-   * Maximum number of node failures that can be tolerated by a component on a 
specific node
-   */
-  String NODE_FAILURE_THRESHOLD =
-  "yarn.node.failure.threshold";
-
-  /**
-   * maximum number of failed containers (in a single role)
-   * before the cluster is deemed to have failed {@value}
-   */
-  String CONTAINER_FAILURE_THRESHOLD =
-  

[30/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
deleted file mode 100644
index f1bf2ad..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.params.ActionAMSuicideArgs;
-import org.apache.slider.common.params.ActionClientArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.slider.common.params.ActionDiagnosticArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionKeytabArgs;
-import org.apache.slider.common.params.ActionNodesArgs;
-import org.apache.slider.common.params.ActionKillContainerArgs;
-import org.apache.slider.common.params.ActionListArgs;
-import org.apache.slider.common.params.ActionRegistryArgs;
-import org.apache.slider.common.params.ActionResolveArgs;
-import org.apache.slider.common.params.ActionResourceArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
-import org.apache.slider.common.params.ActionThawArgs;
-import org.apache.slider.common.params.ActionUpgradeArgs;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.IOException;
-
-/**
- * Interface of those method calls in the slider API that are intended
- * for direct public invocation.
- * 
- * Stability: evolving
- */
-public interface SliderClientAPI extends Service {
-
-  int actionDestroy(String clustername) throws YarnException, IOException;
-
-  /**
-   * AM to commit an asynchronous suicide
-   */
-  int actionAmSuicide(String clustername,
-  ActionAMSuicideArgs args) throws YarnException, IOException;
-
-  /**
-   * Manage keytabs leveraged by slider
-   *
-   * @param keytabInfo the arguments needed to manage the keytab
-   * @throws YarnException Yarn problems
-   * @throws IOException other problems
-   * @throws BadCommandArgumentsException bad arguments.
-   */
-  int actionKeytab(ActionKeytabArgs keytabInfo)
-  throws YarnException, IOException;
-
-  /**
-   * Manage file resources leveraged by slider
-   *
-   * @param resourceInfo the arguments needed to manage the resource
-   * @throws YarnException Yarn problems
-   * @throws IOException other problems
-   * @throws BadCommandArgumentsException bad arguments.
-   */
-  int actionResource(ActionResourceArgs resourceInfo)
-  throws YarnException, IOException;
-
-  /**
-   * Perform client operations such as install or configure
-   *
-   * @param clientInfo the arguments needed for client operations
-   *
-   * @throws SliderException bad arguments.
-   * @throws IOException problems related to package and destination folders
-   */
-  int actionClient(ActionClientArgs clientInfo)
-  throws IOException, YarnException;
-
-  /**
-   * Update the cluster specification
-   *
-   * @param clustername cluster name
-   * @param buildInfo the arguments needed to update the cluster
-   * @throws YarnException Yarn problems
-   * @throws 

[55/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
deleted file mode 100644
index 1049698..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ /dev/null
@@ -1,872 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.RetryNTimes;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
-import 
org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.ServiceMaster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.CommonArgs;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;

[08/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
deleted file mode 100644
index 057f6c5..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.common.tools;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/** Test slider util methods. */
-public class TestSliderUtils {
-  protected static final Logger log =
-  LoggerFactory.getLogger(TestSliderUtils.class);
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder();
-
-  //@Test
-  public void testTruncate() {
-Assert.assertEquals(SliderUtils.truncate(null, 5), null);
-Assert.assertEquals(SliderUtils.truncate("323", -1), "323");
-Assert.assertEquals(SliderUtils.truncate("3232", 5), "3232");
-Assert.assertEquals(SliderUtils.truncate("1234567890", 0), "1234567890");
-Assert.assertEquals(SliderUtils.truncate("123456789012345", 15), 
"123456789012345");
-Assert.assertEquals(SliderUtils.truncate("123456789012345", 14), 
"12345678901...");
-Assert.assertEquals(SliderUtils.truncate("1234567890", 1), "1");
-Assert.assertEquals(SliderUtils.truncate("1234567890", 10), "1234567890");
-Assert.assertEquals(SliderUtils.truncate("", 10), "");
-  }
-
-  //@Test
-  public void testApplicationReportComparison() {
-List instances = getApplicationReports();
-
-SliderUtils.sortApplicationsByMostRecent(instances);
-
-Assert.assertEquals(1000, instances.get(0).getStartTime());
-Assert.assertEquals(1000, instances.get(1).getStartTime());
-Assert.assertEquals(1000, instances.get(2).getStartTime());
-Assert.assertEquals(1000, instances.get(3).getStartTime());
-
-instances = getApplicationReports();
-
-SliderUtils.sortApplicationReport(instances);
-Assert.assertEquals(1000, instances.get(0).getStartTime());
-Assert.assertEquals(1000, instances.get(1).getStartTime());
-Assert.assertEquals(1000, instances.get(2).getStartTime());
-Assert.assertEquals(1000, instances.get(3).getStartTime());
-
-Assert.assertTrue(instances.get(0).getYarnApplicationState() == 
YarnApplicationState.ACCEPTED ||
-  instances.get(0).getYarnApplicationState() == 
YarnApplicationState.RUNNING);
-Assert.assertTrue(instances.get(1).getYarnApplicationState() == 
YarnApplicationState.ACCEPTED ||
-  instances.get(1).getYarnApplicationState() == 
YarnApplicationState.RUNNING);
-Assert.assertTrue(instances.get(2).getYarnApplicationState() == 
YarnApplicationState.ACCEPTED ||
-  instances.get(2).getYarnApplicationState() == 
YarnApplicationState.RUNNING);
-Assert.assertTrue(instances.get(3).getYarnApplicationState() == 
YarnApplicationState.KILLED);
-  }
-
-  private List getApplicationReports() {
-List instances = new ArrayList();
-instances.add(getApplicationReport(1000, 0, "app1", 
YarnApplicationState.ACCEPTED));
-instances.add(getApplicationReport(900, 998, "app1", 

[22/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
deleted file mode 100644
index d96d13e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.services.workflow.ServiceThreadFactory;
-import org.apache.slider.server.services.workflow.WorkflowExecutorService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static 
org.apache.hadoop.yarn.service.conf.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY;
-
-/**
- * A service for launching containers
- */
-public class RoleLaunchService
-extends WorkflowExecutorService {
-  protected static final Logger log =
-LoggerFactory.getLogger(RoleLaunchService.class);
-
-  public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService";
-
-
-  /**
-   * Queue submission API
-   */
-  private  QueueAccess actionQueue;
-
-  /**
-   * Filesystem to use for the launch
-   */
-  private  SliderFileSystem fs;
-
-
-  private Map envVars = new HashMap<>();
-
-  /**
-   * Construct an instance of the launcher
-   * @param queueAccess
-   * @param fs filesystem
-   * @param envVars environment variables
-   */
-  public RoleLaunchService(QueueAccess queueAccess, SliderFileSystem fs,
-  Map envVars) {
-super(ROLE_LAUNCH_SERVICE);
-this.actionQueue = queueAccess;
-this.fs = fs;
-this.envVars = envVars;
-  }
-
-  public RoleLaunchService(SliderFileSystem fs) {
-super(ROLE_LAUNCH_SERVICE);
-this.fs = fs;
-  }
-
-  @Override
-  public void init(Configuration conf) {
-super.init(conf);
-setExecutor(Executors.newCachedThreadPool(
-new ServiceThreadFactory(ROLE_LAUNCH_SERVICE, true)));
-  }
-
-  /**
-   * Start an asychronous launch operation
-   * @param assignment container assignment
-   * @param credentials credentials to use
-   */
-  public void launchRole(ContainerAssignment assignment,
-  Application application, Credentials credentials) {
-  }
-
-  public void launchComponent(Application application,
-  ComponentInstance instance, Container container) {
-RoleLaunchService.RoleLauncher launcher =
-new RoleLaunchService.RoleLauncher(application, instance,
-container);
-execute(launcher);
-  }
-
-  /**
-   * Thread that runs on the AM to launch a container
-   */
-  private class RoleLauncher implements Runnable {
-// Allocated container
-public final Container container;
-public final Application application;
- 

[68/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
new file mode 100644
index 000..c64b1b5
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+  private Integer code = null;
+  private String message = null;
+  private String fields = null;
+
+  /**
+   **/
+  public Error code(Integer code) {
+this.code = code;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("code")
+  public Integer getCode() {
+return code;
+  }
+
+  public void setCode(Integer code) {
+this.code = code;
+  }
+
+  /**
+   **/
+  public Error message(String message) {
+this.message = message;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("message")
+  public String getMessage() {
+return message;
+  }
+
+  public void setMessage(String message) {
+this.message = message;
+  }
+
+  /**
+   **/
+  public Error fields(String fields) {
+this.fields = fields;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("fields")
+  public String getFields() {
+return fields;
+  }
+
+  public void setFields(String fields) {
+this.fields = fields;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != o.getClass()) {
+  return false;
+}
+Error error = (Error) o;
+return Objects.equals(this.code, error.code)
+&& Objects.equals(this.message, error.message)
+&& Objects.equals(this.fields, error.fields);
+  }
+
+  @Override
+  public int hashCode() {
+return Objects.hash(code, message, fields);
+  }
+
+  @Override
+  public String toString() {
+StringBuilder sb = new StringBuilder();
+sb.append("class Error {\n");
+
+sb.append("code: ").append(toIndentedString(code)).append("\n");
+sb.append("message: ").append(toIndentedString(message)).append("\n");
+sb.append("fields: ").append(toIndentedString(fields)).append("\n");
+sb.append("}");
+return sb.toString();
+  }
+
+  /**
+   * Convert the given object to string with each line indented by 4 spaces
+   * (except the first line).
+   */
+  private String toIndentedString(java.lang.Object o) {
+if (o == null) {
+  return "null";
+}
+return o.toString().replace("\n", "\n");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
--
diff --git 

[21/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
deleted file mode 100644
index 06dde67..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ /dev/null
@@ -1,2138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.health.HealthCheckRegistry;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.BlockingService;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import 
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import 
org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceOperations;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import 
org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
-import 

[79/86] [abbrv] hadoop git commit: YARN-7161. Fix special chars in DNS documentation. Contributed by Gour Saha

[jianhe]

YARN-7161. Fix special chars in DNS documentation. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8af82c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8af82c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8af82c8

Branch: refs/heads/yarn-native-services
Commit: e8af82c8516c778672f8808b1ce2359e908bb81d
Parents: 7164b07
Author: Jian He 
Authored: Thu Sep 7 18:17:02 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:25 2017 -0700

--
 .../native-services/NativeServicesDiscovery.md  | 87 ++--
 1 file changed, 45 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8af82c8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
index fa54a09..a927118 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -13,39 +13,42 @@
 -->
 
 # YARN DNS Server
+
+
+
 ## Introduction
 
 The YARN DNS Server provides a standard DNS interface to the information 
posted into the YARN Registry by deployed applications. The DNS service serves 
the following functions:
 
-1. **Exposing existing service­ discovery information via DNS**​­ - 
Information provided in
+1. **Exposing existing service-discovery information via DNS** - Information 
provided in
 the current YARN service registry’s records will be converted into DNS 
entries, thus
 allowing users to discover information about YARN applications using standard 
DNS
 client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
 number for services).
-2. **Enabling Container to IP mappings​­** - Enables discovery of the IPs 
of containers via
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of 
containers via
 standard DNS lookups. Given the availability of the records via DNS, container
-name­based communication will be facilitated (e.g. ‘curl
+name-based communication will be facilitated (e.g. ‘curl
 http://myContainer.myDomain.com/endpoint’).
 
 ## Service Properties
 
 The existing YARN Service Registry is leveraged as the source of information 
for the DNS Service.
 
-The following core functions are supported by the DNS­ Server:
+The following core functions are supported by the DNS-Server:
 
-###Functional properties
+### Functional properties
 
-1. Supports creation of DNS records for end­points of the deployed YARN 
applications
+1. Supports creation of DNS records for end-points of the deployed YARN 
applications
 2. Record names remain unchanged during restart of containers and/or 
applications
 3. Supports reverse lookups (name based on IP).
 4. Supports security using the standards defined by The Domain Name System 
Security
 Extensions (DNSSEC)
 5. Highly available
-6. Scalable ­- The service provides the responsiveness (e.g. low­ latency) 
required to
+6. Scalable - The service provides the responsiveness (e.g. low-latency) 
required to
 respond to DNS queries (timeouts yield attempts to invoke other configured name
 servers).
 
-###Deployment properties
+### Deployment properties
 
 1. Supports integration with existing DNS assets (e.g. a corporate DNS server) 
by acting as
 a DNS server for a Hadoop cluster zone/domain. The server is not intended to 
act as a
@@ -55,59 +58,59 @@ DNS standards. The default port for DNS protocols is in a 
restricted, administra
 range (53), so the port is configurable for deployments in which the service 
may
 not be managed via an administrative account.
 
-##DNS Record Name Structure
+## DNS Record Name Structure
 
 The DNS names of generated records are composed from the following elements 
(labels). Note that these elements must be compatible with DNS conventions (see 
“Preferred Name Syntax” in RFC 1035):
 
-* **domain** -​­ the name of the cluster DNS domain. This name is provided 
as a
+* **domain** - the name of the cluster DNS domain. This name is provided as a
 configuration property. In addition, it is this name that is configured at a 
parent DNS
 server as the zone name for the defined yDNS zone (the zone for which the 
parent DNS
 server will forward requests to yDNS). E.g. yarncluster.com
-* **user­name**​ -­ the name of the application deployer. This 

[07/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
deleted file mode 100644
index 395ff22..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.server.appmaster.model.mock.MockAppState;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY;
-import static 
org.apache.slider.server.appmaster.model.mock.MockFactory.AAROLE_2;
-
-/**
- * Test Anti-affine placement.
- */
-public class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
-implements MockRoles {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(TestMockAppStateAAPlacement.class);
-
-  private static final int NODES = 3;
-
-  /**
-   * The YARN engine has a cluster with very few nodes (3) and lots of
-   * containers, so if AA placement isn't working, there will be affine
-   * placements surfacing.
-   * @return
-   */
-  @Override
-  public MockYarnEngine createYarnEngine() {
-return new MockYarnEngine(NODES, 8);
-  }
-
-  /**
-   * This is the simplest AA allocation: no labels, so allocate anywhere.
-   * @throws Throwable
-   */
-  //@Test
-  public void testAllocateAANoLabel() throws Throwable {
-RoleStatus aaRole = getAaRole();
-
-assertTrue(cloneNodemap().size() > 0);
-
-// want multiple instances, so there will be iterations
-aaRole.setDesired(2);
-
-List ops = appState.reviewRequestAndReleaseNodes();
-AMRMClient.ContainerRequest request = getSingleRequest(ops);
-assertFalse(request.getRelaxLocality());
-assertEquals(request.getNodes().size(), engine.getCluster()
-.getClusterSize());
-assertNull(request.getRacks());
-assertNotNull(request.getCapability());
-
-Container allocated = engine.allocateContainer(request);
-
-// notify the container ane expect
-List assignments = new ArrayList<>();
-List operations = new ArrayList<>();
-

[40/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
new file mode 100644
index 000..e310f45
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
+commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
+
+public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
+
+  @Override
+  public String getActionName() {
+return SliderActions.ACTION_UPDATE;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
index f7b7349..00151f4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.service.client.params;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index d8d8ab4..204149b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -26,59 +26,38 @@ package org.apache.hadoop.yarn.service.client.params;
  * so it is easier to see what arguments are there
  */
 public interface Arguments {
-  String ARG_ADDON = "--addon";
-  String 

[72/86] [abbrv] hadoop git commit: YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang

[jianhe]

YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by 
Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5fad4ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5fad4ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5fad4ee

Branch: refs/heads/yarn-native-services
Commit: d5fad4eee31704bd0fbb8ff5afec0f77f8da1aa5
Parents: 7bfd881
Author: Jian He 
Authored: Tue Aug 29 20:37:38 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:23 2017 -0700

--
 .../java/org/apache/hadoop/yarn/service/api/records/Resource.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5fad4ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
index cec9de9..dfdf92a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty;
 
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -103,6 +104,7 @@ public class Resource extends BaseResource implements 
Cloneable {
 this.memory = memory;
   }
 
+  @JsonIgnore
   public long getMemoryMB() {
 if (this.memory == null) {
   return 0;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[50/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
deleted file mode 100644
index b3fdef9..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Class to bond to a Jersey client, for UGI integration and SPNEGO.
- * 
- *   Usage: create an instance, then when creating a Jersey Client
- *   pass in to the constructor the handler provided by {@link #getHandler()}
- *
- * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey
 docs
- */
-public class UgiJerseyBinding implements
-HttpURLConnectionFactory {
-  private static final Logger log =
-  LoggerFactory.getLogger(UgiJerseyBinding.class);
-
-  private final UrlConnectionOperations operations;
-  private final URLConnectionClientHandler handler;
-
-  /**
-   * Construct an instance
-   * @param operations operations instance
-   */
-  @SuppressWarnings("ThisEscapedInObjectConstruction")
-  public UgiJerseyBinding(UrlConnectionOperations operations) {
-Preconditions.checkArgument(operations != null, "Null operations");
-this.operations = operations;
-handler = new URLConnectionClientHandler(this);
-  }
-
-  /**
-   * Create an instance off the configuration. The SPNEGO policy
-   * is derived from the current UGI settings.
-   * @param conf config
-   */
-  public UgiJerseyBinding(Configuration conf) {
-this(new UrlConnectionOperations(conf));
-  }
-
-  /**
-   * Get a URL connection. 
-   * @param url URL to connect to
-   * @return the connection
-   * @throws IOException any problem. {@link AuthenticationException} 
-   * errors are wrapped
-   */
-  @Override
-  public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
-try {
-  // open a connection handling status codes and so redirections
-  // but as it opens a connection, it's less useful than you think.
-
-  return operations.openConnection(url);
-} catch (AuthenticationException e) {
-  throw new IOException(e);
-}
-  }
-
-  public UrlConnectionOperations getOperations() {
-return operations;
-  }
-
-  public URLConnectionClientHandler getHandler() {
-return handler;
-  }
-  
-  /**
-   * Get the SPNEGO flag (as found in the operations instance
-   * @return the spnego policy
-   */
-  public boolean isUseSpnego() {
-return operations.isUseSpnego();
-  }
-
-
-  /**
-   * Uprate error codes 400 and up into faults; 
-   * 
-   * see {@link ExceptionConverter#convertJerseyException(String, String, 
UniformInterfaceException)}
-   */
-  public static IOException uprateFaults(HttpVerb verb, String url,
- 

[04/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
deleted file mode 100644
index 9632265..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderInternalStateException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.NodeMap;
-import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.slider.utils.SliderTestBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map.Entry;
-
-/**
- * Base for app state tests.
- */
-public abstract class BaseMockAppStateTest extends SliderTestBase implements
-MockRoles {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(BaseMockAppStateTest.class);
-  protected static final List EMPTY_ID_LIST = Collections
-  .emptyList();
-
-  protected final MockFactory factory = MockFactory.INSTANCE;
-  protected MockAppState appState;
-  protected MockYarnEngine engine;
-  protected FileSystem fs;
-  protected SliderFileSystem sliderFileSystem;
-  protected File historyWorkDir;
-  protected Path historyPath;
-  protected MockApplicationId applicationId;
-  protected 

[14/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
deleted file mode 100644
index 0cac430..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.health.HealthCheckRegistry;
-import com.codahale.metrics.servlets.HealthCheckServlet;
-import com.codahale.metrics.servlets.MetricsServlet;
-import com.codahale.metrics.servlets.PingServlet;
-import com.codahale.metrics.servlets.ThreadDumpServlet;
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;
-import com.sun.jersey.api.core.ResourceConfig;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
-import org.apache.hadoop.yarn.webapp.Dispatcher;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig;
-import org.apache.slider.server.appmaster.web.rest.AMWebServices;
-import 
org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-
-/**
- * 
- */
-public class SliderAMWebApp extends WebApp {
-  public static final String BASE_PATH = "slideram";
-  public static final String CONTAINER_STATS = "/stats";
-  public static final String CLUSTER_SPEC = "/spec";
-
-  private final WebAppApi webAppApi;
-
-  public SliderAMWebApp(WebAppApi webAppApi) {
-Preconditions.checkArgument(webAppApi != null, "webAppApi null");
-this.webAppApi = webAppApi;
-  }
-
-  @Override
-  public void setup() {
-Logger.getLogger("com.sun.jersey").setLevel(Level.FINEST);
-// Make one of these to ensure that the jax-b annotations
-// are properly picked up.
-bind(SliderJacksonJaxbJsonProvider.class);
-
-// Get exceptions printed to the screen
-bind(GenericExceptionHandler.class);
-// bind the REST interface
-bind(AMWebServices.class);
-
-//bind(AMAgentWebServices.class);
-route("/", SliderAMController.class);
-route(CONTAINER_STATS, SliderAMController.class, "containerStats");
-route(CLUSTER_SPEC, SliderAMController.class, "specification");
-  }
-
-  @Override
-  public void configureServlets() {
-setup();
-
-serve("/", "/__stop").with(Dispatcher.class);
-
-for (String path : this.getServePathSpecs()) {
-  serve(path).with(Dispatcher.class);
-}
-
-serve(SYSTEM_HEALTHCHECK)
-.with(new HealthCheckServlet(new HealthCheckRegistry()));
-serve(SYSTEM_METRICS).with(new MetricsServlet(new MetricRegistry()));
-serve(SYSTEM_PING).with(new PingServlet());
-serve(SYSTEM_THREADS).with(new ThreadDumpServlet());
-
-String regex = "(?!/ws)";
-serveRegex(regex).with(SliderDefaultWrapperServlet.class); 
-
-Map params = new HashMap<>();
-params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true");
-params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true");
-params.put(ResourceConfig.FEATURE_XMLROOTELEMENT_PROCESSING, "true");
-params.put(ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS, 

[69/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
new file mode 100644
index 000..fb2fd16
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -0,0 +1,654 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.api.ServiceApiConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import 

[31/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
deleted file mode 100644
index 7712191..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ /dev/null
@@ -1,2783 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.slider.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.RegistryPathStatus;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.KerberosDiags;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.client.ipc.SliderClusterOperations;
-import org.apache.slider.common.Constants;
-import 

[18/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
deleted file mode 100644
index 2e40a9b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.rpc;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-public class SliderClusterProtocolProxy implements SliderClusterProtocol {
-
-  private static final RpcController NULL_CONTROLLER = null;
-  private final SliderClusterProtocolPB endpoint;
-  private final InetSocketAddress address;
-
-  public SliderClusterProtocolProxy(SliderClusterProtocolPB endpoint,
-  InetSocketAddress address) {
-Preconditions.checkArgument(endpoint != null, "null endpoint");
-Preconditions.checkNotNull(address != null, "null address");
-this.endpoint = endpoint;
-this.address = address;
-  }
-
-  @Override
-  public String toString() {
-final StringBuilder sb =
-new StringBuilder("SliderClusterProtocolProxy{");
-sb.append("address=").append(address);
-sb.append('}');
-return sb.toString();
-  }
-
-  @Override
-  public ProtocolSignature getProtocolSignature(String protocol,
-  long clientVersion,
-  int clientMethodsHash)
-  throws IOException {
-if (!protocol.equals(RPC.getProtocolName(SliderClusterProtocolPB.class))) {
-  throw new IOException("Serverside implements " +
-RPC.getProtocolName(SliderClusterProtocolPB.class) 
+
-". The following requested protocol is unknown: " +
-protocol);
-}
-
-return ProtocolSignature.getProtocolSignature(clientMethodsHash,
-RPC.getProtocolVersion(
-SliderClusterProtocol.class),
-SliderClusterProtocol.class);
-  }
-
-  @Override
-  public long getProtocolVersion(String protocol, long clientVersion)
-  throws IOException {
-return SliderClusterProtocol.versionID;
-  }
-  
-  private IOException convert(ServiceException se) {
-IOException ioe = ProtobufHelper.getRemoteException(se);
-if (ioe instanceof RemoteException) {
-  RemoteException remoteException = (RemoteException) ioe;
-  return remoteException.unwrapRemoteException();
-}
-return ioe;
-  }
-  
-  @Override public Messages.StopClusterResponseProto stopCluster(
-  Messages.StopClusterRequestProto request)
-  throws IOException, YarnException {
-try {
-  return endpoint.stopCluster(NULL_CONTROLLER, request);
-} catch (ServiceException e) {
-  throw convert(e);
-}
-  }
-
-  @Override
-  public Messages.UpgradeContainersResponseProto upgradeContainers(
-  Messages.UpgradeContainersRequestProto request) throws IOException,
-  YarnException {
-try {
-  return endpoint.upgradeContainers(NULL_CONTROLLER, request);
-} catch (ServiceException e) {
-  throw convert(e);
-}
-  }
-
-  

[66/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
new file mode 100644
index 000..3e53418
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+
+import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
+import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+import java.io.File;
+
+/**
+ * Registry actions
+ * 
+ * --instance {app name}, if  a / is in it, refers underneath?
+ * --dest {destfile}
+ * --list : list instances of slider service
+ * --listfiles 
+ */
+@Parameters(commandNames = {ACTION_REGISTRY},
+commandDescription = DESCRIBE_ACTION_REGISTRY)
+
+public class ActionRegistryArgs extends AbstractActionArgs {
+
+  public static final String USAGE =
+  "Usage: " + SliderActions.ACTION_REGISTRY
+  + " ("
+  + Arguments.ARG_LIST + "|"
+  + Arguments.ARG_LISTCONF + "|"
+  + Arguments.ARG_LISTEXP + "|"
+  + Arguments.ARG_LISTFILES + "|"
+  + Arguments.ARG_GETCONF + "|"
+  + Arguments.ARG_GETEXP + "> "
+  + Arguments.ARG_NAME + "  "
+  + " )"
+  + "[" + Arguments.ARG_VERBOSE + "] "
+  + "[" + Arguments.ARG_USER + "] "
+  + "[" + Arguments.ARG_OUTPUT + "  ] "
+  + "[" + Arguments.ARG_SERVICETYPE + "  ] "
+  + "[" + Arguments.ARG_FORMAT + " ] "
+  + System.getProperty("line.separator")
+  + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
+  ;
+  public ActionRegistryArgs() {
+  }
+
+  public ActionRegistryArgs(String name) {
+this.name = name;
+  }
+
+  @Override
+  public String getActionName() {
+return ACTION_REGISTRY;
+  }
+
+  /**
+   * Get the min #of params expected
+   * @return the min number of params in the {@link #parameters} field
+   */
+  @Override
+  public int getMinParams() {
+return 0;
+  }
+  
+  @Parameter(names = {ARG_LIST}, 
+  description = "list services")
+  public boolean list;
+
+  @Parameter(names = {ARG_LISTCONF}, 
+  description = "list configurations")
+  public boolean listConf;
+
+  @Parameter(names = {ARG_GETCONF},
+  description = "get configuration")
+  public String getConf;
+
+  @Parameter(names = {ARG_LISTEXP},
+ description = "list exports")
+  public boolean listExports;
+
+  @Parameter(names = {ARG_GETEXP},
+ description = "get export")
+  public String getExport;
+
+  @Parameter(names = {ARG_LISTFILES},
+  description = "list files")
+  public String listFiles;
+
+  @Parameter(names = {ARG_GETFILES},
+  description = "get files")
+  public String getFiles;
+
+  //--format 
+  @Parameter(names = ARG_FORMAT,
+  description = "Format for a response: ")
+  public String format = ConfigFormat.XML.toString() ;
+
+  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
+  description = "Output destination")
+  public File out;
+
+  

[02/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
deleted file mode 100644
index a667432..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.junit.Test;
-
-
-public class TestWorkflowServiceTerminatingRunnable extends 
WorkflowServiceTestBase {
-
-  //@Test
-  public void testNoservice() throws Throwable {
-
-try {
-  new ServiceTerminatingRunnable(null, new SimpleRunnable());
-  fail("unexpected ");
-} catch (IllegalArgumentException e) {
-
-  // expected
-}
-  }
-
-
-  //@Test
-  public void testBasicRun() throws Throwable {
-
-WorkflowCompositeService svc = run(new WorkflowCompositeService());
-ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc,
-new SimpleRunnable());
-
-// synchronous in-thread execution
-runnable.run();
-assertStopped(svc);
-  }
-
-  //@Test
-  public void testFailureRun() throws Throwable {
-
-WorkflowCompositeService svc = run(new WorkflowCompositeService());
-ServiceTerminatingRunnable runnable =
-new ServiceTerminatingRunnable(svc, new SimpleRunnable(true));
-
-// synchronous in-thread execution
-runnable.run();
-assertStopped(svc);
-assertNotNull(runnable.getException());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
deleted file mode 100644
index f38bd9d..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.Service;
-import org.junit.Assert;

[39/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
new file mode 100644
index 000..e4eae20
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static 
org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
+
+/**
+ * Launcher of applications: base class
+ */
+public class AbstractLauncher {
+  private static final Logger log =
+LoggerFactory.getLogger(AbstractLauncher.class);
+  public static final String CLASSPATH = "CLASSPATH";
+  /**
+   * Filesystem to use for the launch
+   */
+  protected final CoreFileSystem coreFileSystem;
+  /**
+   * Env vars; set up at final launch stage
+   */
+  protected final Map envVars = new HashMap<>();
+  protected final ContainerLaunchContext containerLaunchContext =
+Records.newRecord(ContainerLaunchContext.class);
+  protected final List commands = new ArrayList<>(20);
+  protected final Map localResources = new HashMap<>();
+  protected final Map mountPaths = new HashMap<>();
+  private final Map serviceData = new HashMap<>();
+  // security
+  protected final Credentials credentials;
+  protected boolean yarnDockerMode = false;
+  protected String dockerImage;
+  protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
+  protected String dockerHostname;
+  protected String runPrivilegedContainer;
+
+
+  /**
+   * Create instance.
+   * @param coreFileSystem filesystem
+   * @param credentials initial set of credentials -null is permitted
+   */
+  public AbstractLauncher(
+  CoreFileSystem coreFileSystem,
+  Credentials credentials) {
+this.coreFileSystem = coreFileSystem;
+this.credentials = credentials != null ? credentials: new Credentials();
+  }
+  
+  public void setYarnDockerMode(boolean yarnDockerMode){
+this.yarnDockerMode = yarnDockerMode;
+  }
+
+  /**
+   * Get the env vars to work on
+   * @return env vars
+   */
+  public Map getEnv() {
+return envVars;
+  }
+
+  /**
+   * Get the launch commands.
+   * @return the live list of commands 
+   */
+  public List getCommands() {
+return commands;
+  }
+
+  public void addLocalResource(String subPath, LocalResource resource) {
+localResources.put(subPath, resource);
+  }
+
+  public void addLocalResource(String subPath, LocalResource resource, String 
mountPath) {
+localResources.put(subPath, 

[46/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
deleted file mode 100644
index 8310530..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test loading example resources.
- */
-@RunWith(value = Parameterized.class)
-public class TestLoadExampleAppJson extends Assert {
-  private String resource;
-
-  public TestLoadExampleAppJson(String resource) {
-this.resource = resource;
-  }
-
-  @Parameterized.Parameters
-  public static Collection filenames() {
-String[][] stringArray = new String[ExampleAppJson
-.ALL_EXAMPLE_RESOURCES.size()][1];
-int i = 0;
-for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
-  stringArray[i++][0] = s;
-}
-return Arrays.asList(stringArray);
-  }
-
-  @Test
-  public void testLoadResource() throws Throwable {
-try {
-  Application application = JSON_SER_DESER.fromResource(resource);
-
-  SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-  FileSystem mockFs = createNiceMock(FileSystem.class);
-  expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-  expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-  new Path("cluster_dir_path")).anyTimes();
-  replay(sfs, mockFs);
-
-  ServiceApiUtil.validateAndResolveApplication(application, sfs,
-  new YarnConfiguration());
-} catch (Exception e) {
-  throw new Exception("exception loading " + resource + ":" + 
e.toString());
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
deleted file mode 100644
index 98c78d3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you 

[33/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
deleted file mode 100644
index e89306c..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Set of configuration properties that can be injected into the application
- * components via envs, files and custom pluggable helper docker containers.
- * Files of several standard formats like xml, properties, json, yaml and
- * templates will be supported.
- **/
-
-@ApiModel(description = "Set of configuration properties that can be injected 
into the application components via envs, files and custom pluggable helper 
docker containers. Files of several standard formats like xml, properties, 
json, yaml and templates will be supported.")
-@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Configuration implements Serializable {
-  private static final long serialVersionUID = -4330788704981074466L;
-
-  private Map properties = new HashMap();
-  private Map env = new HashMap();
-  private List files = new ArrayList();
-
-  /**
-   * A blob of key-value pairs of common application properties.
-   **/
-  public Configuration properties(Map properties) {
-this.properties = properties;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "A blob of key-value pairs of 
common application properties.")
-  @JsonProperty("properties")
-  public Map getProperties() {
-return properties;
-  }
-
-  public void setProperties(Map properties) {
-this.properties = properties;
-  }
-
-  /**
-   * A blob of key-value pairs which will be appended to the default system
-   * properties and handed off to the application at start time. All 
placeholder
-   * references to properties will be substituted before injection.
-   **/
-  public Configuration env(Map env) {
-this.env = env;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "A blob of key-value pairs which 
will be appended to the default system properties and handed off to the 
application at start time. All placeholder references to properties will be 
substituted before injection.")
-  @JsonProperty("env")
-  public Map getEnv() {
-return env;
-  }
-
-  public void setEnv(Map env) {
-this.env = env;
-  }
-
-  /**
-   * Array of list of files that needs to be created and made available as
-   * volumes in the application component containers.
-   **/
-  public Configuration files(List files) {
-this.files = files;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "Array of list of files that 
needs to be created and made available as volumes in the application component 
containers.")
-  

[17/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
deleted file mode 100644
index ba923bc..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ /dev/null
@@ -1,2120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.ServiceApiConstants;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.proto.Messages.ComponentCountProto;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ApplicationState;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.RoleStatistics;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderInternalStateException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.management.MetricsConstants;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;

[13/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
deleted file mode 100644
index 3e9b764..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.publisher;
-
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import org.apache.slider.core.registry.docstore.UriMap;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static 
org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_REGEXP;
-import static 
org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP;
-
-/**
- * This publishes configuration sets
- */
-public class PublisherResource extends AbstractSliderResource {
-  protected static final Logger log =
-  LoggerFactory.getLogger(PublisherResource.class);
-  public static final String EXPORTS_NAME = "exports";
-  public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME;
-  public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + 
"/{exportname}" ;
-  public static final String SET_NAME =
-  "{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
-  public static final String SETNAME = "setname";
-  public static final String CLASSPATH = "/classpath";
-  public static final String CONFIG = "config";
-  
-  public static final String SETNAME_PATTERN = 
-  "{"+ SETNAME+": " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
-  private static final String CONFIG_PATTERN =
-  SETNAME_PATTERN + "/{"+ CONFIG +": " + PUBLISHED_CONFIGURATION_REGEXP + 
"}";
-  private final StateAccessForProviders appState;
-
-  public PublisherResource(WebAppApi slider) {
-super(slider);
-appState = slider.getAppState();
-  }
-
-  private void init(HttpServletResponse res, UriInfo uriInfo) {
-res.setContentType(null);
-log.debug(uriInfo.getRequestUri().toString());
-  }
- 
-  /**
-   * Get a named config set 
-   * @param setname name of the config set
-   * @return the config set
-   * @throws NotFoundException if there was no matching set
-   */
-  private PublishedConfigSet getConfigSet(String setname) {
-PublishedConfigSet configSet =
-appState.getPublishedConfigSet(setname);
-if (configSet == null) {
-  

[10/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
deleted file mode 100644
index da06c13..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
+++ /dev/null
@@ -1,168 +0,0 @@
-{
-  "roleCommand": "{{COMMAND}}",
-  "clusterName": "{{CLUSTER_NAME}}",
-  "hostname": "{{HOST_NAME}}",
-  "hostLevelParams": {
-"java_home": "/usr/jdk64/jdk1.7.0_45"
-  },
-  "commandType": "EXECUTION_COMMAND",
-  "roleParams": {},
-  "serviceName": "{{SERVICE_NAME}}",
-  "role": "{{ROLE_NAME}}",
-  "commandParams": {},
-  "taskId": "{{TASK_ID}}",
-  "public_hostname": "{{HOST_NAME}}",
-  "configurations": {
-"hbase-log4j": {
-  "log4j.threshold": "ALL",
-  "log4j.rootLogger": "${hbase.root.logger}",
-  "log4j.logger.org.apache.zookeeper": "INFO",
-  "log4j.logger.org.apache.hadoop.hbase": "DEBUG",
-  "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": 
"INFO",
-  "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO",
-  "log4j.category.SecurityLogger": "${hbase.security.logger}",
-  "log4j.appender.console": "org.apache.log4j.ConsoleAppender",
-  "log4j.appender.console.target": "System.err",
-  "log4j.appender.console.layout": "org.apache.log4j.PatternLayout",
-  "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p 
[%t] %c{2}: %m%n",
-  "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender",
-  "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout",
-  "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: 
%m%n",
-  "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}",
-  "log4j.appender.RFAS.MaxBackupIndex": 
"${hbase.security.log.maxbackupindex}",
-  "log4j.appender.RFAS.File": 
"${hbase.log.dir}/${hbase.security.log.file}",
-  "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender",
-  "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout",
-  "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] 
%c{2}: %m%n",
-  "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}",
-  "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}",
-  "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}",
-  "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender",
-  "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender",
-  "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout",
-  "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] 
%c{2}: %m%n",
-  "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}",
-  "log4j.appender.DRFA.DatePattern": ".-MM-dd",
-  "log4j.additivity.SecurityLogger": "false",
-  "hbase.security.logger": "INFO,console",
-  "hbase.security.log.maxfilesize": "256MB",
-  "hbase.security.log.maxbackupindex": "20",
-  "hbase.security.log.file": "SecurityAuth.audit",
-  "hbase.root.logger": "INFO,console",
-  "hbase.log.maxfilesize": "256MB",
-  "hbase.log.maxbackupindex": "20",
-  "hbase.log.file": "hbase.log",
-  "hbase.log.dir": "."
-},
-"global": {
-  "hbase_root": "{{HBASE_HOME}}",
-  "hbase_pid_dir": "{{PID_DIR}}",
-  "proxyuser_group": "users",
-  "syncLimit": "5",
-  "hbase_regionserver_heapsize": "{{REGION_SERVER_HEAP_SIZE}}",
-  "rca_enabled": "false",
-  "tickTime": "2000",
-  "hbase_master_heapsize": "{{MASTER_HEAP_SIZE}}",
-  "initLimit": "10",
-  "user_group": "{{GROUP_NAME}}",
-  "hbase_user": "{{USER_NAME}}",
-  "hbase_log_dir": "{{LOG_DIR}}"
-},
-"hdfs-site": {
-  "dfs.namenode.checkpoint.period": "21600",
-  "dfs.namenode.avoid.write.stale.datanode": "true",
-  "dfs.namenode.checkpoint.txns": "100",
-  "dfs.block.access.token.enable": "true",
-  "dfs.support.append": "true",
-  "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}",
-  "dfs.cluster.administrators": " hdfs",
-  "dfs.replication": "3",
-  "ambari.dfs.datanode.http.port": "50075",
-  "dfs.datanode.balance.bandwidthPerSec": "625",
-  "dfs.namenode.safemode.threshold-pct": "1.0f",
-  

[15/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
deleted file mode 100644
index 15333e4..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ /dev/null
@@ -1,1123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.BoolMetric;
-import org.apache.slider.server.appmaster.management.Timestamp;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
-import org.apache.slider.server.avro.LoadedRoleHistory;
-import org.apache.slider.server.avro.NodeEntryRecord;
-import org.apache.slider.server.avro.RoleHistoryHeader;
-import org.apache.slider.server.avro.RoleHistoryWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * The Role History.
- * 
- * Synchronization policy: all public operations are synchronized.
- * Protected methods are in place for testing -no guarantees are made.
- * 
- * Inner classes have no synchronization guarantees; they should be 
manipulated 
- * in these classes and not externally.
- * 
- * Note that as well as some methods marked visible for testing, there
- * is the option for the time generator method, {@link #now()} to
- * be overridden so that a repeatable time series can be used.
- * 
- */
-public class RoleHistory {
-  protected static final Logger log =
-LoggerFactory.getLogger(RoleHistory.class);
-  private final List providerRoles;
-  /** the roles in here are shared with App State */
-  private final Map roleStatusMap = new HashMap<>();
-  private final AbstractClusterServices recordFactory;
-
-  private long startTime;
-
-  /** Time when saved */
-  private final Timestamp saveTime = new Timestamp(0);
-
-  /** If the history was loaded, the time at which the history was saved.
-   * That is: the time the data was valid */
-  private final Timestamp thawedDataTime = new Timestamp(0);
-  
-  private NodeMap nodemap;
-  private int roleSize;
-  private final BoolMetric dirty = new BoolMetric(false);
-  private FileSystem filesystem;
-  private Path historyPath;
-  private RoleHistoryWriter historyWriter = new RoleHistoryWriter();
-
-  /**
-   * When were the nodes updated in a {@link #onNodesUpdated(List)} call?
-   * If zero: never.
-   */
-  private final Timestamp nodesUpdatedTime = new Timestamp(0);
-  private final BoolMetric nodeUpdateReceived = new BoolMetric(false);
-
-  private OutstandingRequestTracker 

[25/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
deleted file mode 100644
index 1fd49ab..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import 
org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static 
org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-
-/**
- * Utils to work with credentials and tokens.
- *
- * Designed to be movable to Hadoop core
- */
-public final class CredentialUtils {
-
-  public static final String JOB_CREDENTIALS_BINARY
-  = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY;
-
-  private CredentialUtils() {
-  }
-
-  private static final Logger LOG =
-  LoggerFactory.getLogger(CredentialUtils.class);
-
-  /**
-   * Save credentials to a byte buffer. Returns null if there were no
-   * credentials to save
-   * @param credentials credential set
-   * @return a byte buffer of serialized tokens
-   * @throws IOException if the credentials could not be written to the stream
-   */
-  public static ByteBuffer marshallCredentials(Credentials credentials) throws 
IOException {
-ByteBuffer buffer = null;
-if (!credentials.getAllTokens().isEmpty()) {
-  DataOutputBuffer dob = new DataOutputBuffer();
-  try {
-credentials.writeTokenStorageToStream(dob);
-  } finally {
-dob.close();
-  }
-  buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
-}
-return buffer;
-  }
-
-  public static File locateEnvCredentials(Map env,
-  Configuration conf,
-  StringBuffer sourceTextOut) throws FileNotFoundException {
-String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
-String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION;
-if (tokenFilename == null) {
-  tokenFilename = 

[38/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 647bfe9..a044838 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -24,30 +24,24 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.AbstractLauncher;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
 import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
+import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,23 +49,18 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.regex.Pattern;
 
-import static org.apache.slider.api.ServiceApiConstants.*;
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+import static 
org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
 
 /**
  * This is a factoring out of methods handy for providers. It's bonded to a log
  * at construction time.
  */
-public class ProviderUtils implements RoleKeys, SliderKeys {
+public class ProviderUtils implements YarnServiceConstants {
 
   protected static final Logger log =
   LoggerFactory.getLogger(ProviderUtils.class);
@@ -174,46 +163,22 @@ public class ProviderUtils implements RoleKeys, 
SliderKeys {
   }
 
   /**
-   * Get resource requirements from a String value. If value isn't specified,
-   * use the default value. If value is greater than max, use the max value.
-   * @param val string value
-   * @param defVal default value
-   * @param maxVal maximum value

[48/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
deleted file mode 100644
index 405f690..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-
-/**
- * Serialized form of an application report which can be persisted
- * and then parsed. It can not be converted back into a
- * real YARN application report
- * 
- * Useful for testing
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-
-public class SerializedApplicationReport {
-
-  public String applicationId;
-  public String applicationAttemptId;
-  public String name;
-  public String applicationType;
-  public String user;
-  public String queue;
-  public String host;
-  public Integer rpcPort;
-  public String state;
-  public String diagnostics;
-  public String url;
-  /**
-   * This value is non-null only when a report is generated from a submission 
context.
-   * The YARN {@link ApplicationReport} structure does not propagate this value
-   * from the RM.
-   */
-  public Long submitTime;
-  public Long startTime;
-  public Long finishTime;
-  public String finalStatus;
-  public String origTrackingUrl;
-  public Float progress;
-  
-  public SerializedApplicationReport() {
-  }
-  
-  public SerializedApplicationReport(ApplicationReport report) {
-this.applicationId = report.getApplicationId().toString();
-ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
-this.applicationAttemptId = attemptId != null ? attemptId.toString() : 
"N/A";
-this.name = report.getName();
-this.applicationType = report.getApplicationType();
-this.user = report.getUser();
-this.queue = report.getQueue();
-this.host = report.getHost();
-this.rpcPort = report.getRpcPort();
-this.state = report.getYarnApplicationState().toString();
-this.diagnostics = report.getDiagnostics();
-this.startTime = report.getStartTime();
-this.finishTime = report.getFinishTime();
-FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
-this.finalStatus = appStatus == null ? "" : appStatus.toString();
-this.progress = report.getProgress();
-this.url = report.getTrackingUrl();
-this.origTrackingUrl= report.getOriginalTrackingUrl();
-  }
-
-  @Override
-  public String toString() {
-try {
-  return ApplicationReportSerDeser.toString(this);
-} catch (IOException e) {
-  return super.toString();
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
--
diff --git 

[03/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
deleted file mode 100644
index 9c5708f..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertNotNull;
-
-/**
- * This is an evolving engine to mock YARN operations.
- */
-public class MockYarnEngine {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(MockYarnEngine.class);
-
-  private MockYarnCluster cluster;
-  private Allocator allocator;
-  private List pending = new ArrayList<>();
-
-  private ApplicationId appId = new MockApplicationId(0, 0);
-
-  private ApplicationAttemptId attemptId = new MockApplicationAttemptId(appId,
-  1);
-
-  @Override
-  public String toString() {
-return "MockYarnEngine " + cluster + " + pending=" + pending.size();
-  }
-
-  public int containerCount() {
-return cluster.containersInUse();
-  }
-
-  public MockYarnEngine(int clusterSize, int containersPerNode) {
-cluster = new MockYarnCluster(clusterSize, containersPerNode);
-allocator = new Allocator(cluster);
-  }
-
-  public MockYarnCluster getCluster() {
-return cluster;
-  }
-
-  public Allocator getAllocator() {
-return allocator;
-  }
-
-  /**
-   * Allocate a container from a request. The containerID will be
-   * unique, nodeId and other fields chosen internally with
-   * no such guarantees; resource and priority copied over
-   * @param request request
-   * @return container
-   */
-  public Container allocateContainer(AMRMClient.ContainerRequest request) {
-MockContainer allocated = allocator.allocate(request);
-if (allocated != null) {
-  MockContainerId id = (MockContainerId)allocated.getId();
-  id.setApplicationAttemptId(attemptId);
-}
-return allocated;
-  }
-
-  MockYarnCluster.MockYarnClusterContainer releaseContainer(ContainerId
-  containerId) {
-return cluster.release(containerId);
-  }
-
-  /**
-   * Process a list of operations -release containers to be released,
-   * allocate those for which there is space (but don't rescan the list after
-   * the scan).
-   * @param ops
-   * @return
-   */
-  public List execute(List ops) {
-return execute(ops, new ArrayList<>());
-  }
-
-  /**
-   * Process a list of operations -release containers to be released,
-   * allocate those for which there is space (but don't rescan the list after
-   * the scan). Unsatisifed entries 

[29/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
deleted file mode 100644
index 51bde7b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_LIST},
-commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
-public class ActionListArgs extends AbstractActionArgs {
-  @Override
-  public String getActionName() {
-return SliderActions.ACTION_LIST;
-  }
-
-  @Parameter(names = {ARG_LIVE},
-  description = "List only live application instances")
-  public boolean live;
-
-  @Parameter(names = {ARG_STATE},
-  description = "list only applications in the specific YARN state")
-  public String state = "";
-  
-  @Parameter(names = {ARG_VERBOSE},
-  description = "print out information in details")
-  public boolean verbose = false;
-
-  @Parameter(names = {ARG_CONTAINERS},
-  description = "List containers of an application instance")
-  public boolean containers;
-
-  @Parameter(names = {ARG_VERSION},
-  description = "Filter containers by app version (used with " +
-ARG_CONTAINERS + ")")
-  public String version;
-
-  @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
-  description = "Filter containers by component names (used with " +
-ARG_CONTAINERS + ")")
-  public Set components = new HashSet<>(0);
-
-  /**
-   * Get the min #of params expected
-   * @return the min number of params in the {@link #parameters} field
-   */
-  public int getMinParams() {
-return 0;
-  }
-
-  @Override
-  public int getMaxParams() {
-return 1;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
deleted file mode 100644
index 012..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed 

[05/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
deleted file mode 100644
index ece65ba..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Testing finding nodes for new instances.
- *
- * This stresses the non-AA codepath
- */
-public class TestRoleHistoryFindNodesForNewInstances extends
-BaseMockAppStateTest {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(TestRoleHistoryFindNodesForNewInstances.class);
-
-  public TestRoleHistoryFindNodesForNewInstances() throws BadConfigException {
-  }
-
-  @Override
-  public String getTestName() {
-return "TestFindNodesForNewInstances";
-  }
-
-  private NodeInstance age1Active4;
-  private NodeInstance age2Active2;
-  private NodeInstance age3Active0;
-  private NodeInstance age4Active1;
-  private NodeInstance age2Active0;
-
-  private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
-
-  private RoleStatus roleStat;
-  private RoleStatus roleStat2;
-
-  @Override
-  public void setup() throws Exception {
-super.setup();
-
-age1Active4 = nodeInstance(1, 4, 0, 0);
-age2Active2 = nodeInstance(2, 2, 0, 1);
-age3Active0 = nodeInstance(3, 0, 0, 0);
-age4Active1 = nodeInstance(4, 1, 0, 0);
-age2Active0 = nodeInstance(2, 0, 0, 0);
-
-roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1,
-age1Active4, age3Active0));
-roleHistory.buildRecentNodeLists();
-
-roleStat = getRole0Status();
-roleStat2 = getRole2Status();
-  }
-
-  public List findNodes(int count) {
-return findNodes(count, roleStat);
-  }
-
-  public List findNodes(int count, RoleStatus roleStatus) {
-List  found = new ArrayList<>();
-for (int i = 0; i < count; i++) {
-  NodeInstance f = roleHistory.findRecentNodeForNewInstance(roleStatus);
-  if (f != null) {
-found.add(f);
-  }
-}
-return found;
-  }
-
-  //@Test
-  public void testFind1NodeR0() throws Throwable {
-NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
-LOG.info("found: {}", found);
-assertTrue(Arrays.asList(age3Active0).contains(found));
-  }
-
-  //@Test
-  public void testFind2NodeR0() throws Throwable {
-NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
-LOG.info("found: {}", found);
-assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found));
-NodeInstance found2 = 

[45/86] [abbrv] hadoop git commit: YARN-5244. Documentation required for DNS Server implementation. Contributed by Jon Maron

[jianhe]

YARN-5244. Documentation required for DNS Server implementation. Contributed by 
Jon Maron


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/538c63ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/538c63ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/538c63ef

Branch: refs/heads/yarn-native-services
Commit: 538c63ef0634f197c622a4b284310768a7425ae5
Parents: 727e6d7
Author: Gour Saha 
Authored: Thu Aug 24 19:05:13 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:22 2017 -0700

--
 hadoop-project/src/site/site.xml|   7 +-
 .../native-services/NativeServicesDiscovery.md  | 127 +++
 .../native-services/NativeServicesIntro.md  |   0
 .../src/site/resources/images/dns_overview.png  | Bin 0 -> 41908 bytes
 .../resources/images/dns_record_creation.jpeg   | Bin 0 -> 51911 bytes
 .../resources/images/dns_record_removal.jpeg| Bin 0 -> 58041 bytes
 6 files changed, 133 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/538c63ef/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index a88f0e3..e6e9d86 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -151,7 +151,12 @@
   
   
 
-
+
+
+  
+  
+
+
 
   
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/538c63ef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
new file mode 100644
index 000..4a048af
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -0,0 +1,127 @@
+# YARN DNS Server
+## Introduction
+
+The YARN DNS Server provides a standard DNS interface to the information 
posted into the YARN Registry by deployed applications. The DNS service serves 
the following functions:
+
+1. **Exposing existing service­ discovery information via DNS**​­ - 
Information provided in
+the current YARN service registry’s records will be converted into DNS 
entries, thus
+allowing users to discover information about YARN applications using standard 
DNS
+client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
+number for services).
+2. **Enabling Container to IP mappings​­** - Enables discovery of the IPs 
of containers via
+standard DNS lookups. Given the availability of the records via DNS, container
+name­based communication will be facilitated (e.g. ‘curl
+http://myContainer.myDomain.com/endpoint’).
+
+## Service Properties
+
+The existing YARN Service Registry is leveraged as the source of information 
for the DNS Service.
+
+The following core functions are supported by the DNS­ Server:
+
+###Functional properties
+
+1. Supports creation of DNS records for end­points of the deployed YARN 
applications
+2. Record names remain unchanged during restart of containers and/or 
applications
+3. Supports reverse lookups (name based on IP).
+4. Supports security using the standards defined by The Domain Name System 
Security
+Extensions (DNSSEC)
+5. Highly available
+6. Scalable ­- The service provides the responsiveness (e.g. low­ latency) 
required to
+respond to DNS queries (timeouts yield attempts to invoke other configured name
+servers).
+
+###Deployment properties
+
+1. Supports integration with existing DNS assets (e.g. a corporate DNS server) 
by acting as
+a DNS server for a Hadoop cluster zone/domain. The server is not intended to 
act as a
+primary DNS server and does not forward requests to other servers.
+2. The DNS Server exposes a port that can receive both TCP and UDP requests per
+DNS standards. The default port for DNS protocols is in a restricted, 
administrative port
+range (53), so the port is configurable for deployments in which the service 
may
+not be managed via an administrative account.
+
+##DNS Record Name Structure
+
+The DNS names of generated records are composed from the following elements 
(labels). Note that these elements must be compatible with DNS conventions (see 
“Preferred Name Syntax” in RFC 1035):
+
+* **domain** -​­ the name of the cluster DNS domain. This name is provided 
as a
+configuration property. In addition, it is this name that is configured at a 
parent DNS
+server as the 

[49/86] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f8fe178/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
deleted file mode 100644
index fe8cce8..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- *
- * Some of the methods take an argument of a map iterable for their sources; 
this allows
- * the same method
- */
-public class ConfigHelper {
-  private static final Logger log = 
LoggerFactory.getLogger(ConfigHelper.class);
-
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Map map,
-  String origin) throws BadConfigException {
-addConfigMap(config, map.entrySet(), origin);
-  }
-
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Iterable> map,
-  String origin) throws BadConfigException {
-for (Map.Entry mapEntry : map) {
-  String key = mapEntry.getKey();
-  String value = mapEntry.getValue();
-  if (value == null) {
-throw new BadConfigException("Null value for property " + key);
-  }
-  config.set(key, value, origin);
-}
-  }
-
-  /**
-   * Convert to an XML string
-   * @param conf configuration
-   * @return conf
-   * @throws IOException
-   */
-  public static String toXml(Configuration conf) throws IOException {
-StringWriter writer = new StringWriter();
-conf.writeXml(writer);
-return writer.toString();
-  }
-
-
-  /**
-   * Register a resource as a default resource.
-   * Do not attempt to use this unless you understand that the
-   * order in which default resources are loaded affects the outcome,
-   * and that subclasses of Configuration often register new default
-   * resources
-   * @param resource the resource name
-   * @return the URL or null
-   */
-  public static URL registerDefaultResource(String resource) {
-URL resURL = getResourceUrl(resource);
-if (resURL != null) {
-  Configuration.addDefaultResource(resource);
-}
-return resURL;
-  }
-
-  /**
-   * Load a configuration from a resource on this classpath.
-   * If the resource is not found, an empty configuration is returned
-   * @param resource the resource name
-   * @return the loaded configuration.
-   */
-  public static Configuration loadFromResource(String resource) {
-Configuration conf = new Configuration(false);
-URL resURL = getResourceUrl(resource);
-if (resURL != null) {
-  log.debug("loaded resources from {}", resURL);
-  conf.addResource(resource);
-} else{
-  log.debug("failed to find {} on 

[32/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
deleted file mode 100644
index b28257f..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.NotFoundException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.retrieve.RegistryRetriever;
-
-import java.io.File;
-import java.io.IOException;
-
-import static 
org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
-import static 
org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-
-public class ClientUtils {
-  public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
-  String user, String name) throws IOException, SliderException {
-return lookupServiceRecord(rops, user, null, name);
-  }
-
-  public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
-  String user, String type, String name) throws IOException,
-  SliderException {
-if (StringUtils.isEmpty(user)) {
-  user = currentUser();
-} else {
-  user = RegistryPathUtils.encodeForRegistry(user);
-}
-if (StringUtils.isEmpty(type)) {
-  type = SliderKeys.APP_TYPE;
-}
-
-String path = servicePath(user, type, name);
-return resolve(rops, path);
-  }
-
-  public static ServiceRecord resolve(RegistryOperations rops, String path)
-  throws IOException, SliderException {
-try {
-  return rops.resolve(path);
-} catch (PathNotFoundException | NoRecordException e) {
-  throw new NotFoundException(e.getPath().toString(), e);
-}
-  }
-
-  public static PublishedConfiguration getConfigFromRegistry(
-  RegistryOperations rops, Configuration configuration,
-  String configName, String appName, String user, boolean external)
-  throws IOException, SliderException {
-ServiceRecord instance = lookupServiceRecord(rops, user, appName);
-
-RegistryRetriever retriever = new RegistryRetriever(configuration, 
instance);
-PublishedConfigSet configurations = retriever.getConfigurations(external);
-
-PublishedConfiguration published = retriever.retrieveConfiguration(
-configurations, configName, external);
-return published;
-  }
-
-  public static String saveOrReturnConfig(PublishedConfiguration published,
-  String format, File destPath, String fileName)
-  throws BadCommandArgumentsException, IOException {
-ConfigFormat configFormat = ConfigFormat.resolve(format);
-if (configFormat == null) {
-  throw new BadCommandArgumentsException(
-  "Unknown/Unsupported format %s 

[36/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
new file mode 100644
index 000..c0712c3
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
@@ -0,0 +1,680 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslPropertiesResolver;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.Cipher;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.InvocationTargetException;
+import java.net.InetAddress;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.security.UserGroupInformation.*;
+import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+
+/**
+ * Kerberos diagnostics
+ * At some point this may move to hadoop core, so please keep use of slider
+ * methods and classes to ~0.
+ *
+ * This operation expands some of the diagnostic output of the security code,
+ * but not all. For completeness
+ *
+ * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
+ * Set the log level for {@code org.apache.hadoop.security=DEBUG}
+ */
+public class KerberosDiags implements Closeable {
+
+  private static final Logger LOG = 
LoggerFactory.getLogger(KerberosDiags.class);
+  public static final String KRB5_CCNAME = "KRB5CCNAME";
+  public static final String JAVA_SECURITY_KRB5_CONF
+= "java.security.krb5.conf";
+  public static final String JAVA_SECURITY_KRB5_REALM
+= "java.security.krb5.realm";
+  public static final String SUN_SECURITY_KRB5_DEBUG
+= "sun.security.krb5.debug";
+  public static final String SUN_SECURITY_SPNEGO_DEBUG
+= "sun.security.spnego.debug";
+  public static final String SUN_SECURITY_JAAS_FILE
+= "java.security.auth.login.config";
+  public static final String KERBEROS_KINIT_COMMAND
+= "hadoop.kerberos.kinit.command";
+  public static final String HADOOP_AUTHENTICATION_IS_DISABLED
+  = "Hadoop authentication is disabled";
+  public static final String UNSET = "(unset)";
+  public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
+
+  private final Configuration conf;
+  private final List services;
+  private final PrintStream out;
+  private final File keytab;
+  private final String principal;
+  private final long minKeyLength;
+  private final boolean securityRequired;
+
+  public static final String CAT_JVM = "JVM";
+  public static final String CAT_JAAS = "JAAS";
+  public static final String CAT_CONFIG = "CONFIG";
+  public static final String CAT_LOGIN = "LOGIN";
+  public static final String CAT_KERBEROS = "KERBEROS";
+  public static 

[09/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
deleted file mode 100644
index 7b0586e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-
-/**
- * Test the argument parsing/validation logic.
- */
-public class TestClientBadArgs extends SliderTestBase {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(TestClientBadArgs.class);
-
-  //@Test
-  public void testNoAction() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Usage: slider COMMAND",
- EMPTY_LIST);
-
-  }
-
-  //@Test
-  public void testUnknownAction() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- "not-a-known-action",
- Arrays.asList("not-a-known-action"));
-  }
-
-  //@Test
-  public void testActionWithoutOptions() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Usage: slider build ",
- Arrays.asList(SliderActions.ACTION_BUILD));
-  }
-
-  //@Test
-  public void testActionWithoutEnoughArgs() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS,
- Arrays.asList(SliderActions.ACTION_START));
-  }
-
-  //@Test
-  public void testActionWithTooManyArgs() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
- Arrays.asList(SliderActions.ACTION_HELP,
- "hello, world"));
-  }
-
-  //@Test
-  public void testBadImageArg() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Unknown option: --image",
-Arrays.asList(SliderActions.ACTION_HELP,
- Arguments.ARG_IMAGE));
-  }
-
-  //@Test
-  public void testRegistryUsage() throws Throwable {
-Throwable exception = launchExpectingException(SliderClient.class,
-createTestConfig(),
-"org.apache.slider.core.exceptions.UsageException: Argument --name " +
-"missing",
-Arrays.asList(SliderActions.ACTION_REGISTRY));
-assertTrue(exception instanceof UsageException);
-LOG.info(exception.toString());
-  }
-
-  //@Test
-  public void testRegistryExportBadUsage1() throws Throwable {
-

[41/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 0ed4860..1049698 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -22,6 +22,8 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40,6 +42,7 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsReque
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
 import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -58,34 +61,32 @@ import 
org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
 import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
 import org.apache.hadoop.yarn.service.ClientAMProtocol;
 import org.apache.hadoop.yarn.service.ServiceMaster;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
 import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
 import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
 import org.apache.hadoop.yarn.service.client.params.Arguments;
 import org.apache.hadoop.yarn.service.client.params.ClientArgs;
 import org.apache.hadoop.yarn.service.client.params.CommonArgs;
 import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
 import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.Times;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.launch.JavaCommandLineBuilder;
-import org.apache.slider.core.registry.SliderRegistryUtils;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.slider.core.zk.ZookeeperUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
+import 

[28/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
deleted file mode 100644
index 64fd8ae..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.xml.sax.SAXException;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.StringWriter;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- * 
- * Some of the methods take an argument of a map iterable for their sources; 
this allows
- * the same method
- */
-public class ConfigHelper {
-  private static final Logger log = 
LoggerFactory.getLogger(ConfigHelper.class);
-
-  /**
-   * Dump the (sorted) configuration
-   * @param conf config
-   * @return the sorted keyset
-   */
-  public static Set dumpConf(Configuration conf) {
-Set keys = sortedConfigKeys(conf);
-for (String key : keys) {
-  log.info("{}={}", key, conf.get(key));
-}
-return keys;
-  }
-
-  /**
-   * Take a configuration and return a sorted set
-   * @param conf config
-   * @return the sorted keyset
-
-   */
-  public static Set sortedConfigKeys(Iterable> conf) {
-TreeSet sorted = new TreeSet();
-for (Map.Entry entry : conf) {
-  sorted.add(entry.getKey());
-}
-return sorted;
-  }
-
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Map map,
-  String origin) throws BadConfigException {
-addConfigMap(config, map.entrySet(), origin);
-  }
-  
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Iterable> map,
-  String origin) throws BadConfigException {
-for (Map.Entry mapEntry : map) {
-  String key = mapEntry.getKey();
-  String value = mapEntry.getValue();
-  if (value == null) {
-throw new BadConfigException("Null value for property " + key);
-  }
-  

[42/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
new file mode 100644
index 000..b57225a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current status of a submitted application, returned as a response to the
+ * GET API.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current status of a submitted application, 
returned as a response to the GET API.")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ApplicationStatus extends BaseResource {
+  private static final long serialVersionUID = -3469885905347851034L;
+
+  private String diagnostics = null;
+  private ApplicationState state = null;
+  private Integer code = null;
+
+  /**
+   * Diagnostic information (if any) for the reason of the current state of the
+   * application. It typically has a non-null value, if the application is in a
+   * non-running state.
+   **/
+  public ApplicationStatus diagnostics(String diagnostics) {
+this.diagnostics = diagnostics;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Diagnostic information (if any) 
for the reason of the current state of the application. It typically has a 
non-null value, if the application is in a non-running state.")
+  @JsonProperty("diagnostics")
+  public String getDiagnostics() {
+return diagnostics;
+  }
+
+  public void setDiagnostics(String diagnostics) {
+this.diagnostics = diagnostics;
+  }
+
+  /**
+   * Application state.
+   **/
+  public ApplicationStatus state(ApplicationState state) {
+this.state = state;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Application state.")
+  @JsonProperty("state")
+  public ApplicationState getState() {
+return state;
+  }
+
+  public void setState(ApplicationState state) {
+this.state = state;
+  }
+
+  /**
+   * An error code specific to a scenario which app owners should be able to 
use
+   * to understand the failure in addition to the diagnostic information.
+   **/
+  public ApplicationStatus code(Integer code) {
+this.code = code;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "An error code specific to a 
scenario which app owners should be able to use to understand the failure in 
addition to the diagnostic information.")
+  @JsonProperty("code")
+  public Integer getCode() {
+return code;
+  }
+
+  public void setCode(Integer code) {
+this.code = code;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != o.getClass()) {
+  return false;
+}
+ApplicationStatus applicationStatus = (ApplicationStatus) o;
+return Objects.equals(this.diagnostics, 

[26/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
deleted file mode 100644
index 86025ee..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.util.VersionInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.Locale;
-import java.util.Properties;
-
-/**
- * Extract the version properties, which will look something like
- * 
- * application.name=${pom.name}
- * application.version=${pom.version}
- * application.build=${buildNumber}
- * application.build.java.version=${java.version}
- * application.build.info=${pom.name}-${pom.version} Built against 
${buildNumber} on ${java.version} by ${user.name}
- * 
- * 
- * the mvn process-resources target will expand the properties
- * and add the resources to target/classes, which will then look something like
- * 
- *   application.name=Slider Core
- *   application.version=0.7.1-SNAPSHOT
- *   application.build=1dd69
- *   application.build.java.version=1.7.0_45
- *   application.build.user=stevel
- *   application.build.info=Slider Core-0.7.1-SNAPSHOT Built against 1dd69 on 
1.7.0_45 by stevel
- * 
- * 
- * Note: the values will change and more properties added.
- */
-public class SliderVersionInfo {
-  private static final Logger log = 
LoggerFactory.getLogger(SliderVersionInfo.class);
-
-  /**
-   * Name of the resource containing the filled-in-at-runtime props
-   */
-  public static final String VERSION_RESOURCE =
-  "org/apache/slider/providers/dynamic/application.properties";
-
-  public static final String APP_NAME = "application.name";
-  public static final String APP_VERSION = "application.version";
-  public static final String APP_BUILD = "application.build";
-  public static final String APP_BUILD_JAVA_VERSION = 
"application.build.java.version";
-  public static final String APP_BUILD_USER = "application.build.user";
-  public static final String APP_BUILD_INFO = "application.build.info";
-  public static final String HADOOP_BUILD_INFO = "hadoop.build.info";
-  public static final String HADOOP_DEPLOYED_INFO = "hadoop.deployed.info";
-
-
-  public static Properties loadVersionProperties()  {
-Properties props = new Properties();
-URL resURL = SliderVersionInfo.class.getClassLoader()
-   .getResource(VERSION_RESOURCE);
-assert resURL != null : "Null resource " + VERSION_RESOURCE;
-
-try {
-  InputStream inStream = resURL.openStream();
-  assert inStream != null : "Null input stream from " + VERSION_RESOURCE;
-  props.load(inStream);
-} catch (IOException e) {
-  log.warn("IOE loading " + VERSION_RESOURCE, e);
-}
-return props;
-  }
-
-  /**
-   * Load the version info and print it
-   * @param logger logger
-   */
-  public static void loadAndPrintVersionInfo(Logger logger) {
-Properties props = loadVersionProperties();
-logger.info(props.getProperty(APP_BUILD_INFO));
-logger.info("Compiled against Hadoop {}",
-props.getProperty(HADOOP_BUILD_INFO));
-logger.info(getHadoopVersionString());
-  }
-  
-  public static String getHadoopVersionString() {
-return String.format(Locale.ENGLISH,
-"Hadoop runtime version %s with source checksum %s and build date %s",
-VersionInfo.getBranch(),
-

[12/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
deleted file mode 100644
index da122da..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.util.Map;
-
-/**
- * Probe for a port being open.
- */
-public class PortProbe extends Probe {
-  protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
-  private final int port;
-  private final int timeout;
-
-  public PortProbe(int port, int timeout) {
-super("Port probe of " + port + " for " + timeout + "ms", null);
-this.port = port;
-this.timeout = timeout;
-  }
-
-  public static PortProbe create(Map props)
-  throws IOException {
-int port = getPropertyInt(props, PORT_PROBE_PORT, null);
-
-if (port >= 65536) {
-  throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
-  "range");
-}
-
-int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
-PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
-
-return new PortProbe(port, timeout);
-  }
-
-  /**
-   * Try to connect to the (host,port); a failure to connect within
-   * the specified timeout is a failure.
-   * @param instance role instance
-   * @return the outcome
-   */
-  @Override
-  public ProbeStatus ping(ComponentInstance instance) {
-ProbeStatus status = new ProbeStatus();
-
-if (instance.getContainerStatus() == null || SliderUtils
-.isEmpty(instance.getContainerStatus().getIPs())) {
-  status.fail(this, new IOException(
-  instance.getCompInstanceName() + ": IP is not available yet"));
-  return status;
-}
-
-String ip = instance.getContainerStatus().getIPs().get(0);
-InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
-Socket socket = new Socket();
-try {
-  if (log.isDebugEnabled()) {
-log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
-.toString() + ", timeout=" + MonitorUtils
-.millisToHumanTime(timeout));
-  }
-  socket.connect(sockAddr, timeout);
-  status.succeed(this);
-} catch (Throwable e) {
-  String error =
-  instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
-  log.debug(error, e);
-  status.fail(this, new IOException(error, e));
-} finally {
-  IOUtils.closeSocket(socket);
-}
-return status;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
 

[44/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/727e6d78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/727e6d78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/727e6d78

Branch: refs/heads/yarn-native-services
Commit: 727e6d78a2c24e089398cfe6982a30badc930fd2
Parents: 4a9bc1f
Author: Billie Rinaldi 
Authored: Wed Aug 23 12:31:47 2017 -0700
Committer: Jian He 
Committed: Mon Sep 25 16:37:21 2017 -0700

--
 .../assemblies/hadoop-yarn-services-dist.xml|   30 +
 .../assemblies/hadoop-yarn-slider-dist.xml  |   30 -
 hadoop-yarn-project/hadoop-yarn/bin/yarn|8 +-
 .../service/webapp/ApplicationApiService.java   |  275 ++
 .../service/webapp/ApplicationApiWebApp.java|  123 +
 .../yarn/services/api/ApplicationApi.java   |   36 -
 .../api/impl/ApplicationApiService.java |  288 --
 .../services/webapp/ApplicationApiWebApp.java   |  125 -
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |3 -
 .../src/main/webapp/WEB-INF/web.xml |2 +-
 .../dev-support/findbugs-exclude.xml|   93 +-
 .../conf/slideram-log4j.properties  |   68 -
 .../conf/yarnservice-log4j.properties   |   62 +
 .../hadoop-yarn-slider-core/pom.xml |   30 +-
 .../slider/server/avro/RoleHistoryRecord.avsc   |  114 -
 .../apache/hadoop/security/KerberosDiags.java   |  677 -
 .../yarn/service/ContainerFailureTracker.java   |4 +-
 .../yarn/service/ContainerLaunchService.java|  101 -
 .../hadoop/yarn/service/ServiceContext.java |6 +-
 .../hadoop/yarn/service/ServiceMaster.java  |9 +-
 .../hadoop/yarn/service/ServiceMonitor.java |  149 -
 .../hadoop/yarn/service/ServiceScheduler.java   |   44 +-
 .../api/constants/ServiceApiConstants.java  |   69 +
 .../yarn/service/api/records/Application.java   |  466 +++
 .../service/api/records/ApplicationState.java   |   33 +
 .../service/api/records/ApplicationStatus.java  |  148 +
 .../yarn/service/api/records/Artifact.java  |  160 +
 .../yarn/service/api/records/BaseResource.java  |   52 +
 .../yarn/service/api/records/Component.java |  412 +++
 .../yarn/service/api/records/ConfigFile.java|  225 ++
 .../yarn/service/api/records/ConfigFormat.java  |   67 +
 .../yarn/service/api/records/Configuration.java |  225 ++
 .../yarn/service/api/records/Container.java |  297 ++
 .../service/api/records/ContainerState.java |   30 +
 .../hadoop/yarn/service/api/records/Error.java  |  129 +
 .../service/api/records/PlacementPolicy.java|  102 +
 .../service/api/records/ReadinessCheck.java |  175 ++
 .../yarn/service/api/records/Resource.java  |  159 +
 .../hadoop/yarn/service/client/ServiceCLI.java  |   14 +-
 .../yarn/service/client/ServiceClient.java  |  249 +-
 .../client/params/AbstractActionArgs.java   |   34 +-
 .../client/params/AbstractArgsDelegate.java |   28 +
 .../AbstractClusterBuildingActionArgs.java  |   58 +
 .../service/client/params/ActionBuildArgs.java  |2 -
 .../service/client/params/ActionClientArgs.java |   71 +
 .../service/client/params/ActionCreateArgs.java |2 -
 .../client/params/ActionDependencyArgs.java |4 +-
 .../service/client/params/ActionExistsArgs.java |   49 +
 .../service/client/params/ActionFlexArgs.java   |7 +-
 .../service/client/params/ActionFreezeArgs.java |   56 +
 .../service/client/params/ActionHelpArgs.java   |   44 +
 .../service/client/params/ActionKDiagArgs.java  |   76 +
 .../service/client/params/ActionKeytabArgs.java |   76 +
 .../service/client/params/ActionListArgs.java   |   76 +
 .../client/params/ActionRegistryArgs.java   |  218 ++
 .../client/params/ActionResolveArgs.java|  153 +
 .../client/params/ActionResourceArgs.java   |   70 +
 .../service/client/params/ActionStatusArgs.java |   51 +
 .../service/client/params/ActionThawArgs.java   |   67 +
 .../service/client/params/ActionTokensArgs.java |   78 +
 .../service/client/params/ActionUpdateArgs.java |   32 +
 .../yarn/service/client/params/ArgOps.java  |4 +-
 .../yarn/service/client/params/Arguments.java   |   37 +-
 .../yarn/service/client/params/ClientArgs.java  |  147 +-
 .../yarn/service/client/params/CommonArgs.java  |   30 +-
 .../client/params/ComponentArgsDelegate.java|4 +-
 .../client/params/DontSplitArguments.java   |   34 +
 .../client/params/LaunchArgsAccessor.java   |   30 +
 .../client/params/LaunchArgsDelegate.java   |   51 +
 .../client/params/OptionArgsDelegate.java   |   66 +
 .../client/params/PathArgumentConverter.java|   34 +
 .../client/params/SliderAMCreateAction.java |4 -
 .../service/client/params/SliderActions.java|   48 +-
 

[20/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
deleted file mode 100644
index 7446e82..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
-import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Kill a specific container
- */
-public class ActionKillContainer extends AsyncAction {
-
-  /**
-   *  container to kill
-   */
-  private final ContainerId containerId;
-
-  /**
-   *  handler for the operation
-   */
-  private final RMOperationHandlerActions operationHandler;
-
-  /**
-   * Kill a container
-   * @param containerId container to kill
-   * @param delay
-   * @param timeUnit
-   * @param operationHandler
-   */
-  public ActionKillContainer(
-  ContainerId containerId,
-  long delay,
-  TimeUnit timeUnit,
-  RMOperationHandlerActions operationHandler) {
-super("kill container", delay, timeUnit, ATTR_CHANGES_APP_SIZE);
-this.operationHandler = operationHandler;
-Preconditions.checkArgument(containerId != null);
-
-this.containerId = containerId;
-  }
-
-  /**
-   * Get the container ID to kill
-   * @return
-   */
-  public ContainerId getContainerId() {
-return containerId;
-  }
-
-  @Override
-  public void execute(SliderAppMaster appMaster,
-  QueueAccess queueService,
-  AppState appState) throws Exception {
-  List opsList = new LinkedList<>();
-ContainerReleaseOperation release = new 
ContainerReleaseOperation(containerId);
-opsList.add(release);
-//now apply the operations
-operationHandler.execute(opsList);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
deleted file mode 100644
index 0d7f7d4..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional 

[27/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
deleted file mode 100644
index fc57c82..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ /dev/null
@@ -1,2548 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.Slider;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.services.utility.PatternValidator;
-import org.apache.slider.server.services.workflow.ForkedProcessService;
-import org.apache.zookeeper.server.util.KerberosUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.Serializable;
-import java.io.StringWriter;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.EnumSet;
-import 

[01/86] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He [Forced Update!]

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/yarn-native-services 3ae5a4721 -> 3f7a50d8d (forced update)


http://git-wip-us.apache.org/repos/asf/hadoop/blob/727e6d78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
deleted file mode 100644
index 5493198..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.utils;
-
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
-import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
-import org.apache.commons.compress.utils.IOUtils;
-import org.junit.Assert;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-/**
- *  Various utility methods
- *  Byte comparison methods are from
- *  org.apache.hadoop.fs.contract.ContractTestUtils
- */
-public class TestUtility {
-  protected static final Logger log =
-  LoggerFactory.getLogger(TestUtility.class);
-
-  public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, 
String prefix) throws IOException {
-for (File file : dirObj.listFiles()) {
-  if (file.isDirectory()) {
-addDir(file, zipFile, prefix + file.getName() + File.separator);
-  } else {
-log.info("Adding to zip - " + prefix + file.getName());
-zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName()));
-IOUtils.copy(new FileInputStream(file), zipFile);
-zipFile.closeArchiveEntry();
-  }
-}
-  }
-
-  public static void zipDir(String zipFile, String dir) throws IOException {
-File dirObj = new File(dir);
-ZipArchiveOutputStream out = new ZipArchiveOutputStream(new 
FileOutputStream(zipFile));
-log.info("Creating : {}", zipFile);
-try {
-  addDir(dirObj, out, "");
-} finally {
-  out.close();
-}
-  }
-
-  public static String createAppPackage(
-  TemporaryFolder folder, String subDir, String pkgName, String srcPath) 
throws IOException {
-String zipFileName;
-File pkgPath = folder.newFolder(subDir);
-File zipFile = new File(pkgPath, pkgName).getAbsoluteFile();
-zipFileName = zipFile.getAbsolutePath();
-TestUtility.zipDir(zipFileName, srcPath);
-log.info("Created temporary zip file at {}", zipFileName);
-return zipFileName;
-  }
-
-
-  /**
-   * Assert that tthe array original[0..len] and received[] are equal.
-   * A failure triggers the logging of the bytes near where the first
-   * difference surfaces.
-   * @param original source data
-   * @param received actual
-   * @param len length of bytes to compare
-   */
-  public static void compareByteArrays(byte[] original,
-  byte[] received,
-  int len) {
-Assert.assertEquals("Number of bytes read != number written",
-len, received.length);
-int errors = 0;
-int first_error_byte = -1;
-for (int i = 0; i < len; i++) {
-  if (original[i] != received[i]) {
-if (errors == 0) {
-  first_error_byte = i;
-}
-errors++;
-  }
-}
-
-if (errors > 0) {
-  String message = String.format(" %d errors in file of length %d",
-  errors, len);
-  log.warn(message);
-  // the range either side of the first error to print
-  // this is a purely arbitrary number, to aid user debugging
-  final int overlap = 10;
-  for (int i = Math.max(0, first_error_byte - overlap);
-   i < 

hadoop git commit: YARN-7240. Add more states and transitions to stabilize the NM Container state machine. (Kartheek Muthyala via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 17b17aed0 -> e7199cb49


YARN-7240. Add more states and transitions to stabilize the NM Container state 
machine. (Kartheek Muthyala via asuresh)

(cherry picked from commit df800f6cf3ea663daf4081ebe784808b08d9366d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7199cb4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7199cb4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7199cb4

Branch: refs/heads/branch-2
Commit: e7199cb494f3d644a4bcf685de9dc33ba36eceb8
Parents: 17b17ae
Author: Arun Suresh 
Authored: Mon Sep 25 14:11:55 2017 -0700
Committer: Arun Suresh 
Committed: Mon Sep 25 16:36:58 2017 -0700

--
 .../containermanager/ContainerManagerImpl.java  |  41 +---
 .../container/ContainerEventType.java   |   6 +-
 .../container/ContainerImpl.java| 174 --
 .../container/ContainerState.java   |   3 +-
 .../container/UpdateContainerTokenEvent.java|  86 +++
 .../scheduler/ContainerScheduler.java   | 114 -
 .../UpdateContainerSchedulerEvent.java  |  46 ++--
 .../BaseContainerManagerTest.java   |   2 +
 .../containermanager/TestContainerManager.java  | 230 ++-
 .../TestContainerSchedulerQueuing.java  | 101 
 10 files changed, 661 insertions(+), 142 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7199cb4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 54ba80a..2252e56 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -20,6 +20,7 @@ package 
org.apache.hadoop.yarn.server.nodemanager.containermanager;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.ByteString;
+import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -142,7 +143,6 @@ import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
 
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.UpdateContainerSchedulerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
 import 
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredApplicationsState;
@@ -1178,29 +1178,6 @@ public class ContainerManagerImpl extends 
CompositeService implements
   + " [" + containerTokenIdentifier.getVersion() + "]");
 }
 
-// Check container state
-org.apache.hadoop.yarn.server.nodemanager.
-containermanager.container.ContainerState currentState =
-container.getContainerState();
-EnumSet allowedStates = EnumSet.of(
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.RUNNING,
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.SCHEDULED,
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.LOCALIZING,
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.REINITIALIZING,
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.RELAUNCHING);
-if (!allowedStates.contains(currentState)) {
-  throw RPCUtil.getRemoteException("Container " + containerId.toString()
-  + " is in " + currentState.name() + 

hadoop git commit: YARN-7240. Add more states and transitions to stabilize the NM Container state machine. (Kartheek Muthyala via asuresh)

[asuresh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 47011d7dd -> df800f6cf


YARN-7240. Add more states and transitions to stabilize the NM Container state 
machine. (Kartheek Muthyala via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df800f6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df800f6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df800f6c

Branch: refs/heads/trunk
Commit: df800f6cf3ea663daf4081ebe784808b08d9366d
Parents: 47011d7
Author: Arun Suresh 
Authored: Mon Sep 25 14:11:55 2017 -0700
Committer: Arun Suresh 
Committed: Mon Sep 25 16:02:55 2017 -0700

--
 .../containermanager/ContainerManagerImpl.java  |  41 +---
 .../container/ContainerEventType.java   |   6 +-
 .../container/ContainerImpl.java| 174 --
 .../container/ContainerState.java   |   3 +-
 .../container/UpdateContainerTokenEvent.java|  86 +++
 .../scheduler/ContainerScheduler.java   | 114 -
 .../UpdateContainerSchedulerEvent.java  |  46 ++--
 .../BaseContainerManagerTest.java   |   2 +
 .../containermanager/TestContainerManager.java  | 229 ++-
 .../TestContainerSchedulerQueuing.java  | 101 
 10 files changed, 660 insertions(+), 142 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/df800f6c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index e497f62..d12892e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -20,6 +20,7 @@ package 
org.apache.hadoop.yarn.server.nodemanager.containermanager;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.ByteString;
+import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -144,7 +145,6 @@ import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerScheduler;
 import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEventType;
 
-import 
org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.UpdateContainerSchedulerEvent;
 import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
 import 
org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredApplicationsState;
@@ -1251,29 +1251,6 @@ public class ContainerManagerImpl extends 
CompositeService implements
   + " [" + containerTokenIdentifier.getVersion() + "]");
 }
 
-// Check container state
-org.apache.hadoop.yarn.server.nodemanager.
-containermanager.container.ContainerState currentState =
-container.getContainerState();
-EnumSet allowedStates = EnumSet.of(
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.RUNNING,
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.SCHEDULED,
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.LOCALIZING,
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.REINITIALIZING,
-org.apache.hadoop.yarn.server.nodemanager.containermanager.container
-.ContainerState.RELAUNCHING);
-if (!allowedStates.contains(currentState)) {
-  throw RPCUtil.getRemoteException("Container " + containerId.toString()
-  + " is in " + currentState.name() + " state."
-  + " Resource can only be changed when a container is 

[2/2] hadoop git commit: HADOOP-14220 Enhance S3GuardTool with bucket-info and set-capacity commands, tests. Contributed by Steve Loughran

[fabbri]

HADOOP-14220 Enhance S3GuardTool with bucket-info and set-capacity commands, 
tests. Contributed by Steve Loughran


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60b966cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60b966cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60b966cb

Branch: refs/heads/branch-3.0
Commit: 60b966cb451ce3affb20ce18297cd5b7488d9bc7
Parents: da6ed36
Author: Aaron Fabbri 
Authored: Mon Sep 25 15:59:38 2017 -0700
Committer: Aaron Fabbri 
Committed: Mon Sep 25 16:01:15 2017 -0700

--
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |  14 +-
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  60 ++-
 .../fs/s3a/s3guard/DynamoDBMetadataStore.java   | 104 +++-
 .../fs/s3a/s3guard/LocalMetadataStore.java  |  25 +-
 .../hadoop/fs/s3a/s3guard/MetadataStore.java|  17 +
 .../fs/s3a/s3guard/NullMetadataStore.java   |  15 +
 .../apache/hadoop/fs/s3a/s3guard/S3Guard.java   |  17 +-
 .../hadoop/fs/s3a/s3guard/S3GuardTool.java  | 539 +++
 .../site/markdown/tools/hadoop-aws/s3guard.md   | 160 +-
 .../hadoop/fs/s3a/ITestS3AConfiguration.java|  10 +
 .../fs/s3a/ITestS3GuardListConsistency.java |   8 +-
 .../hadoop/fs/s3a/ITestS3GuardWriteBack.java|  37 +-
 .../org/apache/hadoop/fs/s3a/S3ATestUtils.java  |  18 -
 .../s3guard/AbstractS3GuardToolTestBase.java| 138 -
 .../s3a/s3guard/ITestS3GuardToolDynamoDB.java   | 167 +-
 .../fs/s3a/s3guard/ITestS3GuardToolLocal.java   | 142 -
 .../hadoop/fs/s3a/s3guard/TestS3GuardCLI.java   | 121 +
 17 files changed, 1405 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/60b966cb/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f4709a7..2171957 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -298,6 +298,10 @@ public class S3AFileSystem extends FileSystem {
   metadataStore = S3Guard.getMetadataStore(this);
   allowAuthoritative = conf.getBoolean(METADATASTORE_AUTHORITATIVE,
   DEFAULT_METADATASTORE_AUTHORITATIVE);
+  if (hasMetadataStore()) {
+LOG.debug("Using metadata store {}, authoritative={}",
+getMetadataStore(), allowAuthoritative);
+  }
 } catch (AmazonClientException e) {
   throw translateException("initializing ", new Path(name), e);
 }
@@ -967,7 +971,7 @@ public class S3AFileSystem extends FileSystem {
* @return the metadata store of this FS instance
*/
   @VisibleForTesting
-  MetadataStore getMetadataStore() {
+  public MetadataStore getMetadataStore() {
 return metadataStore;
   }
 
@@ -2474,9 +2478,11 @@ public class S3AFileSystem extends FileSystem {
 sb.append(", statistics {")
 .append(statistics)
 .append("}");
-sb.append(", metrics {")
-.append(instrumentation.dump("{", "=", "} ", true))
-.append("}");
+if (instrumentation != null) {
+  sb.append(", metrics {")
+  .append(instrumentation.dump("{", "=", "} ", true))
+  .append("}");
+}
 sb.append('}');
 return sb.toString();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/60b966cb/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 9dd5def..6e6f4b6 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -736,9 +736,11 @@ public final class S3AUtils {
 // propagate the value, building a new origin field.
 // to track overwrites, the generic key is overwritten even if
 // already matches the new one.
+String origin = "[" + StringUtils.join(
+source.getPropertySources(key), ", ") +"]";
 final String generic = FS_S3A_PREFIX + stripped;
-LOG.debug("Updating {}", generic);
-dest.set(generic, value, key);
+LOG.debug("Updating {} from {}", generic, origin);
+dest.set(generic, value, key + " via " + origin);
   }
 }
 return dest;
@@ -881,4 +883,58 @@ public final class S3AUtils {

[1/2] hadoop git commit: HADOOP-14220 Enhance S3GuardTool with bucket-info and set-capacity commands, tests. Contributed by Steve Loughran

[fabbri]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 da6ed36ed -> 60b966cb4


http://git-wip-us.apache.org/repos/asf/hadoop/blob/60b966cb/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
index 181cdfb..43cbe93 100644
--- 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
+++ 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
@@ -21,11 +21,12 @@ package org.apache.hadoop.fs.s3a.s3guard;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.PrintStream;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.Callable;
 
 import org.junit.Test;
 
@@ -34,13 +35,16 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
 
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * Test S3Guard related CLI commands against a LocalMetadataStore.
  */
 public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
 
+  private static final String LOCAL_METADATA = "local://metadata";
+
   @Override
   protected MetadataStore newMetadataStore() {
 return new LocalMetadataStore();
@@ -65,10 +69,7 @@ public class ITestS3GuardToolLocal extends 
AbstractS3GuardToolTestBase {
 
 S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
 cmd.setStore(ms);
-
-expectSuccess("Import command did not exit successfully - see output",
-cmd,
-"import", parent.toString());
+exec(cmd, "import", parent.toString());
 
 DirListingMetadata children =
 ms.listChildren(dir);
@@ -80,7 +81,7 @@ public class ITestS3GuardToolLocal extends 
AbstractS3GuardToolTestBase {
   }
 
   @Test
-  public void testDiffCommand() throws IOException {
+  public void testDiffCommand() throws Exception {
 S3AFileSystem fs = getFileSystem();
 MetadataStore ms = getMetadataStore();
 Set filesOnS3 = new HashSet<>(); // files on S3.
@@ -108,13 +109,10 @@ public class ITestS3GuardToolLocal extends 
AbstractS3GuardToolTestBase {
 }
 
 ByteArrayOutputStream buf = new ByteArrayOutputStream();
-PrintStream out = new PrintStream(buf);
 Diff cmd = new Diff(fs.getConf());
 cmd.setStore(ms);
-assertEquals("Diff command did not exit successfully - see output", 
SUCCESS,
-cmd.run(new String[]{"diff", "-meta", "local://metadata",
-testPath.toString()}, out));
-out.close();
+exec(cmd, buf, "diff", "-meta", LOCAL_METADATA,
+testPath.toString());
 
 Set actualOnS3 = new HashSet<>();
 Set actualOnMS = new HashSet<>();
@@ -140,10 +138,128 @@ public class ITestS3GuardToolLocal extends 
AbstractS3GuardToolTestBase {
 }
   }
 }
-String actualOut = out.toString();
+String actualOut = buf.toString();
 assertEquals("Mismatched metadata store outputs: " + actualOut,
 filesOnMS, actualOnMS);
 assertEquals("Mismatched s3 outputs: " + actualOut, filesOnS3, actualOnS3);
 assertFalse("Diff contained duplicates", duplicates);
   }
+
+  @Test
+  public void testDestroyBucketExistsButNoTable() throws Throwable {
+run(Destroy.NAME,
+"-meta", LOCAL_METADATA,
+getLandsatCSVFile());
+  }
+
+  @Test
+  public void testImportNoFilesystem() throws Throwable {
+final Import importer =
+new S3GuardTool.Import(getConfiguration());
+importer.setStore(getMetadataStore());
+intercept(IOException.class,
+new Callable() {
+  @Override
+  public Integer call() throws Exception {
+return importer.run(
+new String[]{
+"import",
+"-meta", LOCAL_METADATA,
+S3A_THIS_BUCKET_DOES_NOT_EXIST
+});
+  }
+});
+  }
+
+  @Test
+  public void testInfoBucketAndRegionNoFS() throws Throwable {
+intercept(FileNotFoundException.class,
+new Callable() {
+  @Override
+  public Integer call() throws Exception {
+return run(BucketInfo.NAME, "-meta",
+LOCAL_METADATA, "-region",
+"any-region", S3A_THIS_BUCKET_DOES_NOT_EXIST);
+  }
+});
+  }
+
+  @Test
+  public void testInitNegativeRead() throws Throwable {
+

[1/2] hadoop git commit: HADOOP-14220 Enhance S3GuardTool with bucket-info and set-capacity commands, tests. Contributed by Steve Loughran

[fabbri]

Repository: hadoop
Updated Branches:
  refs/heads/trunk cde804b91 -> 47011d7dd


http://git-wip-us.apache.org/repos/asf/hadoop/blob/47011d7d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
index 181cdfb..43cbe93 100644
--- 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
+++ 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
@@ -21,11 +21,12 @@ package org.apache.hadoop.fs.s3a.s3guard;
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.PrintStream;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.concurrent.Callable;
 
 import org.junit.Test;
 
@@ -34,13 +35,16 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.s3a.S3AFileSystem;
 import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
 
-import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
 /**
  * Test S3Guard related CLI commands against a LocalMetadataStore.
  */
 public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
 
+  private static final String LOCAL_METADATA = "local://metadata";
+
   @Override
   protected MetadataStore newMetadataStore() {
 return new LocalMetadataStore();
@@ -65,10 +69,7 @@ public class ITestS3GuardToolLocal extends 
AbstractS3GuardToolTestBase {
 
 S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
 cmd.setStore(ms);
-
-expectSuccess("Import command did not exit successfully - see output",
-cmd,
-"import", parent.toString());
+exec(cmd, "import", parent.toString());
 
 DirListingMetadata children =
 ms.listChildren(dir);
@@ -80,7 +81,7 @@ public class ITestS3GuardToolLocal extends 
AbstractS3GuardToolTestBase {
   }
 
   @Test
-  public void testDiffCommand() throws IOException {
+  public void testDiffCommand() throws Exception {
 S3AFileSystem fs = getFileSystem();
 MetadataStore ms = getMetadataStore();
 Set filesOnS3 = new HashSet<>(); // files on S3.
@@ -108,13 +109,10 @@ public class ITestS3GuardToolLocal extends 
AbstractS3GuardToolTestBase {
 }
 
 ByteArrayOutputStream buf = new ByteArrayOutputStream();
-PrintStream out = new PrintStream(buf);
 Diff cmd = new Diff(fs.getConf());
 cmd.setStore(ms);
-assertEquals("Diff command did not exit successfully - see output", 
SUCCESS,
-cmd.run(new String[]{"diff", "-meta", "local://metadata",
-testPath.toString()}, out));
-out.close();
+exec(cmd, buf, "diff", "-meta", LOCAL_METADATA,
+testPath.toString());
 
 Set actualOnS3 = new HashSet<>();
 Set actualOnMS = new HashSet<>();
@@ -140,10 +138,128 @@ public class ITestS3GuardToolLocal extends 
AbstractS3GuardToolTestBase {
 }
   }
 }
-String actualOut = out.toString();
+String actualOut = buf.toString();
 assertEquals("Mismatched metadata store outputs: " + actualOut,
 filesOnMS, actualOnMS);
 assertEquals("Mismatched s3 outputs: " + actualOut, filesOnS3, actualOnS3);
 assertFalse("Diff contained duplicates", duplicates);
   }
+
+  @Test
+  public void testDestroyBucketExistsButNoTable() throws Throwable {
+run(Destroy.NAME,
+"-meta", LOCAL_METADATA,
+getLandsatCSVFile());
+  }
+
+  @Test
+  public void testImportNoFilesystem() throws Throwable {
+final Import importer =
+new S3GuardTool.Import(getConfiguration());
+importer.setStore(getMetadataStore());
+intercept(IOException.class,
+new Callable() {
+  @Override
+  public Integer call() throws Exception {
+return importer.run(
+new String[]{
+"import",
+"-meta", LOCAL_METADATA,
+S3A_THIS_BUCKET_DOES_NOT_EXIST
+});
+  }
+});
+  }
+
+  @Test
+  public void testInfoBucketAndRegionNoFS() throws Throwable {
+intercept(FileNotFoundException.class,
+new Callable() {
+  @Override
+  public Integer call() throws Exception {
+return run(BucketInfo.NAME, "-meta",
+LOCAL_METADATA, "-region",
+"any-region", S3A_THIS_BUCKET_DOES_NOT_EXIST);
+  }
+});
+  }
+
+  @Test
+  public void testInitNegativeRead() throws Throwable {
+

[2/2] hadoop git commit: HADOOP-14220 Enhance S3GuardTool with bucket-info and set-capacity commands, tests. Contributed by Steve Loughran

[fabbri]

HADOOP-14220 Enhance S3GuardTool with bucket-info and set-capacity commands, 
tests. Contributed by Steve Loughran


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47011d7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47011d7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47011d7d

Branch: refs/heads/trunk
Commit: 47011d7dd300b0c74bb6cfe25b918c479d718f4f
Parents: cde804b
Author: Aaron Fabbri 
Authored: Mon Sep 25 15:59:38 2017 -0700
Committer: Aaron Fabbri 
Committed: Mon Sep 25 15:59:38 2017 -0700

--
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java |  14 +-
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  60 ++-
 .../fs/s3a/s3guard/DynamoDBMetadataStore.java   | 104 +++-
 .../fs/s3a/s3guard/LocalMetadataStore.java  |  25 +-
 .../hadoop/fs/s3a/s3guard/MetadataStore.java|  17 +
 .../fs/s3a/s3guard/NullMetadataStore.java   |  15 +
 .../apache/hadoop/fs/s3a/s3guard/S3Guard.java   |  17 +-
 .../hadoop/fs/s3a/s3guard/S3GuardTool.java  | 539 +++
 .../site/markdown/tools/hadoop-aws/s3guard.md   | 160 +-
 .../hadoop/fs/s3a/ITestS3AConfiguration.java|  10 +
 .../fs/s3a/ITestS3GuardListConsistency.java |   8 +-
 .../hadoop/fs/s3a/ITestS3GuardWriteBack.java|  37 +-
 .../org/apache/hadoop/fs/s3a/S3ATestUtils.java  |  18 -
 .../s3guard/AbstractS3GuardToolTestBase.java| 138 -
 .../s3a/s3guard/ITestS3GuardToolDynamoDB.java   | 167 +-
 .../fs/s3a/s3guard/ITestS3GuardToolLocal.java   | 142 -
 .../hadoop/fs/s3a/s3guard/TestS3GuardCLI.java   | 121 +
 17 files changed, 1405 insertions(+), 187 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/47011d7d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f4709a7..2171957 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -298,6 +298,10 @@ public class S3AFileSystem extends FileSystem {
   metadataStore = S3Guard.getMetadataStore(this);
   allowAuthoritative = conf.getBoolean(METADATASTORE_AUTHORITATIVE,
   DEFAULT_METADATASTORE_AUTHORITATIVE);
+  if (hasMetadataStore()) {
+LOG.debug("Using metadata store {}, authoritative={}",
+getMetadataStore(), allowAuthoritative);
+  }
 } catch (AmazonClientException e) {
   throw translateException("initializing ", new Path(name), e);
 }
@@ -967,7 +971,7 @@ public class S3AFileSystem extends FileSystem {
* @return the metadata store of this FS instance
*/
   @VisibleForTesting
-  MetadataStore getMetadataStore() {
+  public MetadataStore getMetadataStore() {
 return metadataStore;
   }
 
@@ -2474,9 +2478,11 @@ public class S3AFileSystem extends FileSystem {
 sb.append(", statistics {")
 .append(statistics)
 .append("}");
-sb.append(", metrics {")
-.append(instrumentation.dump("{", "=", "} ", true))
-.append("}");
+if (instrumentation != null) {
+  sb.append(", metrics {")
+  .append(instrumentation.dump("{", "=", "} ", true))
+  .append("}");
+}
 sb.append('}');
 return sb.toString();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/47011d7d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 9dd5def..6e6f4b6 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -736,9 +736,11 @@ public final class S3AUtils {
 // propagate the value, building a new origin field.
 // to track overwrites, the generic key is overwritten even if
 // already matches the new one.
+String origin = "[" + StringUtils.join(
+source.getPropertySources(key), ", ") +"]";
 final String generic = FS_S3A_PREFIX + stripped;
-LOG.debug("Updating {}", generic);
-dest.set(generic, value, key);
+LOG.debug("Updating {} from {}", generic, origin);
+dest.set(generic, value, key + " via " + origin);
   }
 }
 return dest;
@@ -881,4 +883,58 @@ public final class S3AUtils {
 

hadoop git commit: HADOOP-14903. Add json-smart explicitly to pom.xml. (rchiang)

[rchiang]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 bfad0ca3b -> da6ed36ed


HADOOP-14903. Add json-smart explicitly to pom.xml. (rchiang)

(cherry picked from commit cde804b91f2fed514935d186d89ea7cbbf2701f3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da6ed36e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da6ed36e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da6ed36e

Branch: refs/heads/branch-3.0
Commit: da6ed36edb759bbfcb41d886c7f444e8dddf9f2c
Parents: bfad0ca
Author: Ray Chiang 
Authored: Mon Sep 25 15:30:51 2017 -0700
Committer: Ray Chiang 
Committed: Mon Sep 25 15:32:15 2017 -0700

--
 hadoop-project/pom.xml | 9 +
 1 file changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/da6ed36e/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 1c74c02..a106c17 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1207,6 +1207,15 @@
 
   
   
+
+net.minidev
+json-smart
+2.3
+  
+  
 org.skyscreamer
 jsonassert
 1.3.0


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14903. Add json-smart explicitly to pom.xml. (rchiang)

[rchiang]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 0889e5a8b -> cde804b91


HADOOP-14903. Add json-smart explicitly to pom.xml. (rchiang)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cde804b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cde804b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cde804b9

Branch: refs/heads/trunk
Commit: cde804b91f2fed514935d186d89ea7cbbf2701f3
Parents: 0889e5a
Author: Ray Chiang 
Authored: Mon Sep 25 15:30:51 2017 -0700
Committer: Ray Chiang 
Committed: Mon Sep 25 15:30:51 2017 -0700

--
 hadoop-project/pom.xml | 9 +
 1 file changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cde804b9/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7f657ca..7b342c4 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1207,6 +1207,15 @@
 
   
   
+
+net.minidev
+json-smart
+2.3
+  
+  
 org.skyscreamer
 jsonassert
 1.3.0


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org