hadoop git commit: YARN-6445. [YARN-3926] Performance improvements in resource profile branch with respect to SLS. Contributed by Varun Vasudev.
Repository: hadoop
Updated Branches:
refs/heads/YARN-3926 f3dc4ca00 -> d8bab941e
YARN-6445. [YARN-3926] Performance improvements in resource profile branch with
respect to SLS. Contributed by Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8bab941
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8bab941
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8bab941
Branch: refs/heads/YARN-3926
Commit: d8bab941e3e3f8dc23d2e8d70b3b130dfd68fc84
Parents: f3dc4ca
Author: Sunil G
Authored: Tue Apr 25 11:53:11 2017 +0530
Committer: Sunil G
Committed: Tue Apr 25 11:53:11 2017 +0530
--
.../hadoop/yarn/api/records/Resource.java | 19 +--
.../yarn/api/records/ResourceInformation.java | 51 ++---
.../hadoop/yarn/util/UnitsConversionUtil.java | 34 ++-
.../yarn/conf/TestResourceInformation.java | 4 +-
.../yarn/util/TestUnitsConversionUtil.java | 60 ++--
.../api/records/impl/pb/ResourcePBImpl.java | 23
.../resource/DominantResourceCalculator.java| 54 --
.../hadoop/yarn/util/resource/Resources.java| 18 +++---
.../yarn/util/resource/TestResourceUtils.java | 1 +
.../yarn/util/resource/TestResources.java | 12 +++-
.../resource/ResourceProfilesManagerImpl.java | 3 +-
11 files changed, 157 insertions(+), 122 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 103bcfd..4fa6ea3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -80,12 +80,23 @@ public abstract class Resource implements
Comparable {
@InterfaceStability.Unstable
public static Resource newInstance(Resource resource) {
Resource ret = Resource.newInstance(0, 0);
-for (Map.Entry entry : resource.getResources()
+Resource.copy(resource, ret);
+return ret;
+ }
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public static void copy(Resource source, Resource dest) {
+for (Map.Entry entry : source.getResources()
.entrySet()) {
- ret.setResourceInformation(entry.getKey(),
- ResourceInformation.newInstance(entry.getValue()));
+ try {
+ResourceInformation.copy(entry.getValue(),
+dest.getResourceInformation(entry.getKey()));
+ } catch (YarnException ye) {
+dest.setResourceInformation(entry.getKey(),
+ResourceInformation.newInstance(entry.getValue()));
+ }
}
-return ret;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8bab941/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 7d74efc..d75b441 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -30,9 +30,9 @@ public class ResourceInformation implements
Comparable {
private String name;
private String units;
private ResourceTypes resourceType;
- private Long value;
- private Long minimumAllocation;
- private Long maximumAllocation;
+ private long value;
+ private long minimumAllocation;
+ private long maximumAllocation;
private static final String MEMORY_URI = "memory-mb";
private static final String VCORES_URI = "vcores";
@@ -106,7 +106,7 @@ public class ResourceInformation implements
Comparable {
*
* @return the resource value
*/
- public Long getValue() {
+ public long getValue() {
return value;
}
@@ -115,7 +115,7 @@ public class ResourceInformation implements
Comparable {
*
* @param rValue the resource value
*/
- public void setValue(Long rValue) {
+ public void setValue(long rValue) {
this.value
[47/50] [abbrv] hadoop git commit: YARN-6247. Share a single instance of SubClusterResolver instead of instantiating one per AM. (Botong Huang via Subru)
YARN-6247. Share a single instance of SubClusterResolver instead of
instantiating one per AM. (Botong Huang via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/beaca376
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/beaca376
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/beaca376
Branch: refs/heads/YARN-2915
Commit: beaca376488cd36dc1202f1e48b4580a7e5d444c
Parents: 4e25db6
Author: Subru Krishnan
Authored: Thu Mar 2 18:54:53 2017 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:53 2017 -0700
--
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +++
.../src/main/resources/yarn-default.xml | 7 +++
.../resolver/AbstractSubClusterResolver.java| 6 +--
.../federation/resolver/SubClusterResolver.java | 4 +-
.../utils/FederationStateStoreFacade.java | 48 +---
5 files changed, 59 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/beaca376/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9397c99..cd71563 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2568,6 +2568,12 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
+ public static final String FEDERATION_CLUSTER_RESOLVER_CLASS =
+ FEDERATION_PREFIX + "subcluster-resolver.class";
+ public static final String DEFAULT_FEDERATION_CLUSTER_RESOLVER_CLASS =
+ "org.apache.hadoop.yarn.server.federation.resolver."
+ + "DefaultSubClusterResolverImpl";
+
public static final String DEFAULT_FEDERATION_POLICY_KEY = "*";
public static final String FEDERATION_POLICY_MANAGER = FEDERATION_PREFIX
http://git-wip-us.apache.org/repos/asf/hadoop/blob/beaca376/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 29a9dec..e1b3700 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2663,6 +2663,13 @@
yarn.federation.machine-list
+
+
+ Class name for SubClusterResolver
+
+yarn.federation.subcluster-resolver.class
+
org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/beaca376/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
index 6b4f60c..bccff2d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.yarn.server.federation.resolver;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import java.util.HashMap;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.Map;
/**
@@ -31,9 +31,9 @@ import java.util.Map;
*/
public abstract class AbstractSubClusterResolver implements SubClusterResolver
{
private Map nodeToSubCluster =
- new HashMap();
+ new
[41/50] [abbrv] hadoop git commit: YARN-5467. InputValidator for the FederationStateStore internal APIs. (Giovanni Matteo Fumarola via Subru)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43aafc51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
new file mode 100644
index 000..13175ae
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/utils/TestFederationStateStoreInputValidator.java
@@ -0,0 +1,1265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.utils;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import
org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import
org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import
org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for FederationApplicationInputValidator,
+ * FederationMembershipInputValidator, and FederationPolicyInputValidator.
+ */
+public class TestFederationStateStoreInputValidator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestFederationStateStoreInputValidator.class);
+
+ private static SubClusterId subClusterId;
+ private static String amRMServiceAddress;
+ private static String clientRMServiceAddress;
+ private static String rmAdminServiceAddress;
+ private static String rmWebServiceAddress;
+ private static int lastHeartBeat;
+ private static SubClusterState stateNew;
+ private static SubClusterState stateLost;
+ private static ApplicationId appId;
+ private static int lastStartTime;
+ private static String capability;
+ private static String queue;
+ private static String type;
+ private static ByteBuffer params;
+
+ private static SubClusterId subClusterIdInvalid;
+ private static SubClusterId subClusterIdNull;
+
+ private static int lastHeartBeatNegative;
+ private static int lastStartTimeNegative;
+
+ private static SubClusterState stateNull;
+ private static ApplicationId appIdNull;
+
+ private static String capabilityNull;
+ private static String capabilityEmpty;
+
+ private static String addressNull;
+ private static String addressEmpty;
+ private static String addressWrong;
+ private static String addressWrongPort;
+
+ private static String queueEmpty;
+ private static String q
[16/50] [abbrv] hadoop git commit: YARN-3662. Federation Membership State Store internal APIs.
YARN-3662. Federation Membership State Store internal APIs. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17ddde75 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17ddde75 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17ddde75 Branch: refs/heads/YARN-2915 Commit: 17ddde75aa0ca000a401790a9334dc961e8c3ed0 Parents: 13d48e3 Author: Subru Krishnan Authored: Fri Jul 29 16:53:40 2016 -0700 Committer: Subru Krishnan Committed: Mon Apr 24 18:58:51 2017 -0700 -- .../hadoop-yarn-server-common/pom.xml | 8 + .../store/FederationMembershipStateStore.java | 126 + .../server/federation/store/package-info.java | 17 ++ .../store/records/GetSubClusterInfoRequest.java | 62 + .../records/GetSubClusterInfoResponse.java | 62 + .../records/GetSubClustersInfoRequest.java | 66 + .../records/GetSubClustersInfoResponse.java | 66 + .../records/SubClusterDeregisterRequest.java| 89 +++ .../records/SubClusterDeregisterResponse.java | 42 +++ .../records/SubClusterHeartbeatRequest.java | 149 +++ .../records/SubClusterHeartbeatResponse.java| 45 .../federation/store/records/SubClusterId.java | 100 +++ .../store/records/SubClusterInfo.java | 263 ++ .../records/SubClusterRegisterRequest.java | 74 + .../records/SubClusterRegisterResponse.java | 44 +++ .../store/records/SubClusterState.java | 60 + .../impl/pb/GetSubClusterInfoRequestPBImpl.java | 125 + .../pb/GetSubClusterInfoResponsePBImpl.java | 134 ++ .../pb/GetSubClustersInfoRequestPBImpl.java | 108 .../pb/GetSubClustersInfoResponsePBImpl.java| 184 + .../pb/SubClusterDeregisterRequestPBImpl.java | 156 +++ .../pb/SubClusterDeregisterResponsePBImpl.java | 77 ++ .../pb/SubClusterHeartbeatRequestPBImpl.java| 192 + .../pb/SubClusterHeartbeatResponsePBImpl.java | 77 ++ .../records/impl/pb/SubClusterIdPBImpl.java | 75 ++ .../records/impl/pb/SubClusterInfoPBImpl.java | 267 +++ .../pb/SubClusterRegisterRequestPBImpl.java | 134 ++ .../pb/SubClusterRegisterResponsePBImpl.java| 77 ++ .../store/records/impl/pb/package-info.java | 17 ++ .../federation/store/records/package-info.java | 17 ++ .../proto/yarn_server_federation_protos.proto | 93 +++ .../records/TestFederationProtocolRecords.java | 133 + 32 files changed, 3139 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/17ddde75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index fc23af8..9cc3cae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -58,6 +58,13 @@ org.apache.hadoop hadoop-yarn-common + + + org.apache.hadoop + hadoop-yarn-common + test-jar + test + com.google.guava @@ -146,6 +153,7 @@ yarn_server_common_protos.proto yarn_server_common_service_protos.proto yarn_server_common_service_protos.proto + yarn_server_federation_protos.proto ResourceTracker.proto SCMUploader.proto collectornodemanager_protocol.proto http://git-wip-us.apache.org/repos/asf/hadoop/blob/17ddde75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java new file mode 100644 index 000..378eadc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreement
[49/50] [abbrv] hadoop git commit: YARN-5602. Utils for Federation State and Policy Store. (Giovanni Matteo Fumarola via Subru).
YARN-5602. Utils for Federation State and Policy Store. (Giovanni Matteo
Fumarola via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/988f71c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/988f71c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/988f71c9
Branch: refs/heads/YARN-2915
Commit: 988f71c90760dc6b81892cfd83337c11734fbea6
Parents: 1936169
Author: Subru Krishnan
Authored: Wed Apr 5 15:02:00 2017 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:53 2017 -0700
--
.../FederationStateStoreErrorCode.java | 105 +
.../FederationStateStoreException.java | 45 ++
...derationStateStoreInvalidInputException.java | 48 ++
.../FederationStateStoreRetriableException.java | 44 ++
.../store/exception/package-info.java | 17 ++
.../store/impl/MemoryFederationStateStore.java | 56 +--
.../store/records/SubClusterInfo.java | 62
.../records/impl/pb/SubClusterInfoPBImpl.java | 16 --
...cationHomeSubClusterStoreInputValidator.java | 1 +
...ationMembershipStateStoreInputValidator.java | 1 +
.../FederationPolicyStoreInputValidator.java| 1 +
...derationStateStoreInvalidInputException.java | 48 --
.../store/utils/FederationStateStoreUtils.java | 155 +++
.../utils/FederationStateStoreFacade.java | 23 ++-
.../impl/FederationStateStoreBaseTest.java | 91 ++-
.../impl/TestMemoryFederationStateStore.java| 4 +-
.../TestFederationStateStoreInputValidator.java | 1 +
.../TestFederationStateStoreFacadeRetry.java| 125 +++
18 files changed, 730 insertions(+), 113 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/988f71c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/exception/FederationStateStoreErrorCode.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/exception/FederationStateStoreErrorCode.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/exception/FederationStateStoreErrorCode.java
new file mode 100644
index 000..88e2d3a
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/exception/FederationStateStoreErrorCode.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.exception;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ *
+ * Logical error codes from FederationStateStore.
+ *
+ */
+@Public
+@Unstable
+public enum FederationStateStoreErrorCode {
+
+ MEMBERSHIP_INSERT_FAIL(1101, "Fail to insert a tuple into Membership
table."),
+
+ MEMBERSHIP_DELETE_FAIL(1102, "Fail to delete a tuple from Membership
table."),
+
+ MEMBERSHIP_SINGLE_SELECT_FAIL(1103,
+ "Fail to select a tuple from Membership table."),
+
+ MEMBERSHIP_MULTIPLE_SELECT_FAIL(1104,
+ "Fail to select multiple tuples from Membership table."),
+
+ MEMBERSHIP_UPDATE_DEREGISTER_FAIL(1105,
+ "Fail to update/deregister a tuple in Membership table."),
+
+ MEMBERSHIP_UPDATE_HEARTBEAT_FAIL(1106,
+ "Fail to update/heartbeat a tuple in Membership table."),
+
+ APPLICATIONS_INSERT_FAIL(1201,
+ "Fail to insert a tuple into ApplicationsHomeSubCluster table."),
+
+ APPLICATIONS_DELETE_FAIL(1202,
+ "Fail to delete a tuple from ApplicationsHomeSubCluster table"),
+
+ APPLICATIONS_SINGLE_SELECT_FAIL(1203,
+ "Fail to select a tuple from ApplicationsHomeSubCluster table."),
+
+ APPLICATIONS_MULTIPLE_SEL
[19/50] [abbrv] hadoop git commit: YARN-3664. Federation PolicyStore internal APIs
YARN-3664. Federation PolicyStore internal APIs
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b569ef8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b569ef8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b569ef8b
Branch: refs/heads/YARN-2915
Commit: b569ef8b58c03238f62ce1f906ffbef02e0b932b
Parents: ea75da7
Author: Subru Krishnan
Authored: Fri Aug 5 12:34:58 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:51 2017 -0700
--
.../federation/store/FederationPolicyStore.java | 76
...SubClusterPoliciesConfigurationsRequest.java | 35
...ubClusterPoliciesConfigurationsResponse.java | 66 +++
...GetSubClusterPolicyConfigurationRequest.java | 62 ++
...etSubClusterPolicyConfigurationResponse.java | 65 +++
...SetSubClusterPolicyConfigurationRequest.java | 79
...etSubClusterPolicyConfigurationResponse.java | 36
.../records/SubClusterPolicyConfiguration.java | 130 +
...sterPoliciesConfigurationsRequestPBImpl.java | 95 +
...terPoliciesConfigurationsResponsePBImpl.java | 191 +++
...ClusterPolicyConfigurationRequestPBImpl.java | 103 ++
...lusterPolicyConfigurationResponsePBImpl.java | 143 ++
.../pb/GetSubClustersInfoResponsePBImpl.java| 4 +-
...ClusterPolicyConfigurationRequestPBImpl.java | 159 +++
...lusterPolicyConfigurationResponsePBImpl.java | 93 +
.../pb/SubClusterPolicyConfigurationPBImpl.java | 121
.../proto/yarn_server_federation_protos.proto | 28 +++
.../records/TestFederationProtocolRecords.java | 53 -
18 files changed, 1536 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b569ef8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
new file mode 100644
index 000..9d9bd9b
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationPolicyStore.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import
org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
+
+/**
+ * The FederationPolicyStore provides a key-value interface to access the
+ * policies configured for the system. The key is a "queue" name, i.e., the
+ * system allows to configure a different policy for each queue in the system
+ * (though each policy can make dynamic run-time decisions on a
per-job/per-task
+ * basis). The value is a {@code SubClusterPolicyConfiguration}, a serialized
+ * representation of the policy type and its parameters.
+ */
+@Private
+@Unstable
+p
[45/50] [abbrv] hadoop git commit: YARN-5601. Make the RM epoch base value configurable. Contributed by Subru Krishnan
YARN-5601. Make the RM epoch base value configurable. Contributed by Subru
Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4429d94
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4429d94
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4429d94
Branch: refs/heads/YARN-2915
Commit: e4429d9423f383ac645d18169c4ca97adbb25d9f
Parents: a19a0e2
Author: Jian He
Authored: Fri Sep 2 12:23:57 2016 +0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 5 -
.../java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 3 +++
.../apache/hadoop/yarn/conf/TestYarnConfigurationFields.java | 2 ++
.../hadoop/yarn/server/resourcemanager/ResourceManager.java | 7 +++
.../resourcemanager/recovery/FileSystemRMStateStore.java | 2 +-
.../server/resourcemanager/recovery/LeveldbRMStateStore.java | 2 +-
.../server/resourcemanager/recovery/MemoryRMStateStore.java | 1 +
.../yarn/server/resourcemanager/recovery/RMStateStore.java | 4
.../yarn/server/resourcemanager/recovery/ZKRMStateStore.java | 2 +-
.../resourcemanager/recovery/RMStateStoreTestBase.java | 8 +---
.../server/resourcemanager/recovery/TestFSRMStateStore.java | 1 +
.../resourcemanager/recovery/TestLeveldbRMStateStore.java| 1 +
.../server/resourcemanager/recovery/TestZKRMStateStore.java | 1 +
13 files changed, 32 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4429d94/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 2f5451d..bbd03a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -293,7 +293,10 @@
-
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4429d94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 482eb95..6bc21fd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -143,6 +143,9 @@ public class YarnConfiguration extends Configuration {
public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
+ public static final String RM_EPOCH = RM_PREFIX + "epoch";
+ public static final long DEFAULT_RM_EPOCH = 0L;
+
/** The address of the applications manager interface in the RM.*/
public static final String RM_ADDRESS =
RM_PREFIX + "address";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4429d94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 5e0876f..3f3a06c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -75,6 +75,8 @@ public class TestYarnConfigurationFields extends
TestConfigurationFieldsBase {
.add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
configurationPropsToSkipCompare
.add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
+configurationPropsToSkipCompare
+.add(YarnConfiguration.RM_EPOCH);
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4429d94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
[50/50] [abbrv] hadoop git commit: YARN-6370. Properly handle rack requests for non-active subclusters in LocalityMulticastAMRMProxyPolicy. (Contributed by Botong Huang via curino).
YARN-6370. Properly handle rack requests for non-active subclusters in
LocalityMulticastAMRMProxyPolicy. (Contributed by Botong Huang via curino).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19361696
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19361696
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19361696
Branch: refs/heads/YARN-2915
Commit: 193616968b50544d0059370dbd8a003bd5518251
Parents: d3d5b4c
Author: Carlo Curino
Authored: Wed Mar 22 13:53:47 2017 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:53 2017 -0700
--
.../LocalityMulticastAMRMProxyPolicy.java | 6 ++-
.../TestLocalityMulticastAMRMProxyPolicy.java | 53 +---
2 files changed, 41 insertions(+), 18 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19361696/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
index 6f97a51..454962f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
@@ -261,7 +261,11 @@ public class LocalityMulticastAMRMProxyPolicy extends
AbstractAMRMProxyPolicy {
// If home-subcluster is not active, ignore node/rack request
if (bookkeeper.isActiveAndEnabled(homeSubcluster)) {
-bookkeeper.addLocalizedNodeRR(homeSubcluster, rr);
+if (targetIds != null && targetIds.size() > 0) {
+ bookkeeper.addRackRR(homeSubcluster, rr);
+} else {
+ bookkeeper.addLocalizedNodeRR(homeSubcluster, rr);
+}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("The homeSubCluster (" + homeSubcluster + ") we are "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19361696/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
index 5b3cf74..6e3a2f1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/TestLocalityMulticastAMRMProxyPolicy.java
@@ -339,19 +339,20 @@ public class TestLocalityMulticastAMRMProxyPolicy
validateSplit(response, resourceRequests);
prettyPrintRequests(response);
-// we expect 4 entry for home subcluster (3 for request-id 4, and a part
-// of the broadcast of request-id 2
-checkExpectedAllocation(response, getHomeSubCluster().getId(), 4, 23);
+// we expect 7 entries for home subcluster (2 for request-id 4, 3 for
+// request-id 5, and a part of the broadcast of request-id 2
+checkExpectedAllocation(response, getHomeSubCluster().getId(), 7, 29);
-// for subcluster0 we expect 3 entry from request-id 0, and 3 from
-// request-id 3, as well as part of the request-id 2 broadast
-checkExpectedAllocation(response, "subcluster0", 7, 26);
+// for subcluster0 we expect 10 entries, 3 from request-id 0, and 3 from
+// request-id 3, 3 entries from request-id 5, as well as part of the
+// request-id 2 broadast
+checkExpectedAllocation(response, "subcluster0", 10, 32);
-// we expect 5 entry for subcluster1 (4 from request-id 1, and part
+// we expec
[38/50] [abbrv] hadoop git commit: YARN-5676. Add a HashBasedRouterPolicy, and small policies and test refactoring. (Carlo Curino via Subru).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77697d80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
index 4975a9f..5fa02d6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestRouterPolicyFacade.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import
org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import
org.apache.hadoop.yarn.server.federation.policies.manager.PriorityBroadcastPolicyManager;
+import
org.apache.hadoop.yarn.server.federation.policies.manager.UniformBroadcastPolicyManager;
import
org.apache.hadoop.yarn.server.federation.policies.router.PriorityRouterPolicy;
import
org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77697d80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
deleted file mode 100644
index 542a5ae..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestUniformBroadcastPolicyManager.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import
org.apache.hadoop.yarn.server.federation.policies.amrmproxy.BroadcastAMRMProxyPolicy;
-import
org.apache.hadoop.yarn.server.federation.policies.router.UniformRandomRouterPolicy;
-import org.junit.Before;
-
-/**
- * Simple test of {@link UniformBroadcastPolicyManager}.
- */
-public class TestUniformBroadcastPolicyManager extends BasePolicyManagerTest {
-
- @Before
- public void setup() {
-//config policy
-wfp = new UniformBroadcastPolicyManager();
-wfp.setQueue("queue1");
-
-//set expected params that the base test class will use for tests
-expectedPolicyManager = UniformBroadcastPolicyManager.class;
-expectedAMRMProxyPolicy = BroadcastAMRMProxyPolicy.class;
-expectedRouterPolicy = UniformRandomRouterPolicy.class;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77697d80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/TestWeightedLocalityPolicyManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-se
[35/50] [abbrv] hadoop git commit: YARN-6093. Minor bugs with AMRMtoken renewal and state store availability when using FederationRMFailoverProxyProvider during RM failover. (Botong Huang via Subru).
YARN-6093. Minor bugs with AMRMtoken renewal and state store availability when
using FederationRMFailoverProxyProvider during RM failover. (Botong Huang via
Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec7ab9c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec7ab9c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec7ab9c5
Branch: refs/heads/YARN-2915
Commit: ec7ab9c5bef0f2acaaef5bfb73445fe63b7fed6b
Parents: ba861ac
Author: Subru Krishnan
Authored: Wed Feb 22 13:16:22 2017 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../TestFederationRMFailoverProxyProvider.java | 69 +++
.../FederationRMFailoverProxyProvider.java | 88 +++-
2 files changed, 118 insertions(+), 39 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec7ab9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
index fa3523c..e3f9155 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
@@ -19,17 +19,21 @@ package org.apache.hadoop.yarn.client;
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.io.retry.FailoverProxyProvider.ProxyInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import
org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
+import
org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import
org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
@@ -44,6 +48,10 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
/**
* Unit tests for FederationRMFailoverProxyProvider.
*/
@@ -151,4 +159,65 @@ public class TestFederationRMFailoverProxyProvider {
}
}
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Test
+ public void testUGIForProxyCreation()
+ throws IOException, InterruptedException {
+conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
+
+UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
+UserGroupInformation user1 =
+UserGroupInformation.createProxyUser("user1", currentUser);
+UserGroupInformation user2 =
+UserGroupInformation.createProxyUser("user2", currentUser);
+
+final TestableFederationRMFailoverProxyProvider provider =
+new TestableFederationRMFailoverProxyProvider();
+
+InetSocketAddress addr =
+conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+final ClientRMProxy rmProxy = mock(ClientRMProxy.class);
+when(rmProxy.getRMAddress(any(YarnConfiguration.class), any(Class.class)))
+.thenReturn(addr);
+
+user1.doAs(new PrivilegedExceptionAction() {
+ @Override
+ public Object run() {
+provider.init(conf, rmProxy, ApplicationMasterProtocol.class);
+return null;
+ }
+});
+
+final ProxyInfo currentProxy = provider.getProxy();
+Assert.assertEquals("user1", provider.getLastProxyUGI().getUserName());
+
+user2.doAs(new PrivilegedExceptionAction() {
+ @Override
+ public Object run() {
+provider.
[32/50] [abbrv] hadoop git commit: YARN-3673. Create a FailoverProxy for Federation services. Contributed by Subru Krishnan
YARN-3673. Create a FailoverProxy for Federation services. Contributed by Subru
Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68359b12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68359b12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68359b12
Branch: refs/heads/YARN-2915
Commit: 68359b126b03856d567f26af21c3441c7309e4f8
Parents: 43aafc5
Author: Jian He
Authored: Mon Aug 22 14:43:07 2016 +0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../org/apache/hadoop/yarn/conf/HAUtil.java | 30 ++-
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +
.../TestFederationRMFailoverProxyProvider.java | 154 ++
.../hadoop/yarn/client/ClientRMProxy.java | 4 +-
.../org/apache/hadoop/yarn/client/RMProxy.java | 23 +-
.../src/main/resources/yarn-default.xml | 7 +
.../hadoop-yarn-server-common/pom.xml | 2 -
.../hadoop/yarn/server/api/ServerRMProxy.java | 4 +-
.../failover/FederationProxyProviderUtil.java | 163 ++
.../FederationRMFailoverProxyProvider.java | 211 +++
.../federation/failover/package-info.java | 17 ++
12 files changed, 613 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68359b12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
index 133b377..528b642 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.conf;
-import com.google.common.annotations.VisibleForTesting;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -27,8 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import java.net.InetSocketAddress;
-import java.util.Collection;
+import com.google.common.annotations.VisibleForTesting;
@InterfaceAudience.Private
public class HAUtil {
@@ -45,6 +46,29 @@ public class HAUtil {
}
/**
+ * Returns true if Federation is configured.
+ *
+ * @param conf Configuration
+ * @return true if federation is configured in the configuration; else false.
+ */
+ public static boolean isFederationEnabled(Configuration conf) {
+return conf.getBoolean(YarnConfiguration.FEDERATION_ENABLED,
+YarnConfiguration.DEFAULT_FEDERATION_ENABLED);
+ }
+
+ /**
+ * Returns true if RM failover is enabled in a Federation setting.
+ *
+ * @param conf Configuration
+ * @return if RM failover is enabled in conjunction with Federation in the
+ * configuration; else false.
+ */
+ public static boolean isFederationFailoverEnabled(Configuration conf) {
+return conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
+YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
+ }
+
+ /**
* Returns true if Resource Manager HA is configured.
*
* @param conf Configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68359b12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4eb414f..2960db0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2535,6 +2535,16 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+ public static final String FEDERATION_ENABLED = FEDERATION_PREFIX +
"enabled";
+ public static final boolean DEFAULT_FEDERATION_ENABLED = false;
+
+ public static final String FEDERATION_FAILOVER
[27/50] [abbrv] hadoop git commit: YARN-6190. Validation and synchronization fixes in LocalityMulticastAMRMProxyPolicy. (Botong Huang via curino)
YARN-6190. Validation and synchronization fixes in
LocalityMulticastAMRMProxyPolicy. (Botong Huang via curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e25db6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e25db6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e25db6f
Branch: refs/heads/YARN-2915
Commit: 4e25db6f2f506e2c985f66c3bb1c012b1c6cc0fc
Parents: c19ece6
Author: Carlo Curino
Authored: Tue Feb 28 17:04:20 2017 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../LocalityMulticastAMRMProxyPolicy.java | 63 +---
.../TestLocalityMulticastAMRMProxyPolicy.java | 21 ++-
.../policies/manager/BasePolicyManagerTest.java | 3 -
.../resolver/TestDefaultSubClusterResolver.java | 9 ++-
.../utils/FederationPoliciesTestUtil.java | 6 +-
5 files changed, 73 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e25db6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
index 283f89e..6f97a51 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/LocalityMulticastAMRMProxyPolicy.java
@@ -32,6 +32,7 @@ import
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
import
org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
@@ -143,10 +144,9 @@ public class LocalityMulticastAMRMProxyPolicy extends
AbstractAMRMProxyPolicy {
Map newWeightsConverted = new HashMap<>();
boolean allInactive = true;
WeightedPolicyInfo policy = getPolicyInfo();
-if (policy.getAMRMPolicyWeights() == null
-|| policy.getAMRMPolicyWeights().size() == 0) {
- allInactive = false;
-} else {
+
+if (policy.getAMRMPolicyWeights() != null
+&& policy.getAMRMPolicyWeights().size() > 0) {
for (Map.Entry e : policy.getAMRMPolicyWeights()
.entrySet()) {
if (e.getValue() > 0) {
@@ -180,7 +180,6 @@ public class LocalityMulticastAMRMProxyPolicy extends
AbstractAMRMProxyPolicy {
this.federationFacade =
policyContext.getFederationStateStoreFacade();
-this.bookkeeper = new AllocationBookkeeper();
this.homeSubcluster = policyContext.getHomeSubcluster();
}
@@ -197,7 +196,9 @@ public class LocalityMulticastAMRMProxyPolicy extends
AbstractAMRMProxyPolicy {
List resourceRequests) throws YarnException {
// object used to accumulate statistics about the answer, initialize with
-// active subclusters.
+// active subclusters. Create a new instance per call because this method
+// can be called concurrently.
+bookkeeper = new AllocationBookkeeper();
bookkeeper.reinitialize(federationFacade.getSubClusters(true));
List nonLocalizedRequests =
@@ -238,12 +239,16 @@ public class LocalityMulticastAMRMProxyPolicy extends
AbstractAMRMProxyPolicy {
// we log altogether later
}
if (targetIds != null && targetIds.size() > 0) {
+boolean hasActive = false;
for (SubClusterId tid : targetIds) {
if (bookkeeper.isActiveAndEnabled(tid)) {
bookkeeper.addRackRR(tid, rr);
+hasActive = true;
}
}
-continue;
+if (hasActive) {
+ continue;
+}
}
// Handle node/rack requests that the SubClusterResolver cannot map to
@@ -347,7 +352,7 @@ public class LocalityMulticastAMRMProxyPolicy extends
AbstractAMRMProxyPolicy {
[43/50] [abbrv] hadoop git commit: YARN-3671. Integrate Federation services with ResourceManager. Contributed by Subru Krishnan
YARN-3671. Integrate Federation services with ResourceManager. Contributed by
Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0114ff33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0114ff33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0114ff33
Branch: refs/heads/YARN-2915
Commit: 0114ff33aa64d90d92af70c699212d60357c128d
Parents: 68359b1
Author: Jian He
Authored: Tue Aug 30 12:20:52 2016 +0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../hadoop/yarn/conf/YarnConfiguration.java | 11 +-
.../yarn/conf/TestYarnConfigurationFields.java | 4 +-
.../failover/FederationProxyProviderUtil.java | 2 +-
.../FederationRMFailoverProxyProvider.java | 4 +-
...ationMembershipStateStoreInputValidator.java | 7 +-
.../TestFederationStateStoreInputValidator.java | 10 +-
.../server/resourcemanager/ResourceManager.java | 26 ++
.../FederationStateStoreHeartbeat.java | 108 +++
.../federation/FederationStateStoreService.java | 304 +++
.../federation/package-info.java| 17 ++
.../webapp/dao/ClusterMetricsInfo.java | 5 +-
.../TestFederationRMStateStoreService.java | 170 +++
12 files changed, 648 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0114ff33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2960db0..482eb95 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2542,9 +2542,6 @@ public class YarnConfiguration extends Configuration {
FEDERATION_PREFIX + "failover.enabled";
public static final boolean DEFAULT_FEDERATION_FAILOVER_ENABLED = true;
- public static final String FEDERATION_SUBCLUSTER_ID =
- FEDERATION_PREFIX + "sub-cluster.id";
-
public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
FEDERATION_PREFIX + "state-store.class";
@@ -2557,6 +2554,14 @@ public class YarnConfiguration extends Configuration {
// 5 minutes
public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+ public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
+ FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
+
+ // 5 minutes
+ public static final int
+ DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
+ 5 * 60;
+
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0114ff33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index c4d8f38..5e0876f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -72,9 +72,9 @@ public class TestYarnConfigurationFields extends
TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
configurationPropsToSkipCompare
-.add(YarnConfiguration.FEDERATION_SUBCLUSTER_ID);
-configurationPropsToSkipCompare
.add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
+configurationPropsToSkipCompare
+.add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0114ff33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationProxyProviderUtil.java
--
diff --git
a/hadoop
[22/50] [abbrv] hadoop git commit: YARN-5408. Compose Federation membership/application/policy APIs into an uber FederationStateStore API. (Ellen Hui via Subru).
YARN-5408. Compose Federation membership/application/policy APIs into an uber
FederationStateStore API. (Ellen Hui via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/535e1f88
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/535e1f88
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/535e1f88
Branch: refs/heads/YARN-2915
Commit: 535e1f88fedf72a2dc3079c6d11d442fb6b75c6e
Parents: b569ef8
Author: Subru Krishnan
Authored: Mon Aug 8 14:53:38 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:51 2017 -0700
--
...ederationApplicationHomeSubClusterStore.java | 18 ++
.../store/FederationMembershipStateStore.java | 14 +
.../federation/store/FederationStateStore.java | 64
.../store/impl/MemoryFederationStateStore.java | 19 --
.../impl/FederationStateStoreBaseTest.java | 57 +
.../impl/TestMemoryFederationStateStore.java| 21 +--
6 files changed, 99 insertions(+), 94 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/535e1f88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
index 217ee2e..22bb88a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
@@ -30,7 +30,6 @@ import
org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHom
import
org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
import
org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
import
org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
-import org.apache.hadoop.yarn.server.records.Version;
/**
* FederationApplicationHomeSubClusterStore maintains the state of all
@@ -50,15 +49,6 @@ import org.apache.hadoop.yarn.server.records.Version;
public interface FederationApplicationHomeSubClusterStore {
/**
- * Get the {@link Version} of the underlying federation application state
- * store.
- *
- * @return the {@link Version} of the underlying federation application state
- * store
- */
- Version getApplicationStateStoreVersion();
-
- /**
* Register the home {@code SubClusterId} of the newly submitted
* {@code ApplicationId}. Currently response is empty if the operation was
* successful, if not an exception reporting reason for a failure.
@@ -91,16 +81,16 @@ public interface FederationApplicationHomeSubClusterStore {
* {@code ApplicationId}.
*
* @param request contains the application queried
- * @return {@code ApplicationHomeSubCluster} containing the application's
- * home subcluster
+ * @return {@code ApplicationHomeSubCluster} containing the application's
home
+ * subcluster
* @throws YarnException if the request is invalid/fails
*/
GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
GetApplicationHomeSubClusterRequest request) throws YarnException;
/**
- * Get the {@code ApplicationHomeSubCluster} list representing the mapping
- * of all submitted applications to it's home sub-cluster.
+ * Get the {@code ApplicationHomeSubCluster} list representing the mapping of
+ * all submitted applications to it's home sub-cluster.
*
* @param request empty representing all applications
* @return the mapping of all submitted application to it's home sub-cluster
http://git-wip-us.apache.org/repos/asf/hadoop/blob/535e1f88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationMembershipState
[44/50] [abbrv] hadoop git commit: YARN-5612. Return SubClusterId in FederationStateStoreFacade#addApplicationHomeSubCluster for Router Failover. (Giovanni Matteo Fumarola via Subru).
YARN-5612. Return SubClusterId in
FederationStateStoreFacade#addApplicationHomeSubCluster for Router Failover.
(Giovanni Matteo Fumarola via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a19a0e27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a19a0e27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a19a0e27
Branch: refs/heads/YARN-2915
Commit: a19a0e271f4fe514977f792e3cf83e3fde9d002f
Parents: 0114ff3
Author: Subru Krishnan
Authored: Thu Sep 1 13:55:54 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../utils/FederationStateStoreFacade.java | 11 ---
.../utils/TestFederationStateStoreFacade.java | 30
2 files changed, 37 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a19a0e27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
index f1c8218..66a0b60 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import
org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
import
org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
import
org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
import
org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
@@ -298,13 +299,15 @@ public final class FederationStateStoreFacade {
*
* @param appHomeSubCluster the mapping of the application to it's home
* sub-cluster
+ * @return the stored Subcluster from StateStore
* @throws YarnException if the call to the state store is unsuccessful
*/
- public void addApplicationHomeSubCluster(
+ public SubClusterId addApplicationHomeSubCluster(
ApplicationHomeSubCluster appHomeSubCluster) throws YarnException {
-stateStore.addApplicationHomeSubCluster(
-AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
-return;
+AddApplicationHomeSubClusterResponse response =
+stateStore.addApplicationHomeSubCluster(
+
AddApplicationHomeSubClusterRequest.newInstance(appHomeSubCluster));
+return response.getHomeSubCluster();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a19a0e27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
index 53f4f84..d46bef0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import
org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import
org.apache.hadoop.yarn.server.feder
[33/50] [abbrv] hadoop git commit: YARN-5905. Update the RM webapp host that is reported as part of Federation membership to current primary RM's IP.
YARN-5905. Update the RM webapp host that is reported as part of Federation
membership to current primary RM's IP.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/202e9a04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/202e9a04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/202e9a04
Branch: refs/heads/YARN-2915
Commit: 202e9a04a6a166157ef890067c8e4fa38ef2a792
Parents: 77697d8
Author: Subru Krishnan
Authored: Tue Nov 22 18:30:40 2016 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../federation/FederationStateStoreService.java | 4 ++--
.../federation/TestFederationRMStateStoreService.java| 11 ++-
2 files changed, 12 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/202e9a04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
index 9a01d7e..530184f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java
@@ -177,8 +177,8 @@ public class FederationStateStoreService extends
AbstractService
config.getSocketAddr(YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_PORT));
-String webAppAddress =
-WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(config);
+String webAppAddress = getServiceAddress(NetUtils
+.createSocketAddr(WebAppUtils.getRMWebAppURLWithScheme(config)));
SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId,
amRMAddress, clientRMAddress, rmAdminAddress, webAppAddress,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/202e9a04/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
index 30f69b5..d92a793 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/federation/TestFederationRMStateStoreService.java
@@ -19,6 +19,7 @@ package
org.apache.hadoop.yarn.server.resourcemanager.federation;
import java.io.IOException;
import java.io.StringReader;
+import java.net.UnknownHostException;
import javax.xml.bind.JAXBException;
@@ -157,12 +158,20 @@ public class TestFederationRMStateStoreService {
}
private String checkSubClusterInfo(SubClusterState state)
- throws YarnException {
+ throws YarnException, UnknownHostException {
Assert.assertNotNull(stateStore.getSubCluster(request));
SubClusterInfo response =
stateStore.getSubCluster(request).getSubClusterInfo();
Assert.assertEquals(state, response.getState());
Assert.assertTrue(response.getLastHeartBeat() >= lastHearbeatTS);
+String expectedAddress =
+(response.getClientRMServiceAddress().split(":"))[0];
+Assert.assertEquals(expectedAddress,
+(response.getAMRMServiceAddress().split(":"))[0]);
+Assert.assertEquals(expectedAddress,
+(response.getRMAdminServiceAddress().split(":"))[0]);
+Assert.assertEquals(expectedAddress,
+(respon
[24/50] [abbrv] hadoop git commit: YARN-5324. Stateless Federation router policies implementation. (Carlo Curino via Subru).
YARN-5324. Stateless Federation router policies implementation. (Carlo Curino
via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55a3f1cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55a3f1cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55a3f1cc
Branch: refs/heads/YARN-2915
Commit: 55a3f1ccf5acdbb675e3a49af553209e29dae5e0
Parents: 7b74a66
Author: Subru Krishnan
Authored: Thu Sep 22 17:06:57 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../policies/FederationPolicyConfigurator.java | 91 ---
.../FederationPolicyInitializationContext.java | 11 +-
.../policies/FederationPolicyManager.java | 126 +
.../policies/FederationPolicyWriter.java| 45
.../policies/dao/WeightedPolicyInfo.java| 253 +++
.../federation/policies/dao/package-info.java | 20 ++
.../router/BaseWeightedRouterPolicy.java| 150 +++
.../policies/router/LoadBasedRouterPolicy.java | 109
.../policies/router/PriorityRouterPolicy.java | 66 +
.../router/UniformRandomRouterPolicy.java | 85 +++
.../router/WeightedRandomRouterPolicy.java | 79 ++
.../store/records/SubClusterIdInfo.java | 75 ++
.../policies/BaseFederationPoliciesTest.java| 155
...ionPolicyInitializationContextValidator.java | 17 +-
.../router/TestLoadBasedRouterPolicy.java | 109
.../router/TestPriorityRouterPolicy.java| 87 +++
.../router/TestUniformRandomRouterPolicy.java | 65 +
.../router/TestWeightedRandomRouterPolicy.java | 127 ++
.../utils/FederationPoliciesTestUtil.java | 82 +-
19 files changed, 1604 insertions(+), 148 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a3f1cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
deleted file mode 100644
index fdc3857..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations
under
- * the License.
- */
-
-package org.apache.hadoop.yarn.server.federation.policies;
-
-import
org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
-
-
-import
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
-
-import org.apache.hadoop.yarn.server.federation.policies.router
-.FederationRouterPolicy;
-
-/**
- * Implementors of this interface are capable to instantiate and (re)initalize
- * {@link FederationAMRMProxyPolicy} and {@link FederationRouterPolicy} based
on
- * a {@link FederationPolicyInitializationContext}. The reason to bind these
two
- * policies together is to make sure we remain consistent across the router and
- * amrmproxy policy decisions.
- */
-public interface FederationPolicyConfigurator {
-
- /**
- * If the current instance is compatible, this method returns the same
- * instance of {@link FederationAMRMProxyPolicy} reinitialized with the
- * current context, otherwise a new instance initialized with the current
- * context is provided. If the instance is compatible with the current class
- * the implementors should attempt to reinitalize (retaining state). To
affect
- * a complete policy reset oldInstance should be null.
- *
- * @p
[29/50] [abbrv] hadoop git commit: YARN-5634. Simplify initialization/use of RouterPolicy via a RouterPolicyFacade. (Carlo Curino via Subru).
YARN-5634. Simplify initialization/use of RouterPolicy via a
RouterPolicyFacade. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebdac260
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebdac260
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebdac260
Branch: refs/heads/YARN-2915
Commit: ebdac26009ca0b37080336c96a0b3cc4e29c924f
Parents: cdcd3ce
Author: Subru Krishnan
Authored: Wed Nov 16 19:39:25 2016 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../dev-support/findbugs-exclude.xml| 9 +
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../yarn/conf/TestYarnConfigurationFields.java | 12 +
...ionPolicyInitializationContextValidator.java | 2 +-
.../PriorityBroadcastPolicyManager.java | 66 +
.../federation/policies/RouterPolicyFacade.java | 266 +++
.../policies/dao/WeightedPolicyInfo.java| 6 +-
.../utils/FederationStateStoreFacade.java | 16 +-
.../TestPriorityBroadcastPolicyManager.java | 72 +
.../policies/TestRouterPolicyFacade.java| 220 +++
.../utils/FederationStateStoreTestUtil.java | 22 +-
11 files changed, 693 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdac260/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index bbd03a9..ee51094 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -310,6 +310,15 @@
+
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdac260/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6bc21fd..285e3c5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2568,6 +2568,19 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
+ public static final String DEFAULT_FEDERATION_POLICY_KEY = "*";
+
+ public static final String FEDERATION_POLICY_MANAGER = FEDERATION_PREFIX
+ + "policy-manager";
+
+ public static final String DEFAULT_FEDERATION_POLICY_MANAGER = "org.apache"
+ +
".hadoop.yarn.server.federation.policies.UniformBroadcastPolicyManager";
+
+ public static final String FEDERATION_POLICY_MANAGER_PARAMS =
+ FEDERATION_PREFIX + "policy-manager-params";
+
+ public static final String DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS = "";
+
// Other Configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebdac260/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 3f3a06c..6e33c0a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -78,6 +78,18 @@ public class TestYarnConfigurationFields extends
TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration.RM_EPOCH);
+// Federation policies configs to be ignored
+configurationPropsToSkipCompare
+.add(YarnConfiguration.FEDERATION_POLICY_MANAGER);
+configurationPropsToSkipCompare
+.add(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS);
+configurationPropsToSkipCompare
+.add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_KEY);
+con
[46/50] [abbrv] hadoop git commit: YARN-6281. Cleanup when AMRMProxy fails to initialize a new interceptor chain. (Botong Huang via Subru)
YARN-6281. Cleanup when AMRMProxy fails to initialize a new interceptor chain.
(Botong Huang via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3d5b4cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3d5b4cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3d5b4cd
Branch: refs/heads/YARN-2915
Commit: d3d5b4cd1abce52fe2f1196012230fa5b66591b4
Parents: beaca37
Author: Subru Krishnan
Authored: Fri Mar 10 18:13:29 2017 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:53 2017 -0700
--
.../nodemanager/amrmproxy/AMRMProxyService.java | 25 ++--
.../amrmproxy/BaseAMRMProxyTest.java| 21 -
.../amrmproxy/TestAMRMProxyService.java | 31
3 files changed, 61 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3d5b4cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index 5e91a20..c17d8ca 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -297,11 +297,16 @@ public class AMRMProxyService extends AbstractService
implements
+ " ApplicationId:" + applicationAttemptId + " for the user: "
+ user);
-RequestInterceptor interceptorChain =
-this.createRequestInterceptorChain();
-interceptorChain.init(createApplicationMasterContext(
-applicationAttemptId, user, amrmToken, localToken));
-chainWrapper.init(interceptorChain, applicationAttemptId);
+try {
+ RequestInterceptor interceptorChain =
+ this.createRequestInterceptorChain();
+ interceptorChain.init(createApplicationMasterContext(this.nmContext,
+ applicationAttemptId, user, amrmToken, localToken));
+ chainWrapper.init(interceptorChain, applicationAttemptId);
+} catch (Exception e) {
+ this.applPipelineMap.remove(applicationAttemptId.getApplicationId());
+ throw e;
+}
}
/**
@@ -317,8 +322,10 @@ public class AMRMProxyService extends AbstractService
implements
this.applPipelineMap.remove(applicationId);
if (pipeline == null) {
- LOG.info("Request to stop an application that does not exist. Id:"
- + applicationId);
+ LOG.info(
+ "No interceptor pipeline for application {},"
+ + " likely because its AM is not run in this node.",
+ applicationId);
} else {
LOG.info("Stopping the request processing pipeline for application: "
+ applicationId);
@@ -387,11 +394,11 @@ public class AMRMProxyService extends AbstractService
implements
}
private AMRMProxyApplicationContext createApplicationMasterContext(
- ApplicationAttemptId applicationAttemptId, String user,
+ Context context, ApplicationAttemptId applicationAttemptId, String user,
Token amrmToken,
Token localToken) {
AMRMProxyApplicationContextImpl appContext =
-new AMRMProxyApplicationContextImpl(this.nmContext, getConfig(),
+new AMRMProxyApplicationContextImpl(context, getConfig(),
applicationAttemptId, user, amrmToken, localToken);
return appContext;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3d5b4cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 7f96947..6f5009e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nod
[20/50] [abbrv] hadoop git commit: YARN-5307. Federation Application State Store internal APIs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea75da73/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
new file mode 100644
index 000..8b72a1e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/GetApplicationsHomeSubClusterResponsePBImpl.java
@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import
org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.ApplicationHomeSubClusterProto;
+import
org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProto;
+import
org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.GetApplicationsHomeSubClusterResponseProtoOrBuilder;
+import
org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of
+ * {@link GetApplicationsHomeSubClusterResponse}.
+ */
+@Private
+@Unstable
+public class GetApplicationsHomeSubClusterResponsePBImpl
+extends GetApplicationsHomeSubClusterResponse {
+
+ private GetApplicationsHomeSubClusterResponseProto proto =
+ GetApplicationsHomeSubClusterResponseProto.getDefaultInstance();
+ private GetApplicationsHomeSubClusterResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private List appsHomeSubCluster;
+
+ public GetApplicationsHomeSubClusterResponsePBImpl() {
+builder = GetApplicationsHomeSubClusterResponseProto.newBuilder();
+ }
+
+ public GetApplicationsHomeSubClusterResponsePBImpl(
+ GetApplicationsHomeSubClusterResponseProto proto) {
+this.proto = proto;
+viaProto = true;
+ }
+
+ public GetApplicationsHomeSubClusterResponseProto getProto() {
+mergeLocalToProto();
+proto = viaProto ? proto : builder.build();
+viaProto = true;
+return proto;
+ }
+
+ private void mergeLocalToProto() {
+if (viaProto) {
+ maybeInitBuilder();
+}
+mergeLocalToBuilder();
+proto = builder.build();
+viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+if (viaProto || builder == null) {
+ builder = GetApplicationsHomeSubClusterResponseProto.newBuilder(proto);
+}
+viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+if (this.appsHomeSubCluster != null) {
+ addSubClustersInfoToProto();
+}
+ }
+
+ @Override
+ public int hashCode() {
+return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+if (other == null) {
+ return false;
+}
+if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+}
+return false;
+ }
+
+ @Override
+ public String toString() {
+return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public List getAppsHomeSubClusters() {
+initSubClustersInfoList();
+return appsHomeSubCluster;
+ }
+
+ @Override
+ public void setAppsHomeSubClusters(
+ List appsHomeSubClusters) {
+maybeInitBuilder();
+if (a
[42/50] [abbrv] hadoop git commit: YARN-5467. InputValidator for the FederationStateStore internal APIs. (Giovanni Matteo Fumarola via Subru)
YARN-5467. InputValidator for the FederationStateStore internal APIs. (Giovanni
Matteo Fumarola via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43aafc51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43aafc51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43aafc51
Branch: refs/heads/YARN-2915
Commit: 43aafc51d321586f3fa3143c8364582f5d1f525c
Parents: 024f0d4
Author: Subru Krishnan
Authored: Wed Aug 17 12:07:06 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../store/impl/MemoryFederationStateStore.java | 30 +
...cationHomeSubClusterStoreInputValidator.java | 183 +++
...ationMembershipStateStoreInputValidator.java | 317 +
.../FederationPolicyStoreInputValidator.java| 144 ++
...derationStateStoreInvalidInputException.java | 48 +
.../federation/store/utils/package-info.java| 17 +
.../impl/FederationStateStoreBaseTest.java |6 +-
.../TestFederationStateStoreInputValidator.java | 1265 ++
8 files changed, 2007 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43aafc51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
index 8144435..6e564dc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -57,6 +57,9 @@ import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegister
import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
import
org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
import
org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import
org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator;
+import
org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator;
+import
org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.util.MonotonicClock;
@@ -88,6 +91,8 @@ public class MemoryFederationStateStore implements
FederationStateStore {
@Override
public SubClusterRegisterResponse registerSubCluster(
SubClusterRegisterRequest request) throws YarnException {
+FederationMembershipStateStoreInputValidator
+.validateSubClusterRegisterRequest(request);
SubClusterInfo subClusterInfo = request.getSubClusterInfo();
membership.put(subClusterInfo.getSubClusterId(), subClusterInfo);
return SubClusterRegisterResponse.newInstance();
@@ -96,6 +101,8 @@ public class MemoryFederationStateStore implements
FederationStateStore {
@Override
public SubClusterDeregisterResponse deregisterSubCluster(
SubClusterDeregisterRequest request) throws YarnException {
+FederationMembershipStateStoreInputValidator
+.validateSubClusterDeregisterRequest(request);
SubClusterInfo subClusterInfo = membership.get(request.getSubClusterId());
if (subClusterInfo == null) {
throw new YarnException(
@@ -111,6 +118,8 @@ public class MemoryFederationStateStore implements
FederationStateStore {
public SubClusterHeartbeatResponse subClusterHeartbeat(
SubClusterHeartbeatRequest request) throws YarnException {
+FederationMembershipStateStoreInputValidator
+.validateSubClusterHeartbeatRequest(request);
SubClusterId subClusterId = request.getSubClusterId();
SubClusterInfo subClusterInfo = membership.get(subClusterId);
@@ -129,6 +138,9 @@ public class MemoryFederationStateStore implements
FederationStateStore {
@Override
public GetSubClusterInfoResponse getSubCluster(
GetSubClusterInfoRequest request) throws YarnException {
+
+FederationMembershipStateStoreInputValidator
+.validateGetSubClusterInfoRequest(request);
SubClus
[39/50] [abbrv] hadoop git commit: YARN-5676. Add a HashBasedRouterPolicy, and small policies and test refactoring. (Carlo Curino via Subru).
YARN-5676. Add a HashBasedRouterPolicy, and small policies and test
refactoring. (Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77697d80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77697d80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77697d80
Branch: refs/heads/YARN-2915
Commit: 77697d80394e5aff277e30c0d8943aa7ffdedfc1
Parents: ebdac26
Author: Subru Krishnan
Authored: Tue Nov 22 15:02:22 2016 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +-
.../policies/AbstractPolicyManager.java | 175 -
.../policies/FederationPolicyManager.java | 117
.../PriorityBroadcastPolicyManager.java | 66 ---
.../federation/policies/RouterPolicyFacade.java | 1 +
.../policies/UniformBroadcastPolicyManager.java | 56 --
.../policies/WeightedLocalityPolicyManager.java | 67 ---
.../policies/manager/AbstractPolicyManager.java | 190 +++
.../manager/FederationPolicyManager.java| 118
.../manager/HashBroadcastPolicyManager.java | 38
.../manager/PriorityBroadcastPolicyManager.java | 66 +++
.../manager/UniformBroadcastPolicyManager.java | 44 +
.../manager/WeightedLocalityPolicyManager.java | 67 +++
.../policies/manager/package-info.java | 19 ++
.../policies/router/AbstractRouterPolicy.java | 19 ++
.../policies/router/HashBasedRouterPolicy.java | 81
.../policies/router/LoadBasedRouterPolicy.java | 3 +
.../policies/router/PriorityRouterPolicy.java | 3 +
.../router/UniformRandomRouterPolicy.java | 10 +-
.../router/WeightedRandomRouterPolicy.java | 3 +
.../policies/BaseFederationPoliciesTest.java| 17 +-
.../policies/BasePolicyManagerTest.java | 108 ---
...ionPolicyInitializationContextValidator.java | 1 +
.../TestPriorityBroadcastPolicyManager.java | 72 ---
.../policies/TestRouterPolicyFacade.java| 2 +
.../TestUniformBroadcastPolicyManager.java | 40
.../TestWeightedLocalityPolicyManager.java | 79
.../policies/manager/BasePolicyManagerTest.java | 104 ++
.../TestHashBasedBroadcastPolicyManager.java| 40
.../TestPriorityBroadcastPolicyManager.java | 72 +++
.../TestUniformBroadcastPolicyManager.java | 40
.../TestWeightedLocalityPolicyManager.java | 79
.../policies/router/BaseRouterPoliciesTest.java | 51 +
.../router/TestHashBasedRouterPolicy.java | 83
.../router/TestLoadBasedRouterPolicy.java | 3 +-
.../router/TestPriorityRouterPolicy.java| 3 +-
.../router/TestUniformRandomRouterPolicy.java | 3 +-
.../router/TestWeightedRandomRouterPolicy.java | 15 +-
38 files changed, 1160 insertions(+), 798 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77697d80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 285e3c5..9397c99 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2574,7 +2574,8 @@ public class YarnConfiguration extends Configuration {
+ "policy-manager";
public static final String DEFAULT_FEDERATION_POLICY_MANAGER = "org.apache"
- +
".hadoop.yarn.server.federation.policies.UniformBroadcastPolicyManager";
+ + ".hadoop.yarn.server.federation.policies"
+ + ".manager.UniformBroadcastPolicyManager";
public static final String FEDERATION_POLICY_MANAGER_PARAMS =
FEDERATION_PREFIX + "policy-manager-params";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77697d80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/fed
[26/50] [abbrv] hadoop git commit: YARN-5325. Stateless ARMRMProxy policies implementation. (Carlo Curino via Subru).
YARN-5325. Stateless ARMRMProxy policies implementation. (Carlo Curino via Subru). Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14bf8ecd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14bf8ecd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14bf8ecd Branch: refs/heads/YARN-2915 Commit: 14bf8ecd200f63a1522bfda081c104850cca Parents: 55a3f1c Author: Subru Krishnan Authored: Thu Oct 13 17:59:13 2016 -0700 Committer: Subru Krishnan Committed: Mon Apr 24 18:58:52 2017 -0700 -- .../AbstractConfigurableFederationPolicy.java | 155 + .../policies/ConfigurableFederationPolicy.java | 9 +- .../FederationPolicyInitializationContext.java | 37 +- ...ionPolicyInitializationContextValidator.java | 28 +- .../policies/FederationPolicyManager.java | 59 +- .../amrmproxy/AbstractAMRMProxyPolicy.java | 47 ++ .../amrmproxy/BroadcastAMRMProxyPolicy.java | 85 +++ .../amrmproxy/FederationAMRMProxyPolicy.java| 25 +- .../LocalityMulticastAMRMProxyPolicy.java | 583 +++ .../policies/amrmproxy/package-info.java| 1 - .../policies/dao/WeightedPolicyInfo.java| 180 +++--- .../federation/policies/dao/package-info.java | 1 - .../policies/exceptions/package-info.java | 1 - .../federation/policies/package-info.java | 1 - .../policies/router/AbstractRouterPolicy.java | 47 ++ .../router/BaseWeightedRouterPolicy.java| 150 - .../policies/router/FederationRouterPolicy.java | 5 +- .../policies/router/LoadBasedRouterPolicy.java | 36 +- .../policies/router/PriorityRouterPolicy.java | 19 +- .../router/UniformRandomRouterPolicy.java | 28 +- .../router/WeightedRandomRouterPolicy.java | 32 +- .../policies/router/package-info.java | 1 - .../resolver/AbstractSubClusterResolver.java| 4 +- .../policies/BaseFederationPoliciesTest.java| 28 +- ...ionPolicyInitializationContextValidator.java | 25 +- .../TestBroadcastAMRMProxyFederationPolicy.java | 112 .../TestLocalityMulticastAMRMProxyPolicy.java | 566 ++ .../router/TestLoadBasedRouterPolicy.java | 18 +- .../router/TestPriorityRouterPolicy.java| 15 +- .../router/TestWeightedRandomRouterPolicy.java | 35 +- .../utils/FederationPoliciesTestUtil.java | 64 ++ .../src/test/resources/nodes| 6 +- 32 files changed, 1950 insertions(+), 453 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bf8ecd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java new file mode 100644 index 000..4cb9bbe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractConfigurableFederationPolicy.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.federation.policies; + +import java.util.Map; + +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo; +import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException; +import org.apache.hadoop.yarn.server.federation.policies.exceptions.NoActiveSubclustersException; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.
[36/50] [abbrv] hadoop git commit: YARN-5872. Add AlwayReject policies for router and amrmproxy. (Carlo Curino via Subru).
YARN-5872. Add AlwayReject policies for router and amrmproxy. (Carlo Curino via
Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba861ac1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba861ac1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba861ac1
Branch: refs/heads/YARN-2915
Commit: ba861ac1db68ade0b91d561be5bb4a5f4383018b
Parents: 202e9a0
Author: Subru Krishnan
Authored: Tue Nov 22 18:37:30 2016 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../amrmproxy/RejectAMRMProxyPolicy.java| 67 +
.../manager/RejectAllPolicyManager.java | 40 ++
.../policies/router/RejectRouterPolicy.java | 66 +
.../amrmproxy/TestRejectAMRMProxyPolicy.java| 78
.../manager/TestRejectAllPolicyManager.java | 40 ++
.../policies/router/TestRejectRouterPolicy.java | 63
6 files changed, 354 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba861ac1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
new file mode 100644
index 000..3783df6
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/amrmproxy/RejectAMRMProxyPolicy.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies.amrmproxy;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContextValidator;
+import
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyException;
+import
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+
+/**
+ * An implementation of the {@link FederationAMRMProxyPolicy} that simply
+ * rejects all requests. Useful to prevent apps from accessing any sub-cluster.
+ */
+public class RejectAMRMProxyPolicy extends AbstractAMRMProxyPolicy {
+
+ private Set knownClusterIds = new HashSet<>();
+
+ @Override
+ public void reinitialize(FederationPolicyInitializationContext policyContext)
+ throws FederationPolicyInitializationException {
+// overrides initialize to avoid weight checks that do no apply for
+// this policy.
+FederationPolicyInitializationContextValidator.validate(policyContext,
+this.getClass().getCanonicalName());
+setPolicyContext(policyContext);
+ }
+
+ @Override
+ public Map> splitResourceRequests(
+ List resourceRequests) throws YarnException {
+throw new FederationPolicyException("The policy configured for this queue "
++ "rejects all routing requests by construction.");
+ }
+
+ @Override
+ public void notifyOfResponse(SubClusterId subClusterId,
+ AllocateResponse response) throws YarnException {
+// This might be invoked for applications started with a previous policy,
+// do nothing for this policy.
[08/50] [abbrv] hadoop git commit: HDFS-11529. Add libHDFS API to return last exception. Contributed by Sailesh Mukil.
HDFS-11529. Add libHDFS API to return last exception. Contributed by Sailesh
Mukil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fda86ef2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fda86ef2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fda86ef2
Branch: refs/heads/YARN-2915
Commit: fda86ef2a32026c02d9b5d4cca1ecb7b4decd872
Parents: 20e3ae2
Author: John Zhuge
Authored: Sat Apr 22 02:56:48 2017 -0700
Committer: John Zhuge
Committed: Sat Apr 22 02:56:48 2017 -0700
--
.../src/main/native/libhdfs-tests/expect.h | 18
.../libhdfs-tests/test_libhdfs_threaded.c | 6 ++
.../src/main/native/libhdfs/exception.c | 75 +++-
.../src/main/native/libhdfs/exception.h | 16 +++-
.../src/main/native/libhdfs/hdfs.c | 9 ++
.../src/main/native/libhdfs/include/hdfs/hdfs.h | 32 +++
.../src/main/native/libhdfs/jni_helper.c| 95 +---
.../src/main/native/libhdfs/jni_helper.h| 35
.../libhdfs/os/posix/thread_local_storage.c | 54 ---
.../native/libhdfs/os/thread_local_storage.h| 61 +
.../libhdfs/os/windows/thread_local_storage.c | 43 +++--
11 files changed, 366 insertions(+), 78 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fda86ef2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h
index 49aa285..528c96f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/expect.h
@@ -163,6 +163,24 @@ struct hdfsFile_internal;
ret = -errno; \
} while (ret == -EINTR);
+#define EXPECT_STR_CONTAINS(str, substr) \
+do { \
+char *_my_ret_ = (str); \
+int _my_errno_ = errno; \
+if ((str) == NULL) { \
+fprintf(stderr, "TEST_ERROR: failed on %s:%d with NULL return " \
+ "return value (errno: %d): expected substring: %s\n", \
+ __FILE__, __LINE__, _my_errno_, (substr)); \
+return -1; \
+} \
+if (strstr((str), (substr)) == NULL) { \
+fprintf(stderr, "TEST_ERROR: failed on %s:%d with return " \
+ "value %s (errno: %d): expected substring: %s\n", \
+ __FILE__, __LINE__, _my_ret_, _my_errno_, (substr)); \
+return -1; \
+} \
+} while (0);
+
/**
* Test that an HDFS file has the given statistics.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fda86ef2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
index ee1b080..f80200d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
@@ -165,6 +165,12 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti,
hdfsFS fs,
/* There should not be any file to open for reading. */
EXPECT_NULL(hdfsOpenFile(fs, paths->file1, O_RDONLY, 0, 0, 0));
+/* Check if the exceptions are stored in the TLS */
+EXPECT_STR_CONTAINS(hdfsGetLastExceptionRootCause(),
+"File does not exist");
+EXPECT_STR_CONTAINS(hdfsGetLastExceptionStackTrace(),
+"java.io.FileNotFoundException");
+
/* hdfsOpenFile should not accept mode = 3 */
EXPECT_NULL(hdfsOpenFile(fs, paths->file1, 3, 0, 0, 0));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fda86ef2/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
index 35e9d2d..dbaf1f6 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
+++
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c
@@ -110,15 +110,50 @@ void getExceptionInfo(const char *excName, int
noPrintFlags,
}
}
+/**
[03/50] [abbrv] hadoop git commit: HDFS-11604. Define and parse erasure code policies. Contributed by Lin Zeng
HDFS-11604. Define and parse erasure code policies. Contributed by Lin Zeng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0803388
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0803388
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0803388
Branch: refs/heads/YARN-2915
Commit: b0803388fc5ec03b774aa003f52232deb8db6f69
Parents: de69d6e
Author: Kai Zheng
Authored: Fri Apr 21 13:33:33 2017 +0800
Committer: Kai Zheng
Committed: Fri Apr 21 13:33:33 2017 +0800
--
.../apache/hadoop/hdfs/util/ECPolicyLoader.java | 328 +++
.../hadoop/hdfs/util/TestECPolicyLoader.java| 313 ++
.../src/main/conf/user_ec_policies.xml.template | 71
.../src/site/markdown/HDFSErasureCoding.md | 10 +
4 files changed, 722 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0803388/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
new file mode 100644
index 000..e75f091
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
@@ -0,0 +1,328 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.util;
+
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.w3c.dom.Node;
+import org.w3c.dom.Text;
+import org.w3c.dom.Element;
+import org.w3c.dom.Document;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+import java.util.List;
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.Collections;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A EC policy loading tool that loads user defined EC policies from XML file.
+ */
+@InterfaceAudience.Private
+public class ECPolicyLoader {
+
+ private static final Logger LOG
+ = LoggerFactory.getLogger(ECPolicyLoader.class);
+
+ private static final int LAYOUT_VERSION = 1;
+
+ /**
+ * Load user defined EC policies from a XML configuration file.
+ * @param policyFilePath path of EC policy file
+ * @return all valid EC policies in EC policy file
+ */
+ public List loadPolicy(String policyFilePath) {
+File policyFile = getPolicyFile(policyFilePath);
+if (policyFile == null) {
+ LOG.warn("Not found any EC policy file");
+ return Collections.emptyList();
+}
+
+try {
+ return loadECPolicies(policyFile);
+} catch (ParserConfigurationException | IOException | SAXException e) {
+ throw new RuntimeException("Failed to load EC policy file: "
+ + policyFile);
+}
+ }
+
+ /**
+ * Load EC policies from a XML configuration file.
+ * @param policyFile EC policy file
+ * @return list of EC policies
+ * @throws ParserConfigurationException if ParserConfigurationException
happen
+ * @throws IOException if no such EC policy file
+ * @throws SAXException if the xml file has some invalid elements
+ */
+ private List loadECPolicies(File policyFile)
+ throws ParserConfigurationException, IOException, SAXException {
+
+LOG.info("Loading EC policy file " + policyFile);
+
+// Read and parse the EC policy file.
+DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+dbf.setIgnoringComments(true);
+DocumentBuilder builder = dbf.newDocumentBuilder();
+Document doc = builder.parse(policyFil
[21/50] [abbrv] hadoop git commit: YARN-5307. Federation Application State Store internal APIs
YARN-5307. Federation Application State Store internal APIs Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea75da73 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea75da73 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea75da73 Branch: refs/heads/YARN-2915 Commit: ea75da734db836b16b23055b5224e30b59976e9b Parents: fe7fd95 Author: Subru Krishnan Authored: Fri Aug 5 11:52:44 2016 -0700 Committer: Subru Krishnan Committed: Mon Apr 24 18:58:51 2017 -0700 -- ...ederationApplicationHomeSubClusterStore.java | 126 .../AddApplicationHomeSubClusterRequest.java| 72 +++ .../AddApplicationHomeSubClusterResponse.java | 44 + .../records/ApplicationHomeSubCluster.java | 124 .../DeleteApplicationHomeSubClusterRequest.java | 65 +++ ...DeleteApplicationHomeSubClusterResponse.java | 43 + .../GetApplicationHomeSubClusterRequest.java| 64 +++ .../GetApplicationHomeSubClusterResponse.java | 73 +++ .../GetApplicationsHomeSubClusterRequest.java | 40 .../GetApplicationsHomeSubClusterResponse.java | 75 .../UpdateApplicationHomeSubClusterRequest.java | 74 ...UpdateApplicationHomeSubClusterResponse.java | 43 + ...dApplicationHomeSubClusterRequestPBImpl.java | 132 + ...ApplicationHomeSubClusterResponsePBImpl.java | 78 .../pb/ApplicationHomeSubClusterPBImpl.java | 167 ...eApplicationHomeSubClusterRequestPBImpl.java | 130 + ...ApplicationHomeSubClusterResponsePBImpl.java | 78 ...tApplicationHomeSubClusterRequestPBImpl.java | 135 + ...ApplicationHomeSubClusterResponsePBImpl.java | 132 + ...ApplicationsHomeSubClusterRequestPBImpl.java | 78 ...pplicationsHomeSubClusterResponsePBImpl.java | 190 +++ .../pb/GetSubClustersInfoResponsePBImpl.java| 6 +- ...eApplicationHomeSubClusterRequestPBImpl.java | 132 + ...ApplicationHomeSubClusterResponsePBImpl.java | 78 .../proto/yarn_server_federation_protos.proto | 45 - .../records/TestFederationProtocolRecords.java | 81 26 files changed, 2301 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea75da73/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java new file mode 100644 index 000..217ee2e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.yarn.server.federation.store; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; +i
[06/50] [abbrv] hadoop git commit: MAPREDUCE-6871. Allow users to specify racks and nodes for strict locality for AMs (rkanter)
MAPREDUCE-6871. Allow users to specify racks and nodes for strict locality for
AMs (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3721cfe1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3721cfe1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3721cfe1
Branch: refs/heads/YARN-2915
Commit: 3721cfe1fbd98c5b6aa46aefdfcf62276c28c4a4
Parents: 5078df7
Author: Robert Kanter
Authored: Fri Apr 21 16:12:01 2017 -0700
Committer: Robert Kanter
Committed: Fri Apr 21 16:12:01 2017 -0700
--
.../apache/hadoop/mapreduce/MRJobConfig.java| 6 +
.../org/apache/hadoop/mapred/YARNRunner.java| 132 ---
.../apache/hadoop/mapred/TestYARNRunner.java| 167 +++
3 files changed, 279 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3721cfe1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 18bf139..cfc1bcc 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -91,6 +91,12 @@ public interface MRJobConfig {
*/
public static final String REDUCE_NODE_LABEL_EXP =
"mapreduce.reduce.node-label-expression";
+ /**
+ * Specify strict locality on a comma-separated list of racks and/or nodes.
+ * Syntax: /rack or /rack/node or node (assumes /default-rack)
+ */
+ public static final String AM_STRICT_LOCALITY =
"mapreduce.job.am.strict-locality";
+
public static final String RESERVATION_ID = "mapreduce.job.reservation.id";
public static final String JOB_TAGS = "mapreduce.job.tags";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3721cfe1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
index 2339c79..1baa467 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
@@ -22,13 +22,14 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Vector;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -101,6 +102,25 @@ public class YARNRunner implements ClientProtocol {
private static final Log LOG = LogFactory.getLog(YARNRunner.class);
+ private static final String RACK_GROUP = "rack";
+ private static final String NODE_IF_RACK_GROUP = "node1";
+ private static final String NODE_IF_NO_RACK_GROUP = "node2";
+
+ /**
+ * Matches any of the following patterns with capturing groups:
+ *
+ * /rack
+ * /rack/node
+ * node (assumes /default-rack)
+ *
+ * The groups can be retrieved using the RACK_GROUP, NODE_IF_RACK_GROUP,
+ * and/or NODE_IF_NO_RACK_GROUP group keys.
+ */
+ private static final Pattern RACK_NODE_PATTERN =
+ Pattern.compile(
+ String.format("(?<%s>[^/]+?)|(?<%s>/[^/]+?)(?:/(?<%s>[^/]+?))?",
+ NODE_IF_NO_RACK_GROUP, RACK_GROUP, NODE_IF_RACK_GROUP));
+
private final static RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
@@ -503,20 +523,6 @@ public class YARNRunner implements ClientProtocol {
throws IOException {
ApplicationId applicationId = resMgrDelegate.getApplicationId();
-// Setup resource requirements
-Resource capability = recordFactory.newRecordInstance(Resource.cla
[34/50] [abbrv] hadoop git commit: YARN-5323. Policies APIs for Federation Router and AMRMProxy policies. (Carlo Curino via Subru).
YARN-5323. Policies APIs for Federation Router and AMRMProxy policies. (Carlo
Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b74a666
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b74a666
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b74a666
Branch: refs/heads/YARN-2915
Commit: 7b74a666f70eb4265404be90f52875c1bb9cdc2a
Parents: e4429d9
Author: Subru Krishnan
Authored: Wed Sep 7 17:33:34 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../policies/ConfigurableFederationPolicy.java | 44 +++
.../policies/FederationPolicyConfigurator.java | 91 +
.../FederationPolicyInitializationContext.java | 109
...ionPolicyInitializationContextValidator.java | 82
.../policies/FederationPolicyWriter.java| 45 +++
.../amrmproxy/FederationAMRMProxyPolicy.java| 66 ++
.../policies/amrmproxy/package-info.java| 20 +++
.../exceptions/FederationPolicyException.java | 33 +
...FederationPolicyInitializationException.java | 33 +
.../NoActiveSubclustersException.java | 27
.../exceptions/UnknownSubclusterException.java | 28
.../policies/exceptions/package-info.java | 20 +++
.../federation/policies/package-info.java | 20 +++
.../policies/router/FederationRouterPolicy.java | 45 +++
.../policies/router/package-info.java | 20 +++
...ionPolicyInitializationContextValidator.java | 128 +++
.../utils/FederationPoliciesTestUtil.java | 83
17 files changed, 894 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b74a666/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
new file mode 100644
index 000..fd6ceea
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/ConfigurableFederationPolicy.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+
+/**
+ * This interface provides a general method to reinitialize a policy. The
+ * semantics are try-n-swap, so in case of an exception is thrown the
+ * implmentation must ensure the previous state and configuration is preserved.
+ */
+public interface ConfigurableFederationPolicy {
+
+ /**
+ * This method is invoked to initialize of update the configuration of
+ * policies. The implementor should provide try-n-swap semantics, and retain
+ * state if possible.
+ *
+ * @param federationPolicyInitializationContext the new context to provide to
+ * implementor.
+ *
+ * @throws FederationPolicyInitializationException in case the initialization
+ * fails.
+ */
+ void reinitialize(
+ FederationPolicyInitializationContext
+ federationPolicyInitializationContext)
+ throws FederationPolicyInitializationException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b74a666/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/FederationPolicyConfigurator.java
--
[40/50] [abbrv] hadoop git commit: YARN-3672. Create Facade for Federation State and Policy Store. Contributed by Subru Krishnan
YARN-3672. Create Facade for Federation State and Policy Store. Contributed by
Subru Krishnan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/024f0d4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/024f0d4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/024f0d4a
Branch: refs/heads/YARN-2915
Commit: 024f0d4a64934c4ea08c1178d3b28a6ae4ab4678
Parents: 367b08a
Author: Jian He
Authored: Wed Aug 17 11:13:19 2016 +0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
hadoop-project/pom.xml | 13 +
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +
.../src/main/resources/yarn-default.xml | 20 +-
.../hadoop-yarn-server-common/pom.xml | 10 +
.../utils/FederationStateStoreFacade.java | 532 +++
.../server/federation/utils/package-info.java | 17 +
.../utils/FederationStateStoreTestUtil.java | 149 ++
.../utils/TestFederationStateStoreFacade.java | 148 ++
9 files changed, 905 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/024f0d4a/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8c855cc..3683a0f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -98,6 +98,9 @@
2.0.0-M21
1.0.0-M33
+1.0.0
+3.0.3
+
1.8
@@ -1256,6 +1259,16 @@
kerb-simplekdc
1.0.0-RC2
+
+ javax.cache
+ cache-api
+ ${jcache.version}
+
+
+ org.ehcache
+ ehcache
+ ${ehcache.version}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/024f0d4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4d1d56b..4eb414f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2534,6 +2534,19 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+
+ public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
+ FEDERATION_PREFIX + "state-store.class";
+
+ public static final String DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS =
+
"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore";
+
+ public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
+ FEDERATION_PREFIX + "cache-ttl.secs";
+
+ // 5 minutes
+ public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
public static final String FEDERATION_MACHINE_LIST =
FEDERATION_PREFIX + "machine-list";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/024f0d4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 3da4bab..bfc2534 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -68,6 +68,10 @@ public class TestYarnConfigurationFields extends
TestConfigurationFieldsBase {
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
+// Federation default configs to be ignored
+configurationPropsToSkipCompare
+.add(YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS);
+
// Ignore blacklisting nodes for AM failures feature since it is still a
// "work in progress"
configurationPropsToSkipCompare.add(YarnConfiguration.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0
[09/50] [abbrv] hadoop git commit: YARN-6291. Introduce query parameters (sort, filter, etc.) for tables to keep state on refresh/navigation in new YARN UI. Contributed by Gergely Novák.
YARN-6291. Introduce query parameters (sort, filter, etc.) for tables to keep
state on refresh/navigation in new YARN UI. Contributed by Gergely Novák.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/561718e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/561718e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/561718e0
Branch: refs/heads/YARN-2915
Commit: 561718e05de51c0cb7c17295d7713d52408918eb
Parents: fda86ef
Author: Sunil G
Authored: Mon Apr 24 13:28:57 2017 +0530
Committer: Sunil G
Committed: Mon Apr 24 13:28:57 2017 +0530
--
.../src/main/webapp/app/controllers/yarn-apps/apps.js | 10 ++
.../src/main/webapp/app/controllers/yarn-apps/services.js | 10 ++
.../src/main/webapp/app/controllers/yarn-nodes/table.js | 8
.../src/main/webapp/app/templates/yarn-apps/apps.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-apps/services.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-nodes/table.hbs| 2 +-
6 files changed, 31 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/561718e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
index 0b0be20..44a7c1c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
@@ -16,7 +16,17 @@
* limitations under the License.
*/
+import Ember from 'ember';
+import ColumnDef from 'em-table/utils/column-definition';
+import TableDefinition from 'em-table/utils/table-definition';
import AppTableController from '../app-table-columns';
export default AppTableController.extend({
+ queryParams: ['searchText', 'sortColumnId', 'sortOrder', 'pageNum',
'rowCount'],
+ tableDefinition: TableDefinition.create(),
+ searchText: Ember.computed.alias('tableDefinition.searchText'),
+ sortColumnId: Ember.computed.alias('tableDefinition.sortColumnId'),
+ sortOrder: Ember.computed.alias('tableDefinition.sortOrder'),
+ pageNum: Ember.computed.alias('tableDefinition.pageNum'),
+ rowCount: Ember.computed.alias('tableDefinition.rowCount')
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/561718e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
index 0b0be20..44a7c1c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
@@ -16,7 +16,17 @@
* limitations under the License.
*/
+import Ember from 'ember';
+import ColumnDef from 'em-table/utils/column-definition';
+import TableDefinition from 'em-table/utils/table-definition';
import AppTableController from '../app-table-columns';
export default AppTableController.extend({
+ queryParams: ['searchText', 'sortColumnId', 'sortOrder', 'pageNum',
'rowCount'],
+ tableDefinition: TableDefinition.create(),
+ searchText: Ember.computed.alias('tableDefinition.searchText'),
+ sortColumnId: Ember.computed.alias('tableDefinition.sortColumnId'),
+ sortOrder: Ember.computed.alias('tableDefinition.sortOrder'),
+ pageNum: Ember.computed.alias('tableDefinition.pageNum'),
+ rowCount: Ember.computed.alias('tableDefinition.rowCount')
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/561718e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
index 3b7b0f6..3fae596 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
@@ -19,8 +19,16 @@
import Ember from 'ember';
import ColumnDef from 'em-table/utils/column-definition';
+import TableDefini
[31/50] [abbrv] hadoop git commit: YARN-5391. PolicyManager to tie together Router/AMRM Federation policies. (Carlo Curino via Subru).
YARN-5391. PolicyManager to tie together Router/AMRM Federation policies.
(Carlo Curino via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdcd3cec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdcd3cec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdcd3cec
Branch: refs/heads/YARN-2915
Commit: cdcd3cec8bd2c861d9e0f4d4bbd7871cca3da293
Parents: 14bf8ec
Author: Subru Krishnan
Authored: Tue Nov 1 19:54:18 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
.../policies/AbstractPolicyManager.java | 175 +++
.../FederationPolicyInitializationContext.java | 3 +-
.../policies/UniformBroadcastPolicyManager.java | 56 ++
.../policies/WeightedLocalityPolicyManager.java | 67 +++
.../records/SubClusterPolicyConfiguration.java | 13 ++
.../policies/BasePolicyManagerTest.java | 108
...ionPolicyInitializationContextValidator.java | 5 +-
.../TestUniformBroadcastPolicyManager.java | 40 +
.../TestWeightedLocalityPolicyManager.java | 79 +
.../utils/FederationPoliciesTestUtil.java | 2 +-
10 files changed, 545 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdcd3cec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
new file mode 100644
index 000..e77f2e3
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/AbstractPolicyManager.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.policies;
+
+import
org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
+import
org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
+import
org.apache.hadoop.yarn.server.federation.policies.router.FederationRouterPolicy;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides basic implementation for common methods that multiple
+ * policies will need to implement.
+ */
+public abstract class AbstractPolicyManager implements
+FederationPolicyManager {
+
+ private String queue;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class routerFederationPolicy;
+ @SuppressWarnings("checkstyle:visibilitymodifier")
+ protected Class amrmProxyFederationPolicy;
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(AbstractPolicyManager.class);
+ /**
+ * This default implementation validates the
+ * {@link FederationPolicyInitializationContext},
+ * then checks whether it needs to reinstantiate the class (null or
+ * mismatching type), and reinitialize the policy.
+ *
+ * @param federationPolicyContext the current context
+ * @param oldInstance the existing (possibly null) instance.
+ *
+ * @return a valid and fully reinitalized {@link FederationAMRMProxyPolicy}
+ * instance
+ *
+ * @throws FederationPolicyInitializationException if the reinitalization is
+ * not valid, and ensure
+ * previous state is
preserved
+ */
+ public FederationAMRMProxyPolicy getAMRMPolicy(
+ FederationPolicyInitializat
[23/50] [abbrv] hadoop git commit: YARN-5324. Stateless Federation router policies implementation. (Carlo Curino via Subru).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a3f1cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
index 8c2115b..f901329 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/FederationPoliciesTestUtil.java
@@ -19,13 +19,20 @@ package org.apache.hadoop.yarn.server.federation.utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import
org.apache.hadoop.yarn.server.federation.policies.ConfigurableFederationPolicy;
+import
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
+import
org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
import
org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
-import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.*;
import java.net.URL;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
@@ -41,6 +48,41 @@ public final class FederationPoliciesTestUtil {
// disabled.
}
+
+ public static void initializePolicyContext(
+ FederationPolicyInitializationContext fpc, ConfigurableFederationPolicy
+ policy, WeightedPolicyInfo policyInfo,
+ Map activeSubclusters)
+ throws YarnException {
+ByteBuffer buf = policyInfo.toByteBuffer();
+fpc.setSubClusterPolicyConfiguration(SubClusterPolicyConfiguration
+.newInstance("queue1", policy.getClass().getCanonicalName(), buf));
+FederationStateStoreFacade facade = FederationStateStoreFacade
+.getInstance();
+FederationStateStore fss = mock(FederationStateStore.class);
+
+if (activeSubclusters == null) {
+ activeSubclusters = new HashMap();
+}
+GetSubClustersInfoResponse response = GetSubClustersInfoResponse
+.newInstance(new
ArrayList(activeSubclusters.values()));
+
+when(fss.getSubClusters(any())).thenReturn(response);
+facade.reinitialize(fss, new Configuration());
+fpc.setFederationStateStoreFacade(facade);
+policy.reinitialize(fpc);
+ }
+
+ public static void initializePolicyContext(
+ ConfigurableFederationPolicy policy,
+ WeightedPolicyInfo policyInfo, Map activeSubclusters) throws YarnException {
+FederationPolicyInitializationContext context =
+new FederationPolicyInitializationContext(null, initResolver(),
+initFacade());
+initializePolicyContext(context, policy, policyInfo, activeSubclusters);
+ }
+
/**
* Initialize a {@link SubClusterResolver}.
*
@@ -66,18 +108,52 @@ public final class FederationPoliciesTestUtil {
* Initialiaze a main-memory {@link FederationStateStoreFacade} used for
* testing, wiht a mock resolver.
*
+ * @param subClusterInfos the list of subclusters to be served on
+ *getSubClusters invocations.
+ *
* @return the facade.
*
* @throws YarnException in case the initialization is not successful.
*/
- public static FederationStateStoreFacade initFacade() throws YarnException {
+
+ public static FederationStateStoreFacade initFacade(
+ List subClusterInfos, SubClusterPolicyConfiguration
+ policyConfiguration) throws YarnException {
FederationStateStoreFacade goodFacade = FederationStateStoreFacade
.getInstance();
FederationStateStore fss = mock(FederationStateStore.class);
GetSubClustersInfoResponse response = GetSubClustersInfoResponse
-.newInstance(new ArrayList<>());
+.newInstance(subClusterInfos);
when(fss.getSubClusters(any())).thenReturn(response);
+
+List configurations = new ArrayList<>();
+configurations.add(policyConfiguration);
+
+GetSubClusterPoliciesConfigurationsResponse policiesResponse =
+
[10/50] [abbrv] hadoop git commit: YARN-5617. AMs only intended to run one attempt can be run more than once? Contributed by Jason Lowe.
YARN-5617. AMs only intended to run one attempt can be run more than once?
Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52adf719
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52adf719
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52adf719
Branch: refs/heads/YARN-2915
Commit: 52adf719143c20f4f2af369c6c40dd98677e7410
Parents: 561718e
Author: Eric Payne
Authored: Mon Apr 24 12:03:28 2017 -0500
Committer: Eric Payne
Committed: Mon Apr 24 12:03:28 2017 -0500
--
.../server/resourcemanager/rmapp/RMAppImpl.java | 24 ++--
.../applicationsmanager/TestAMRestart.java | 138 +++
2 files changed, 124 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52adf719/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 531844c..78df913 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1517,14 +1517,22 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public RMAppState transition(RMAppImpl app, RMAppEvent event) {
int numberOfFailure = app.getNumFailedAppAttempts();
- LOG.info("The number of failed attempts"
- + (app.attemptFailuresValidityInterval > 0 ? " in previous "
- + app.attemptFailuresValidityInterval + " milliseconds " : " ")
- + "is " + numberOfFailure + ". The max attempts is "
- + app.maxAppAttempts);
-
- if (app.attemptFailuresValidityInterval > 0) {
-removeExcessAttempts(app);
+ if (app.maxAppAttempts == 1) {
+// If the user explicitly set the attempts to 1 then there are likely
+// correctness issues if the AM restarts for any reason.
+LOG.info("Max app attempts is 1 for " + app.applicationId
++ ", preventing further attempts.");
+numberOfFailure = app.maxAppAttempts;
+ } else {
+LOG.info("The number of failed attempts"
++ (app.attemptFailuresValidityInterval > 0 ? " in previous "
++ app.attemptFailuresValidityInterval + " milliseconds " : " ")
++ "is " + numberOfFailure + ". The max attempts is "
++ app.maxAppAttempts);
+
+if (app.attemptFailuresValidityInterval > 0) {
+ removeExcessAttempts(app);
+}
}
if (!app.submissionContext.getUnmanagedAM()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52adf719/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index f5da5b5..f178884 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -378,8 +378,7 @@ public class TestAMRestart {
YarnConfiguration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
-// explicitly set max-am-retry count as 1.
-conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
+conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
conf.set(YarnConfiguration.RM_STORE
[25/50] [abbrv] hadoop git commit: YARN-5325. Stateless ARMRMProxy policies implementation. (Carlo Curino via Subru).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bf8ecd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
index e57709f..5de749f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/LoadBasedRouterPolicy.java
@@ -17,8 +17,8 @@
package org.apache.hadoop.yarn.server.federation.policies.router;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.util.Map;
+
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import
org.apache.hadoop.yarn.server.federation.policies.FederationPolicyInitializationContext;
@@ -30,34 +30,27 @@ import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
-import java.util.Map;
-
/**
* This implements a simple load-balancing policy. The policy "weights" are
* binary 0/1 values that enable/disable each sub-cluster, and the policy peaks
* the sub-cluster with the least load to forward this application.
*/
-public class LoadBasedRouterPolicy
-extends BaseWeightedRouterPolicy {
-
- private static final Log LOG =
- LogFactory.getLog(LoadBasedRouterPolicy.class);
+public class LoadBasedRouterPolicy extends AbstractRouterPolicy {
@Override
- public void reinitialize(FederationPolicyInitializationContext
- federationPolicyContext)
+ public void reinitialize(FederationPolicyInitializationContext policyContext)
throws FederationPolicyInitializationException {
// remember old policyInfo
WeightedPolicyInfo tempPolicy = getPolicyInfo();
-//attempt new initialization
-super.reinitialize(federationPolicyContext);
+// attempt new initialization
+super.reinitialize(policyContext);
-//check extra constraints
+// check extra constraints
for (Float weight : getPolicyInfo().getRouterPolicyWeights().values()) {
if (weight != 0 && weight != 1) {
-//reset to old policyInfo if check fails
+// reset to old policyInfo if check fails
setPolicyInfo(tempPolicy);
throw new FederationPolicyInitializationException(
this.getClass().getCanonicalName()
@@ -69,18 +62,16 @@ public class LoadBasedRouterPolicy
@Override
public SubClusterId getHomeSubcluster(
- ApplicationSubmissionContext appSubmissionContext)
- throws YarnException {
+ ApplicationSubmissionContext appSubmissionContext) throws YarnException {
Map activeSubclusters =
getActiveSubclusters();
-Map weights = getPolicyInfo()
-.getRouterPolicyWeights();
+Map weights =
+getPolicyInfo().getRouterPolicyWeights();
SubClusterIdInfo chosen = null;
long currBestMem = -1;
-for (Map.Entry entry :
-activeSubclusters
+for (Map.Entry entry : activeSubclusters
.entrySet()) {
SubClusterIdInfo id = new SubClusterIdInfo(entry.getKey());
if (weights.containsKey(id) && weights.get(id) > 0) {
@@ -95,8 +86,7 @@ public class LoadBasedRouterPolicy
return chosen.toId();
}
- private long getAvailableMemory(SubClusterInfo value)
- throws YarnException {
+ private long getAvailableMemory(SubClusterInfo value) throws YarnException {
try {
long mem = -1;
JSONObject obj = new JSONObject(value.getCapability());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14bf8ecd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/policies/router/PriorityRouterPolicy.java
index a8ac5f7..bc3a1f7 100644
---
a/hadoop-yarn-pro
[37/50] [abbrv] hadoop git commit: YARN-5519. Add SubClusterId in AddApplicationHomeSubClusterResponse for Router Failover. (Ellen Hui via Subru)
YARN-5519. Add SubClusterId in AddApplicationHomeSubClusterResponse for Router
Failover. (Ellen Hui via Subru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/367b08a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/367b08a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/367b08a2
Branch: refs/heads/YARN-2915
Commit: 367b08a2f4e3a018ed7a435baeafc6a4a265fe04
Parents: f95ad21
Author: Subru Krishnan
Authored: Mon Aug 15 14:47:02 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
...ederationApplicationHomeSubClusterStore.java | 21 +++---
.../store/impl/MemoryFederationStateStore.java | 22 +++---
.../AddApplicationHomeSubClusterResponse.java | 29 ++--
...ApplicationHomeSubClusterResponsePBImpl.java | 39 +++
.../proto/yarn_server_federation_protos.proto | 1 +
.../impl/FederationStateStoreBaseTest.java | 71 +---
6 files changed, 120 insertions(+), 63 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/367b08a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
index 22bb88a..ace2457 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationApplicationHomeSubClusterStore.java
@@ -51,15 +51,20 @@ public interface FederationApplicationHomeSubClusterStore {
/**
* Register the home {@code SubClusterId} of the newly submitted
* {@code ApplicationId}. Currently response is empty if the operation was
- * successful, if not an exception reporting reason for a failure.
+ * successful, if not an exception reporting reason for a failure. If a
+ * mapping for the application already existed, the {@code SubClusterId} in
+ * this response will return the existing mapping which might be different
+ * from that in the {@code AddApplicationHomeSubClusterRequest}.
*
* @param request the request to register a new application with its home
* sub-cluster
- * @return empty on successful registration of the application in the
- * StateStore, if not an exception reporting reason for a failure
+ * @return upon successful registration of the application in the StateStore,
+ * {@code AddApplicationHomeSubClusterRequest} containing the home
+ * sub-cluster of the application. Otherwise, an exception reporting
+ * reason for a failure
* @throws YarnException if the request is invalid/fails
*/
- AddApplicationHomeSubClusterResponse addApplicationHomeSubClusterMap(
+ AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
AddApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -73,7 +78,7 @@ public interface FederationApplicationHomeSubClusterStore {
* not an exception reporting reason for a failure
* @throws YarnException if the request is invalid/fails
*/
- UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubClusterMap(
+ UpdateApplicationHomeSubClusterResponse updateApplicationHomeSubCluster(
UpdateApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -85,7 +90,7 @@ public interface FederationApplicationHomeSubClusterStore {
* subcluster
* @throws YarnException if the request is invalid/fails
*/
- GetApplicationHomeSubClusterResponse getApplicationHomeSubClusterMap(
+ GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
GetApplicationHomeSubClusterRequest request) throws YarnException;
/**
@@ -96,7 +101,7 @@ public interface FederationApplicationHomeSubClusterStore {
* @return the mapping of all submitted application to it's home sub-cluster
* @throws YarnException if the request is invalid/fails
*/
- GetApplicationsHomeSubClusterResponse getApplicationsHomeSubClusterMap(
+ GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request)
[14/50] [abbrv] hadoop git commit: YARN-5390. Federation Subcluster Resolver. Contributed by Ellen Hui.
YARN-5390. Federation Subcluster Resolver. Contributed by Ellen Hui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe7fd959
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe7fd959
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe7fd959
Branch: refs/heads/YARN-2915
Commit: fe7fd959e69a6ad3025f290f22777efaaec89462
Parents: 1960090
Author: Subru Krishnan
Authored: Thu Aug 4 15:58:31 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:51 2017 -0700
--
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +
.../src/main/resources/yarn-default.xml | 7 +
.../hadoop-yarn-server-common/pom.xml | 10 +
.../resolver/AbstractSubClusterResolver.java| 67 +++
.../resolver/DefaultSubClusterResolverImpl.java | 164 +
.../federation/resolver/SubClusterResolver.java | 58 ++
.../federation/resolver/package-info.java | 17 ++
.../resolver/TestDefaultSubClusterResolver.java | 184 +++
.../src/test/resources/nodes| 4 +
.../src/test/resources/nodes-malformed | 3 +
10 files changed, 522 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7fd959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index fa4d2e3..4d1d56b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2530,6 +2530,14 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 20;
+ // Federation Configs
+
+
+ public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
+ public static final String FEDERATION_MACHINE_LIST =
+ FEDERATION_PREFIX + "machine-list";
+
+
// Other Configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7fd959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index bdd4de5..fc478bd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2651,6 +2651,13 @@
+
+ Machine list file to be loaded by the FederationSubCluster Resolver
+
+yarn.federation.machine-list
+
+
+
The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7fd959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 9cc3cae..6d2fbef 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -177,6 +177,16 @@
+
+org.apache.rat
+apache-rat-plugin
+
+
+src/test/resources/nodes
+src/test/resources/nodes-malformed
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe7fd959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/resolver/AbstractSubClusterResolver.java
b/hadoop-yarn-project/hadoop-yarn/
[15/50] [abbrv] hadoop git commit: YARN-3662. Federation Membership State Store internal APIs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17ddde75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
new file mode 100644
index 000..d4c5451
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/SubClusterDeregisterRequestPBImpl.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import
org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProto;
+import
org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterDeregisterRequestProtoOrBuilder;
+import
org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import
org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterStateProto;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Protocol buffer based implementation of {@link SubClusterDeregisterRequest}.
+ */
+@Private
+@Unstable
+public class SubClusterDeregisterRequestPBImpl
+extends SubClusterDeregisterRequest {
+
+ private SubClusterDeregisterRequestProto proto =
+ SubClusterDeregisterRequestProto.getDefaultInstance();
+ private SubClusterDeregisterRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public SubClusterDeregisterRequestPBImpl() {
+builder = SubClusterDeregisterRequestProto.newBuilder();
+ }
+
+ public SubClusterDeregisterRequestPBImpl(
+ SubClusterDeregisterRequestProto proto) {
+this.proto = proto;
+viaProto = true;
+ }
+
+ public SubClusterDeregisterRequestProto getProto() {
+mergeLocalToProto();
+proto = viaProto ? proto : builder.build();
+viaProto = true;
+return proto;
+ }
+
+ private void mergeLocalToProto() {
+if (viaProto) {
+ maybeInitBuilder();
+}
+mergeLocalToBuilder();
+proto = builder.build();
+viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+if (viaProto || builder == null) {
+ builder = SubClusterDeregisterRequestProto.newBuilder(proto);
+}
+viaProto = false;
+ }
+
+ private void mergeLocalToBuilder() {
+ }
+
+ @Override
+ public int hashCode() {
+return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+if (other == null) {
+ return false;
+}
+if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+}
+return false;
+ }
+
+ @Override
+ public String toString() {
+return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public SubClusterId getSubClusterId() {
+SubClusterDeregisterRequestProtoOrBuilder p = viaProto ? proto : builder;
+if (!p.hasSubClusterId()) {
+ return null;
+}
+return convertFromProtoFormat(p.getSubClusterId());
+ }
+
+ @Override
+ public void setSubClusterId(SubClusterId subClusterId) {
+maybeInitBuilder();
+if (subClusterId == null) {
+ builder.clearSubClusterId();
+ return;
+}
+builder.setSubClusterId(convertToProtoFormat(s
[17/50] [abbrv] hadoop git commit: YARN-5300. Exclude generated federation protobuf sources from YARN Javadoc/findbugs build
YARN-5300. Exclude generated federation protobuf sources from YARN Javadoc/findbugs build Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13d48e30 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13d48e30 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13d48e30 Branch: refs/heads/YARN-2915 Commit: 13d48e30403809ad73041a457d94f34323903500 Parents: 2ba21d6 Author: Subru Krishnan Authored: Tue Jul 19 15:08:25 2016 -0700 Committer: Subru Krishnan Committed: Mon Apr 24 18:58:51 2017 -0700 -- hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml | 3 +++ hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d48e30/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index c090749..2f5451d 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -21,6 +21,9 @@ + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d48e30/hadoop-yarn-project/hadoop-yarn/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml index c43588a..99b8b5f 100644 --- a/hadoop-yarn-project/hadoop-yarn/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/pom.xml @@ -75,7 +75,7 @@ org.apache.maven.plugins maven-javadoc-plugin - org.apache.hadoop.yarn.proto + org.apache.hadoop.yarn.proto:org.apache.hadoop.yarn.federation.proto - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-6203: Occasional test failure in TestWeightedRandomRouterPolicy (curino)
YARN-6203: Occasional test failure in TestWeightedRandomRouterPolicy (curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86c84e1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86c84e1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86c84e1b
Branch: refs/heads/YARN-2915
Commit: 86c84e1b19bfb7352cd266aa232b7fd68d990c35
Parents: 988f71c
Author: Carlo Curino
Authored: Thu Apr 13 12:09:39 2017 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:53 2017 -0700
--
.../router/TestWeightedRandomRouterPolicy.java| 18 +++---
1 file changed, 11 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/86c84e1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
index 34cc5f8..09173e6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/policies/router/TestWeightedRandomRouterPolicy.java
@@ -50,8 +50,9 @@ public class TestWeightedRandomRouterPolicy extends
BaseRouterPoliciesTest {
Map routerWeights = new HashMap<>();
Map amrmWeights = new HashMap<>();
-// simulate 20 subclusters with a 5% chance of being inactive
-for (int i = 0; i < 20; i++) {
+float numSubClusters = 20;
+// simulate N subclusters each with a 5% chance of being inactive
+for (int i = 0; i < numSubClusters; i++) {
SubClusterIdInfo sc = new SubClusterIdInfo("sc" + i);
// with 5% omit a subcluster
if (getRand().nextFloat() < 0.95f) {
@@ -60,8 +61,12 @@ public class TestWeightedRandomRouterPolicy extends
BaseRouterPoliciesTest {
when(sci.getSubClusterId()).thenReturn(sc.toId());
getActiveSubclusters().put(sc.toId(), sci);
}
- // 5% chance we omit one of the weights
- float weight = getRand().nextFloat();
+
+ // 80% of the weight is evenly spread, 20% is randomly generated
+ float weight =
+ (0.8f * 1f / numSubClusters) + (0.2f * getRand().nextFloat());
+
+ // also 5% chance we omit one of the weights
if (i <= 5 || getRand().nextFloat() > 0.05f) {
routerWeights.put(sc, weight);
amrmWeights.put(sc, weight);
@@ -89,7 +94,7 @@ public class TestWeightedRandomRouterPolicy extends
BaseRouterPoliciesTest {
counter.put(id.toId(), new AtomicLong(0));
}
-float numberOfDraws = 10;
+float numberOfDraws = 1;
for (float i = 0; i < numberOfDraws; i++) {
SubClusterId chosenId = ((FederationRouterPolicy) getPolicy())
@@ -118,8 +123,7 @@ public class TestWeightedRandomRouterPolicy extends
BaseRouterPoliciesTest {
Assert.assertTrue(
"Id " + counterEntry.getKey() + " Actual weight: " + actualWeight
+ " expected weight: " + expectedWeight,
-expectedWeight == 0 || (actualWeight / expectedWeight) < 1.2
-&& (actualWeight / expectedWeight) > 0.8);
+Math.abs(actualWeight - expectedWeight) < 0.01);
} else {
Assert
.assertTrue(
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-5410. Bootstrap Router server module. (Giovanni Matteo Fumarola via Subru).
YARN-5410. Bootstrap Router server module. (Giovanni Matteo Fumarola via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c19ece6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c19ece6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c19ece6e
Branch: refs/heads/YARN-2915
Commit: c19ece6e66eaed989ab7734dc8041d899130149e
Parents: ec7ab9c
Author: Subru Krishnan
Authored: Fri Feb 24 12:08:53 2017 -0800
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:52 2017 -0700
--
hadoop-project/pom.xml | 6 ++
.../hadoop-yarn-server-router/pom.xml | 62
.../hadoop/yarn/server/router/Router.java | 38
.../hadoop/yarn/server/router/package-info.java | 20 +++
.../hadoop/yarn/server/router/TestRouter.java | 26
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 +
hadoop-yarn-project/pom.xml | 4 ++
7 files changed, 157 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c19ece6e/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
old mode 100644
new mode 100755
index 3683a0f..34291ff
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -402,6 +402,12 @@
org.apache.hadoop
+hadoop-yarn-server-router
+${project.version}
+
+
+
+org.apache.hadoop
hadoop-mapreduce-client-jobclient
${project.version}
test-jar
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c19ece6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
new file mode 100644
index 000..25afa5c
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/pom.xml
@@ -0,0 +1,62 @@
+
+
+http://maven.apache.org/POM/4.0.0";
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+ http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+
+hadoop-yarn-server
+org.apache.hadoop
+3.0.0-alpha3-SNAPSHOT
+
+ 4.0.0
+ org.apache.hadoop
+ hadoop-yarn-server-router
+ 3.0.0-alpha3-SNAPSHOT
+ Apache Hadoop YARN Router
+
+
+
+${project.parent.parent.basedir}
+
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-api
+
+
+
+ org.apache.hadoop
+ hadoop-common
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-common
+
+
+
+ org.apache.hadoop
+ hadoop-yarn-server-common
+
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c19ece6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
new file mode 100644
index 000..7be8a59
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/Router.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.router;
+
+/**
+ * The router is a stateless YARN component which is the entry point to the
+ * cluster. It can be deployed on multiple nodes behind a Virtual IP (VIP) with
+ * a LoadBalancer.
+ *
+ * The Router exposes the A
[05/50] [abbrv] hadoop git commit: HDFS-11689. New exception thrown by DFSClient%isHDFSEncryptionEnabled broke hacky hive code. Contributed by Yongjun Zhang.
HDFS-11689. New exception thrown by DFSClient%isHDFSEncryptionEnabled broke
hacky hive code. Contributed by Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5078df7b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5078df7b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5078df7b
Branch: refs/heads/YARN-2915
Commit: 5078df7be317e635615c05c5da3285798993ff1f
Parents: a22fe02
Author: Yongjun Zhang
Authored: Fri Apr 21 13:36:31 2017 -0700
Committer: Yongjun Zhang
Committed: Fri Apr 21 13:36:31 2017 -0700
--
.../java/org/apache/hadoop/hdfs/DFSClient.java| 18 --
.../apache/hadoop/hdfs/DistributedFileSystem.java | 8 +---
2 files changed, 17 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5078df7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ef49950..7773891 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2944,10 +2944,24 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
/**
* Probe for encryption enabled on this filesystem.
+ * Note (see HDFS-11689):
+ * Not to throw exception in this method since it would break hive.
+ * Hive accesses this method and assumes no exception would be thrown.
+ * Hive should not access DFSClient since it is InterfaceAudience.Private.
+ * Deprecated annotation is added to trigger build warning at hive side.
+ * Request has been made to Hive to remove access to DFSClient.
* @return true if encryption is enabled
*/
- public boolean isHDFSEncryptionEnabled() throws IOException{
-return getKeyProviderUri() != null;
+ @Deprecated
+ public boolean isHDFSEncryptionEnabled() {
+boolean result = false;
+try {
+ result = (getKeyProviderUri() != null);
+} catch (IOException ioe) {
+ DFSClient.LOG.warn("Exception while checking whether encryption zone "
++ "is supported, assumes it is not supported", ioe);
+}
+return result;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5078df7b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index e1c2105..cfe7240 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2570,13 +2570,7 @@ public class DistributedFileSystem extends FileSystem {
*/
@Override
public Path getTrashRoot(Path path) {
-try {
- if ((path == null) || !dfs.isHDFSEncryptionEnabled()) {
-return super.getTrashRoot(path);
- }
-} catch (IOException ioe) {
- DFSClient.LOG.warn("Exception while checking whether encryption zone is "
- + "supported", ioe);
+if ((path == null) || !dfs.isHDFSEncryptionEnabled()) {
return super.getTrashRoot(path);
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[01/50] [abbrv] hadoop git commit: YARN-6363. Extending SLS: Synthetic Load Generator. (Carlo Curino via wangda) [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-2915 47451e0e3 -> 86c84e1b1 (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de69d6e8/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
--
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
new file mode 100644
index 000..3d2ec94
--- /dev/null
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/synthetic/SynthTraceJobProducer.java
@@ -0,0 +1,316 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.sls.synthetic;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.math3.random.JDKRandomGenerator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.tools.rumen.JobStory;
+import org.apache.hadoop.tools.rumen.JobStoryProducer;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.codehaus.jackson.JsonParser.Feature.INTERN_FIELD_NAMES;
+import static
org.codehaus.jackson.map.DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES;
+
+/**
+ * This is a JobStoryProducer that operates from distribution of different
+ * workloads. The .json input file is used to determine how many jobs, which
+ * size, number of maps/reducers and their duration, as well as the temporal
+ * distributed of submissions. For each parameter we control avg and stdev, and
+ * generate values via normal or log-normal distributions.
+ */
+public class SynthTraceJobProducer implements JobStoryProducer {
+
+ @SuppressWarnings("StaticVariableName")
+ private static final Log LOG =
LogFactory.getLog(SynthTraceJobProducer.class);
+
+ private final Configuration conf;
+ private final AtomicInteger numJobs;
+ private final Trace trace;
+ private final long seed;
+
+ private int totalWeight;
+ private final List weightList;
+ private final Map workloads;
+
+ private final Queue listStoryParams;
+
+ private final JDKRandomGenerator rand;
+
+ public static final String SLS_SYNTHETIC_TRACE_FILE =
+ "sls.synthetic" + ".trace_file";
+
+ public SynthTraceJobProducer(Configuration conf) throws IOException {
+this(conf, new Path(conf.get(SLS_SYNTHETIC_TRACE_FILE)));
+ }
+
+ public SynthTraceJobProducer(Configuration conf, Path path)
+ throws IOException {
+
+LOG.info("SynthTraceJobProducer");
+
+this.conf = conf;
+this.rand = new JDKRandomGenerator();
+workloads = new HashMap();
+weightList = new ArrayList();
+
+ObjectMapper mapper = new ObjectMapper();
+mapper.configure(INTERN_FIELD_NAMES, true);
+mapper.configure(FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+FileSystem ifs = path.getFileSystem(conf);
+FSDataInputStream fileIn = ifs.open(path);
+
+this.trace = mapper.readValue(fileIn, Trace.class);
+seed = trace.rand_seed;
+rand.setSeed(seed);
+
+this.numJobs = new AtomicInteger(trace.num_jobs);
+
+for (int workloadId = 0; workloadId < trace.workloads
+.size(); workloadId++) {
+ SynthWorkload workload = new SynthWorkload(workloadId, trace);
+ for (int classId =
+ 0; classId < trace.workloads.get(workloadId).job_classes
+ .size(); classId++) {
+SynthJobClass cls = new SynthJobClass(rand, trace, workload, classId);
+workload.add(cls);
+ }
+ workloads.put(workloadId, workload);
+}
+
+for (int i = 0; i < workloads.size(); i++) {
+ double w = workloads.get(i).getWorkloadWeight();
+ totalWeight += w;
+ weightList.add(w);
+}
+
+// create priority queue to keep start-time sorted
+listStoryPa
[30/50] [abbrv] hadoop git commit: YARN-5407. In-memory based implementation of the FederationApplicationStateStore/FederationPolicyStateStore. (Ellen Hui via Subru)
YARN-5407. In-memory based implementation of the FederationApplicationStateStore/FederationPolicyStateStore. (Ellen Hui via Subru) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f95ad216 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f95ad216 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f95ad216 Branch: refs/heads/YARN-2915 Commit: f95ad216695a7667aabc18c0145778e969d4863a Parents: 535e1f8 Author: Subru Krishnan Authored: Tue Aug 9 16:07:55 2016 -0700 Committer: Subru Krishnan Committed: Mon Apr 24 18:58:52 2017 -0700 -- .../store/impl/MemoryFederationStateStore.java | 158 +++- ...SubClusterPoliciesConfigurationsRequest.java | 2 +- ...ubClusterPoliciesConfigurationsResponse.java | 2 +- ...GetSubClusterPolicyConfigurationRequest.java | 3 +- ...etSubClusterPolicyConfigurationResponse.java | 2 +- ...SetSubClusterPolicyConfigurationRequest.java | 20 +- ...etSubClusterPolicyConfigurationResponse.java | 2 +- .../records/SubClusterPolicyConfiguration.java | 27 +- ...tApplicationHomeSubClusterRequestPBImpl.java | 4 + ...ClusterPolicyConfigurationRequestPBImpl.java | 17 - .../pb/SubClusterPolicyConfigurationPBImpl.java | 17 + .../proto/yarn_server_federation_protos.proto | 8 +- .../impl/FederationStateStoreBaseTest.java | 367 ++- .../impl/TestMemoryFederationStateStore.java| 4 +- 14 files changed, 558 insertions(+), 75 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f95ad216/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index cea4ac2..a540dff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -20,35 +20,72 @@ package org.apache.hadoop.yarn.server.federation.store.impl; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; -import org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; +import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClus
[18/50] [abbrv] hadoop git commit: YARN-5406. In-memory based implementation of the FederationMembershipStateStore. Contributed by Ellen Hui.
YARN-5406. In-memory based implementation of the
FederationMembershipStateStore. Contributed by Ellen Hui.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19600907
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19600907
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19600907
Branch: refs/heads/YARN-2915
Commit: 19600907e1255c0fcec3e74667ad52f1e8f2f035
Parents: 17ddde7
Author: Subru Krishnan
Authored: Thu Aug 4 15:54:38 2016 -0700
Committer: Subru Krishnan
Committed: Mon Apr 24 18:58:51 2017 -0700
--
.../store/impl/MemoryFederationStateStore.java | 138
.../federation/store/impl/package-info.java | 17 ++
.../records/GetSubClustersInfoRequest.java | 4 +
.../store/records/SubClusterState.java | 4 +
.../impl/FederationStateStoreBaseTest.java | 221 +++
.../impl/TestMemoryFederationStateStore.java| 49
6 files changed, 433 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19600907/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
new file mode 100644
index 000..7fdc4a9
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import
org.apache.hadoop.yarn.server.federation.store.FederationMembershipStateStore;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import
org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.util.MonotonicClock;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * In-memory implementation of FederationMembershipStateStore.
+ */
+public class MemoryFederationStateStore
+implements FederationMembershipStateStore {
+
+ private final Map membership =
+ new ConcurrentHashMap();
+ private final MonotonicClock clock = new MonotonicClock();
+
+ @Override
+ public Version getMembershipStateStoreVersion() {
+return null;
+ }
+
+ @Override
+ public SubClusterRegisterResponse registerSubCluster(
+ SubClusterRegisterRequest request) throws YarnEx
[11/50] [abbrv] hadoop git commit: YARN-6500. Do not mount inaccessible cgroups directories in CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
YARN-6500. Do not mount inaccessible cgroups directories in
CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ac50e13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ac50e13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ac50e13
Branch: refs/heads/YARN-2915
Commit: 8ac50e1322cb3f84bd998635924d85846aa47c94
Parents: 52adf71
Author: Haibo Chen
Authored: Mon Apr 24 11:37:52 2017 -0700
Committer: Haibo Chen
Committed: Mon Apr 24 11:37:52 2017 -0700
--
.../linux/resources/CGroupsHandlerImpl.java | 3 +-
.../util/CgroupsLCEResourcesHandler.java| 13 ++--
.../linux/resources/TestCGroupsHandlerImpl.java | 32
.../util/TestCgroupsLCEResourcesHandler.java| 29 ++
4 files changed, 73 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index d5295c5..0f4c17e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -232,7 +232,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
* @param entries map of paths to mount options
* @return the first mount path that has the requested subsystem
*/
- private static String findControllerInMtab(String controller,
+ @VisibleForTesting
+ static String findControllerInMtab(String controller,
Map> entries) {
for (Map.Entry> e : entries.entrySet()) {
if (e.getValue().contains(controller)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index f04fcd2..cb4dcf6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -428,11 +428,18 @@ public class CgroupsLCEResourcesHandler implements
LCEResourcesHandler {
return ret;
}
- private String findControllerInMtab(String controller,
+ @VisibleForTesting
+ String findControllerInMtab(String controller,
Map> entries) {
for (Entry> e : entries.entrySet()) {
- if (e.getValue().contains(controller))
-return e.getKey();
+ if (e.getValue().contains(controller)) {
+if (new File(e.getKey()).canRead()) {
+ return e.getKey();
+} else {
+ LOG.warn(String.format(
+ "Skipping inaccessible cgroup mount point %s", e.getKey()));
+}
+ }
}
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/r
[02/50] [abbrv] hadoop git commit: YARN-6363. Extending SLS: Synthetic Load Generator. (Carlo Curino via wangda)
YARN-6363. Extending SLS: Synthetic Load Generator. (Carlo Curino via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de69d6e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de69d6e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de69d6e8
Branch: refs/heads/YARN-2915
Commit: de69d6e81128470dd5d2fd865d4b3a79188f740b
Parents: 667966c
Author: Wangda Tan
Authored: Thu Apr 20 21:54:18 2017 -0700
Committer: Wangda Tan
Committed: Thu Apr 20 21:54:30 2017 -0700
--
.../org/apache/hadoop/tools/rumen/TaskInfo.java | 29 +-
.../apache/hadoop/tools/rumen/ZombieJob.java| 9 +-
hadoop-tools/hadoop-sls/pom.xml | 3 +
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 36 +-
.../hadoop/yarn/sls/ReservationClientUtil.java | 78 +++
.../org/apache/hadoop/yarn/sls/SLSRunner.java | 569 +--
.../hadoop/yarn/sls/appmaster/AMSimulator.java | 89 ++-
.../yarn/sls/appmaster/MRAMSimulator.java | 9 +-
.../sls/resourcemanager/MockAMLauncher.java | 5 +
.../sls/scheduler/SLSCapacityScheduler.java | 24 +-
.../yarn/sls/scheduler/SLSFairScheduler.java| 22 +-
.../hadoop/yarn/sls/scheduler/TaskRunner.java | 9 +-
.../hadoop/yarn/sls/synthetic/SynthJob.java | 306 ++
.../yarn/sls/synthetic/SynthJobClass.java | 180 ++
.../sls/synthetic/SynthTraceJobProducer.java| 316 ++
.../hadoop/yarn/sls/synthetic/SynthUtils.java | 101
.../yarn/sls/synthetic/SynthWorkload.java | 121
.../hadoop/yarn/sls/synthetic/package-info.java | 22 +
.../apache/hadoop/yarn/sls/utils/SLSUtils.java | 9 +
.../src/site/markdown/SchedulerLoadSimulator.md | 122 +++-
.../hadoop/yarn/sls/BaseSLSRunnerTest.java | 120
.../apache/hadoop/yarn/sls/TestSLSRunner.java | 90 +--
.../hadoop/yarn/sls/TestSynthJobGeneration.java | 96
.../yarn/sls/appmaster/TestAMSimulator.java | 2 +-
.../yarn/sls/scheduler/TestTaskRunner.java | 2 +-
.../src/test/resources/capacity-scheduler.xml | 10 +
.../src/test/resources/fair-scheduler.xml | 8 +-
.../hadoop-sls/src/test/resources/inputsls.json | 55 ++
.../hadoop-sls/src/test/resources/nodes.json| 84 +++
.../src/test/resources/sls-runner.xml | 6 +-
.../hadoop-sls/src/test/resources/syn.json | 53 ++
.../hadoop-sls/src/test/resources/yarn-site.xml | 10 +-
32 files changed, 2303 insertions(+), 292 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de69d6e8/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
--
diff --git
a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
index 9aa6373..6159f85 100644
---
a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
+++
b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/TaskInfo.java
@@ -23,21 +23,37 @@ public class TaskInfo {
private final long bytesOut;
private final int recsOut;
private final long maxMemory;
+ private final long maxVcores;
private final ResourceUsageMetrics metrics;
+
+ public TaskInfo(long bytesIn, int recsIn, long bytesOut, int recsOut,
+ long maxMemory) {
+this(bytesIn, recsIn, bytesOut, recsOut, maxMemory, 1,
+new ResourceUsageMetrics());
+ }
+
public TaskInfo(long bytesIn, int recsIn, long bytesOut, int recsOut,
- long maxMemory) {
-this(bytesIn, recsIn, bytesOut, recsOut, maxMemory,
+ long maxMemory, ResourceUsageMetrics
+ metrics) {
+this(bytesIn, recsIn, bytesOut, recsOut, maxMemory, 1, metrics);
+ }
+
+ public TaskInfo(long bytesIn, int recsIn, long bytesOut, int recsOut,
+ long maxMemory, long maxVcores) {
+this(bytesIn, recsIn, bytesOut, recsOut, maxMemory, maxVcores,
new ResourceUsageMetrics());
}
public TaskInfo(long bytesIn, int recsIn, long bytesOut, int recsOut,
- long maxMemory, ResourceUsageMetrics metrics) {
+ long maxMemory, long maxVcores, ResourceUsageMetrics
+ metrics) {
this.bytesIn = bytesIn;
this.recsIn = recsIn;
this.bytesOut = bytesOut;
this.recsOut = recsOut;
this.maxMemory = maxMemory;
+this.maxVcores = maxVcores;
this.metrics = metrics;
}
@@ -79,6 +95,13 @@ public class TaskInfo {
}
/**
+ * @return Vcores used by the task.
+ */
+ public long getTaskVCores() {
+return maxVcores;
+ }
+
+ /**
* @return Resource usage metrics
*/
public ResourceUsageMetrics getResourceUsage
[07/50] [abbrv] hadoop git commit: HDFS-11402. HDFS Snapshots should capture point-in-time copies of OPEN files. (Manoj Govindassamy via Yongjun Zhang)
HDFS-11402. HDFS Snapshots should capture point-in-time copies of OPEN files.
(Manoj Govindassamy via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20e3ae26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20e3ae26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20e3ae26
Branch: refs/heads/YARN-2915
Commit: 20e3ae260b40cd6ef657b2a629a02219d68f162f
Parents: 3721cfe
Author: Yongjun Zhang
Authored: Fri Apr 21 20:19:20 2017 -0700
Committer: Yongjun Zhang
Committed: Fri Apr 21 20:35:48 2017 -0700
--
.../hdfs/client/HdfsClientConfigKeys.java | 4 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
.../hdfs/server/namenode/FSDirSnapshotOp.java | 7 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 5 +-
.../hdfs/server/namenode/FSNamesystem.java | 2 +-
.../hdfs/server/namenode/INodeDirectory.java| 8 +-
.../hdfs/server/namenode/INodesInPath.java | 78 -
.../hdfs/server/namenode/LeaseManager.java | 103 ++-
.../snapshot/DirectorySnapshottableFeature.java | 15 +-
.../namenode/snapshot/SnapshotManager.java | 24 +-
.../src/main/resources/hdfs-default.xml | 17 ++
.../hdfs/server/namenode/TestLeaseManager.java | 286 +-
.../snapshot/TestOpenFilesWithSnapshot.java | 298 +++
.../snapshot/TestSnapshotDiffReport.java| 188 ++--
.../namenode/snapshot/TestSnapshotManager.java | 11 +-
15 files changed, 994 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20e3ae26/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index c152a4b..fbc8d89 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -171,6 +171,10 @@ public interface HdfsClientConfigKeys {
"dfs.data.transfer.client.tcpnodelay";
boolean DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT = true;
+ String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES =
+ "dfs.namenode.snapshot.capture.openfiles";
+ boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT = false;
+
/**
* These are deprecated config keys to client code.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20e3ae26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 6ff7e5a..3fa383b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -352,6 +352,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
public static final String
DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY =
"dfs.namenode.startup.delay.block.deletion.sec";
public static final long
DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT = 0L;
+ public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES =
+ HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES;
+ public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =
+ HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT;
+
// Whether to enable datanode's stale state detection and usage for reads
public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY =
"dfs.namenode.avoid.read.stale.datanode";
public static final boolean
DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20e3ae26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSnapshotOp.java
index ff076e4..9dd75bc 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/na
[13/50] [abbrv] hadoop git commit: YARN-6392. Add submit time to Application Summary log. (Zhihai Xu via wangda)
YARN-6392. Add submit time to Application Summary log. (Zhihai Xu via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ba21d63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ba21d63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ba21d63
Branch: refs/heads/YARN-2915
Commit: 2ba21d63767e11535d3210dc58a03b41e83df949
Parents: 5f934f8
Author: Wangda Tan
Authored: Mon Apr 24 13:17:35 2017 -0700
Committer: Wangda Tan
Committed: Mon Apr 24 13:17:35 2017 -0700
--
.../apache/hadoop/yarn/server/resourcemanager/RMAppManager.java| 1 +
.../apache/hadoop/yarn/server/resourcemanager/TestAppManager.java | 2 ++
2 files changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba21d63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index e0cff7b..e49c3ed 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -178,6 +178,7 @@ public class RMAppManager implements
EventHandler,
.add("state", app.getState())
.add("trackingUrl", trackingUrl)
.add("appMasterHost", host)
+ .add("submitTime", app.getSubmitTime())
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
.add("finalStatus", app.getFinalApplicationStatus())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba21d63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index eb69efa..db26a87 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -826,6 +826,7 @@ public class TestAppManager{
when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
+when(app.getSubmitTime()).thenReturn(1000L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
10, 1, 16384, 64, 0, 0);
@@ -842,6 +843,7 @@ public class TestAppManager{
Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+Assert.assertTrue(msg.contains("submitTime=1000"));
Assert.assertTrue(msg.contains("memorySeconds=16384"));
Assert.assertTrue(msg.contains("vcoreSeconds=64"));
Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-14261. Some refactoring work for erasure coding raw coder. Contributed by Lin Zeng.
HADOOP-14261. Some refactoring work for erasure coding raw coder. Contributed
by Lin Zeng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a22fe02f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a22fe02f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a22fe02f
Branch: refs/heads/YARN-2915
Commit: a22fe02fba66280a8e994282e9ead23d9e20669a
Parents: b080338
Author: Andrew Wang
Authored: Fri Apr 21 11:35:30 2017 -0700
Committer: Andrew Wang
Committed: Fri Apr 21 11:35:36 2017 -0700
--
.../apache/hadoop/io/erasurecode/CodecUtil.java | 4 +-
.../rawcoder/RSLegacyRawDecoder.java| 270 +++
.../rawcoder/RSLegacyRawEncoder.java| 129 +
.../RSLegacyRawErasureCoderFactory.java | 38 +++
.../rawcoder/RSRawDecoderLegacy.java| 270 ---
.../rawcoder/RSRawEncoderLegacy.java| 129 -
.../RSRawErasureCoderFactoryLegacy.java | 38 ---
.../src/main/resources/core-default.xml | 2 +-
.../erasurecode/TestCodecRawCoderMapping.java | 12 +-
.../rawcoder/RawErasureCoderBenchmark.java | 2 +-
.../erasurecode/rawcoder/TestDummyRawCoder.java | 4 +-
.../rawcoder/TestNativeRSRawCoder.java | 4 +-
.../rawcoder/TestNativeXORRawCoder.java | 4 +-
.../rawcoder/TestRSLegacyRawCoder.java | 33 +++
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 4 +-
.../rawcoder/TestRSRawCoderInteroperable1.java | 4 +-
.../rawcoder/TestRSRawCoderInteroperable2.java | 4 +-
.../rawcoder/TestRSRawCoderLegacy.java | 33 ---
.../erasurecode/rawcoder/TestRawCoderBase.java | 16 +-
.../erasurecode/rawcoder/TestXORRawCoder.java | 4 +-
.../rawcoder/TestXORRawCoderInteroperable1.java | 4 +-
.../rawcoder/TestXORRawCoderInteroperable2.java | 4 +-
22 files changed, 504 insertions(+), 508 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22fe02f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 0c66df6..c8b6a68 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
import
org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
import
org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
-import
org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactoryLegacy;
+import
org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
@@ -80,7 +80,7 @@ public final class CodecUtil {
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT =
- RSRawErasureCoderFactoryLegacy.class.getCanonicalName();
+ RSLegacyRawErasureCoderFactory.class.getCanonicalName();
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
IO_ERASURECODE_CODEC + "rs.rawcoders";
public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22fe02f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
new file mode 100644
index 000..cfd7d29
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawDecoder.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The A
[12/50] [abbrv] hadoop git commit: HADOOP-14305 S3A SSE tests won't run in parallel: Bad request in directory GetFileStatus. Contributed by Steve Moist.
HADOOP-14305 S3A SSE tests won't run in parallel: Bad request in directory GetFileStatus. Contributed by Steve Moist. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f934f83 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f934f83 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f934f83 Branch: refs/heads/YARN-2915 Commit: 5f934f838619ff73fcd80ad5fd6aee4c2134371a Parents: 8ac50e1 Author: Steve Loughran Authored: Mon Apr 24 20:15:42 2017 +0100 Committer: Steve Loughran Committed: Mon Apr 24 20:33:19 2017 +0100 -- hadoop-tools/hadoop-aws/pom.xml | 4 +- .../src/site/markdown/tools/hadoop-aws/index.md | 81 + .../fs/s3a/AbstractTestS3AEncryption.java | 4 + .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 308 ++- 4 files changed, 383 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f934f83/hadoop-tools/hadoop-aws/pom.xml -- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 03a3665..1084881 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -184,7 +184,7 @@ **/ITestS3AContractRootDir.java **/ITestS3NContractRootDir.java **/ITestS3AFileContextStatistics.java -**/ITestS3AEncryptionSSE*.java +**/ITestS3AEncryptionSSEC*.java **/ITestS3AHuge*.java @@ -214,7 +214,7 @@ **/ITestS3NContractRootDir.java **/ITestS3AFileContextStatistics.java **/ITestS3AHuge*.java -**/ITestS3AEncryptionSSE*.java +**/ITestS3AEncryptionSSEC*.java http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f934f83/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md -- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 2cde730..ceda79e 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1468,6 +1468,52 @@ basis. to set fadvise policies on input streams. Once implemented, this will become the supported mechanism used for configuring the input IO policy. + +### Encrypting objects with S3A + +Currently, S3A only supports S3's Server Side Encryption for at rest data encryption. +It is *encouraged* to read up on the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html) +for S3 Server Side Encryption before using these options as each behave differently +and the documentation will be more up to date on its behavior. When configuring +an encryption method in the `core-site.xml`, this will apply cluster wide. Any +new files written will be encrypted with this encryption configuration. Any +existing files when read, will decrypt using the existing method (if possible) +and will not be re-encrypted with the new method. It is also possible if mixing +multiple keys that the user does not have access to decrypt the object. It is +**NOT** advised to mix and match encryption types in a bucket, and is *strongly* +recommended to just one type and key per bucket. + +SSE-S3 is where S3 will manage the encryption keys for each object. The parameter +for `fs.s3a.server-side-encryption-algorithm` is `AES256`. + +SSE-KMS is where the user specifies a Customer Master Key(CMK) that is used to +encrypt the objects. The user may specify a specific CMK or leave the +`fs.s3a.server-side-encryption-key` empty to use the default auto-generated key +in AWS IAM. Each CMK configured in AWS IAM is region specific, and cannot be +used in a in a S3 bucket in a different region. There is can also be policies +assigned to the CMK that prohibit or restrict its use for users causing S3A +requests to fail. + +SSE-C is where the user specifies an actual base64 encoded AES-256 key supplied +and managed by the user. + + SSE-C Warning + +It is strongly recommended to fully understand how SSE-C works in the S3 +environment before using this encryption type. Please refer to the Server Side +Encryption documentation available from AWS. SSE-C is only recommended for +advanced users with advanced encryption use cases. Failure to properly manage +encryption keys can cause data loss. Currently, the AWS S3 API(and thus S3A) +only supports one encryption key and cannot support de
hadoop git commit: YARN-6392. Add submit time to Application Summary log. (Zhihai Xu via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 49a2f286f -> 503dd650b
YARN-6392. Add submit time to Application Summary log. (Zhihai Xu via wangda)
(cherry picked from commit 2ba21d63767e11535d3210dc58a03b41e83df949)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/503dd650
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/503dd650
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/503dd650
Branch: refs/heads/branch-2
Commit: 503dd650bff138d2eba30218525cf3aeefa6c205
Parents: 49a2f28
Author: Wangda Tan
Authored: Mon Apr 24 13:17:35 2017 -0700
Committer: Wangda Tan
Committed: Mon Apr 24 13:18:47 2017 -0700
--
.../apache/hadoop/yarn/server/resourcemanager/RMAppManager.java| 1 +
.../apache/hadoop/yarn/server/resourcemanager/TestAppManager.java | 2 ++
2 files changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/503dd650/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 0e6aa39..11610ff 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -178,6 +178,7 @@ public class RMAppManager implements
EventHandler,
.add("state", app.getState())
.add("trackingUrl", trackingUrl)
.add("appMasterHost", host)
+ .add("submitTime", app.getSubmitTime())
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
.add("finalStatus", app.getFinalApplicationStatus())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/503dd650/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 8ec3eae..ea3117f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -821,6 +821,7 @@ public class TestAppManager{
when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
+when(app.getSubmitTime()).thenReturn(1000L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
10, 1, 16384, 64, 0, 0);
@@ -837,6 +838,7 @@ public class TestAppManager{
Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+Assert.assertTrue(msg.contains("submitTime=1000"));
Assert.assertTrue(msg.contains("memorySeconds=16384"));
Assert.assertTrue(msg.contains("vcoreSeconds=64"));
Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-6392. Add submit time to Application Summary log. (Zhihai Xu via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 5f934f838 -> 2ba21d637
YARN-6392. Add submit time to Application Summary log. (Zhihai Xu via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ba21d63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ba21d63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ba21d63
Branch: refs/heads/trunk
Commit: 2ba21d63767e11535d3210dc58a03b41e83df949
Parents: 5f934f8
Author: Wangda Tan
Authored: Mon Apr 24 13:17:35 2017 -0700
Committer: Wangda Tan
Committed: Mon Apr 24 13:17:35 2017 -0700
--
.../apache/hadoop/yarn/server/resourcemanager/RMAppManager.java| 1 +
.../apache/hadoop/yarn/server/resourcemanager/TestAppManager.java | 2 ++
2 files changed, 3 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba21d63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index e0cff7b..e49c3ed 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -178,6 +178,7 @@ public class RMAppManager implements
EventHandler,
.add("state", app.getState())
.add("trackingUrl", trackingUrl)
.add("appMasterHost", host)
+ .add("submitTime", app.getSubmitTime())
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
.add("finalStatus", app.getFinalApplicationStatus())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ba21d63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index eb69efa..db26a87 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -826,6 +826,7 @@ public class TestAppManager{
when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
+when(app.getSubmitTime()).thenReturn(1000L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
10, 1, 16384, 64, 0, 0);
@@ -842,6 +843,7 @@ public class TestAppManager{
Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+Assert.assertTrue(msg.contains("submitTime=1000"));
Assert.assertTrue(msg.contains("memorySeconds=16384"));
Assert.assertTrue(msg.contains("vcoreSeconds=64"));
Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: HADOOP-14305 S3A SSE tests won't run in parallel: Bad request in directory GetFileStatus. Contributed by Steve Moist.
Repository: hadoop Updated Branches: refs/heads/trunk 8ac50e132 -> 5f934f838 HADOOP-14305 S3A SSE tests won't run in parallel: Bad request in directory GetFileStatus. Contributed by Steve Moist. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f934f83 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f934f83 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f934f83 Branch: refs/heads/trunk Commit: 5f934f838619ff73fcd80ad5fd6aee4c2134371a Parents: 8ac50e1 Author: Steve Loughran Authored: Mon Apr 24 20:15:42 2017 +0100 Committer: Steve Loughran Committed: Mon Apr 24 20:33:19 2017 +0100 -- hadoop-tools/hadoop-aws/pom.xml | 4 +- .../src/site/markdown/tools/hadoop-aws/index.md | 81 + .../fs/s3a/AbstractTestS3AEncryption.java | 4 + .../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 308 ++- 4 files changed, 383 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f934f83/hadoop-tools/hadoop-aws/pom.xml -- diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index 03a3665..1084881 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -184,7 +184,7 @@ **/ITestS3AContractRootDir.java **/ITestS3NContractRootDir.java **/ITestS3AFileContextStatistics.java -**/ITestS3AEncryptionSSE*.java +**/ITestS3AEncryptionSSEC*.java **/ITestS3AHuge*.java @@ -214,7 +214,7 @@ **/ITestS3NContractRootDir.java **/ITestS3AFileContextStatistics.java **/ITestS3AHuge*.java -**/ITestS3AEncryptionSSE*.java +**/ITestS3AEncryptionSSEC*.java http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f934f83/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md -- diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 2cde730..ceda79e 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1468,6 +1468,52 @@ basis. to set fadvise policies on input streams. Once implemented, this will become the supported mechanism used for configuring the input IO policy. + +### Encrypting objects with S3A + +Currently, S3A only supports S3's Server Side Encryption for at rest data encryption. +It is *encouraged* to read up on the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html) +for S3 Server Side Encryption before using these options as each behave differently +and the documentation will be more up to date on its behavior. When configuring +an encryption method in the `core-site.xml`, this will apply cluster wide. Any +new files written will be encrypted with this encryption configuration. Any +existing files when read, will decrypt using the existing method (if possible) +and will not be re-encrypted with the new method. It is also possible if mixing +multiple keys that the user does not have access to decrypt the object. It is +**NOT** advised to mix and match encryption types in a bucket, and is *strongly* +recommended to just one type and key per bucket. + +SSE-S3 is where S3 will manage the encryption keys for each object. The parameter +for `fs.s3a.server-side-encryption-algorithm` is `AES256`. + +SSE-KMS is where the user specifies a Customer Master Key(CMK) that is used to +encrypt the objects. The user may specify a specific CMK or leave the +`fs.s3a.server-side-encryption-key` empty to use the default auto-generated key +in AWS IAM. Each CMK configured in AWS IAM is region specific, and cannot be +used in a in a S3 bucket in a different region. There is can also be policies +assigned to the CMK that prohibit or restrict its use for users causing S3A +requests to fail. + +SSE-C is where the user specifies an actual base64 encoded AES-256 key supplied +and managed by the user. + + SSE-C Warning + +It is strongly recommended to fully understand how SSE-C works in the S3 +environment before using this encryption type. Please refer to the Server Side +Encryption documentation available from AWS. SSE-C is only recommended for +advanced users with advanced encryption use cases. Failure to properly manage +encryption keys can cause data loss. Currently, the AWS
hadoop git commit: YARN-6500. Do not mount inaccessible cgroups directories in CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 421e51cb9 -> 49a2f286f
YARN-6500. Do not mount inaccessible cgroups directories in
CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
(cherry picked from commit 8ac50e1322cb3f84bd998635924d85846aa47c94)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49a2f286
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49a2f286
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49a2f286
Branch: refs/heads/branch-2
Commit: 49a2f286f72bab644fd6aa55b5bc51ae47120e1f
Parents: 421e51c
Author: Haibo Chen
Authored: Mon Apr 24 11:37:52 2017 -0700
Committer: Haibo Chen
Committed: Mon Apr 24 11:39:55 2017 -0700
--
.../linux/resources/CGroupsHandlerImpl.java | 3 +-
.../util/CgroupsLCEResourcesHandler.java| 13 ++--
.../linux/resources/TestCGroupsHandlerImpl.java | 32
.../util/TestCgroupsLCEResourcesHandler.java| 29 ++
4 files changed, 73 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index d5295c5..0f4c17e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -232,7 +232,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
* @param entries map of paths to mount options
* @return the first mount path that has the requested subsystem
*/
- private static String findControllerInMtab(String controller,
+ @VisibleForTesting
+ static String findControllerInMtab(String controller,
Map> entries) {
for (Map.Entry> e : entries.entrySet()) {
if (e.getValue().contains(controller)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index f04fcd2..cb4dcf6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -428,11 +428,18 @@ public class CgroupsLCEResourcesHandler implements
LCEResourcesHandler {
return ret;
}
- private String findControllerInMtab(String controller,
+ @VisibleForTesting
+ String findControllerInMtab(String controller,
Map> entries) {
for (Entry> e : entries.entrySet()) {
- if (e.getValue().contains(controller))
-return e.getKey();
+ if (e.getValue().contains(controller)) {
+if (new File(e.getKey()).canRead()) {
+ return e.getKey();
+} else {
+ LOG.warn(String.format(
+ "Skipping inaccessible cgroup mount point %s", e.getKey()));
+}
+ }
}
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49a2f286/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
--
diff --git
a/hadoop-ya
hadoop git commit: YARN-6500. Do not mount inaccessible cgroups directories in CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
Repository: hadoop
Updated Branches:
refs/heads/trunk 52adf7191 -> 8ac50e132
YARN-6500. Do not mount inaccessible cgroups directories in
CgroupsLCEResourcesHandler. (Miklos Szegedi via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ac50e13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ac50e13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ac50e13
Branch: refs/heads/trunk
Commit: 8ac50e1322cb3f84bd998635924d85846aa47c94
Parents: 52adf71
Author: Haibo Chen
Authored: Mon Apr 24 11:37:52 2017 -0700
Committer: Haibo Chen
Committed: Mon Apr 24 11:37:52 2017 -0700
--
.../linux/resources/CGroupsHandlerImpl.java | 3 +-
.../util/CgroupsLCEResourcesHandler.java| 13 ++--
.../linux/resources/TestCGroupsHandlerImpl.java | 32
.../util/TestCgroupsLCEResourcesHandler.java| 29 ++
4 files changed, 73 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index d5295c5..0f4c17e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -232,7 +232,8 @@ class CGroupsHandlerImpl implements CGroupsHandler {
* @param entries map of paths to mount options
* @return the first mount path that has the requested subsystem
*/
- private static String findControllerInMtab(String controller,
+ @VisibleForTesting
+ static String findControllerInMtab(String controller,
Map> entries) {
for (Map.Entry> e : entries.entrySet()) {
if (e.getValue().contains(controller)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index f04fcd2..cb4dcf6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -428,11 +428,18 @@ public class CgroupsLCEResourcesHandler implements
LCEResourcesHandler {
return ret;
}
- private String findControllerInMtab(String controller,
+ @VisibleForTesting
+ String findControllerInMtab(String controller,
Map> entries) {
for (Entry> e : entries.entrySet()) {
- if (e.getValue().contains(controller))
-return e.getKey();
+ if (e.getValue().contains(controller)) {
+if (new File(e.getKey()).canRead()) {
+ return e.getKey();
+} else {
+ LOG.warn(String.format(
+ "Skipping inaccessible cgroup mount point %s", e.getKey()));
+}
+ }
}
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ac50e13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src
hadoop git commit: YARN-5617. AMs only intended to run one attempt can be run more than once? Contributed by Jason Lowe.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a9495b206 -> 421e51cb9
YARN-5617. AMs only intended to run one attempt can be run more than once?
Contributed by Jason Lowe.
(cherry picked from commit 52adf719143c20f4f2af369c6c40dd98677e7410)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/421e51cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/421e51cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/421e51cb
Branch: refs/heads/branch-2
Commit: 421e51cb9c13e58f527f531495ddf2f62ae45b69
Parents: a9495b2
Author: Eric Payne
Authored: Mon Apr 24 12:03:28 2017 -0500
Committer: Eric Payne
Committed: Mon Apr 24 12:08:35 2017 -0500
--
.../server/resourcemanager/rmapp/RMAppImpl.java | 24 ++--
.../applicationsmanager/TestAMRestart.java | 138 +++
2 files changed, 124 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/421e51cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 41f6e04..89af924 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1432,14 +1432,22 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public RMAppState transition(RMAppImpl app, RMAppEvent event) {
int numberOfFailure = app.getNumFailedAppAttempts();
- LOG.info("The number of failed attempts"
- + (app.attemptFailuresValidityInterval > 0 ? " in previous "
- + app.attemptFailuresValidityInterval + " milliseconds " : " ")
- + "is " + numberOfFailure + ". The max attempts is "
- + app.maxAppAttempts);
-
- if (app.attemptFailuresValidityInterval > 0) {
-removeExcessAttempts(app);
+ if (app.maxAppAttempts == 1) {
+// If the user explicitly set the attempts to 1 then there are likely
+// correctness issues if the AM restarts for any reason.
+LOG.info("Max app attempts is 1 for " + app.applicationId
++ ", preventing further attempts.");
+numberOfFailure = app.maxAppAttempts;
+ } else {
+LOG.info("The number of failed attempts"
++ (app.attemptFailuresValidityInterval > 0 ? " in previous "
++ app.attemptFailuresValidityInterval + " milliseconds " : " ")
++ "is " + numberOfFailure + ". The max attempts is "
++ app.maxAppAttempts);
+
+if (app.attemptFailuresValidityInterval > 0) {
+ removeExcessAttempts(app);
+}
}
if (!app.submissionContext.getUnmanagedAM()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/421e51cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index f5da5b5..f178884 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -378,8 +378,7 @@ public class TestAMRestart {
YarnConfiguration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
-// explicitly set max-am-retry count as 1.
-conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
+conf.
hadoop git commit: YARN-5617. AMs only intended to run one attempt can be run more than once? Contributed by Jason Lowe.
Repository: hadoop
Updated Branches:
refs/heads/trunk 561718e05 -> 52adf7191
YARN-5617. AMs only intended to run one attempt can be run more than once?
Contributed by Jason Lowe.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52adf719
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52adf719
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52adf719
Branch: refs/heads/trunk
Commit: 52adf719143c20f4f2af369c6c40dd98677e7410
Parents: 561718e
Author: Eric Payne
Authored: Mon Apr 24 12:03:28 2017 -0500
Committer: Eric Payne
Committed: Mon Apr 24 12:03:28 2017 -0500
--
.../server/resourcemanager/rmapp/RMAppImpl.java | 24 ++--
.../applicationsmanager/TestAMRestart.java | 138 +++
2 files changed, 124 insertions(+), 38 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52adf719/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 531844c..78df913 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1517,14 +1517,22 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public RMAppState transition(RMAppImpl app, RMAppEvent event) {
int numberOfFailure = app.getNumFailedAppAttempts();
- LOG.info("The number of failed attempts"
- + (app.attemptFailuresValidityInterval > 0 ? " in previous "
- + app.attemptFailuresValidityInterval + " milliseconds " : " ")
- + "is " + numberOfFailure + ". The max attempts is "
- + app.maxAppAttempts);
-
- if (app.attemptFailuresValidityInterval > 0) {
-removeExcessAttempts(app);
+ if (app.maxAppAttempts == 1) {
+// If the user explicitly set the attempts to 1 then there are likely
+// correctness issues if the AM restarts for any reason.
+LOG.info("Max app attempts is 1 for " + app.applicationId
++ ", preventing further attempts.");
+numberOfFailure = app.maxAppAttempts;
+ } else {
+LOG.info("The number of failed attempts"
++ (app.attemptFailuresValidityInterval > 0 ? " in previous "
++ app.attemptFailuresValidityInterval + " milliseconds " : " ")
++ "is " + numberOfFailure + ". The max attempts is "
++ app.maxAppAttempts);
+
+if (app.attemptFailuresValidityInterval > 0) {
+ removeExcessAttempts(app);
+}
}
if (!app.submissionContext.getUnmanagedAM()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52adf719/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
index f5da5b5..f178884 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
@@ -378,8 +378,7 @@ public class TestAMRestart {
YarnConfiguration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
ResourceScheduler.class);
-// explicitly set max-am-retry count as 1.
-conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
+conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
conf.setBoolean(YarnCo
hadoop git commit: YARN-6291. Introduce query parameters (sort, filter, etc.) for tables to keep state on refresh/navigation in new YARN UI. Contributed by Gergely Novák.
Repository: hadoop
Updated Branches:
refs/heads/trunk fda86ef2a -> 561718e05
YARN-6291. Introduce query parameters (sort, filter, etc.) for tables to keep
state on refresh/navigation in new YARN UI. Contributed by Gergely Novák.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/561718e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/561718e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/561718e0
Branch: refs/heads/trunk
Commit: 561718e05de51c0cb7c17295d7713d52408918eb
Parents: fda86ef
Author: Sunil G
Authored: Mon Apr 24 13:28:57 2017 +0530
Committer: Sunil G
Committed: Mon Apr 24 13:28:57 2017 +0530
--
.../src/main/webapp/app/controllers/yarn-apps/apps.js | 10 ++
.../src/main/webapp/app/controllers/yarn-apps/services.js | 10 ++
.../src/main/webapp/app/controllers/yarn-nodes/table.js | 8
.../src/main/webapp/app/templates/yarn-apps/apps.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-apps/services.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-nodes/table.hbs| 2 +-
6 files changed, 31 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/561718e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
index 0b0be20..44a7c1c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js
@@ -16,7 +16,17 @@
* limitations under the License.
*/
+import Ember from 'ember';
+import ColumnDef from 'em-table/utils/column-definition';
+import TableDefinition from 'em-table/utils/table-definition';
import AppTableController from '../app-table-columns';
export default AppTableController.extend({
+ queryParams: ['searchText', 'sortColumnId', 'sortOrder', 'pageNum',
'rowCount'],
+ tableDefinition: TableDefinition.create(),
+ searchText: Ember.computed.alias('tableDefinition.searchText'),
+ sortColumnId: Ember.computed.alias('tableDefinition.sortColumnId'),
+ sortOrder: Ember.computed.alias('tableDefinition.sortOrder'),
+ pageNum: Ember.computed.alias('tableDefinition.pageNum'),
+ rowCount: Ember.computed.alias('tableDefinition.rowCount')
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/561718e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
index 0b0be20..44a7c1c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/services.js
@@ -16,7 +16,17 @@
* limitations under the License.
*/
+import Ember from 'ember';
+import ColumnDef from 'em-table/utils/column-definition';
+import TableDefinition from 'em-table/utils/table-definition';
import AppTableController from '../app-table-columns';
export default AppTableController.extend({
+ queryParams: ['searchText', 'sortColumnId', 'sortOrder', 'pageNum',
'rowCount'],
+ tableDefinition: TableDefinition.create(),
+ searchText: Ember.computed.alias('tableDefinition.searchText'),
+ sortColumnId: Ember.computed.alias('tableDefinition.sortColumnId'),
+ sortOrder: Ember.computed.alias('tableDefinition.sortOrder'),
+ pageNum: Ember.computed.alias('tableDefinition.pageNum'),
+ rowCount: Ember.computed.alias('tableDefinition.rowCount')
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/561718e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
index 3b7b0f6..3fae596 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
@@ -19,8 +19,16 @@
import Ember from 'ember';
