[hadoop] branch trunk updated: YARN-10512. CS Flexible Auto Queue Creation: Modify RM /scheduler endpoint to include mode of operation for CS. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4b5bc05 YARN-10512. CS Flexible Auto Queue Creation: Modify RM
/scheduler endpoint to include mode of operation for CS. Contributed by Szilard
Nemeth.
4b5bc05 is described below
commit 4b5bc05a788ddacf99dd0374e70d87369dd03ae5
Author: Sunil G
AuthorDate: Wed Jan 20 12:28:59 2021 +0530
YARN-10512. CS Flexible Auto Queue Creation: Modify RM /scheduler endpoint
to include mode of operation for CS. Contributed by Szilard Nemeth.
---
.../scheduler/capacity/AbstractCSQueue.java| 2 +-
.../webapp/dao/CapacitySchedulerInfo.java | 8 +
.../webapp/dao/CapacitySchedulerQueueInfo.java | 8 +
.../dao/helper/CapacitySchedulerInfoHelper.java| 44
.../webapp/dao/helper/package-info.java| 27 ++
.../CapacitySchedulerConfigGeneratorForTest.java | 2 +-
.../webapp/TestRMWebServicesCapacitySched.java | 6 +-
...estRMWebServicesCapacitySchedDynamicConfig.java | 274 +
.../TestRMWebServicesForCSWithPartitions.java | 2 +-
9 files changed, 367 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fd144f2..c02e1d3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -128,7 +128,7 @@ public abstract class AbstractCSQueue implements CSQueue {
// either at this level or anywhere in the queue's hierarchy.
private volatile boolean defaultAppLifetimeWasSpecifiedInConfig = false;
- protected enum CapacityConfigType {
+ public enum CapacityConfigType {
// FIXME, from what I can see, Percentage mode can almost apply to weighted
// and percentage mode at the same time, there's only small area need to be
// changed, we need to rename "PERCENTAGE" to "PERCENTAGE" and "WEIGHT"
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 8de7529..9794acd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -32,6 +32,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.helper.CapacitySchedulerInfoHelper;
import java.util.ArrayList;
import java.util.Map;
@@ -53,6 +54,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
protected QueueAclsInfo queueAcls;
protected int queuePriority;
protected String orderingPolicyInfo;
+ protected String mode;
@XmlTransient
static final float EPSILON = 1e-8f;
@@ -98,6 +100,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
orderingPolicyInfo = ((ParentQueue) parent).getQueueOrderingPolicy()
.getConfigName();
}
+mode = CapacitySchedulerInfoHelper.getMode(parent);
}
public float getCapacity() {
@@ -173,4 +176,9 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
}
return queuesInfo;
}
+
+ public String getMode() {
+return mode;
+ }
+
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-ya
[hadoop] branch branch-3.2.2 updated: HADOOP-17329. mvn site commands fails due to MetricsSystemImpl changes. Contributed by Xiaoqiao He.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2.2 by this push:
new df8b54b HADOOP-17329. mvn site commands fails due to
MetricsSystemImpl changes. Contributed by Xiaoqiao He.
df8b54b is described below
commit df8b54bb4ef2a0438388df085089207d0f098dad
Author: Sunil G
AuthorDate: Thu Oct 29 07:19:36 2020 +0530
HADOOP-17329. mvn site commands fails due to MetricsSystemImpl changes.
Contributed by Xiaoqiao He.
(cherry picked from commit f17e067d527b0089124a8ca69b36fb20ed118c55)
---
.../dev-support/jdiff-workaround.patch | 29 +++---
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
index 8f87d40..2bd7b63 100644
--- a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
@@ -1,11 +1,12 @@
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-index a277abd..ed7c709 100644
+index a277abd6e13..1d131d5db6e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-@@ -43,18 +43,6 @@
+@@ -42,18 +42,6 @@
+ @InterfaceAudience.Private
public abstract MetricsSystem init(String prefix);
- /**
+- /**
- * Register a metrics source
- * @paramthe actual type of the source object
- * @param source object to register
@@ -17,15 +18,11 @@ index a277abd..ed7c709 100644
- */
- public abstract T register(String name, String desc, T source);
-
-- /**
+ /**
* Unregister a metrics source
* @param name of the source. This is the name you use to call register()
-*/
-@@ -77,18 +65,19 @@
-*/
- @InterfaceAudience.Private
+@@ -79,16 +67,16 @@
public abstract MetricsSource getSource(String name);
-+
/**
- * Register a metrics sink
@@ -50,10 +47,10 @@ index a277abd..ed7c709 100644
/**
* Register a callback interface for JMX events
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
-index 6986edb..eeea81f 100644
+index a6edf08e5a7..5b87be1ec67 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
-@@ -270,27 +270,6 @@ void registerSource(String name, String desc,
MetricsSource source) {
+@@ -269,31 +269,6 @@ void registerSource(String name, String desc,
MetricsSource source) {
LOG.debug("Registered source "+ name);
}
@@ -61,7 +58,11 @@ index 6986edb..eeea81f 100644
- T register(final String name, final String description, final T sink) {
-LOG.debug(name +", "+ description);
-if (allSinks.containsKey(name)) {
-- LOG.warn("Sink "+ name +" already exists!");
+- if(sinks.get(name) == null) {
+-registerSink(name, description, sink);
+- } else {
+-LOG.warn("Sink "+ name +" already exists!");
+- }
- return sink;
-}
-allSinks.put(name, sink);
@@ -82,10 +83,10 @@ index 6986edb..eeea81f 100644
checkNotNull(config, "config");
MetricsConfig conf = sinkConfigs.get(name);
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
-index c19d238..f8412f1 100644
+index 7bc772f062a..f012c877d18 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
-@@ -130,8 +130,8 @@ public void testTagsForPrefix() throws Exception {
+@@ -139,8 +139,8 @@ public void testTagsForPrefix() throws Exception {
GangliaMetricsTestHelper.setDatagramSocket(gsink31, mockds31);
// register the sinks
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: HADOOP-17329. mvn site commands fails due to MetricsSystemImpl changes. Contributed by Xiaoqiao He.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 14a4606 HADOOP-17329. mvn site commands fails due to
MetricsSystemImpl changes. Contributed by Xiaoqiao He.
14a4606 is described below
commit 14a4606a9a47cf2c7726c804d1e6043d1dd12381
Author: Sunil G
AuthorDate: Thu Oct 29 07:19:36 2020 +0530
HADOOP-17329. mvn site commands fails due to MetricsSystemImpl changes.
Contributed by Xiaoqiao He.
(cherry picked from commit f17e067d527b0089124a8ca69b36fb20ed118c55)
---
.../dev-support/jdiff-workaround.patch | 29 +++---
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
index 8f87d40..2bd7b63 100644
--- a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
@@ -1,11 +1,12 @@
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-index a277abd..ed7c709 100644
+index a277abd6e13..1d131d5db6e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-@@ -43,18 +43,6 @@
+@@ -42,18 +42,6 @@
+ @InterfaceAudience.Private
public abstract MetricsSystem init(String prefix);
- /**
+- /**
- * Register a metrics source
- * @paramthe actual type of the source object
- * @param source object to register
@@ -17,15 +18,11 @@ index a277abd..ed7c709 100644
- */
- public abstract T register(String name, String desc, T source);
-
-- /**
+ /**
* Unregister a metrics source
* @param name of the source. This is the name you use to call register()
-*/
-@@ -77,18 +65,19 @@
-*/
- @InterfaceAudience.Private
+@@ -79,16 +67,16 @@
public abstract MetricsSource getSource(String name);
-+
/**
- * Register a metrics sink
@@ -50,10 +47,10 @@ index a277abd..ed7c709 100644
/**
* Register a callback interface for JMX events
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
-index 6986edb..eeea81f 100644
+index a6edf08e5a7..5b87be1ec67 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
-@@ -270,27 +270,6 @@ void registerSource(String name, String desc,
MetricsSource source) {
+@@ -269,31 +269,6 @@ void registerSource(String name, String desc,
MetricsSource source) {
LOG.debug("Registered source "+ name);
}
@@ -61,7 +58,11 @@ index 6986edb..eeea81f 100644
- T register(final String name, final String description, final T sink) {
-LOG.debug(name +", "+ description);
-if (allSinks.containsKey(name)) {
-- LOG.warn("Sink "+ name +" already exists!");
+- if(sinks.get(name) == null) {
+-registerSink(name, description, sink);
+- } else {
+-LOG.warn("Sink "+ name +" already exists!");
+- }
- return sink;
-}
-allSinks.put(name, sink);
@@ -82,10 +83,10 @@ index 6986edb..eeea81f 100644
checkNotNull(config, "config");
MetricsConfig conf = sinkConfigs.get(name);
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
-index c19d238..f8412f1 100644
+index 7bc772f062a..f012c877d18 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
-@@ -130,8 +130,8 @@ public void testTagsForPrefix() throws Exception {
+@@ -139,8 +139,8 @@ public void testTagsForPrefix() throws Exception {
GangliaMetricsTestHelper.setDatagramSocket(gsink31, mockds31);
// register the sinks
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HADOOP-17329. mvn site commands fails due to MetricsSystemImpl changes. Contributed by Xiaoqiao He.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 91a3d29 HADOOP-17329. mvn site commands fails due to
MetricsSystemImpl changes. Contributed by Xiaoqiao He.
91a3d29 is described below
commit 91a3d298b9a70a1ebecdea28e5cf5142c78ca591
Author: Sunil G
AuthorDate: Thu Oct 29 07:19:36 2020 +0530
HADOOP-17329. mvn site commands fails due to MetricsSystemImpl changes.
Contributed by Xiaoqiao He.
(cherry picked from commit f17e067d527b0089124a8ca69b36fb20ed118c55)
---
.../dev-support/jdiff-workaround.patch | 29 +++---
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
index 8f87d40..2bd7b63 100644
--- a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
@@ -1,11 +1,12 @@
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-index a277abd..ed7c709 100644
+index a277abd6e13..1d131d5db6e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-@@ -43,18 +43,6 @@
+@@ -42,18 +42,6 @@
+ @InterfaceAudience.Private
public abstract MetricsSystem init(String prefix);
- /**
+- /**
- * Register a metrics source
- * @paramthe actual type of the source object
- * @param source object to register
@@ -17,15 +18,11 @@ index a277abd..ed7c709 100644
- */
- public abstract T register(String name, String desc, T source);
-
-- /**
+ /**
* Unregister a metrics source
* @param name of the source. This is the name you use to call register()
-*/
-@@ -77,18 +65,19 @@
-*/
- @InterfaceAudience.Private
+@@ -79,16 +67,16 @@
public abstract MetricsSource getSource(String name);
-+
/**
- * Register a metrics sink
@@ -50,10 +47,10 @@ index a277abd..ed7c709 100644
/**
* Register a callback interface for JMX events
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
-index 6986edb..eeea81f 100644
+index a6edf08e5a7..5b87be1ec67 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
-@@ -270,27 +270,6 @@ void registerSource(String name, String desc,
MetricsSource source) {
+@@ -269,31 +269,6 @@ void registerSource(String name, String desc,
MetricsSource source) {
LOG.debug("Registered source "+ name);
}
@@ -61,7 +58,11 @@ index 6986edb..eeea81f 100644
- T register(final String name, final String description, final T sink) {
-LOG.debug(name +", "+ description);
-if (allSinks.containsKey(name)) {
-- LOG.warn("Sink "+ name +" already exists!");
+- if(sinks.get(name) == null) {
+-registerSink(name, description, sink);
+- } else {
+-LOG.warn("Sink "+ name +" already exists!");
+- }
- return sink;
-}
-allSinks.put(name, sink);
@@ -82,10 +83,10 @@ index 6986edb..eeea81f 100644
checkNotNull(config, "config");
MetricsConfig conf = sinkConfigs.get(name);
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
-index c19d238..f8412f1 100644
+index 7bc772f062a..f012c877d18 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
-@@ -130,8 +130,8 @@ public void testTagsForPrefix() throws Exception {
+@@ -139,8 +139,8 @@ public void testTagsForPrefix() throws Exception {
GangliaMetricsTestHelper.setDatagramSocket(gsink31, mockds31);
// register the sinks
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HADOOP-17329. mvn site commands fails due to MetricsSystemImpl changes. Contributed by Xiaoqiao He.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new f17e067 HADOOP-17329. mvn site commands fails due to
MetricsSystemImpl changes. Contributed by Xiaoqiao He.
f17e067 is described below
commit f17e067d527b0089124a8ca69b36fb20ed118c55
Author: Sunil G
AuthorDate: Thu Oct 29 07:19:36 2020 +0530
HADOOP-17329. mvn site commands fails due to MetricsSystemImpl changes.
Contributed by Xiaoqiao He.
---
.../dev-support/jdiff-workaround.patch | 29 +++---
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
index 8f87d40..2bd7b63 100644
--- a/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
+++ b/hadoop-common-project/hadoop-common/dev-support/jdiff-workaround.patch
@@ -1,11 +1,12 @@
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-index a277abd..ed7c709 100644
+index a277abd6e13..1d131d5db6e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/MetricsSystem.java
-@@ -43,18 +43,6 @@
+@@ -42,18 +42,6 @@
+ @InterfaceAudience.Private
public abstract MetricsSystem init(String prefix);
- /**
+- /**
- * Register a metrics source
- * @paramthe actual type of the source object
- * @param source object to register
@@ -17,15 +18,11 @@ index a277abd..ed7c709 100644
- */
- public abstract T register(String name, String desc, T source);
-
-- /**
+ /**
* Unregister a metrics source
* @param name of the source. This is the name you use to call register()
-*/
-@@ -77,18 +65,19 @@
-*/
- @InterfaceAudience.Private
+@@ -79,16 +67,16 @@
public abstract MetricsSource getSource(String name);
-+
/**
- * Register a metrics sink
@@ -50,10 +47,10 @@ index a277abd..ed7c709 100644
/**
* Register a callback interface for JMX events
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
-index 6986edb..eeea81f 100644
+index a6edf08e5a7..5b87be1ec67 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSystemImpl.java
-@@ -270,27 +270,6 @@ void registerSource(String name, String desc,
MetricsSource source) {
+@@ -269,31 +269,6 @@ void registerSource(String name, String desc,
MetricsSource source) {
LOG.debug("Registered source "+ name);
}
@@ -61,7 +58,11 @@ index 6986edb..eeea81f 100644
- T register(final String name, final String description, final T sink) {
-LOG.debug(name +", "+ description);
-if (allSinks.containsKey(name)) {
-- LOG.warn("Sink "+ name +" already exists!");
+- if(sinks.get(name) == null) {
+-registerSink(name, description, sink);
+- } else {
+-LOG.warn("Sink "+ name +" already exists!");
+- }
- return sink;
-}
-allSinks.put(name, sink);
@@ -82,10 +83,10 @@ index 6986edb..eeea81f 100644
checkNotNull(config, "config");
MetricsConfig conf = sinkConfigs.get(name);
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
-index c19d238..f8412f1 100644
+index 7bc772f062a..f012c877d18 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGangliaMetrics.java
-@@ -130,8 +130,8 @@ public void testTagsForPrefix() throws Exception {
+@@ -139,8 +139,8 @@ public void testTagsForPrefix() throws Exception {
GangliaMetricsTestHelper.setDatagramSocket(gsink31, mockds31);
// register the sinks
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10453. Add partition resource info to get-node-labels and label-mappings api responses. Contributed Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7435604 YARN-10453. Add partition resource info to get-node-labels
and label-mappings api responses. Contributed Akhil PB.
7435604 is described below
commit 7435604a91a49f5c5717083fbaee74dd8ec1c426
Author: Sunil G
AuthorDate: Tue Oct 20 15:53:44 2020 +0530
YARN-10453. Add partition resource info to get-node-labels and
label-mappings api responses. Contributed Akhil PB.
---
.../server/resourcemanager/webapp/NodeIDsInfo.java | 16 ++
.../resourcemanager/webapp/RMWebServices.java | 18 +-
.../resourcemanager/webapp/dao/NodeLabelInfo.java | 10
.../{NodeIDsInfo.java => dao/PartitionInfo.java} | 30 --
.../webapp/TestRMWebServicesNodeLabels.java| 67 +-
5 files changed, 118 insertions(+), 23 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
index 5f45b96..b492793 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
+
import java.util.ArrayList;
import java.util.List;
@@ -39,6 +43,9 @@ public class NodeIDsInfo {
@XmlElement(name="nodes")
protected ArrayList nodeIDsList = new ArrayList();
+ @XmlElement(name = "partitionInfo")
+ private PartitionInfo partitionInfo;
+
public NodeIDsInfo() {
} // JAXB needs this
@@ -46,7 +53,16 @@ public class NodeIDsInfo {
this.nodeIDsList.addAll(nodeIdsList);
}
+ public NodeIDsInfo(List nodeIdsList, Resource resource) {
+this(nodeIdsList);
+this.partitionInfo = new PartitionInfo(new ResourceInfo(resource));
+ }
+
public ArrayList getNodeIDs() {
return nodeIDsList;
}
+
+ public PartitionInfo getPartitionInfo() {
+return partitionInfo;
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 02ad3ab..2a725cf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -184,6 +184,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntr
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.RMQueueAclInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo;
@@ -209,6 +210,7 @@ import
org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
import org.apache.hadoop.yarn.util.AdHocLogDumper;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -1296,8 +1298,10 @@ public class RMWebServices extends WebServices
implements RMWebServiceProtocol {
for (NodeId nodeId : entry.getValue()) {
nodeIdStrList.add(nodeId.toString());
}
+ Resource resource = rm.getRMContex
[hadoop] branch branch-3.2 updated: Revert "YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke."
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 94723bf Revert "YARN-10396. Max applications calculation per queue
disregards queue level settings in absolute mode. Contributed by Benjamin Teke."
94723bf is described below
commit 94723bff64ecb976b6f2d9d9a5b20b8c6eb4c6c6
Author: Sunil G
AuthorDate: Thu Aug 20 19:15:10 2020 +0530
Revert "YARN-10396. Max applications calculation per queue disregards queue
level settings in absolute mode. Contributed by Benjamin Teke."
This reverts commit 2a40a33dfecb17eba42f67c0151be9b1e86740aa.
---
.../scheduler/capacity/ParentQueue.java| 14 +--
.../scheduler/capacity/TestParentQueue.java| 134 +++--
2 files changed, 16 insertions(+), 132 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index ee74f78..531e520 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1119,18 +1119,8 @@ public class ParentQueue extends AbstractCSQueue {
if (childQueue instanceof LeafQueue) {
LeafQueue leafQueue = (LeafQueue) childQueue;
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- int maxApplications =
- conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath());
- if (maxApplications < 0) {
-int maxGlobalPerQueueApps =
conf.getGlobalMaximumApplicationsPerQueue();
-if (maxGlobalPerQueueApps > 0) {
- maxApplications = (int) (maxGlobalPerQueueApps *
- childQueue.getQueueCapacities().getAbsoluteCapacity(label));
-} else {
- maxApplications = (int) (conf.getMaximumSystemApplications()
- *
childQueue.getQueueCapacities().getAbsoluteCapacity(label));
-}
- }
+ int maxApplications = (int) (conf.getMaximumSystemApplications()
+ * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
leafQueue.setMaxApplications(maxApplications);
int maxApplicationsPerUser = Math.min(maxApplications,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 6cd1573..7a012a8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -110,17 +110,15 @@ public class TestParentQueue {
private static final String A = "a";
private static final String B = "b";
- private static final String Q_A =
- CapacitySchedulerConfiguration.ROOT + "." + A;
- private static final String Q_B =
- CapacitySchedulerConfiguration.ROOT + "." + B;
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
+final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
+final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 70);
LOG.info("Setup top-level queues a and b");
@@ -132,9 +130,11 @@ public class TestParentQueue {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
+final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setMinimumResourceRequirement("", Q_A,
QUEUE_A_RESOURCE);
+final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setMinim
[hadoop] branch branch-3.1 updated: Revert "YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke."
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 2f7e584 Revert "YARN-10396. Max applications calculation per queue
disregards queue level settings in absolute mode. Contributed by Benjamin Teke."
2f7e584 is described below
commit 2f7e584c21f6ad8456d14046ab2bfa8d5977dbf7
Author: Sunil G
AuthorDate: Thu Aug 20 19:13:03 2020 +0530
Revert "YARN-10396. Max applications calculation per queue disregards queue
level settings in absolute mode. Contributed by Benjamin Teke."
This reverts commit 7369fc31b456e2b90893212f499a30755fab5cc9.
---
.../scheduler/capacity/ParentQueue.java| 14 +--
.../scheduler/capacity/TestParentQueue.java| 134 +++--
2 files changed, 16 insertions(+), 132 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 2201728..e080d1c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1114,18 +1114,8 @@ public class ParentQueue extends AbstractCSQueue {
if (childQueue instanceof LeafQueue) {
LeafQueue leafQueue = (LeafQueue) childQueue;
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- int maxApplications =
- conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath());
- if (maxApplications < 0) {
-int maxGlobalPerQueueApps =
conf.getGlobalMaximumApplicationsPerQueue();
-if (maxGlobalPerQueueApps > 0) {
- maxApplications = (int) (maxGlobalPerQueueApps *
- childQueue.getQueueCapacities().getAbsoluteCapacity(label));
-} else {
- maxApplications = (int) (conf.getMaximumSystemApplications()
- *
childQueue.getQueueCapacities().getAbsoluteCapacity(label));
-}
- }
+ int maxApplications = (int) (conf.getMaximumSystemApplications()
+ * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
leafQueue.setMaxApplications(maxApplications);
int maxApplicationsPerUser = Math.min(maxApplications,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 896bcb1..fe66aba 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -109,17 +109,15 @@ public class TestParentQueue {
private static final String A = "a";
private static final String B = "b";
- private static final String Q_A =
- CapacitySchedulerConfiguration.ROOT + "." + A;
- private static final String Q_B =
- CapacitySchedulerConfiguration.ROOT + "." + B;
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
+final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
+final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 70);
LOG.info("Setup top-level queues a and b");
@@ -131,9 +129,11 @@ public class TestParentQueue {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
+final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setMinimumResourceRequirement("", Q_A,
QUEUE_A_RESOURCE);
+final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setMinim
[hadoop] branch branch-3.1 updated: YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 7369fc3 YARN-10396. Max applications calculation per queue disregards
queue level settings in absolute mode. Contributed by Benjamin Teke.
7369fc3 is described below
commit 7369fc31b456e2b90893212f499a30755fab5cc9
Author: Sunil G
AuthorDate: Wed Aug 19 11:54:48 2020 +0530
YARN-10396. Max applications calculation per queue disregards queue level
settings in absolute mode. Contributed by Benjamin Teke.
(cherry picked from commit 82ec28f4421c162a505ba5e5b329e4be199878a7)
---
.../scheduler/capacity/ParentQueue.java| 14 ++-
.../scheduler/capacity/TestParentQueue.java| 134 ++---
2 files changed, 132 insertions(+), 16 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index e080d1c..2201728 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1114,8 +1114,18 @@ public class ParentQueue extends AbstractCSQueue {
if (childQueue instanceof LeafQueue) {
LeafQueue leafQueue = (LeafQueue) childQueue;
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- int maxApplications = (int) (conf.getMaximumSystemApplications()
- * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+ int maxApplications =
+ conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath());
+ if (maxApplications < 0) {
+int maxGlobalPerQueueApps =
conf.getGlobalMaximumApplicationsPerQueue();
+if (maxGlobalPerQueueApps > 0) {
+ maxApplications = (int) (maxGlobalPerQueueApps *
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+} else {
+ maxApplications = (int) (conf.getMaximumSystemApplications()
+ *
childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+}
+ }
leafQueue.setMaxApplications(maxApplications);
int maxApplicationsPerUser = Math.min(maxApplications,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index fe66aba..896bcb1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -109,15 +109,17 @@ public class TestParentQueue {
private static final String A = "a";
private static final String B = "b";
+ private static final String Q_A =
+ CapacitySchedulerConfiguration.ROOT + "." + A;
+ private static final String Q_B =
+ CapacitySchedulerConfiguration.ROOT + "." + B;
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 70);
LOG.info("Setup top-level queues a and b");
@@ -129,11 +131,9 @@ public class TestParentQueue {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setMinimumResourceRequirement("", Q_A,
QUEUE_A_RESOURCE);
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setMinimumResourceRequiremen
[hadoop] branch branch-3.2 updated: YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 2a40a33 YARN-10396. Max applications calculation per queue disregards
queue level settings in absolute mode. Contributed by Benjamin Teke.
2a40a33 is described below
commit 2a40a33dfecb17eba42f67c0151be9b1e86740aa
Author: Sunil G
AuthorDate: Wed Aug 19 11:54:48 2020 +0530
YARN-10396. Max applications calculation per queue disregards queue level
settings in absolute mode. Contributed by Benjamin Teke.
(cherry picked from commit 82ec28f4421c162a505ba5e5b329e4be199878a7)
---
.../scheduler/capacity/ParentQueue.java| 14 ++-
.../scheduler/capacity/TestParentQueue.java| 134 ++---
2 files changed, 132 insertions(+), 16 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 531e520..ee74f78 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1119,8 +1119,18 @@ public class ParentQueue extends AbstractCSQueue {
if (childQueue instanceof LeafQueue) {
LeafQueue leafQueue = (LeafQueue) childQueue;
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- int maxApplications = (int) (conf.getMaximumSystemApplications()
- * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+ int maxApplications =
+ conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath());
+ if (maxApplications < 0) {
+int maxGlobalPerQueueApps =
conf.getGlobalMaximumApplicationsPerQueue();
+if (maxGlobalPerQueueApps > 0) {
+ maxApplications = (int) (maxGlobalPerQueueApps *
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+} else {
+ maxApplications = (int) (conf.getMaximumSystemApplications()
+ *
childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+}
+ }
leafQueue.setMaxApplications(maxApplications);
int maxApplicationsPerUser = Math.min(maxApplications,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 7a012a8..6cd1573 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -110,15 +110,17 @@ public class TestParentQueue {
private static final String A = "a";
private static final String B = "b";
+ private static final String Q_A =
+ CapacitySchedulerConfiguration.ROOT + "." + A;
+ private static final String Q_B =
+ CapacitySchedulerConfiguration.ROOT + "." + B;
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 70);
LOG.info("Setup top-level queues a and b");
@@ -130,11 +132,9 @@ public class TestParentQueue {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setMinimumResourceRequirement("", Q_A,
QUEUE_A_RESOURCE);
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setMinimumResourceRequiremen
[hadoop] branch branch-3.3 updated: YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new ea1c048 YARN-10396. Max applications calculation per queue disregards
queue level settings in absolute mode. Contributed by Benjamin Teke.
ea1c048 is described below
commit ea1c048a61163dad837a8009168ba4fcb60d0ca5
Author: Sunil G
AuthorDate: Wed Aug 19 11:54:48 2020 +0530
YARN-10396. Max applications calculation per queue disregards queue level
settings in absolute mode. Contributed by Benjamin Teke.
(cherry picked from commit 82ec28f4421c162a505ba5e5b329e4be199878a7)
---
.../scheduler/capacity/ParentQueue.java| 14 ++-
.../scheduler/capacity/TestParentQueue.java| 134 ++---
2 files changed, 132 insertions(+), 16 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 07377d8..1f31d8b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1124,8 +1124,18 @@ public class ParentQueue extends AbstractCSQueue {
if (childQueue instanceof LeafQueue) {
LeafQueue leafQueue = (LeafQueue) childQueue;
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- int maxApplications = (int) (conf.getMaximumSystemApplications()
- * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+ int maxApplications =
+ conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath());
+ if (maxApplications < 0) {
+int maxGlobalPerQueueApps =
conf.getGlobalMaximumApplicationsPerQueue();
+if (maxGlobalPerQueueApps > 0) {
+ maxApplications = (int) (maxGlobalPerQueueApps *
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+} else {
+ maxApplications = (int) (conf.getMaximumSystemApplications()
+ *
childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+}
+ }
leafQueue.setMaxApplications(maxApplications);
int maxApplicationsPerUser = Math.min(maxApplications,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 0560d59..9ed0388 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -108,15 +108,17 @@ public class TestParentQueue {
private static final String A = "a";
private static final String B = "b";
+ private static final String Q_A =
+ CapacitySchedulerConfiguration.ROOT + "." + A;
+ private static final String Q_B =
+ CapacitySchedulerConfiguration.ROOT + "." + B;
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 70);
LOG.info("Setup top-level queues a and b");
@@ -128,11 +130,9 @@ public class TestParentQueue {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setMinimumResourceRequirement("", Q_A,
QUEUE_A_RESOURCE);
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setMinimumResourceRequiremen
[hadoop] branch trunk updated: YARN-10396. Max applications calculation per queue disregards queue level settings in absolute mode. Contributed by Benjamin Teke.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 82ec28f YARN-10396. Max applications calculation per queue disregards
queue level settings in absolute mode. Contributed by Benjamin Teke.
82ec28f is described below
commit 82ec28f4421c162a505ba5e5b329e4be199878a7
Author: Sunil G
AuthorDate: Wed Aug 19 11:54:48 2020 +0530
YARN-10396. Max applications calculation per queue disregards queue level
settings in absolute mode. Contributed by Benjamin Teke.
---
.../scheduler/capacity/ParentQueue.java| 14 ++-
.../scheduler/capacity/TestParentQueue.java| 134 ++---
2 files changed, 132 insertions(+), 16 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index bbb80ba..923e6875 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1123,8 +1123,18 @@ public class ParentQueue extends AbstractCSQueue {
if (childQueue instanceof LeafQueue) {
LeafQueue leafQueue = (LeafQueue) childQueue;
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- int maxApplications = (int) (conf.getMaximumSystemApplications()
- * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+ int maxApplications =
+ conf.getMaximumApplicationsPerQueue(childQueue.getQueuePath());
+ if (maxApplications < 0) {
+int maxGlobalPerQueueApps =
conf.getGlobalMaximumApplicationsPerQueue();
+if (maxGlobalPerQueueApps > 0) {
+ maxApplications = (int) (maxGlobalPerQueueApps *
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+} else {
+ maxApplications = (int) (conf.getMaximumSystemApplications()
+ *
childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+}
+ }
leafQueue.setMaxApplications(maxApplications);
int maxApplicationsPerUser = Math.min(maxApplications,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 0560d59..9ed0388 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -108,15 +108,17 @@ public class TestParentQueue {
private static final String A = "a";
private static final String B = "b";
+ private static final String Q_A =
+ CapacitySchedulerConfiguration.ROOT + "." + A;
+ private static final String Q_B =
+ CapacitySchedulerConfiguration.ROOT + "." + B;
private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 70);
LOG.info("Setup top-level queues a and b");
@@ -128,11 +130,9 @@ public class TestParentQueue {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
-final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setMinimumResourceRequirement("", Q_A,
QUEUE_A_RESOURCE);
-final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setMinimumResourceRequirement("", Q_B,
QUEUE_B_RESOURCE);
@@ -368,9 +368,7 @@ public class TestParentQueue {
pub
[hadoop] branch trunk updated: YARN-10333. YarnClient obtain Delegation Token for Log Aggregation Path. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 5dd270e YARN-10333. YarnClient obtain Delegation Token for Log
Aggregation Path. Contributed by Prabhu Joseph.
5dd270e is described below
commit 5dd270e2085c8e8c3428287ed6f0c541a5548a31
Author: Sunil G
AuthorDate: Thu Jul 9 12:50:25 2020 +0530
YARN-10333. YarnClient obtain Delegation Token for Log Aggregation Path.
Contributed by Prabhu Joseph.
---
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 13 +++
.../yarn/client/api/impl/YarnClientImpl.java | 56 +
.../yarn/client/api/impl/TestYarnClientImpl.java | 127 +
3 files changed, 196 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index 15e0eaf..e563118 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -39,6 +39,19 @@
+ org.apache.hadoop
+ hadoop-hdfs
+ test
+
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ test-jar
+ test
+
+
+
com.google.guava
guava
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 14133ba..ebc4e76 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -30,9 +30,12 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.Future;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
@@ -131,6 +134,8 @@ import
org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import
org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController;
+import
org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import
org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -314,6 +319,16 @@ public class YarnClientImpl extends YarnClient {
addTimelineDelegationToken(appContext.getAMContainerSpec());
}
+// Automatically add the DT for Log Aggregation path
+// This is useful when a separate storage is used for log aggregation
+try {
+ if (isSecurityEnabled()) {
+addLogAggregationDelegationToken(appContext.getAMContainerSpec());
+ }
+} catch (Exception e) {
+ LOG.warn("Failed to obtain delegation token for Log Aggregation Path",
e);
+}
+
//TODO: YARN-1763:Handle RM failovers during the submitApplication call.
rmClient.submitApplication(request);
@@ -373,6 +388,47 @@ public class YarnClientImpl extends YarnClient {
return applicationId;
}
+ private void addLogAggregationDelegationToken(
+ ContainerLaunchContext clc) throws YarnException, IOException {
+Credentials credentials = new Credentials();
+DataInputByteBuffer dibb = new DataInputByteBuffer();
+ByteBuffer tokens = clc.getTokens();
+if (tokens != null) {
+ dibb.reset(tokens);
+ credentials.readTokenStorageStream(dibb);
+ tokens.rewind();
+}
+
+Configuration conf = getConfig();
+String masterPrincipal = YarnClientUtils.getRmPrincipal(conf);
+if (StringUtils.isEmpty(masterPrincipal)) {
+ throw new IOException(
+ "Can't get Master Kerberos principal for use as renewer");
+}
+LOG.debug("Delegation Token Renewer: " + masterPrincipal);
+
+LogAggregationFileControllerFactory factory =
+new LogAggregationFileControllerFactory(conf);
+LogAggregationFileController fileController =
+factory.getFileControllerForWrite();
+Path remoteRootLogDir = fileController.getRemoteRootLogDir();
+FileSystem fs = remoteRootLogDi
[hadoop] branch trunk updated: YARN-10154. CS Dynamic Queues cannot be configured with absolute resources. Contributed by Manikandan R.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2fe122e YARN-10154. CS Dynamic Queues cannot be configured with
absolute resources. Contributed by Manikandan R.
2fe122e is described below
commit 2fe122e322afd35dfae6e45b3e7fdaddd20a411f
Author: Sunil G
AuthorDate: Fri Apr 17 18:51:05 2020 +0530
YARN-10154. CS Dynamic Queues cannot be configured with absolute resources.
Contributed by Manikandan R.
---
.../capacity/AbstractAutoCreatedLeafQueue.java | 17 ++
.../scheduler/capacity/AbstractCSQueue.java| 18 +-
.../scheduler/capacity/AutoCreatedLeafQueue.java | 6 +-
.../capacity/CapacitySchedulerConfiguration.java | 46 +++-
.../scheduler/capacity/ManagedParentQueue.java | 65 -
.../scheduler/capacity/ParentQueue.java| 5 +-
.../GuaranteedOrZeroCapacityOverTimePolicy.java| 15 ++
.../TestAbsoluteResourceConfiguration.java | 104 +++-
.../TestAbsoluteResourceWithAutoQueue.java | 277 +
.../TestCapacitySchedulerAutoCreatedQueueBase.java | 25 +-
.../src/site/markdown/CapacityScheduler.md | 21 +-
11 files changed, 573 insertions(+), 26 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
index bed2aa5..b022b12 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractAutoCreatedLeafQueue.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler
.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common
@@ -67,6 +68,22 @@ public class AbstractAutoCreatedLeafQueue extends LeafQueue {
setEntitlement(NO_LABEL, entitlement);
}
+ @Override
+ protected Resource getMinimumAbsoluteResource(String queuePath,
+ String label) {
+return super.getMinimumAbsoluteResource(csContext.getConfiguration()
+
.getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
+label);
+ }
+
+ @Override
+ protected Resource getMaximumAbsoluteResource(String queuePath,
+ String label) {
+return super.getMaximumAbsoluteResource(csContext.getConfiguration()
+
.getAutoCreatedQueueTemplateConfPrefix(this.getParent().getQueuePath()),
+label);
+ }
+
/**
* This methods to change capacity for a queue and adjusts its
* absoluteCapacity
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 9177430..bcb255e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -524,16 +524,26 @@ public abstract class AbstractCSQueue implements CSQueue {
return unionInheritedWeights;
}
+ protected Resource getMinimumAbsoluteResource(String queuePath, String
label) {
+Resource minResource = csContext.getConfiguration()
+.getMinimumResourceRequirement(label, queuePath, resourceTypes);
+return minResource;
+ }
+
+ protected Resource getMaximumAbsoluteResource(String queuePath, String
label) {
+Resource maxResource = csContext.getConfiguration()
+.getMaximumResourceRequirement(label, queuePath, resourceTypes);
+return maxResource;
+ }
+
protected void updateConfigurableResourceRequirement(String queuePath,
Resource clusterResource) {
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
Set configuredNodelabels
[hadoop] branch branch-3.3 updated: YARN-10226. NPE in Capacity Scheduler while using %primary_group queue mapping. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 94df6b5 YARN-10226. NPE in Capacity Scheduler while using
%primary_group queue mapping. Contributed by Peter Bacsko.
94df6b5 is described below
commit 94df6b513bbf5826c830c1ff588d26461d10cb7c
Author: Sunil G
AuthorDate: Thu Apr 9 18:51:42 2020 +0530
YARN-10226. NPE in Capacity Scheduler while using %primary_group queue
mapping. Contributed by Peter Bacsko.
(cherry picked from commit 6fb29d0f1710cbef021e58846eb5c67a8d0b1f87)
---
.../placement/UserGroupMappingPlacementRule.java | 18 ++
.../placement/TestUserGroupMappingPlacementRule.java | 1 +
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index 4f97582..391fb34 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -161,17 +161,19 @@ public class UserGroupMappingPlacementRule extends
PlacementRule {
String group =
CapacitySchedulerConfiguration.ROOT + "." + getPrimaryGroup(user);
-CSQueue parent = queueManager.getQueue(mapping.getParentQueue());
+String parent = mapping.getParentQueue();
+CSQueue groupQueue = queueManager.getQueue(group);
-if (parent instanceof ManagedParentQueue) {
- return getPlacementContext(mapping, group);
-} else {
- CSQueue queue = this.queueManager.getQueue(group);
- if ( queue != null) {
-return getPlacementContext(mapping, queue.getQueuePath());
+if (parent != null) {
+ CSQueue parentQueue = queueManager.getQueue(parent);
+
+ if (parentQueue instanceof ManagedParentQueue) {
+return getPlacementContext(mapping, group);
} else {
-return null;
+return groupQueue == null ? null : getPlacementContext(mapping, group);
}
+} else {
+ return groupQueue == null ? null : getPlacementContext(mapping, group);
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
index 95a0a80..1d7b6b7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
@@ -93,6 +93,7 @@ public class TestUserGroupMappingPlacementRule {
when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
when(queueManager.getQueue("managedParent")).thenReturn(managedParent);
+when(queueManager.getQueue(null)).thenThrow(new NullPointerException());
when(queueManager.getQueue("root.agroup")).thenReturn(agroup);
when(queueManager.getQueue("root.bsubgroup2")).thenReturn(bsubgroup2);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10226. NPE in Capacity Scheduler while using %primary_group queue mapping. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6fb29d0 YARN-10226. NPE in Capacity Scheduler while using
%primary_group queue mapping. Contributed by Peter Bacsko.
6fb29d0 is described below
commit 6fb29d0f1710cbef021e58846eb5c67a8d0b1f87
Author: Sunil G
AuthorDate: Thu Apr 9 18:51:42 2020 +0530
YARN-10226. NPE in Capacity Scheduler while using %primary_group queue
mapping. Contributed by Peter Bacsko.
---
.../placement/UserGroupMappingPlacementRule.java | 18 ++
.../placement/TestUserGroupMappingPlacementRule.java | 1 +
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index 4f97582..391fb34 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -161,17 +161,19 @@ public class UserGroupMappingPlacementRule extends
PlacementRule {
String group =
CapacitySchedulerConfiguration.ROOT + "." + getPrimaryGroup(user);
-CSQueue parent = queueManager.getQueue(mapping.getParentQueue());
+String parent = mapping.getParentQueue();
+CSQueue groupQueue = queueManager.getQueue(group);
-if (parent instanceof ManagedParentQueue) {
- return getPlacementContext(mapping, group);
-} else {
- CSQueue queue = this.queueManager.getQueue(group);
- if ( queue != null) {
-return getPlacementContext(mapping, queue.getQueuePath());
+if (parent != null) {
+ CSQueue parentQueue = queueManager.getQueue(parent);
+
+ if (parentQueue instanceof ManagedParentQueue) {
+return getPlacementContext(mapping, group);
} else {
-return null;
+return groupQueue == null ? null : getPlacementContext(mapping, group);
}
+} else {
+ return groupQueue == null ? null : getPlacementContext(mapping, group);
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
index 95a0a80..1d7b6b7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
@@ -93,6 +93,7 @@ public class TestUserGroupMappingPlacementRule {
when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
when(queueManager.getQueue("managedParent")).thenReturn(managedParent);
+when(queueManager.getQueue(null)).thenThrow(new NullPointerException());
when(queueManager.getQueue("root.agroup")).thenReturn(agroup);
when(queueManager.getQueue("root.bsubgroup2")).thenReturn(bsubgroup2);
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10191. FS-CS converter: call System.exit function call for every code path in main method. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0fd8bf5 YARN-10191. FS-CS converter: call System.exit function call
for every code path in main method. Contributed by Peter Bacsko.
0fd8bf5 is described below
commit 0fd8bf5f6b2cd7af351a5a9ebdc98cb12d6a82aa
Author: Sunil G
AuthorDate: Thu Mar 12 15:03:12 2020 +0530
YARN-10191. FS-CS converter: call System.exit function call for every code
path in main method. Contributed by Peter Bacsko.
---
.../fair/converter/FSConfigToCSConfigConverterMain.java | 3 ++-
.../fair/converter/TestFSConfigToCSConfigConverterMain.java | 9 -
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
index af1998c..3953dcd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigToCSConfigConverterMain.java
@@ -42,8 +42,9 @@ public class FSConfigToCSConfigConverterMain {
LOG.error(FATAL,
"Error while starting FS configuration conversion, " +
"see previous error messages for details!");
-System.exit(exitCode);
}
+
+ System.exit(exitCode);
} catch (Throwable t) {
LOG.error(FATAL,
"Error while starting FS configuration conversion!", t);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
index 0e2be72..383a4bc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverterMain.java
@@ -22,6 +22,7 @@ import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.conve
import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.YARN_SITE_XML;
import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.FSConfigConverterTestCommons.setupFSConfigConversionFiles;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.IOException;
@@ -40,11 +41,13 @@ import org.junit.Test;
public class TestFSConfigToCSConfigConverterMain {
private FSConfigConverterTestCommons converterTestCommons;
private SecurityManager originalSecurityManager;
+ private ExitHandlerSecurityManager exitHandlerSecurityManager;
@Before
public void setUp() throws Exception {
originalSecurityManager = System.getSecurityManager();
-System.setSecurityManager(new ExitHandlerSecurityManager());
+exitHandlerSecurityManager = new ExitHandlerSecurityManager();
+System.setSecurityManager(exitHandlerSecurityManager);
converterTestCommons = new FSConfigConverterTestCommons();
converterTestCommons.setUp();
}
@@ -82,6 +85,7 @@ public class TestFSConfigToCSConfigConverterMain {
assertTrue("capacity-scheduler.xml was not generated", csConfigExists);
assertTrue("yarn-site.xml was not generated", yarnSiteConfigExists);
+assertEquals("Exit code", 0, exitHandlerSecurityManager.exitCode);
}
@Test
@@ -141,12 +145,15 @@ public class TestFSConfigToCSConfigConverterMain {
}
class ExitHandlerSecurityManager extends SecurityManager {
+int exitCode = Integer.MIN_VALUE;
+
@Override
public void checkExit(int status) {
if (status != 0) {
throw new IllegalStateException(
"Exit cod
[hadoop] branch branch-3.1 updated: YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new a6124cd YARN-10139. ValidateAndGetSchedulerConfiguration API fails
when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.
a6124cd is described below
commit a6124cd2b8955306e6acc7a0249871f17e0565e0
Author: Sunil G
AuthorDate: Wed Feb 19 11:17:22 2020 +0530
YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max
allocation > default 8GB. Contributed by Prabhu Joseph.
(cherry picked from commit 6526f95bd281fc011f8776d21ff933087c5924de)
---
.../resourcemanager/webapp/RMWebServices.java | 13 +--
.../TestRMWebServicesConfigurationMutation.java| 26 ++
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 09b8ea1..e2c8cbb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -31,6 +31,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -2353,14 +2354,22 @@ public class RMWebServices extends WebServices
implements RMWebServiceProtocol {
((MutableConfScheduler) scheduler).getMutableConfProvider();
Configuration schedulerConf = mutableConfigurationProvider
.getConfiguration();
-Configuration newConfig = mutableConfigurationProvider
+Configuration newSchedulerConf = mutableConfigurationProvider
.applyChanges(schedulerConf, mutationInfo);
Configuration yarnConf = ((CapacityScheduler) scheduler).getConf();
+
+Configuration newConfig = new Configuration(yarnConf);
+Iterator> iter = newSchedulerConf.iterator();
+Entry e = null;
+while (iter.hasNext()) {
+ e = iter.next();
+ newConfig.set(e.getKey(), e.getValue());
+}
CapacitySchedulerConfigValidator.validateCSConfiguration(yarnConf,
newConfig, rm.getRMContext());
return Response.status(Status.OK)
-.entity(new ConfInfo(newConfig))
+.entity(new ConfInfo(newSchedulerConf))
.build();
} catch (Exception e) {
String errorMsg = "CapacityScheduler configuration validation failed:"
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index accebb9..128559d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -693,6 +693,32 @@ public class TestRMWebServicesConfigurationMutation
extends JerseyTestBase {
newCSConf.getMaximumSystemApplications());
}
+ @Test
+ public void testValidateWithClusterMaxAllocation() throws Exception {
+WebResource r = resource();
+int clusterMax = YarnConfiguration.
+DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB * 2;
+conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+clusterMax);
+
+SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+Map updateParam = new HashMap<>();
+updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_APPLICATIONS_SUFFIX,
+"100");
+QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+
+ClientResponse response =
+r.path("ws").path("v1").path
[hadoop] branch branch-3.2 updated: YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new de63115 YARN-10139. ValidateAndGetSchedulerConfiguration API fails
when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.
de63115 is described below
commit de63115a2aa76341bc60c4b3cdc2a264666da288
Author: Sunil G
AuthorDate: Wed Feb 19 11:17:22 2020 +0530
YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max
allocation > default 8GB. Contributed by Prabhu Joseph.
(cherry picked from commit 6526f95bd281fc011f8776d21ff933087c5924de)
---
.../resourcemanager/webapp/RMWebServices.java | 13 +--
.../TestRMWebServicesConfigurationMutation.java| 26 ++
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 9f8d0b9..02d9cfa 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -31,6 +31,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -2391,14 +2392,22 @@ public class RMWebServices extends WebServices
implements RMWebServiceProtocol {
((MutableConfScheduler) scheduler).getMutableConfProvider();
Configuration schedulerConf = mutableConfigurationProvider
.getConfiguration();
-Configuration newConfig = mutableConfigurationProvider
+Configuration newSchedulerConf = mutableConfigurationProvider
.applyChanges(schedulerConf, mutationInfo);
Configuration yarnConf = ((CapacityScheduler) scheduler).getConf();
+
+Configuration newConfig = new Configuration(yarnConf);
+Iterator> iter = newSchedulerConf.iterator();
+Entry e = null;
+while (iter.hasNext()) {
+ e = iter.next();
+ newConfig.set(e.getKey(), e.getValue());
+}
CapacitySchedulerConfigValidator.validateCSConfiguration(yarnConf,
newConfig, rm.getRMContext());
return Response.status(Status.OK)
-.entity(new ConfInfo(newConfig))
+.entity(new ConfInfo(newSchedulerConf))
.build();
} catch (Exception e) {
String errorMsg = "CapacityScheduler configuration validation failed:"
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index e2f82a0..d12c2ff 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -731,6 +731,32 @@ public class TestRMWebServicesConfigurationMutation
extends JerseyTestBase {
newCSConf.getMaximumSystemApplications());
}
+ @Test
+ public void testValidateWithClusterMaxAllocation() throws Exception {
+WebResource r = resource();
+int clusterMax = YarnConfiguration.
+DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB * 2;
+conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+clusterMax);
+
+SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+Map updateParam = new HashMap<>();
+updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_APPLICATIONS_SUFFIX,
+"100");
+QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+
+ClientResponse response =
+r.path("ws").path("v1").path
[hadoop] branch trunk updated: YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6526f95 YARN-10139. ValidateAndGetSchedulerConfiguration API fails
when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.
6526f95 is described below
commit 6526f95bd281fc011f8776d21ff933087c5924de
Author: Sunil G
AuthorDate: Wed Feb 19 11:17:22 2020 +0530
YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max
allocation > default 8GB. Contributed by Prabhu Joseph.
---
.../resourcemanager/webapp/RMWebServices.java | 13 +--
.../TestRMWebServicesConfigurationMutation.java| 26 ++
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index c6858f9..dfdaba9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -31,6 +31,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -2639,14 +2640,22 @@ public class RMWebServices extends WebServices
implements RMWebServiceProtocol {
((MutableConfScheduler) scheduler).getMutableConfProvider();
Configuration schedulerConf = mutableConfigurationProvider
.getConfiguration();
-Configuration newConfig = mutableConfigurationProvider
+Configuration newSchedulerConf = mutableConfigurationProvider
.applyChanges(schedulerConf, mutationInfo);
Configuration yarnConf = ((CapacityScheduler) scheduler).getConf();
+
+Configuration newConfig = new Configuration(yarnConf);
+Iterator> iter = newSchedulerConf.iterator();
+Entry e = null;
+while (iter.hasNext()) {
+ e = iter.next();
+ newConfig.set(e.getKey(), e.getValue());
+}
CapacitySchedulerConfigValidator.validateCSConfiguration(yarnConf,
newConfig, rm.getRMContext());
return Response.status(Status.OK)
-.entity(new ConfInfo(newConfig))
+.entity(new ConfInfo(newSchedulerConf))
.build();
} catch (Exception e) {
String errorMsg = "CapacityScheduler configuration validation failed:"
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 4bb50ea..34b7c12 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -763,6 +763,32 @@ public class TestRMWebServicesConfigurationMutation
extends JerseyTestBase {
newCSConf.getMaximumSystemApplications());
}
+ @Test
+ public void testValidateWithClusterMaxAllocation() throws Exception {
+WebResource r = resource();
+int clusterMax = YarnConfiguration.
+DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB * 2;
+conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+clusterMax);
+
+SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+Map updateParam = new HashMap<>();
+updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_APPLICATIONS_SUFFIX,
+"100");
+QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+
+ClientResponse response =
+r.path("ws").path("v1").path("cluster")
+.path(RMWSConsts.SCHEDULER_CONF_VALIDATE)
+.queryP
[hadoop] branch branch-3.2 updated: YARN-10109. Allow stop and convert from leaf to parent queue in a single Mutation API call. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 95b1cbc YARN-10109. Allow stop and convert from leaf to parent queue
in a single Mutation API call. Contributed by Prabhu Joseph
95b1cbc is described below
commit 95b1cbcbd44654c31c8cf25faee3b5f8b4fd9995
Author: Sunil G
AuthorDate: Sun Feb 9 21:14:53 2020 +0530
YARN-10109. Allow stop and convert from leaf to parent queue in a single
Mutation API call. Contributed by Prabhu Joseph
(cherry picked from commit 28f730b317b23038bf9bd0775dd2cdb96518b13b)
---
.../capacity/CapacitySchedulerConfigValidator.java | 21 --
.../TestRMWebServicesConfigurationMutation.java| 33 ++
2 files changed, 45 insertions(+), 9 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index 525ea43..3957c5f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -122,18 +122,20 @@ public final class CapacitySchedulerConfigValidator {
String queueName = e.getKey();
CSQueue oldQueue = e.getValue();
CSQueue newQueue = newQueues.get(queueName);
-if (null == newQueue) {
- // old queue doesn't exist in the new XML
- String configPrefix = newConf.getQueuePrefix(
- oldQueue.getQueuePath());
- QueueState newQueueState = null;
+String configPrefix = newConf.getQueuePrefix(
+oldQueue.getQueuePath());
+String state = newConf.get(configPrefix + "state");
+QueueState newQueueState = null;
+if (state != null) {
try {
-newQueueState = QueueState.valueOf(
-newConf.get(configPrefix + "state"));
+newQueueState = QueueState.valueOf(state);
} catch (Exception ex) {
LOG.warn("Not a valid queue state for queue "
-+ oldQueue.getQueuePath());
++ oldQueue.getQueuePath());
}
+}
+if (null == newQueue) {
+ // old queue doesn't exist in the new XML
if (oldQueue.getState() == QueueState.STOPPED ||
newQueueState == QueueState.STOPPED) {
LOG.info("Deleting Queue " + queueName + ", as it is not"
@@ -169,7 +171,8 @@ public final class CapacitySchedulerConfigValidator {
+ " is set to true");
} else if (oldQueue instanceof LeafQueue
&& newQueue instanceof ParentQueue) {
- if (oldQueue.getState() == QueueState.STOPPED) {
+ if (oldQueue.getState() == QueueState.STOPPED ||
+ newQueueState == QueueState.STOPPED) {
LOG.info("Converting the leaf queue: " + oldQueue.getQueuePath()
+ " to parent queue.");
} else{
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 24a71fe..e2f82a0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -459,6 +459,39 @@ public class TestRMWebServicesConfigurationMutation
extends JerseyTestBase {
}
@Test
+ public void testStopWithConvertLeafToParentQueue() throws Exception {
+WebResource r = resource();
+ClientResponse response;
+
+// Set state of queues to STOPPED.
+SchedConfUpdateInfo upda
[hadoop] branch trunk updated: YARN-10109. Allow stop and convert from leaf to parent queue in a single Mutation API call. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 28f730b YARN-10109. Allow stop and convert from leaf to parent queue
in a single Mutation API call. Contributed by Prabhu Joseph
28f730b is described below
commit 28f730b317b23038bf9bd0775dd2cdb96518b13b
Author: Sunil G
AuthorDate: Sun Feb 9 21:14:53 2020 +0530
YARN-10109. Allow stop and convert from leaf to parent queue in a single
Mutation API call. Contributed by Prabhu Joseph
---
.../capacity/CapacitySchedulerConfigValidator.java | 21 --
.../TestRMWebServicesConfigurationMutation.java| 33 ++
2 files changed, 45 insertions(+), 9 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index 525ea43..3957c5f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -122,18 +122,20 @@ public final class CapacitySchedulerConfigValidator {
String queueName = e.getKey();
CSQueue oldQueue = e.getValue();
CSQueue newQueue = newQueues.get(queueName);
-if (null == newQueue) {
- // old queue doesn't exist in the new XML
- String configPrefix = newConf.getQueuePrefix(
- oldQueue.getQueuePath());
- QueueState newQueueState = null;
+String configPrefix = newConf.getQueuePrefix(
+oldQueue.getQueuePath());
+String state = newConf.get(configPrefix + "state");
+QueueState newQueueState = null;
+if (state != null) {
try {
-newQueueState = QueueState.valueOf(
-newConf.get(configPrefix + "state"));
+newQueueState = QueueState.valueOf(state);
} catch (Exception ex) {
LOG.warn("Not a valid queue state for queue "
-+ oldQueue.getQueuePath());
++ oldQueue.getQueuePath());
}
+}
+if (null == newQueue) {
+ // old queue doesn't exist in the new XML
if (oldQueue.getState() == QueueState.STOPPED ||
newQueueState == QueueState.STOPPED) {
LOG.info("Deleting Queue " + queueName + ", as it is not"
@@ -169,7 +171,8 @@ public final class CapacitySchedulerConfigValidator {
+ " is set to true");
} else if (oldQueue instanceof LeafQueue
&& newQueue instanceof ParentQueue) {
- if (oldQueue.getState() == QueueState.STOPPED) {
+ if (oldQueue.getState() == QueueState.STOPPED ||
+ newQueueState == QueueState.STOPPED) {
LOG.info("Converting the leaf queue: " + oldQueue.getQueuePath()
+ " to parent queue.");
} else{
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index b92fbfb..4bb50ea 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -491,6 +491,39 @@ public class TestRMWebServicesConfigurationMutation
extends JerseyTestBase {
}
@Test
+ public void testStopWithConvertLeafToParentQueue() throws Exception {
+WebResource r = resource();
+ClientResponse response;
+
+// Set state of queues to STOPPED.
+SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+Map stoppedParam = new HashMap<>();
+stoppedParam.put(CapacitySched
[hadoop] branch branch-3.1 updated: YARN-8373. RM Received RMFatalEvent of type CRITICAL_THREAD_CRASH. Contributed by Wilfred Spiegelenburg.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new af89b5b YARN-8373. RM Received RMFatalEvent of type
CRITICAL_THREAD_CRASH. Contributed by Wilfred Spiegelenburg.
af89b5b is described below
commit af89b5b0864f8e861e9879393b37efad5b714585
Author: Sunil G
AuthorDate: Wed Jan 15 17:07:34 2020 +0530
YARN-8373. RM Received RMFatalEvent of type CRITICAL_THREAD_CRASH.
Contributed by Wilfred Spiegelenburg.
---
.../scheduler/ClusterNodeTracker.java| 20 ++--
.../scheduler/fair/FairScheduler.java| 15 +--
.../scheduler/fair/TestContinuousScheduling.java | 2 --
3 files changed, 19 insertions(+), 18 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index be7b3a2..704893a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -33,11 +33,11 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -363,21 +363,21 @@ public class ClusterNodeTracker {
/**
* Convenience method to sort nodes.
+ * Nodes can change while being sorted. Using a standard sort will fail
+ * without locking each node, the TreeSet handles this without locks.
*
- * Note that the sort is performed without holding a lock. We are sorting
- * here instead of on the caller to allow for future optimizations (e.g.
- * sort once every x milliseconds).
+ * @param comparator the comparator to sort the nodes with
+ * @return sorted set of nodes in the form of a TreeSet
*/
- public List sortedNodeList(Comparator comparator) {
-List sortedList = null;
+ public TreeSet sortedNodeSet(Comparator comparator) {
+TreeSet sortedSet = new TreeSet<>(comparator);
readLock.lock();
try {
- sortedList = new ArrayList(nodes.values());
+ sortedSet.addAll(nodes.values());
} finally {
readLock.unlock();
}
-Collections.sort(sortedList, comparator);
-return sortedList;
+return sortedSet;
}
/**
@@ -431,4 +431,4 @@ public class ClusterNodeTracker {
}
return retNodes;
}
-}
\ No newline at end of file
+}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index caa6aa0..013cf9d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -103,6 +103,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -945,15 +946,17 @@ public class FairScheduler extends
@Deprecated
void continuousSchedulingAttempt() throws InterruptedException {
long start = getClock().getTime();
-List nodeIdList;
-// Hold a lock to prevent comparator order changes due to changes of node
-// unallocated resources
-synchronized (this) {
- nodeIdList = nodeTracker.sortedNodeList(nodeAvailableResourceComparator);
+TreeSet nodeIdSet;
+// Hold a lock to prevent node changes as much as possible.
+readLock.lock();
+try {
+ nodeIdSet = nodeTracker.sortedN
[hadoop] branch branch-3.2 updated: YARN-8373. RM Received RMFatalEvent of type CRITICAL_THREAD_CRASH. Contributed by Wilfred Spiegelenburg.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new c1ec516 YARN-8373. RM Received RMFatalEvent of type
CRITICAL_THREAD_CRASH. Contributed by Wilfred Spiegelenburg.
c1ec516 is described below
commit c1ec51696cc7d9ba2e5d9a010f1381e3c71115e8
Author: Sunil G
AuthorDate: Tue Nov 19 14:10:41 2019 +0530
YARN-8373. RM Received RMFatalEvent of type CRITICAL_THREAD_CRASH.
Contributed by Wilfred Spiegelenburg.
(cherry picked from commit ea68756c0cd3e6e20b8e7045a8b7edd180ea4708)
---
.../resourcemanager/scheduler/ClusterNodeTracker.java | 18 +-
.../resourcemanager/scheduler/fair/FairScheduler.java | 15 +--
.../scheduler/fair/TestContinuousScheduling.java | 2 --
3 files changed, 18 insertions(+), 17 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index 19f9c68..16a5e0a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -386,21 +386,21 @@ public class ClusterNodeTracker {
/**
* Convenience method to sort nodes.
+ * Nodes can change while being sorted. Using a standard sort will fail
+ * without locking each node, the TreeSet handles this without locks.
*
- * Note that the sort is performed without holding a lock. We are sorting
- * here instead of on the caller to allow for future optimizations (e.g.
- * sort once every x milliseconds).
+ * @param comparator the comparator to sort the nodes with
+ * @return sorted set of nodes in the form of a TreeSet
*/
- public List sortedNodeList(Comparator comparator) {
-List sortedList = null;
+ public TreeSet sortedNodeSet(Comparator comparator) {
+TreeSet sortedSet = new TreeSet<>(comparator);
readLock.lock();
try {
- sortedList = new ArrayList(nodes.values());
+ sortedSet.addAll(nodes.values());
} finally {
readLock.unlock();
}
-Collections.sort(sortedList, comparator);
-return sortedList;
+return sortedSet;
}
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 8ffdc5b..a60e4e0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -111,6 +111,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -1036,15 +1037,17 @@ public class FairScheduler extends
@Deprecated
void continuousSchedulingAttempt() throws InterruptedException {
long start = getClock().getTime();
-List nodeIdList;
-// Hold a lock to prevent comparator order changes due to changes of node
-// unallocated resources
-synchronized (this) {
- nodeIdList = nodeTracker.sortedNodeList(nodeAvailableResourceComparator);
+TreeSet nodeIdSet;
+// Hold a lock to prevent node changes as much as possible.
+readLock.lock();
+try {
+ nodeIdSet = nodeTracker.sortedN
[hadoop] branch trunk updated: YARN-8373. RM Received RMFatalEvent of type CRITICAL_THREAD_CRASH. Contributed by Wilfred Spiegelenburg.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new ea68756 YARN-8373. RM Received RMFatalEvent of type
CRITICAL_THREAD_CRASH. Contributed by Wilfred Spiegelenburg.
ea68756 is described below
commit ea68756c0cd3e6e20b8e7045a8b7edd180ea4708
Author: Sunil G
AuthorDate: Tue Nov 19 14:10:41 2019 +0530
YARN-8373. RM Received RMFatalEvent of type CRITICAL_THREAD_CRASH.
Contributed by Wilfred Spiegelenburg.
---
.../resourcemanager/scheduler/ClusterNodeTracker.java | 18 +-
.../resourcemanager/scheduler/fair/FairScheduler.java | 15 +--
.../scheduler/fair/TestContinuousScheduling.java | 2 --
3 files changed, 18 insertions(+), 17 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
index f4491df..50c45fc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java
@@ -33,12 +33,12 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -386,21 +386,21 @@ public class ClusterNodeTracker {
/**
* Convenience method to sort nodes.
+ * Nodes can change while being sorted. Using a standard sort will fail
+ * without locking each node, the TreeSet handles this without locks.
*
- * Note that the sort is performed without holding a lock. We are sorting
- * here instead of on the caller to allow for future optimizations (e.g.
- * sort once every x milliseconds).
+ * @param comparator the comparator to sort the nodes with
+ * @return sorted set of nodes in the form of a TreeSet
*/
- public List sortedNodeList(Comparator comparator) {
-List sortedList = null;
+ public TreeSet sortedNodeSet(Comparator comparator) {
+TreeSet sortedSet = new TreeSet<>(comparator);
readLock.lock();
try {
- sortedList = new ArrayList(nodes.values());
+ sortedSet.addAll(nodes.values());
} finally {
readLock.unlock();
}
-Collections.sort(sortedList, comparator);
-return sortedList;
+return sortedSet;
}
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 04bbe0f..958eddc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -113,6 +113,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -1001,15 +1002,17 @@ public class FairScheduler extends
@Deprecated
void continuousSchedulingAttempt() throws InterruptedException {
long start = getClock().getTime();
-List nodeIdList;
-// Hold a lock to prevent comparator order changes due to changes of node
-// unallocated resources
-synchronized (this) {
- nodeIdList = nodeTracker.sortedNodeList(nodeAvailableResourceComparator);
+TreeSet nodeIdSet;
+// Hold a lock to prevent node changes as much as possible.
+readLock.lock();
+try {
+ nodeIdSet = nodeTracker.sortedNodeSet(nodeAvailableResourceComparator);
+} finally {
+ readLock.unlock();
}
// iterate all
[hadoop] branch branch-3.1 updated: YARN-9984. FSPreemptionThread can cause NullPointerException while app is unregistered with containers running on a node. Contributed by Wilfred Spiegelenburg.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new f24d1fa YARN-9984. FSPreemptionThread can cause NullPointerException
while app is unregistered with containers running on a node. Contributed by
Wilfred Spiegelenburg.
f24d1fa is described below
commit f24d1fac35447124e01f1a457bf9dac02b32d54c
Author: Sunil G
AuthorDate: Tue Nov 19 14:03:02 2019 +0530
YARN-9984. FSPreemptionThread can cause NullPointerException while app is
unregistered with containers running on a node. Contributed by Wilfred
Spiegelenburg.
(cherry picked from commit 215f2052fc3b7e366e8bd1bd332663966fa9206c)
---
.../scheduler/fair/FSPreemptionThread.java | 17 ++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index 6ed90f8..b3e066e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -140,9 +140,13 @@ class FSPreemptionThread extends Thread {
for (RMContainer container : containers) {
FSAppAttempt app = scheduler.getSchedulerApp(
container.getApplicationAttemptId());
- LOG.info("Preempting container " + container +
- " from queue " + app.getQueueName());
- app.trackContainerForPreemption(container);
+ LOG.info("Preempting container " + container + " from queue: "
+ + (app != null ? app.getQueueName() : "unknown"));
+ // If the app has unregistered while building the container list
+ // the app might be null, skip notifying the app
+ if (app != null) {
+app.trackContainerForPreemption(container);
+ }
}
}
}
@@ -204,6 +208,13 @@ class FSPreemptionThread extends Thread {
for (RMContainer container : containersToCheck) {
FSAppAttempt app =
scheduler.getSchedulerApp(container.getApplicationAttemptId());
+ // If the app has unregistered while building the container list the app
+ // might be null, just skip this container: it should be cleaned up soon
+ if (app == null) {
+LOG.info("Found container " + container + " on node "
++ node.getNodeName() + "without app, skipping preemption");
+continue;
+ }
ApplicationId appId = app.getApplicationId();
if (app.canContainerBePreempted(container,
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-9984. FSPreemptionThread can cause NullPointerException while app is unregistered with containers running on a node. Contributed by Wilfred Spiegelenburg.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 049279b YARN-9984. FSPreemptionThread can cause NullPointerException
while app is unregistered with containers running on a node. Contributed by
Wilfred Spiegelenburg.
049279b is described below
commit 049279bb66b036b4767fe6a0283a6f334ac91440
Author: Sunil G
AuthorDate: Tue Nov 19 14:03:02 2019 +0530
YARN-9984. FSPreemptionThread can cause NullPointerException while app is
unregistered with containers running on a node. Contributed by Wilfred
Spiegelenburg.
(cherry picked from commit 215f2052fc3b7e366e8bd1bd332663966fa9206c)
---
.../scheduler/fair/FSPreemptionThread.java | 17 ++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index 7ccfd72..fe445b6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -141,9 +141,13 @@ class FSPreemptionThread extends Thread {
for (RMContainer container : containers) {
FSAppAttempt app = scheduler.getSchedulerApp(
container.getApplicationAttemptId());
- LOG.info("Preempting container " + container +
- " from queue " + app.getQueueName());
- app.trackContainerForPreemption(container);
+ LOG.info("Preempting container " + container + " from queue: "
+ + (app != null ? app.getQueueName() : "unknown"));
+ // If the app has unregistered while building the container list
+ // the app might be null, skip notifying the app
+ if (app != null) {
+app.trackContainerForPreemption(container);
+ }
}
}
}
@@ -205,6 +209,13 @@ class FSPreemptionThread extends Thread {
for (RMContainer container : containersToCheck) {
FSAppAttempt app =
scheduler.getSchedulerApp(container.getApplicationAttemptId());
+ // If the app has unregistered while building the container list the app
+ // might be null, just skip this container: it should be cleaned up soon
+ if (app == null) {
+LOG.info("Found container " + container + " on node "
++ node.getNodeName() + "without app, skipping preemption");
+continue;
+ }
ApplicationId appId = app.getApplicationId();
if (app.canContainerBePreempted(container,
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9984. FSPreemptionThread can cause NullPointerException while app is unregistered with containers running on a node. Contributed by Wilfred Spiegelenburg.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 215f205 YARN-9984. FSPreemptionThread can cause NullPointerException
while app is unregistered with containers running on a node. Contributed by
Wilfred Spiegelenburg.
215f205 is described below
commit 215f2052fc3b7e366e8bd1bd332663966fa9206c
Author: Sunil G
AuthorDate: Tue Nov 19 14:03:02 2019 +0530
YARN-9984. FSPreemptionThread can cause NullPointerException while app is
unregistered with containers running on a node. Contributed by Wilfred
Spiegelenburg.
---
.../scheduler/fair/FSPreemptionThread.java | 17 ++---
1 file changed, 14 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
index e664725..221bb17 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSPreemptionThread.java
@@ -125,9 +125,13 @@ class FSPreemptionThread extends Thread {
for (RMContainer container : containers) {
FSAppAttempt app = scheduler.getSchedulerApp(
container.getApplicationAttemptId());
- LOG.info("Preempting container " + container +
- " from queue " + app.getQueueName());
- app.trackContainerForPreemption(container);
+ LOG.info("Preempting container " + container + " from queue: "
+ + (app != null ? app.getQueueName() : "unknown"));
+ // If the app has unregistered while building the container list
+ // the app might be null, skip notifying the app
+ if (app != null) {
+app.trackContainerForPreemption(container);
+ }
}
}
}
@@ -189,6 +193,13 @@ class FSPreemptionThread extends Thread {
for (RMContainer container : containersToCheck) {
FSAppAttempt app =
scheduler.getSchedulerApp(container.getApplicationAttemptId());
+ // If the app has unregistered while building the container list the app
+ // might be null, just skip this container: it should be cleaned up soon
+ if (app == null) {
+LOG.info("Found container " + container + " on node "
++ node.getNodeName() + "without app, skipping preemption");
+continue;
+ }
ApplicationId appId = app.getApplicationId();
if (app.canContainerBePreempted(container,
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-9950. Unset Ordering Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 597b315 YARN-9950. Unset Ordering Policy of Leaf/Parent queue
converted from Parent/Leaf queue respectively. Contributed by Prabhu Joseph.
597b315 is described below
commit 597b315811188597d66e7fb264fd8b46628d1404
Author: Sunil G
AuthorDate: Mon Nov 4 23:27:33 2019 +0530
YARN-9950. Unset Ordering Policy of Leaf/Parent queue converted from
Parent/Leaf queue respectively. Contributed by Prabhu Joseph.
(cherry picked from commit 51e7d1b37eb06165657befdae82c015055c58334)
---
.../conf/MutableCSConfigurationProvider.java | 17
.../TestRMWebServicesConfigurationMutation.java| 100 +
2 files changed, 117 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 992eea7..7eab65f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -42,6 +42,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
+
/**
* CS configuration provider which implements
* {@link MutableConfigurationProvider} for modifying capacity scheduler
@@ -278,6 +280,13 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
+ CapacitySchedulerConfiguration.QUEUES;
if (siblingQueues.size() == 0) {
confUpdate.put(queuesConfig, null);
+ // Unset Ordering Policy of Leaf Queue converted from
+ // Parent Queue after removeQueue
+ String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + ORDERING_POLICY;
+ proposedConf.unset(queueOrderingPolicy);
+ confUpdate.put(queueOrderingPolicy, null);
} else {
confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues));
}
@@ -326,6 +335,14 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
}
confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
}
+ // Unset Ordering Policy of Parent Queue converted from
+ // Leaf Queue after addQueue
+ String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + parentQueue + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+ if (siblingQueues.size() == 1) {
+proposedConf.unset(queueOrderingPolicy);
+confUpdate.put(queueOrderingPolicy, null);
+ }
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 773b4bf..24a71fe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -60,6 +60,7 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
+import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
/**
* Test scheduler configuration mutation via REST API.
@@ -334,6 +335,105 @@ public class TestRMWebServicesConfigurationMutation
extends JerseyTestBase {
}
@Test
+ public void testUnsetParentQueueOrderingPolicy() throws Exception {
+WebResource r
[hadoop] branch trunk updated: YARN-9950. Unset Ordering Policy of Leaf/Parent queue converted from Parent/Leaf queue respectively. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 51e7d1b YARN-9950. Unset Ordering Policy of Leaf/Parent queue
converted from Parent/Leaf queue respectively. Contributed by Prabhu Joseph.
51e7d1b is described below
commit 51e7d1b37eb06165657befdae82c015055c58334
Author: Sunil G
AuthorDate: Mon Nov 4 23:27:33 2019 +0530
YARN-9950. Unset Ordering Policy of Leaf/Parent queue converted from
Parent/Leaf queue respectively. Contributed by Prabhu Joseph.
---
.../conf/MutableCSConfigurationProvider.java | 17
.../TestRMWebServicesConfigurationMutation.java| 100 +
2 files changed, 117 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 494c18c..c8d0a0c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -42,6 +42,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
+
/**
* CS configuration provider which implements
* {@link MutableConfigurationProvider} for modifying capacity scheduler
@@ -259,6 +261,13 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
+ CapacitySchedulerConfiguration.QUEUES;
if (siblingQueues.size() == 0) {
confUpdate.put(queuesConfig, null);
+ // Unset Ordering Policy of Leaf Queue converted from
+ // Parent Queue after removeQueue
+ String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + parentQueuePath + CapacitySchedulerConfiguration.DOT
+ + ORDERING_POLICY;
+ proposedConf.unset(queueOrderingPolicy);
+ confUpdate.put(queueOrderingPolicy, null);
} else {
confUpdate.put(queuesConfig, Joiner.on(',').join(siblingQueues));
}
@@ -307,6 +316,14 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
}
confUpdate.put(keyPrefix + kv.getKey(), kv.getValue());
}
+ // Unset Ordering Policy of Parent Queue converted from
+ // Leaf Queue after addQueue
+ String queueOrderingPolicy = CapacitySchedulerConfiguration.PREFIX
+ + parentQueue + CapacitySchedulerConfiguration.DOT + ORDERING_POLICY;
+ if (siblingQueues.size() == 1) {
+proposedConf.unset(queueOrderingPolicy);
+confUpdate.put(queueOrderingPolicy, null);
+ }
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 773b4bf..24a71fe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -60,6 +60,7 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
+import static
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.ORDERING_POLICY;
/**
* Test scheduler configuration mutation via REST API.
@@ -334,6 +335,105 @@ public class TestRMWebServicesConfigurationMutation
extends JerseyTestBase {
}
@Test
+ public void testUnsetParentQueueOrderingPolicy() throws Exception {
+WebResource r = resource();
+ClientResponse response;
+
+// Update ordering policy of Leaf Queue
[hadoop] branch branch-3.2 updated: YARN-9949. Add missing queue configs for root queue in RMWebService#CapacitySchedulerInfo. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 11c763c YARN-9949. Add missing queue configs for root queue in
RMWebService#CapacitySchedulerInfo. Contributed by Prabhu Joseph.
11c763c is described below
commit 11c763c22055fea367b19b338a3d8067f9386ba4
Author: Sunil G
AuthorDate: Sun Nov 3 08:47:27 2019 +0530
YARN-9949. Add missing queue configs for root queue in
RMWebService#CapacitySchedulerInfo. Contributed by Prabhu Joseph.
(cherry picked from commit d462308e0476053d1aae65eca2752eacce797e22)
---
.../capacity/CapacitySchedulerConfiguration.java | 8
.../policy/AbstractComparatorOrderingPolicy.java | 3 ++
.../scheduler/policy/FairOrderingPolicy.java | 6 +++
.../scheduler/policy/FifoOrderingPolicy.java | 6 +++
.../policy/FifoOrderingPolicyForPendingApps.java | 6 +++
.../FifoOrderingPolicyWithExclusivePartitions.java | 7 +++
.../scheduler/policy/OrderingPolicy.java | 6 +++
.../webapp/CapacitySchedulerPage.java | 2 +-
.../webapp/dao/CapacitySchedulerInfo.java | 51 ++
.../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 10 -
.../webapp/TestRMWebServicesCapacitySched.java | 42 +-
.../TestRMWebServicesForCSWithPartitions.java | 2 +-
12 files changed, 145 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 760636e..f98a129 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -50,6 +50,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNo
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodePolicySpec;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyForPendingApps;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyWithExclusivePartitions;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
@@ -163,6 +164,9 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
public static final String FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY
= "fifo-with-partitions";
+ public static final String FIFO_FOR_PENDING_APPS
+ = "fifo-for-pending-apps";
+
public static final String DEFAULT_APP_ORDERING_POLICY =
FIFO_APP_ORDERING_POLICY;
@@ -573,6 +577,10 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
if (policyType.trim().equals(FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY)) {
policyType = FifoOrderingPolicyWithExclusivePartitions.class.getName();
}
+if (policyType.trim().equals(FIFO_FOR_PENDING_APPS)) {
+ policyType = FifoOrderingPolicyForPendingApps.class.getName();
+}
+
try {
orderingPolicy = (OrderingPolicy)
Class.forName(policyType).newInstance();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java
index 4ee4450..b5e870b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java
@@ -13
[hadoop] branch trunk updated: YARN-9949. Add missing queue configs for root queue in RMWebService#CapacitySchedulerInfo. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d462308 YARN-9949. Add missing queue configs for root queue in
RMWebService#CapacitySchedulerInfo. Contributed by Prabhu Joseph.
d462308 is described below
commit d462308e0476053d1aae65eca2752eacce797e22
Author: Sunil G
AuthorDate: Sun Nov 3 08:47:27 2019 +0530
YARN-9949. Add missing queue configs for root queue in
RMWebService#CapacitySchedulerInfo. Contributed by Prabhu Joseph.
---
.../capacity/CapacitySchedulerConfiguration.java | 8
.../policy/AbstractComparatorOrderingPolicy.java | 3 ++
.../scheduler/policy/FairOrderingPolicy.java | 6 +++
.../scheduler/policy/FifoOrderingPolicy.java | 6 +++
.../policy/FifoOrderingPolicyForPendingApps.java | 6 +++
.../FifoOrderingPolicyWithExclusivePartitions.java | 7 +++
.../scheduler/policy/OrderingPolicy.java | 6 +++
.../webapp/CapacitySchedulerPage.java | 2 +-
.../webapp/dao/CapacitySchedulerInfo.java | 51 ++
.../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 11 -
.../webapp/TestRMWebServicesCapacitySched.java | 42 +-
.../TestRMWebServicesForCSWithPartitions.java | 2 +-
12 files changed, 146 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index a72e807..adf5e6b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -51,6 +51,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNo
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodePolicySpec;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderingPolicy;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyForPendingApps;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicyWithExclusivePartitions;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
@@ -166,6 +167,9 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
public static final String FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY
= "fifo-with-partitions";
+ public static final String FIFO_FOR_PENDING_APPS
+ = "fifo-for-pending-apps";
+
public static final String DEFAULT_APP_ORDERING_POLICY =
FIFO_APP_ORDERING_POLICY;
@@ -579,6 +583,10 @@ public class CapacitySchedulerConfiguration extends
ReservationSchedulerConfigur
if (policyType.trim().equals(FIFO_WITH_PARTITIONS_APP_ORDERING_POLICY)) {
policyType = FifoOrderingPolicyWithExclusivePartitions.class.getName();
}
+if (policyType.trim().equals(FIFO_FOR_PENDING_APPS)) {
+ policyType = FifoOrderingPolicyForPendingApps.class.getName();
+}
+
try {
orderingPolicy = (OrderingPolicy)
Class.forName(policyType).newInstance();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java
index 4ee4450..b5e870b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/AbstractComparatorOrderingPolicy.java
@@ -139,4 +139,7 @@ public abstract class AbstractComparatorOrderingPolicy extends Abs
[hadoop] branch trunk updated: YARN-9937. Add missing queue configs in RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9a2e43e YARN-9937. Add missing queue configs in
RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph.
9a2e43e is described below
commit 9a2e43e29e14905d98edd8512d24363dec407634
Author: Sunil G
AuthorDate: Thu Oct 31 00:34:27 2019 +0530
YARN-9937. Add missing queue configs in
RMWebService#CapacitySchedulerQueueInfo. Contributed by Prabhu Joseph.
---
.../webapp/JAXBContextResolver.java| 3 +-
.../webapp/dao/CapacitySchedulerInfo.java | 11 +++--
.../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 32 +
.../webapp/dao/CapacitySchedulerQueueInfo.java | 55 +-
.../resourcemanager/webapp/dao/QueueAclInfo.java | 48 +++
.../resourcemanager/webapp/dao/QueueAclsInfo.java | 47 ++
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
7 files changed, 181 insertions(+), 19 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
index e0c6647..f6eb2ad 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
@@ -56,7 +56,8 @@ public class JAXBContextResolver implements
ContextResolver {
StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class,
FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class,
AppTimeoutInfo.class, ResourceInformationsInfo.class,
-ActivitiesInfo.class, AppActivitiesInfo.class};
+ActivitiesInfo.class, AppActivitiesInfo.class,
+QueueAclsInfo.class, QueueAclInfo.class};
// these dao classes need root unwrapping
final Class[] rootUnwrappedTypes =
{ NewApplication.class, ApplicationSubmissionContextInfo.class,
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index ca42041..b8a5426 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -61,7 +61,7 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
parent.getQueueResourceQuotas(), false);
-queues = getQueues(parent);
+queues = getQueues(cs, parent);
health = new CapacitySchedulerHealthInfo(cs);
}
@@ -89,7 +89,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
return this.queues;
}
- protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
+ protected CapacitySchedulerQueueInfoList getQueues(
+ CapacityScheduler cs, CSQueue parent) {
CapacitySchedulerQueueInfoList queuesInfo =
new CapacitySchedulerQueueInfoList();
// JAXB marashalling leads to situation where the "type" field injected
@@ -112,10 +113,10 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
for (CSQueue queue : childQueues) {
CapacitySchedulerQueueInfo info;
if (queue instanceof LeafQueue) {
-info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue);
+info = new CapacitySchedulerLeafQueueInfo(cs, (LeafQueue) queue);
} else {
-info = new CapacitySchedulerQueueInfo(queue);
-info.queues = getQueues(queue);
+info = new CapacitySchedulerQueueInfo(cs, queue);
+info.queues = getQueues(cs, queue);
}
queuesInfo.addToQueueInfoList(info);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
[hadoop] branch trunk updated: YARN-9909. Offline format of YarnConfigurationStore. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3990ffa YARN-9909. Offline format of YarnConfigurationStore.
Contributed by Prabhu Joseph.
3990ffa is described below
commit 3990ffa0834e6b24314a565a92e92d6e8fa37a1e
Author: Sunil G
AuthorDate: Thu Oct 17 06:43:06 2019 +0530
YARN-9909. Offline format of YarnConfigurationStore. Contributed by Prabhu
Joseph.
---
.../server/resourcemanager/ResourceManager.java| 48 +-
.../conf/MutableCSConfigurationProvider.java | 21 +-
.../conf/YarnConfigurationStoreFactory.java| 22 --
.../resourcemanager/TestRMStoreCommands.java | 35
.../src/site/markdown/YarnCommands.md | 1 +
5 files changed, 102 insertions(+), 25 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 351519b..228c1b5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -105,12 +105,15 @@ import
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStoreFactory;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.MemoryPlacementConstraintManager;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManagerService;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.ProxyCAManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
@@ -1563,6 +1566,8 @@ public class ResourceManager extends CompositeService
if (argv.length >= 1) {
if (argv[0].equals("-format-state-store")) {
deleteRMStateStore(conf);
+} else if (argv[0].equals("-format-conf-store")) {
+ deleteRMConfStore(conf);
} else if (argv[0].equals("-remove-application-from-state-store")
&& argv.length == 2) {
removeApplication(conf, argv[1]);
@@ -1671,6 +1676,45 @@ public class ResourceManager extends CompositeService
}
}
+ /**
+ * Deletes the YarnConfigurationStore
+ *
+ * @param conf
+ * @throws Exception
+ */
+ @VisibleForTesting
+ static void deleteRMConfStore(Configuration conf) throws Exception {
+ResourceManager rm = new ResourceManager();
+rm.conf = conf;
+ResourceScheduler scheduler = rm.createScheduler();
+RMContextImpl rmContext = new RMContextImpl();
+rmContext.setResourceManager(rm);
+
+boolean isConfigurationMutable = false;
+String confProviderStr = conf.get(
+YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+switch (confProviderStr) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
+ case YarnConfiguration.FS_CONFIGURATION_STORE:
+isConfigurationMutable = true;
+break;
+ default:
+}
+
+if (scheduler instanceof MutableConfScheduler && isConfigurationMutable) {
+ YarnConfigurationStore confStore = YarnConfigurationStoreFactory
+ .getStore(conf);
+ confStore.initialize(conf, conf, rmContext);
+ confStore.
[hadoop] branch branch-3.2 updated: YARN-9900. Revert to previous state when Invalid Config is applied and Refresh Support in SchedulerConfig Format. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 9672b81 YARN-9900. Revert to previous state when Invalid Config is
applied and Refresh Support in SchedulerConfig Format. Contributed by Prabhu
Joseph.
9672b81 is described below
commit 9672b81fa33f6a21e1e34fe188f09905206eca06
Author: Sunil G
AuthorDate: Wed Oct 16 18:14:31 2019 +0530
YARN-9900. Revert to previous state when Invalid Config is applied and
Refresh Support in SchedulerConfig Format. Contributed by Prabhu Joseph.
(cherry picked from commit 090f73a9aaeb4c03032a78c0a837478b527c27e0)
---
.../scheduler/MutableConfigurationProvider.java| 2 ++
.../conf/MutableCSConfigurationProvider.java | 16 ++
.../resourcemanager/webapp/RMWebServices.java | 7 +
.../TestRMWebServicesConfigurationMutation.java| 35 --
4 files changed, 57 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index eff8aa8..03902e3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -73,6 +73,8 @@ public interface MutableConfigurationProvider {
void formatConfigurationInStore(Configuration conf) throws Exception;
+ void revertToOldConfig(Configuration config) throws Exception;
+
/**
* Closes the configuration provider, releasing any required resources.
* @throws IOException on failure to close
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 1f78d09..992eea7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -166,6 +166,7 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
formatLock.writeLock().lock();
try {
confStore.format();
+ oldConf = new Configuration(schedConf);
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
this.schedConf = new Configuration(false);
@@ -185,6 +186,21 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
}
@Override
+ public void revertToOldConfig(Configuration config) throws Exception {
+formatLock.writeLock().lock();
+try {
+ schedConf = oldConf;
+ confStore.format();
+ confStore.initialize(config, oldConf, rmContext);
+ confStore.checkVersion();
+} catch (Exception e) {
+ throw new IOException(e);
+} finally {
+ formatLock.writeLock().unlock();
+}
+ }
+
+ @Override
public void confirmPendingMutation(boolean isValid) throws Exception {
formatLock.readLock().lock();
try {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 6ba8f43..4c3af9f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2337,6 +2337,13
[hadoop] branch trunk updated: YARN-9900. Revert to previous state when Invalid Config is applied and Refresh Support in SchedulerConfig Format. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 090f73a YARN-9900. Revert to previous state when Invalid Config is
applied and Refresh Support in SchedulerConfig Format. Contributed by Prabhu
Joseph.
090f73a is described below
commit 090f73a9aaeb4c03032a78c0a837478b527c27e0
Author: Sunil G
AuthorDate: Wed Oct 16 18:14:31 2019 +0530
YARN-9900. Revert to previous state when Invalid Config is applied and
Refresh Support in SchedulerConfig Format. Contributed by Prabhu Joseph.
---
.../scheduler/MutableConfigurationProvider.java| 2 ++
.../conf/MutableCSConfigurationProvider.java | 16 ++
.../resourcemanager/webapp/RMWebServices.java | 7 +
.../TestRMWebServicesConfigurationMutation.java| 35 --
4 files changed, 57 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index eff8aa8..03902e3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -73,6 +73,8 @@ public interface MutableConfigurationProvider {
void formatConfigurationInStore(Configuration conf) throws Exception;
+ void revertToOldConfig(Configuration config) throws Exception;
+
/**
* Closes the configuration provider, releasing any required resources.
* @throws IOException on failure to close
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index f464b2c..ba72763 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -166,6 +166,7 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
formatLock.writeLock().lock();
try {
confStore.format();
+ oldConf = new Configuration(schedConf);
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
this.schedConf = new Configuration(false);
@@ -185,6 +186,21 @@ public class MutableCSConfigurationProvider implements
CSConfigurationProvider,
}
@Override
+ public void revertToOldConfig(Configuration config) throws Exception {
+formatLock.writeLock().lock();
+try {
+ schedConf = oldConf;
+ confStore.format();
+ confStore.initialize(config, oldConf, rmContext);
+ confStore.checkVersion();
+} catch (Exception e) {
+ throw new IOException(e);
+} finally {
+ formatLock.writeLock().unlock();
+}
+ }
+
+ @Override
public void confirmPendingMutation(boolean isValid) throws Exception {
formatLock.readLock().lock();
try {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index bb77dbd..d263830 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2582,6 +2582,13 @@ public class RMWebServices extends WebServices
implements RMWebServiceProtocol
[hadoop] branch branch-3.2 updated: YARN-9873. Mutation API Config Change need to update Version Number. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 9cb3ab0 YARN-9873. Mutation API Config Change need to update Version
Number. Contributed by Prabhu Joseph
9cb3ab0 is described below
commit 9cb3ab058fee99387da413dc87ca686ea01425f1
Author: Sunil G
AuthorDate: Wed Oct 9 15:53:14 2019 +0530
YARN-9873. Mutation API Config Change need to update Version Number.
Contributed by Prabhu Joseph
(cherry picked from commit be901f49628a553cfe6a3942478cb528cce2b266)
---
.../scheduler/MutableConfigurationProvider.java| 6 +++
.../conf/FSSchedulerConfigurationStore.java| 43 -
.../capacity/conf/InMemoryConfigurationStore.java | 8
.../capacity/conf/LeveldbConfigurationStore.java | 54 +++---
.../conf/MutableCSConfigurationProvider.java | 5 ++
.../capacity/conf/YarnConfigurationStore.java | 6 +++
.../capacity/conf/ZKConfigurationStore.java| 18
.../server/resourcemanager/webapp/RMWSConsts.java | 3 ++
.../resourcemanager/webapp/RMWebServices.java | 36 ++-
.../webapp/dao/ConfigVersionInfo.java | 44 ++
.../conf/TestFSSchedulerConfigurationStore.java| 12 +++--
.../capacity/conf/TestZKConfigurationStore.java| 15 ++
.../TestRMWebServicesConfigurationMutation.java| 19
13 files changed, 258 insertions(+), 11 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 9e843df..eff8aa8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -65,6 +65,12 @@ public interface MutableConfigurationProvider {
*/
Configuration getConfiguration();
+ /**
+ * Get the last updated scheduler config version.
+ * @return Last updated scheduler config version.
+ */
+ long getConfigVersion() throws Exception;
+
void formatConfigurationInStore(Configuration conf) throws Exception;
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index ddc5c8a..ea9a4b9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -29,6 +29,7 @@ import com.google.gson.GsonBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -62,6 +63,7 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
private volatile Configuration schedConf;
private volatile Configuration oldConf;
private Path tempConfigPath;
+ private Path configVersionFile;
@Override
public void initialize(Configuration conf, Configuration vSchedConf,
@@ -99,9 +101,17 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
}
}
+this.configVersionFile = new Path(schedulerConfPathStr, "ConfigVersion");
+if (!fileSystem.exists(configVersionFile)) {
+ fileSystem.createNewFile(configVersionFile);
+ writeConfigVersion(0L);
+}
+
// create capacity-schedule.xml.ts file if not existing
if (this.getConfigFileInputStream() == null) {
writeConfigurationToFileSystem(vSchedConf);
+ long configVersion = getConfigVersion() + 1L;
+ writeConfigVersion(con
[hadoop] branch trunk updated: YARN-9873. Mutation API Config Change need to update Version Number. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new be901f4 YARN-9873. Mutation API Config Change need to update Version
Number. Contributed by Prabhu Joseph
be901f4 is described below
commit be901f49628a553cfe6a3942478cb528cce2b266
Author: Sunil G
AuthorDate: Wed Oct 9 15:53:14 2019 +0530
YARN-9873. Mutation API Config Change need to update Version Number.
Contributed by Prabhu Joseph
---
.../scheduler/MutableConfigurationProvider.java| 6 +++
.../conf/FSSchedulerConfigurationStore.java| 43 -
.../capacity/conf/InMemoryConfigurationStore.java | 8
.../capacity/conf/LeveldbConfigurationStore.java | 54 +++---
.../conf/MutableCSConfigurationProvider.java | 5 ++
.../capacity/conf/YarnConfigurationStore.java | 6 +++
.../capacity/conf/ZKConfigurationStore.java| 18
.../server/resourcemanager/webapp/RMWSConsts.java | 3 ++
.../resourcemanager/webapp/RMWebServices.java | 36 ++-
.../webapp/dao/ConfigVersionInfo.java | 44 ++
.../conf/TestFSSchedulerConfigurationStore.java| 12 +++--
.../capacity/conf/TestZKConfigurationStore.java| 15 ++
.../TestRMWebServicesConfigurationMutation.java| 19
13 files changed, 258 insertions(+), 11 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 9e843df..eff8aa8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -65,6 +65,12 @@ public interface MutableConfigurationProvider {
*/
Configuration getConfiguration();
+ /**
+ * Get the last updated scheduler config version.
+ * @return Last updated scheduler config version.
+ */
+ long getConfigVersion() throws Exception;
+
void formatConfigurationInStore(Configuration conf) throws Exception;
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 80053be..464ef14 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -29,6 +29,7 @@ import com.google.gson.GsonBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -62,6 +63,7 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
private volatile Configuration schedConf;
private volatile Configuration oldConf;
private Path tempConfigPath;
+ private Path configVersionFile;
@Override
public void initialize(Configuration conf, Configuration vSchedConf,
@@ -99,9 +101,17 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
}
}
+this.configVersionFile = new Path(schedulerConfPathStr, "ConfigVersion");
+if (!fileSystem.exists(configVersionFile)) {
+ fileSystem.createNewFile(configVersionFile);
+ writeConfigVersion(0L);
+}
+
// create capacity-schedule.xml.ts file if not existing
if (this.getConfigFileInputStream() == null) {
writeConfigurationToFileSystem(vSchedConf);
+ long configVersion = getConfigVersion() + 1L;
+ writeConfigVersion(configVersion);
}
this.schedConf = this.getConfigurationFromFileSystem();
@@ -141,6 +151,8 @@ public class FSSchedulerConfigurationSto
[hadoop] branch branch-3.2 updated: Revert "YARN-9873. Mutation API Config Change updates Version Number. Contributed by Prabhu Joseph"
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 5704f15 Revert "YARN-9873. Mutation API Config Change updates Version
Number. Contributed by Prabhu Joseph"
5704f15 is described below
commit 5704f1558974e97ace213c75fe99d5f327d33db9
Author: Sunil G
AuthorDate: Sat Oct 5 09:16:04 2019 +0530
Revert "YARN-9873. Mutation API Config Change updates Version Number.
Contributed by Prabhu Joseph"
This reverts commit 3a0afcfb7f196bee8e534a3e80211bfa077b2aee.
---
.../scheduler/MutableConfigurationProvider.java| 6
.../conf/FSSchedulerConfigurationStore.java| 7 -
.../capacity/conf/InMemoryConfigurationStore.java | 8 -
.../capacity/conf/LeveldbConfigurationStore.java | 23 +--
.../conf/MutableCSConfigurationProvider.java | 5
.../capacity/conf/YarnConfigurationStore.java | 6
.../capacity/conf/ZKConfigurationStore.java| 19
.../server/resourcemanager/webapp/RMWSConsts.java | 3 --
.../resourcemanager/webapp/RMWebServices.java | 34 +-
.../capacity/conf/TestZKConfigurationStore.java| 15 --
10 files changed, 2 insertions(+), 124 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index eff8aa8..9e843df 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -65,12 +65,6 @@ public interface MutableConfigurationProvider {
*/
Configuration getConfiguration();
- /**
- * Get the last updated scheduler config version.
- * @return Last updated scheduler config version.
- */
- long getConfigVersion() throws Exception;
-
void formatConfigurationInStore(Configuration conf) throws Exception;
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 14cd0a4..ddc5c8a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -148,13 +148,6 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
tempConfigPath = null;
}
- @Override
- public long getConfigVersion() throws Exception {
-String version = getLatestConfigPath().getName().
-substring(YarnConfiguration.CS_CONFIGURATION_FILE.length() + 1);
-return Long.parseLong(version);
- }
-
private void finalizeFileSystemFile() throws IOException {
// call confirmMutation() make sure tempConfigPath is not null
Path finalConfigPath = getFinalConfigPath(tempConfigPath);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 59d140e..4871443 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -33,13 +33,11 @@ public class InMemoryConfigurationStore extends
Ya
[hadoop] branch trunk updated: Revert "YARN-9873. Mutation API Config Change updates Version Number. Contributed by Prabhu Joseph"
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new fb1ecff Revert "YARN-9873. Mutation API Config Change updates Version
Number. Contributed by Prabhu Joseph"
fb1ecff is described below
commit fb1ecff6a26875c7f2b86ef07d7b9145c469377e
Author: Sunil G
AuthorDate: Sat Oct 5 09:15:17 2019 +0530
Revert "YARN-9873. Mutation API Config Change updates Version Number.
Contributed by Prabhu Joseph"
This reverts commit 4510970e2f7728d036c750b596985e5ffa357b60.
---
.../scheduler/MutableConfigurationProvider.java| 6
.../conf/FSSchedulerConfigurationStore.java| 7 -
.../capacity/conf/InMemoryConfigurationStore.java | 8 -
.../capacity/conf/LeveldbConfigurationStore.java | 23 +--
.../conf/MutableCSConfigurationProvider.java | 5
.../capacity/conf/YarnConfigurationStore.java | 6
.../capacity/conf/ZKConfigurationStore.java| 19
.../server/resourcemanager/webapp/RMWSConsts.java | 3 --
.../resourcemanager/webapp/RMWebServices.java | 34 +-
.../capacity/conf/TestZKConfigurationStore.java| 15 --
10 files changed, 2 insertions(+), 124 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index eff8aa8..9e843df 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -65,12 +65,6 @@ public interface MutableConfigurationProvider {
*/
Configuration getConfiguration();
- /**
- * Get the last updated scheduler config version.
- * @return Last updated scheduler config version.
- */
- long getConfigVersion() throws Exception;
-
void formatConfigurationInStore(Configuration conf) throws Exception;
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index f59fa0a..80053be 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -148,13 +148,6 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
tempConfigPath = null;
}
- @Override
- public long getConfigVersion() throws Exception {
-String version = getLatestConfigPath().getName().
-substring(YarnConfiguration.CS_CONFIGURATION_FILE.length() + 1);
-return Long.parseLong(version);
- }
-
private void finalizeFileSystemFile() throws IOException {
// call confirmMutation() make sure tempConfigPath is not null
Path finalConfigPath = getFinalConfigPath(tempConfigPath);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 59d140e..4871443 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -33,13 +33,11 @@ public class InMemoryConfigurationStore extends
Ya
[hadoop] branch branch-3.2 updated: YARN-9873. Mutation API Config Change updates Version Number. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 3a0afcf YARN-9873. Mutation API Config Change updates Version Number.
Contributed by Prabhu Joseph
3a0afcf is described below
commit 3a0afcfb7f196bee8e534a3e80211bfa077b2aee
Author: Sunil G
AuthorDate: Fri Oct 4 21:49:07 2019 +0530
YARN-9873. Mutation API Config Change updates Version Number. Contributed
by Prabhu Joseph
(cherry picked from commit 4510970e2f7728d036c750b596985e5ffa357b60)
---
.../scheduler/MutableConfigurationProvider.java| 6
.../conf/FSSchedulerConfigurationStore.java| 7 +
.../capacity/conf/InMemoryConfigurationStore.java | 8 +
.../capacity/conf/LeveldbConfigurationStore.java | 23 ++-
.../conf/MutableCSConfigurationProvider.java | 5
.../capacity/conf/YarnConfigurationStore.java | 6
.../capacity/conf/ZKConfigurationStore.java| 19
.../server/resourcemanager/webapp/RMWSConsts.java | 3 ++
.../resourcemanager/webapp/RMWebServices.java | 34 +-
.../capacity/conf/TestZKConfigurationStore.java| 15 ++
10 files changed, 124 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 9e843df..eff8aa8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -65,6 +65,12 @@ public interface MutableConfigurationProvider {
*/
Configuration getConfiguration();
+ /**
+ * Get the last updated scheduler config version.
+ * @return Last updated scheduler config version.
+ */
+ long getConfigVersion() throws Exception;
+
void formatConfigurationInStore(Configuration conf) throws Exception;
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index ddc5c8a..14cd0a4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -148,6 +148,13 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
tempConfigPath = null;
}
+ @Override
+ public long getConfigVersion() throws Exception {
+String version = getLatestConfigPath().getName().
+substring(YarnConfiguration.CS_CONFIGURATION_FILE.length() + 1);
+return Long.parseLong(version);
+ }
+
private void finalizeFileSystemFile() throws IOException {
// call confirmMutation() make sure tempConfigPath is not null
Path finalConfigPath = getFinalConfigPath(tempConfigPath);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 4871443..59d140e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -33,11 +33,13 @@ public class InMemoryConfigurationStore extends
YarnConfigurationStore
[hadoop] branch trunk updated: YARN-9873. Mutation API Config Change updates Version Number. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4510970 YARN-9873. Mutation API Config Change updates Version Number.
Contributed by Prabhu Joseph
4510970 is described below
commit 4510970e2f7728d036c750b596985e5ffa357b60
Author: Sunil G
AuthorDate: Fri Oct 4 21:49:07 2019 +0530
YARN-9873. Mutation API Config Change updates Version Number. Contributed
by Prabhu Joseph
---
.../scheduler/MutableConfigurationProvider.java| 6
.../conf/FSSchedulerConfigurationStore.java| 7 +
.../capacity/conf/InMemoryConfigurationStore.java | 8 +
.../capacity/conf/LeveldbConfigurationStore.java | 23 ++-
.../conf/MutableCSConfigurationProvider.java | 5
.../capacity/conf/YarnConfigurationStore.java | 6
.../capacity/conf/ZKConfigurationStore.java| 19
.../server/resourcemanager/webapp/RMWSConsts.java | 3 ++
.../resourcemanager/webapp/RMWebServices.java | 34 +-
.../capacity/conf/TestZKConfigurationStore.java| 15 ++
10 files changed, 124 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 9e843df..eff8aa8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -65,6 +65,12 @@ public interface MutableConfigurationProvider {
*/
Configuration getConfiguration();
+ /**
+ * Get the last updated scheduler config version.
+ * @return Last updated scheduler config version.
+ */
+ long getConfigVersion() throws Exception;
+
void formatConfigurationInStore(Configuration conf) throws Exception;
/**
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 80053be..f59fa0a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -148,6 +148,13 @@ public class FSSchedulerConfigurationStore extends
YarnConfigurationStore {
tempConfigPath = null;
}
+ @Override
+ public long getConfigVersion() throws Exception {
+String version = getLatestConfigPath().getName().
+substring(YarnConfiguration.CS_CONFIGURATION_FILE.length() + 1);
+return Long.parseLong(version);
+ }
+
private void finalizeFileSystemFile() throws IOException {
// call confirmMutation() make sure tempConfigPath is not null
Path finalConfigPath = getFinalConfigPath(tempConfigPath);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 4871443..59d140e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -33,11 +33,13 @@ public class InMemoryConfigurationStore extends
YarnConfigurationStore {
private Configuration schedConf;
private LogMutation pendingMutation;
+ private long
[hadoop] branch trunk updated: YARN-9792. Document examples of SchedulerConf with Node Labels. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 0d2d6f9 YARN-9792. Document examples of SchedulerConf with Node Labels. Contributed by Prabhu Joseph 0d2d6f9 is described below commit 0d2d6f92679a0c1e169fd90706bc5ded06d088b7 Author: Sunil G AuthorDate: Wed Oct 2 23:43:47 2019 +0530 YARN-9792. Document examples of SchedulerConf with Node Labels. Contributed by Prabhu Joseph --- .../src/site/markdown/ResourceManagerRest.md | 81 ++ 1 file changed, 81 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md index a6d7971..8f0b464 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md @@ -5292,6 +5292,87 @@ Response Header: Content-Type: application/xml Transfer-Encoding: chunked +**Adding Node Labels to a queue** + +Assuming we are using the capacity scheduler and the current queue configuration is two queues root.default, and root.a, this example adds a Node Label x to the queue root.a. Create a Node Label x and assign the nodes with below commands. + +```yarn rmadmin -addToClusterNodeLabels "x(exclusive=true)"``` + +```yarn rmadmin -replaceLabelsOnNode "=x"``` + +HTTP Request: + +```xml + Accept: application/xml + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf + Content-Type: application/xml + + + + root.a + + + accessible-node-labels + x + + + accessible-node-labels.x.capacity + 100 + + + + + root + + + accessible-node-labels.x.capacity + 100 + + + + +``` + + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml + Transfer-Encoding: chunked + +**Removing Node Labels from a queue** + +Assuming we are using the capacity scheduler and the current queue configuration is two queues root.default, and root.a and Node Label x is assigned to queue root.a. This example unsets the Node Label x from the queue root.a and removes it. + +HTTP Request: + +```xml + Accept: application/xml + PUT http://rm-http-address:port/ws/v1/cluster/scheduler-conf + Content-Type: application/xml + + + + root.a + + + accessible-node-labels + + + + + +``` + + +Response Header: + + HTTP/1.1 200 OK + Content-Type: application/xml + Transfer-Encoding: chunked + +```yarn rmadmin -removeFromClusterNodeLabels x``` + Cluster Container Signal API - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: YARN-9801. SchedConfCli does not work wiwith https mode. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 933d81f YARN-9801. SchedConfCli does not work wiwith https mode.
Contributed by Prabhu Joseph
933d81f is described below
commit 933d81f0ce11961626f11c93c2f144cce0d742bf
Author: Sunil G
AuthorDate: Tue Oct 1 20:06:21 2019 +0530
YARN-9801. SchedConfCli does not work wiwith https mode. Contributed by
Prabhu Joseph
(cherry picked from commit 99cd7572f113cbbb8f5ef89af60af3734fe8cfa7)
---
.../hadoop/yarn/client/cli/SchedConfCLI.java | 62 --
1 file changed, 58 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index 273b50d..9030e1b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -23,6 +23,8 @@ import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.MissingArgumentException;
@@ -31,6 +33,8 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -41,6 +45,9 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -156,7 +163,12 @@ public class SchedConfCLI extends Configured implements
Tool {
@VisibleForTesting
int formatSchedulerConf(String webAppAddress, WebResource resource)
throws Exception {
-Client webServiceClient = Client.create();
+Configuration conf = getConf();
+SSLFactory clientSslFactory = null;
+if (YarnConfiguration.useHttps(conf)) {
+ clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+}
+Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
resource = (resource != null) ? resource :
webServiceClient.resource(webAppAddress);
@@ -194,14 +206,24 @@ public class SchedConfCLI extends Configured implements
Tool {
if (response != null) {
response.close();
}
- webServiceClient.destroy();
+ if (webServiceClient != null) {
+webServiceClient.destroy();
+ }
+ if (clientSslFactory != null) {
+clientSslFactory.destroy();
+ }
}
}
@VisibleForTesting
int updateSchedulerConfOnRMNode(String webAppAddress,
SchedConfUpdateInfo updateInfo) throws Exception {
-Client webServiceClient = Client.create();
+Configuration conf = getConf();
+SSLFactory clientSslFactory = null;
+if (YarnConfiguration.useHttps(conf)) {
+ clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+}
+Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
WebResource resource = webServiceClient.resource(webAppAddress);
@@ -236,10 +258,42 @@ public class SchedConfCLI extends Configured implements
Tool {
if (response != null) {
response.close();
}
- webServiceClient.destroy();
+ if (webServiceClient != null) {
+webServiceClient.destroy();
+ }
+ if (clientSslFactory != null) {
+clientSslFactory.destroy();
+ }
}
}
+ private Client createWebServiceClient(SSLFactory clientSslFactory) {
+Client webServiceClient = new Client(new URLConnectionClientHandler(
+new HttpURLConnectionFactory() {
+@Override
+public HttpURLConnection getHttpURLConnection(URL url)
+throws IOException {
+ AuthenticatedURL.Token token = new
[hadoop] branch branch-3.2 updated: YARN-9801. SchedConfCli does not work wiwith https mode. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 312cfa9 YARN-9801. SchedConfCli does not work wiwith https mode.
Contributed by Prabhu Joseph
312cfa9 is described below
commit 312cfa994a7e9c76eadb18d14570fb645657edcc
Author: Sunil G
AuthorDate: Tue Oct 1 20:06:21 2019 +0530
YARN-9801. SchedConfCli does not work wiwith https mode. Contributed by
Prabhu Joseph
(cherry picked from commit 99cd7572f113cbbb8f5ef89af60af3734fe8cfa7)
---
.../hadoop/yarn/client/cli/SchedConfCLI.java | 62 --
1 file changed, 58 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index daf4add..146db9e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -23,6 +23,8 @@ import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.MissingArgumentException;
@@ -31,6 +33,8 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -41,6 +45,9 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -156,7 +163,12 @@ public class SchedConfCLI extends Configured implements
Tool {
@VisibleForTesting
int formatSchedulerConf(String webAppAddress, WebResource resource)
throws Exception {
-Client webServiceClient = Client.create();
+Configuration conf = getConf();
+SSLFactory clientSslFactory = null;
+if (YarnConfiguration.useHttps(conf)) {
+ clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+}
+Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
resource = (resource != null) ? resource :
webServiceClient.resource(webAppAddress);
@@ -194,14 +206,24 @@ public class SchedConfCLI extends Configured implements
Tool {
if (response != null) {
response.close();
}
- webServiceClient.destroy();
+ if (webServiceClient != null) {
+webServiceClient.destroy();
+ }
+ if (clientSslFactory != null) {
+clientSslFactory.destroy();
+ }
}
}
@VisibleForTesting
int updateSchedulerConfOnRMNode(String webAppAddress,
SchedConfUpdateInfo updateInfo) throws Exception {
-Client webServiceClient = Client.create();
+Configuration conf = getConf();
+SSLFactory clientSslFactory = null;
+if (YarnConfiguration.useHttps(conf)) {
+ clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+}
+Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
WebResource resource = webServiceClient.resource(webAppAddress);
@@ -236,10 +258,42 @@ public class SchedConfCLI extends Configured implements
Tool {
if (response != null) {
response.close();
}
- webServiceClient.destroy();
+ if (webServiceClient != null) {
+webServiceClient.destroy();
+ }
+ if (clientSslFactory != null) {
+clientSslFactory.destroy();
+ }
}
}
+ private Client createWebServiceClient(SSLFactory clientSslFactory) {
+Client webServiceClient = new Client(new URLConnectionClientHandler(
+new HttpURLConnectionFactory() {
+@Override
+public HttpURLConnection getHttpURLConnection(URL url)
+throws IOException {
+ AuthenticatedURL.Token token = new
[hadoop] branch trunk updated: YARN-9801. SchedConfCli does not work wiwith https mode. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 99cd757 YARN-9801. SchedConfCli does not work wiwith https mode.
Contributed by Prabhu Joseph
99cd757 is described below
commit 99cd7572f113cbbb8f5ef89af60af3734fe8cfa7
Author: Sunil G
AuthorDate: Tue Oct 1 20:06:21 2019 +0530
YARN-9801. SchedConfCli does not work wiwith https mode. Contributed by
Prabhu Joseph
---
.../hadoop/yarn/client/cli/SchedConfCLI.java | 62 --
1 file changed, 58 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index daf4add..146db9e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -23,6 +23,8 @@ import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.WebResource.Builder;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.MissingArgumentException;
@@ -31,6 +33,8 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -41,6 +45,9 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -156,7 +163,12 @@ public class SchedConfCLI extends Configured implements
Tool {
@VisibleForTesting
int formatSchedulerConf(String webAppAddress, WebResource resource)
throws Exception {
-Client webServiceClient = Client.create();
+Configuration conf = getConf();
+SSLFactory clientSslFactory = null;
+if (YarnConfiguration.useHttps(conf)) {
+ clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+}
+Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
resource = (resource != null) ? resource :
webServiceClient.resource(webAppAddress);
@@ -194,14 +206,24 @@ public class SchedConfCLI extends Configured implements
Tool {
if (response != null) {
response.close();
}
- webServiceClient.destroy();
+ if (webServiceClient != null) {
+webServiceClient.destroy();
+ }
+ if (clientSslFactory != null) {
+clientSslFactory.destroy();
+ }
}
}
@VisibleForTesting
int updateSchedulerConfOnRMNode(String webAppAddress,
SchedConfUpdateInfo updateInfo) throws Exception {
-Client webServiceClient = Client.create();
+Configuration conf = getConf();
+SSLFactory clientSslFactory = null;
+if (YarnConfiguration.useHttps(conf)) {
+ clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+}
+Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
WebResource resource = webServiceClient.resource(webAppAddress);
@@ -236,10 +258,42 @@ public class SchedConfCLI extends Configured implements
Tool {
if (response != null) {
response.close();
}
- webServiceClient.destroy();
+ if (webServiceClient != null) {
+webServiceClient.destroy();
+ }
+ if (clientSslFactory != null) {
+clientSslFactory.destroy();
+ }
}
}
+ private Client createWebServiceClient(SSLFactory clientSslFactory) {
+Client webServiceClient = new Client(new URLConnectionClientHandler(
+new HttpURLConnectionFactory() {
+@Override
+public HttpURLConnection getHttpURLConnection(URL url)
+throws IOException {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ AuthenticatedURL aUrl;
+ HttpURLConnection
[hadoop] branch branch-3.1 updated: YARN-9864. Format CS Configuration present in Configuration Store. Contributeed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 71792f212 YARN-9864. Format CS Configuration present in Configuration
Store. Contributeed by Prabhu Joseph
71792f212 is described below
commit 71792f21229bddf79e2299ecaf1af88afffc0ef2
Author: Sunil G
AuthorDate: Tue Oct 1 20:03:35 2019 +0530
YARN-9864. Format CS Configuration present in Configuration Store.
Contributeed by Prabhu Joseph
---
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 10 ++
.../org/apache/hadoop/yarn/client/cli/LogsCLI.java | 44 +++--
.../hadoop/yarn/client/cli/SchedConfCLI.java | 125 --
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 187 -
.../hadoop/yarn/webapp/util/WebAppUtils.java | 47 +-
.../yarn/webapp/util/YarnWebServiceUtils.java | 40 +++--
.../hadoop/yarn/conf/TestYarnConfiguration.java| 6 +
.../scheduler/MutableConfigurationProvider.java| 2 +
.../capacity/conf/InMemoryConfigurationStore.java | 5 +
.../capacity/conf/LeveldbConfigurationStore.java | 7 +
.../conf/MutableCSConfigurationProvider.java | 46 -
.../capacity/conf/YarnConfigurationStore.java | 6 +
.../capacity/conf/ZKConfigurationStore.java| 5 +
.../server/resourcemanager/webapp/RMWSConsts.java | 6 +
.../resourcemanager/webapp/RMWebServices.java | 40 -
.../conf/TestMutableCSConfigurationProvider.java | 4 +
.../capacity/conf/TestZKConfigurationStore.java| 9 +
.../TestRMWebServicesConfigurationMutation.java| 13 ++
18 files changed, 547 insertions(+), 55 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index 299bd18..26391db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -65,6 +65,16 @@
hadoop-annotations
+ com.sun.jersey.jersey-test-framework
+ jersey-test-framework-core
+ test
+
+
+ com.sun.jersey.jersey-test-framework
+ jersey-test-framework-grizzly2
+ test
+
+
org.mockito
mockito-all
test
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index 33f9450..d134ba3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -25,6 +25,7 @@ import com.sun.jersey.api.client.ClientRequest;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.WebResource.Builder;
import com.sun.jersey.api.client.filter.ClientFilter;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
@@ -155,6 +156,9 @@ public class LogsCLI extends Configured implements Tool {
if (yarnClient != null) {
yarnClient.close();
}
+ if (webServiceClient != null) {
+webServiceClient.destroy();
+ }
}
}
@@ -418,24 +422,34 @@ public class LogsCLI extends Configured implements Tool {
}
protected List getAMContainerInfoForRMWebService(
- Configuration conf, String appId) throws ClientHandlerException,
- UniformInterfaceException, JSONException {
-String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf);
-
-WebResource webResource = webServiceClient.resource(webAppAddress);
+ Configuration conf, String appId) throws Exception {
+return WebAppUtils.execOnActiveRM(conf, this::getAMContainerInfoFromRM,
+appId);
+ }
-ClientResponse response =
-webResource.path("ws").path("v1").path("cluster").path("apps")
- .path(appId).path("appattempts").accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
-JSONObject json =
-response.getEntity(JSONObject.class).getJSONObject("appAttempts");
-JSONArray requests = json.getJSONArray("appAttempt");
+ private List getAMContainerInfoFromRM(
+ String webAppAddress, String appId) throws ClientHandlerException,
+ UniformInterfaceException, JSONException {
List amContainersList = new ArrayList();
-for (int i = 0; i < requests.length(); i++) {
- amContainersList.add(requests.getJSONObject(i)
[hadoop] branch branch-3.2 updated: YARN-9864. Format CS Configuration present in Configuration Store. Contributeed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 52f815d YARN-9864. Format CS Configuration present in Configuration
Store. Contributeed by Prabhu Joseph
52f815d is described below
commit 52f815d39d9e5585770ae0a62cb1c328fea96e94
Author: Sunil G
AuthorDate: Tue Oct 1 09:08:41 2019 +0530
YARN-9864. Format CS Configuration present in Configuration Store.
Contributeed by Prabhu Joseph
(cherry picked from commit 137546a78a45adbfb5ef338c1c2984a97b3cecc2)
---
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 10 ++
.../hadoop/yarn/client/cli/SchedConfCLI.java | 89 +-
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 187 -
.../scheduler/MutableConfigurationProvider.java| 2 +
.../conf/FSSchedulerConfigurationStore.java| 5 +
.../capacity/conf/InMemoryConfigurationStore.java | 5 +
.../capacity/conf/LeveldbConfigurationStore.java | 7 +
.../conf/MutableCSConfigurationProvider.java | 46 -
.../capacity/conf/YarnConfigurationStore.java | 7 +
.../capacity/conf/ZKConfigurationStore.java| 5 +
.../server/resourcemanager/webapp/RMWSConsts.java | 3 +
.../resourcemanager/webapp/RMWebServices.java | 31
.../conf/TestFSSchedulerConfigurationStore.java| 16 ++
.../conf/TestMutableCSConfigurationProvider.java | 8 +
.../capacity/conf/TestZKConfigurationStore.java| 9 +
.../TestRMWebServicesConfigurationMutation.java| 13 ++
16 files changed, 431 insertions(+), 12 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index c21c381..5568734 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -65,6 +65,16 @@
mockito-all
test
+
+ com.sun.jersey.jersey-test-framework
+ jersey-test-framework-core
+ test
+
+
+ com.sun.jersey.jersey-test-framework
+ jersey-test-framework-grizzly2
+ test
+
org.apache.hadoop
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index be54553..daf4add 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.client.cli;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.WebResource.Builder;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
@@ -30,6 +31,7 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
@@ -56,6 +58,7 @@ public class SchedConfCLI extends Configured implements Tool {
private static final String REMOVE_QUEUES_OPTION = "removeQueues";
private static final String UPDATE_QUEUES_OPTION = "updateQueues";
private static final String GLOBAL_OPTIONS = "globalUpdates";
+ private static final String FORMAT_CONF = "formatConfig";
private static final String HELP_CMD = "help";
private static final String CONF_ERR_MSG = "Specify configuration key " +
@@ -83,6 +86,9 @@ public class SchedConfCLI extends Configured implements Tool {
"Update queue configurations");
opts.addOption("global", GLOBAL_OPTIONS, true,
"Update global scheduler configurations");
+opts.addOption("format", FORMAT_CONF, false,
+"Format Scheduler Configuration and reload from" +
+" capacity-scheduler.xml");
opts.addOption("h", HELP_CMD, false, "Displays help for all commands.");
int exitCode = -1;
@@ -101,6 +107,7 @@ public class SchedConfCLI extends Configured implements
Tool {
}
boolean hasOption = false;
+boolean format = false;
SchedConfUpdateInfo updateInfo
[hadoop] branch trunk updated: YARN-9864. Format CS Configuration present in Configuration Store. Contributeed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 137546a YARN-9864. Format CS Configuration present in Configuration
Store. Contributeed by Prabhu Joseph
137546a is described below
commit 137546a78a45adbfb5ef338c1c2984a97b3cecc2
Author: Sunil G
AuthorDate: Tue Oct 1 09:08:41 2019 +0530
YARN-9864. Format CS Configuration present in Configuration Store.
Contributeed by Prabhu Joseph
---
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 10 ++
.../hadoop/yarn/client/cli/SchedConfCLI.java | 89 +-
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 187 -
.../scheduler/MutableConfigurationProvider.java| 2 +
.../conf/FSSchedulerConfigurationStore.java| 5 +
.../capacity/conf/InMemoryConfigurationStore.java | 5 +
.../capacity/conf/LeveldbConfigurationStore.java | 7 +
.../conf/MutableCSConfigurationProvider.java | 46 -
.../capacity/conf/YarnConfigurationStore.java | 7 +
.../capacity/conf/ZKConfigurationStore.java| 5 +
.../server/resourcemanager/webapp/RMWSConsts.java | 3 +
.../resourcemanager/webapp/RMWebServices.java | 31
.../conf/TestFSSchedulerConfigurationStore.java| 16 ++
.../conf/TestMutableCSConfigurationProvider.java | 8 +
.../capacity/conf/TestZKConfigurationStore.java| 9 +
.../TestRMWebServicesConfigurationMutation.java| 13 ++
16 files changed, 431 insertions(+), 12 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index 3efbddd..88e5e24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -65,6 +65,16 @@
mockito-core
test
+
+ com.sun.jersey.jersey-test-framework
+ jersey-test-framework-core
+ test
+
+
+ com.sun.jersey.jersey-test-framework
+ jersey-test-framework-grizzly2
+ test
+
org.apache.hadoop
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index be54553..daf4add 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.client.cli;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.WebResource.Builder;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
@@ -30,6 +31,7 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
@@ -56,6 +58,7 @@ public class SchedConfCLI extends Configured implements Tool {
private static final String REMOVE_QUEUES_OPTION = "removeQueues";
private static final String UPDATE_QUEUES_OPTION = "updateQueues";
private static final String GLOBAL_OPTIONS = "globalUpdates";
+ private static final String FORMAT_CONF = "formatConfig";
private static final String HELP_CMD = "help";
private static final String CONF_ERR_MSG = "Specify configuration key " +
@@ -83,6 +86,9 @@ public class SchedConfCLI extends Configured implements Tool {
"Update queue configurations");
opts.addOption("global", GLOBAL_OPTIONS, true,
"Update global scheduler configurations");
+opts.addOption("format", FORMAT_CONF, false,
+"Format Scheduler Configuration and reload from" +
+" capacity-scheduler.xml");
opts.addOption("h", HELP_CMD, false, "Displays help for all commands.");
int exitCode = -1;
@@ -101,6 +107,7 @@ public class SchedConfCLI extends Configured implements
Tool {
}
boolean hasOption = false;
+boolean format = false;
SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
try {
if (parsedCli.hasOption(ADD_QUEUES_OPTIO
[hadoop] branch trunk updated: YARN-9814. JobHistoryServer can't delete aggregated files, if remote app root directory is created by NodeManager. Contributed by Adam Antal.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 01d7924 YARN-9814. JobHistoryServer can't delete aggregated files, if
remote app root directory is created by NodeManager. Contributed by Adam Antal.
01d7924 is described below
commit 01d79244732c7f60dff3cd7181647c0460955491
Author: Sunil G
AuthorDate: Wed Sep 18 09:27:41 2019 +0530
YARN-9814. JobHistoryServer can't delete aggregated files, if remote app
root directory is created by NodeManager. Contributed by Adam Antal.
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 6 ++
.../LogAggregationFileController.java | 25 --
.../src/main/resources/yarn-default.xml| 8 ++
.../TestLogAggregationFileController.java | 91 ++
4 files changed, 122 insertions(+), 8 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9c62827..7b05905 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1479,6 +1479,12 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_NM_REMOTE_APP_LOG_DIR_INCLUDE_OLDER =
true;
+ /**
+ * Specifies the group of the aggregated log directory.
+ */
+ public static final String NM_REMOTE_APP_LOG_DIR_GROUPNAME =
+ NM_PREFIX + "remote-app-log-dir.groupname";
+
public static final String YARN_LOG_SERVER_URL =
YARN_PREFIX + "log.server.url";
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
index 661e321..001f4f5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
@@ -52,7 +52,6 @@ import
org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.webapp.View.ViewContext;
@@ -346,13 +345,23 @@ public abstract class LogAggregationFileController {
}
UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
-String primaryGroupName = null;
-try {
- primaryGroupName = loginUser.getPrimaryGroupName();
-} catch (IOException e) {
- LOG.warn("No primary group found. The remote root log directory" +
- " will be created with the HDFS superuser being its group " +
- "owner. JobHistoryServer may be unable to read the directory.");
+String primaryGroupName = conf.get(
+YarnConfiguration.NM_REMOTE_APP_LOG_DIR_GROUPNAME);
+if (primaryGroupName == null || primaryGroupName.isEmpty()) {
+ try {
+primaryGroupName = loginUser.getPrimaryGroupName();
+ } catch (IOException e) {
+LOG.warn("No primary group found. The remote root log directory" +
+" will be created with the HDFS superuser being its " +
+"group owner. JobHistoryServer may be unable to read " +
+"the directory.");
+ }
+} else {
+ if (LOG.isDebugEnabled()) {
+LOG.debug("The group of remote root log directory has been " +
+"determined by the configuration and set to " +
+primaryGroupName);
+ }
}
// set owner on the remote directory only if the primary group exists
if (primaryGroupName != null) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index b856536..55e908d 100644
[hadoop] branch branch-3.1 updated: YARN-9833. Race condition when DirectoryCollection.checkDirs() runs during container launch. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 45cf3de YARN-9833. Race condition when
DirectoryCollection.checkDirs() runs during container launch. Contributed by
Peter Bacsko.
45cf3de is described below
commit 45cf3de2e9bf836bbf643aec5076962c1d1de761
Author: Sunil G
AuthorDate: Tue Sep 17 19:41:38 2019 +0530
YARN-9833. Race condition when DirectoryCollection.checkDirs() runs during
container launch. Contributed by Peter Bacsko.
(cherry picked from commit c474e24c0b73c0f52a7d5af2495355f4a0799344)
---
.../apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
index 7a840b1..d7282f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
/**
* Manages a list of local storage directories.
@@ -230,7 +231,7 @@ public class DirectoryCollection {
List getGoodDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(localDirs);
+ return ImmutableList.copyOf(localDirs);
} finally {
this.readLock.unlock();
}
@@ -242,7 +243,7 @@ public class DirectoryCollection {
List getFailedDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(
+ return ImmutableList.copyOf(
DirectoryCollection.concat(errorDirs, fullDirs));
} finally {
this.readLock.unlock();
@@ -255,7 +256,7 @@ public class DirectoryCollection {
List getFullDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(fullDirs);
+ return ImmutableList.copyOf(fullDirs);
} finally {
this.readLock.unlock();
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-9833. Race condition when DirectoryCollection.checkDirs() runs during container launch. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 3e0025d YARN-9833. Race condition when
DirectoryCollection.checkDirs() runs during container launch. Contributed by
Peter Bacsko.
3e0025d is described below
commit 3e0025d8776e0a1797b9261f4b1776c7c5a304a8
Author: Sunil G
AuthorDate: Tue Sep 17 19:41:38 2019 +0530
YARN-9833. Race condition when DirectoryCollection.checkDirs() runs during
container launch. Contributed by Peter Bacsko.
(cherry picked from commit c474e24c0b73c0f52a7d5af2495355f4a0799344)
---
.../apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
index d0a0219..3462aa4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
/**
* Manages a list of local storage directories.
@@ -227,7 +228,7 @@ public class DirectoryCollection {
List getGoodDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(localDirs);
+ return ImmutableList.copyOf(localDirs);
} finally {
this.readLock.unlock();
}
@@ -239,7 +240,7 @@ public class DirectoryCollection {
List getFailedDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(
+ return ImmutableList.copyOf(
DirectoryCollection.concat(errorDirs, fullDirs));
} finally {
this.readLock.unlock();
@@ -252,7 +253,7 @@ public class DirectoryCollection {
List getFullDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(fullDirs);
+ return ImmutableList.copyOf(fullDirs);
} finally {
this.readLock.unlock();
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9833. Race condition when DirectoryCollection.checkDirs() runs during container launch. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c474e24 YARN-9833. Race condition when
DirectoryCollection.checkDirs() runs during container launch. Contributed by
Peter Bacsko.
c474e24 is described below
commit c474e24c0b73c0f52a7d5af2495355f4a0799344
Author: Sunil G
AuthorDate: Tue Sep 17 19:41:38 2019 +0530
YARN-9833. Race condition when DirectoryCollection.checkDirs() runs during
container launch. Contributed by Peter Bacsko.
---
.../apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
index d0a0219..3462aa4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
/**
* Manages a list of local storage directories.
@@ -227,7 +228,7 @@ public class DirectoryCollection {
List getGoodDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(localDirs);
+ return ImmutableList.copyOf(localDirs);
} finally {
this.readLock.unlock();
}
@@ -239,7 +240,7 @@ public class DirectoryCollection {
List getFailedDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(
+ return ImmutableList.copyOf(
DirectoryCollection.concat(errorDirs, fullDirs));
} finally {
this.readLock.unlock();
@@ -252,7 +253,7 @@ public class DirectoryCollection {
List getFullDirs() {
this.readLock.lock();
try {
- return Collections.unmodifiableList(fullDirs);
+ return ImmutableList.copyOf(fullDirs);
} finally {
this.readLock.unlock();
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9758. Upgrade JQuery to latest version for YARN UI. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 094d736 YARN-9758. Upgrade JQuery to latest version for YARN UI.
Contributed by Akhil PB.
094d736 is described below
commit 094d73608fdd455a30084658ad8ac40d12886736
Author: Sunil G
AuthorDate: Tue Aug 20 18:47:44 2019 +0530
YARN-9758. Upgrade JQuery to latest version for YARN UI. Contributed by
Akhil PB.
---
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +-
.../src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java | 2 +-
.../src/main/resources/webapps/static/jquery/jquery-3.3.1.min.js| 2 --
.../src/main/resources/webapps/static/jquery/jquery-3.4.1.min.js| 2 ++
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 26268c3..304fed2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -242,7 +242,7 @@
src/main/resources/webapps/static/dt-1.10.18/js/jquery.dataTables.min.js
src/main/resources/webapps/static/jt/jquery.jstree.js
src/main/resources/webapps/static/jquery/jquery-ui-1.12.1.custom.min.js
-
src/main/resources/webapps/static/jquery/jquery-3.3.1.min.js
+
src/main/resources/webapps/static/jquery/jquery-3.4.1.min.js
src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css
src/test/resources/application_1440536969523_0001.har/_index
src/test/resources/application_1440536969523_0001.har/part-0
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
index 0754c36..4438da0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
@@ -69,7 +69,7 @@ public class JQueryUI extends HtmlBlock {
.link(root_url("static/dt-1.10.18/css/jquery.dataTables.css"))
.link(root_url("static/dt-1.10.18/css/jui-dt.css"))
.link(root_url("static/dt-1.10.18/css/custom_datatable.css"))
-.script(root_url("static/jquery/jquery-3.3.1.min.js"))
+.script(root_url("static/jquery/jquery-3.4.1.min.js"))
.script(root_url("static/jquery/jquery-ui-1.12.1.custom.min.js"))
.script(root_url("static/dt-1.10.18/js/jquery.dataTables.min.js"))
.script(root_url("static/yarn.dt.plugins.js"))
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.3.1.min.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.3.1.min.js
deleted file mode 100644
index 4d9b3a2..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.3.1.min.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! jQuery v3.3.1 | (c) JS Foundation and other contributors |
jquery.org/license */
-!function(e,t){"use strict";"object"==typeof module&&"object"==typeof
module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw
new Error("jQuery requires a window with a document");return
t(e)}:t(e)}("undefined"!=typeof window?window:this,function(e,t){"use
strict";var
n=[],r=e.document,i=Object.getPrototypeOf,o=n.slice,a=n.concat,s=n.push,u=n.indexOf,l={},c=l.toString,f=l.hasOwnProperty,p=f.toString,d=p.call(Object),h={},g=function
e(t){return"function"==typeof [...]
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.4.1.min.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.4.1.min.js
new file mode 100644
index 000..a1c07fd
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery/jquery-3.4.1.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v3.4.1 | (c) JS Foundation and other contributors |
jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof
module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw
new Error("jQuery requires a window with a document");return
t(e)}:t(e)}("un
[hadoop] branch trunk updated: YARN-2599. Standby RM should expose jmx endpoint. Contributed by Rohith Sharma K S.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0e0ddfa YARN-2599. Standby RM should expose jmx endpoint. Contributed
by Rohith Sharma K S.
0e0ddfa is described below
commit 0e0ddfaf242cf2c66b3ab95de919efecb1137505
Author: Sunil G
AuthorDate: Sat Aug 17 15:43:19 2019 +0530
YARN-2599. Standby RM should expose jmx endpoint. Contributed by Rohith
Sharma K S.
---
.../test/java/org/apache/hadoop/yarn/client/TestRMFailover.java| 7 ---
.../hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java | 5 +++--
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index 8d9a1b8..25377f1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -294,10 +294,8 @@ public class TestRMFailover extends ClientBaseWithFixes {
redirectURL = getRedirectURL(rm2Url + "/metrics");
assertEquals(redirectURL,rm1Url + "/metrics");
-redirectURL = getRedirectURL(rm2Url + "/jmx?param1=value1+x¶m2=y");
-assertEquals(rm1Url + "/jmx?param1=value1+x¶m2=y", redirectURL);
-// standby RM links /conf, /stacks, /logLevel, /static, /logs,
+// standby RM links /conf, /stacks, /logLevel, /static, /logs, /jmx
// /cluster/cluster as well as webService
// /ws/v1/cluster/info should not be redirected to active RM
redirectURL = getRedirectURL(rm2Url + "/cluster/cluster");
@@ -318,6 +316,9 @@ public class TestRMFailover extends ClientBaseWithFixes {
redirectURL = getRedirectURL(rm2Url + "/logs");
assertNull(redirectURL);
+redirectURL = getRedirectURL(rm2Url + "/jmx?param1=value1+x¶m2=y");
+assertNull(redirectURL);
+
redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info");
assertNull(redirectURL);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index 40b3834..d197be4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -69,7 +69,8 @@ public class RMWebAppFilter extends GuiceContainer {
// define a set of URIs which do not need to do redirection
private static final Set NON_REDIRECTED_URIS = Sets.newHashSet(
- "/conf", "/stacks", "/logLevel", "/logs", IsActiveServlet.PATH_SPEC);
+ "/conf", "/stacks", "/logLevel", "/logs", IsActiveServlet.PATH_SPEC,
+ "/jmx");
private String path;
private boolean ahsEnabled;
private String ahsPageURLPrefix;
@@ -274,4 +275,4 @@ public class RMWebAppFilter extends GuiceContainer {
long baseTime = BASIC_SLEEP_TIME * (1L << retries);
return (int) (baseTime * (randnum.nextDouble() + 0.5));
}
-}
\ No newline at end of file
+}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: YARN-9729. [UI2] Fix error message for logs when ATSv2 is offline. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 58ad5ad YARN-9729. [UI2] Fix error message for logs when ATSv2 is
offline. Contributed by Zoltan Siegl.
58ad5ad is described below
commit 58ad5ad4931018a8af3ba9c2a293693be7f6b06a
Author: Sunil G
AuthorDate: Sun Aug 11 11:46:47 2019 +0530
YARN-9729. [UI2] Fix error message for logs when ATSv2 is offline.
Contributed by Zoltan Siegl.
(cherry picked from commit 1c5b28659fe1310030d14e0be40dcd77b25056d6)
---
.../main/webapp/app/controllers/yarn-app/logs.js | 9 ++-
.../src/main/webapp/app/routes/yarn-app/logs.js| 5 +-
.../main/webapp/app/templates/yarn-app/logs.hbs| 71 --
3 files changed, 52 insertions(+), 33 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
index e1a8ec8..d7eac3f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
@@ -263,5 +263,12 @@ export default Ember.Controller.extend({
isLogAggregationNotSucceeded: Ember.computed("model.app", function() {
const logAggregationStatus = this.get("model.app.logAggregationStatus");
return logAggregationStatus !== "SUCCEEDED";
- })
+ }),
+
+ isTimelineUnHealthy: function() {
+ if (this.model && this.model.timelineHealth) {
+return this.model.timelineHealth.get('isTimelineUnHealthy');
+ }
+ return true;
+}.property('model.timelineHealth')
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
index b73bc80..9719f80 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
@@ -31,7 +31,10 @@ export default AbstractRoute.extend(AppAttemptMixin, {
attempts: this.fetchAttemptListFromRMorATS(app_id,
this.store).catch(function() {
return [];
}),
- app: this.fetchAppInfoFromRMorATS(app_id, this.store)
+ app: this.fetchAppInfoFromRMorATS(app_id, this.store),
+ timelineHealth: this.store.queryRecord('timeline-health',
{}).catch(function() {
+return null;
+ })
});
},
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
index 10fd73e..b6dd318 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
@@ -32,43 +32,52 @@
{{/if}}
-{{#if attemptList}}
+{{#if isTimelineUnHealthy}}
-
- Choose attempt to fetch containers
-
-
- None
- {{#each attemptList as |attempt|}}
-{{attempt.id}}
- {{/each}}
-
-
+
+
+ Logs are unavailable because Application Timeline Service
seems unhealthy.
-{{#if attemptContainerList}}
-
-Choose container to fetch logs
-
-
-None
-{{#each attemptContainerList as |container|}}
- {{container.id}}
-{{/each}}
-
+
+{{else}}
+{{#if attemptList}}
+
+
+Choose attempt to fetch containers
+
+
+None
+{{#each attemptList as |attempt|}}
+{{attempt.id}}
+{{/each}}
+
+
+
+{{#if attemptContainerList}}
+
+Choose container to fetch logs
+
+
+None
+{{#each attemptContainerList as
|
[hadoop] branch branch-3.2 updated: YARN-9729. [UI2] Fix error message for logs when ATSv2 is offline. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 02b4635 YARN-9729. [UI2] Fix error message for logs when ATSv2 is
offline. Contributed by Zoltan Siegl.
02b4635 is described below
commit 02b4635ff034af565b9903cd00db9beb0d038e69
Author: Sunil G
AuthorDate: Sun Aug 11 11:46:47 2019 +0530
YARN-9729. [UI2] Fix error message for logs when ATSv2 is offline.
Contributed by Zoltan Siegl.
(cherry picked from commit 1c5b28659fe1310030d14e0be40dcd77b25056d6)
---
.../main/webapp/app/controllers/yarn-app/logs.js | 9 ++-
.../src/main/webapp/app/routes/yarn-app/logs.js| 5 +-
.../main/webapp/app/templates/yarn-app/logs.hbs| 71 --
3 files changed, 52 insertions(+), 33 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
index e1a8ec8..d7eac3f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
@@ -263,5 +263,12 @@ export default Ember.Controller.extend({
isLogAggregationNotSucceeded: Ember.computed("model.app", function() {
const logAggregationStatus = this.get("model.app.logAggregationStatus");
return logAggregationStatus !== "SUCCEEDED";
- })
+ }),
+
+ isTimelineUnHealthy: function() {
+ if (this.model && this.model.timelineHealth) {
+return this.model.timelineHealth.get('isTimelineUnHealthy');
+ }
+ return true;
+}.property('model.timelineHealth')
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
index b73bc80..9719f80 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
@@ -31,7 +31,10 @@ export default AbstractRoute.extend(AppAttemptMixin, {
attempts: this.fetchAttemptListFromRMorATS(app_id,
this.store).catch(function() {
return [];
}),
- app: this.fetchAppInfoFromRMorATS(app_id, this.store)
+ app: this.fetchAppInfoFromRMorATS(app_id, this.store),
+ timelineHealth: this.store.queryRecord('timeline-health',
{}).catch(function() {
+return null;
+ })
});
},
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
index 10fd73e..b6dd318 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
@@ -32,43 +32,52 @@
{{/if}}
-{{#if attemptList}}
+{{#if isTimelineUnHealthy}}
-
- Choose attempt to fetch containers
-
-
- None
- {{#each attemptList as |attempt|}}
-{{attempt.id}}
- {{/each}}
-
-
+
+
+ Logs are unavailable because Application Timeline Service
seems unhealthy.
-{{#if attemptContainerList}}
-
-Choose container to fetch logs
-
-
-None
-{{#each attemptContainerList as |container|}}
- {{container.id}}
-{{/each}}
-
+
+{{else}}
+{{#if attemptList}}
+
+
+Choose attempt to fetch containers
+
+
+None
+{{#each attemptList as |attempt|}}
+{{attempt.id}}
+{{/each}}
+
+
+
+{{#if attemptContainerList}}
+
+Choose container to fetch logs
+
+
+None
+{{#each attemptContainerList as
|
[hadoop] branch trunk updated: YARN-9729. [UI2] Fix error message for logs when ATSv2 is offline. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1c5b286 YARN-9729. [UI2] Fix error message for logs when ATSv2 is
offline. Contributed by Zoltan Siegl.
1c5b286 is described below
commit 1c5b28659fe1310030d14e0be40dcd77b25056d6
Author: Sunil G
AuthorDate: Sun Aug 11 11:46:47 2019 +0530
YARN-9729. [UI2] Fix error message for logs when ATSv2 is offline.
Contributed by Zoltan Siegl.
---
.../main/webapp/app/controllers/yarn-app/logs.js | 9 ++-
.../src/main/webapp/app/routes/yarn-app/logs.js| 5 +-
.../main/webapp/app/templates/yarn-app/logs.hbs| 71 --
3 files changed, 52 insertions(+), 33 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
index e1a8ec8..d7eac3f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
@@ -263,5 +263,12 @@ export default Ember.Controller.extend({
isLogAggregationNotSucceeded: Ember.computed("model.app", function() {
const logAggregationStatus = this.get("model.app.logAggregationStatus");
return logAggregationStatus !== "SUCCEEDED";
- })
+ }),
+
+ isTimelineUnHealthy: function() {
+ if (this.model && this.model.timelineHealth) {
+return this.model.timelineHealth.get('isTimelineUnHealthy');
+ }
+ return true;
+}.property('model.timelineHealth')
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
index b73bc80..9719f80 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
@@ -31,7 +31,10 @@ export default AbstractRoute.extend(AppAttemptMixin, {
attempts: this.fetchAttemptListFromRMorATS(app_id,
this.store).catch(function() {
return [];
}),
- app: this.fetchAppInfoFromRMorATS(app_id, this.store)
+ app: this.fetchAppInfoFromRMorATS(app_id, this.store),
+ timelineHealth: this.store.queryRecord('timeline-health',
{}).catch(function() {
+return null;
+ })
});
},
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
index 10fd73e..b6dd318 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
@@ -32,43 +32,52 @@
{{/if}}
-{{#if attemptList}}
+{{#if isTimelineUnHealthy}}
-
- Choose attempt to fetch containers
-
-
- None
- {{#each attemptList as |attempt|}}
-{{attempt.id}}
- {{/each}}
-
-
+
+
+ Logs are unavailable because Application Timeline Service
seems unhealthy.
-{{#if attemptContainerList}}
-
-Choose container to fetch logs
-
-
-None
-{{#each attemptContainerList as |container|}}
- {{container.id}}
-{{/each}}
-
+
+{{else}}
+{{#if attemptList}}
+
+
+Choose attempt to fetch containers
+
+
+None
+{{#each attemptList as |attempt|}}
+{{attempt.id}}
+{{/each}}
+
+
+
+{{#if attemptContainerList}}
+
+Choose container to fetch logs
+
+
+None
+{{#each attemptContainerList as
|container|}}
+
[hadoop] branch trunk updated: Revert "YARN-9729. [UI2] Fix error message for logs without ATSv2. Contributed by Zoltan Siegl."
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e69db45 Revert "YARN-9729. [UI2] Fix error message for logs without
ATSv2. Contributed by Zoltan Siegl."
e69db45 is described below
commit e69db45e500f83a166e35a3fcc29a8029ecb3c5c
Author: Sunil G
AuthorDate: Sat Aug 10 21:10:22 2019 +0530
Revert "YARN-9729. [UI2] Fix error message for logs without ATSv2.
Contributed by Zoltan Siegl."
This reverts commit 50cec5a704d11ffc5e0333d22dbea7cf6837c2fc.
---
.../main/webapp/app/controllers/yarn-app/logs.js | 9 +--
.../src/main/webapp/app/routes/yarn-app/logs.js| 5 +-
.../main/webapp/app/templates/yarn-app/logs.hbs| 73 ++
3 files changed, 34 insertions(+), 53 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
index d7eac3f..e1a8ec8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
@@ -263,12 +263,5 @@ export default Ember.Controller.extend({
isLogAggregationNotSucceeded: Ember.computed("model.app", function() {
const logAggregationStatus = this.get("model.app.logAggregationStatus");
return logAggregationStatus !== "SUCCEEDED";
- }),
-
- isTimelineUnHealthy: function() {
- if (this.model && this.model.timelineHealth) {
-return this.model.timelineHealth.get('isTimelineUnHealthy');
- }
- return true;
-}.property('model.timelineHealth')
+ })
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
index 9719f80..b73bc80 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
@@ -31,10 +31,7 @@ export default AbstractRoute.extend(AppAttemptMixin, {
attempts: this.fetchAttemptListFromRMorATS(app_id,
this.store).catch(function() {
return [];
}),
- app: this.fetchAppInfoFromRMorATS(app_id, this.store),
- timelineHealth: this.store.queryRecord('timeline-health',
{}).catch(function() {
-return null;
- })
+ app: this.fetchAppInfoFromRMorATS(app_id, this.store)
});
},
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
index 72147b9..10fd73e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
@@ -32,52 +32,43 @@
{{/if}}
-{{#if isTimelineUnHealthy}}
-
-
-
-Logs are unavailable because Application Timeline
Service seems unhealthy.
-
+{{#if attemptList}}
+
+
+ Choose attempt to fetch containers
+
+
+ None
+ {{#each attemptList as |attempt|}}
+{{attempt.id}}
+ {{/each}}
+
+
-{{else}}
-{{#if attemptList}}
-
-
-Choose attempt to fetch containers
-
-
-None
-{{#each attemptList as |attempt|}}
-{{attempt.id}}
-{{/each}}
-
-
-
-{{#if attemptContainerList}}
-
-Choose container to fetch logs
-
-
-None
-{{#each attemptContainerList as
|container|}}
-{{container.id}}
-{{/each}}
-
-
-
-{{else}}
-{{#if (and selectedAttemptId (not
_isLoading
[hadoop] branch trunk updated: YARN-9729. [UI2] Fix error message for logs without ATSv2. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 50cec5a YARN-9729. [UI2] Fix error message for logs without ATSv2.
Contributed by Zoltan Siegl.
50cec5a is described below
commit 50cec5a704d11ffc5e0333d22dbea7cf6837c2fc
Author: Sunil G
AuthorDate: Sat Aug 10 21:06:25 2019 +0530
YARN-9729. [UI2] Fix error message for logs without ATSv2. Contributed by
Zoltan Siegl.
---
.../main/webapp/app/controllers/yarn-app/logs.js | 9 ++-
.../src/main/webapp/app/routes/yarn-app/logs.js| 5 +-
.../main/webapp/app/templates/yarn-app/logs.hbs| 73 --
3 files changed, 53 insertions(+), 34 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
index e1a8ec8..d7eac3f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js
@@ -263,5 +263,12 @@ export default Ember.Controller.extend({
isLogAggregationNotSucceeded: Ember.computed("model.app", function() {
const logAggregationStatus = this.get("model.app.logAggregationStatus");
return logAggregationStatus !== "SUCCEEDED";
- })
+ }),
+
+ isTimelineUnHealthy: function() {
+ if (this.model && this.model.timelineHealth) {
+return this.model.timelineHealth.get('isTimelineUnHealthy');
+ }
+ return true;
+}.property('model.timelineHealth')
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
index b73bc80..9719f80 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js
@@ -31,7 +31,10 @@ export default AbstractRoute.extend(AppAttemptMixin, {
attempts: this.fetchAttemptListFromRMorATS(app_id,
this.store).catch(function() {
return [];
}),
- app: this.fetchAppInfoFromRMorATS(app_id, this.store)
+ app: this.fetchAppInfoFromRMorATS(app_id, this.store),
+ timelineHealth: this.store.queryRecord('timeline-health',
{}).catch(function() {
+return null;
+ })
});
},
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
index 10fd73e..72147b9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-app/logs.hbs
@@ -32,43 +32,52 @@
{{/if}}
-{{#if attemptList}}
-
-
- Choose attempt to fetch containers
-
-
- None
- {{#each attemptList as |attempt|}}
-{{attempt.id}}
- {{/each}}
-
-
+{{#if isTimelineUnHealthy}}
+
+
+
+Logs are unavailable because Application Timeline
Service seems unhealthy.
+
-{{#if attemptContainerList}}
-
-Choose container to fetch logs
-
-
-None
-{{#each attemptContainerList as |container|}}
- {{container.id}}
-{{/each}}
-
+{{else}}
+{{#if attemptList}}
+
+
+Choose attempt to fetch containers
+
+
+None
+{{#each attemptList as |attempt|}}
+{{attempt.id}}
+{{/each}}
+
+
+
+{{#if attemptContainerList}}
+
+Choose container to fetch logs
+
+
+None
+{{#each attemptContainerList as
|container|}}
+
[hadoop] branch branch-3.2 updated: YARN-9715. [UI2] yarn-container-log URI need to be encoded to avoid potential misuses. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 9fb6c6e YARN-9715. [UI2] yarn-container-log URI need to be encoded to
avoid potential misuses. Contributed by Akhil PB.
9fb6c6e is described below
commit 9fb6c6e2a1f563feb337716800836460f4576da3
Author: Sunil G
AuthorDate: Fri Aug 9 12:25:10 2019 +0530
YARN-9715. [UI2] yarn-container-log URI need to be encoded to avoid
potential misuses. Contributed by Akhil PB.
(cherry picked from commit acffec7a92be540aa8531dbe06a3ea7bb813ab93)
---
.../src/main/webapp/app/controllers/yarn-container-log.js | 5 +++--
.../src/main/webapp/app/controllers/yarn-node-app.js | 5 +
.../src/main/webapp/app/controllers/yarn-node-apps.js | 5 +
.../src/main/webapp/app/controllers/yarn-node-container.js| 5 +
.../src/main/webapp/app/controllers/yarn-node-containers.js | 5 +
.../src/main/webapp/app/routes/yarn-container-log.js | 8 +---
.../hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js| 8 +---
.../src/main/webapp/app/templates/yarn-node-app.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-apps.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-container.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-containers.hbs| 4 ++--
11 files changed, 38 insertions(+), 13 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
index db7470d..d23b2b3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
@@ -23,6 +23,7 @@ export default Ember.Controller.extend({
breadcrumbs: Ember.computed('model.nodeInfo', 'model.containerInfo',
function () {
var nodeInfo = this.get('model.nodeInfo'),
containerInfo = this.get('model.containerInfo');
+var nodeAddr = encodeURIComponent(nodeInfo.addr);
return [{
text: "Home",
routeName: 'application'
@@ -31,10 +32,10 @@ export default Ember.Controller.extend({
routeName: 'yarn-nodes.table'
}, {
text: `Node [ ${nodeInfo.id} ]`,
- href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeAddr}/info`,
}, {
text: `Container [ ${containerInfo.id} ]`,
- href:
`#/yarn-node-container/${nodeInfo.id}/${nodeInfo.addr}/${containerInfo.id}`,
+ href:
`#/yarn-node-container/${nodeInfo.id}/${nodeAddr}/${containerInfo.id}`,
}, {
text: "Log",
}];
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
index e0d58ec..5ecf706 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
@@ -35,6 +35,11 @@ export default Ember.Controller.extend({
}, {
text: `Application [ ${nodeInfo.appId} ]`,
}];
+ }),
+
+ encodedNodeAddr: Ember.computed("model.nodeInfo", function() {
+var nodeInfo = this.get("model.nodeInfo");
+return encodeURIComponent(nodeInfo.addr);
})
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
index ddc8256..460f186 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
@@ -35,6 +35,11 @@ export default Ember.Controller.extend({
}, {
text: "Applications",
}];
+ }),
+
+ encodedNodeAddr: Ember.computed("model.nodeInfo", function() {
+var nodeInfo = this.get("model.nodeInfo");
+return encodeURIComponent(nodeInfo.addr);
})
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
index b8e7e36..bac26b2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
+++
[hadoop] branch branch-3.1 updated: YARN-9715. [UI2] yarn-container-log URI need to be encoded to avoid potential misuses. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 1c4364c YARN-9715. [UI2] yarn-container-log URI need to be encoded to
avoid potential misuses. Contributed by Akhil PB.
1c4364c is described below
commit 1c4364cb9a67b4694758cf2e1f0adc1b6ec7954b
Author: Sunil G
AuthorDate: Fri Aug 9 12:25:10 2019 +0530
YARN-9715. [UI2] yarn-container-log URI need to be encoded to avoid
potential misuses. Contributed by Akhil PB.
(cherry picked from commit acffec7a92be540aa8531dbe06a3ea7bb813ab93)
---
.../src/main/webapp/app/controllers/yarn-container-log.js | 5 +++--
.../src/main/webapp/app/controllers/yarn-node-app.js | 5 +
.../src/main/webapp/app/controllers/yarn-node-apps.js | 5 +
.../src/main/webapp/app/controllers/yarn-node-container.js| 5 +
.../src/main/webapp/app/controllers/yarn-node-containers.js | 5 +
.../src/main/webapp/app/routes/yarn-container-log.js | 8 +---
.../hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js| 8 +---
.../src/main/webapp/app/templates/yarn-node-app.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-apps.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-container.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-containers.hbs| 4 ++--
11 files changed, 38 insertions(+), 13 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
index db7470d..d23b2b3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
@@ -23,6 +23,7 @@ export default Ember.Controller.extend({
breadcrumbs: Ember.computed('model.nodeInfo', 'model.containerInfo',
function () {
var nodeInfo = this.get('model.nodeInfo'),
containerInfo = this.get('model.containerInfo');
+var nodeAddr = encodeURIComponent(nodeInfo.addr);
return [{
text: "Home",
routeName: 'application'
@@ -31,10 +32,10 @@ export default Ember.Controller.extend({
routeName: 'yarn-nodes.table'
}, {
text: `Node [ ${nodeInfo.id} ]`,
- href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeAddr}/info`,
}, {
text: `Container [ ${containerInfo.id} ]`,
- href:
`#/yarn-node-container/${nodeInfo.id}/${nodeInfo.addr}/${containerInfo.id}`,
+ href:
`#/yarn-node-container/${nodeInfo.id}/${nodeAddr}/${containerInfo.id}`,
}, {
text: "Log",
}];
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
index e0d58ec..5ecf706 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
@@ -35,6 +35,11 @@ export default Ember.Controller.extend({
}, {
text: `Application [ ${nodeInfo.appId} ]`,
}];
+ }),
+
+ encodedNodeAddr: Ember.computed("model.nodeInfo", function() {
+var nodeInfo = this.get("model.nodeInfo");
+return encodeURIComponent(nodeInfo.addr);
})
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
index ddc8256..460f186 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
@@ -35,6 +35,11 @@ export default Ember.Controller.extend({
}, {
text: "Applications",
}];
+ }),
+
+ encodedNodeAddr: Ember.computed("model.nodeInfo", function() {
+var nodeInfo = this.get("model.nodeInfo");
+return encodeURIComponent(nodeInfo.addr);
})
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
index b8e7e36..bac26b2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
+++
[hadoop] branch trunk updated: YARN-9715. [UI2] yarn-container-log URI need to be encoded to avoid potential misuses. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new acffec7 YARN-9715. [UI2] yarn-container-log URI need to be encoded to
avoid potential misuses. Contributed by Akhil PB.
acffec7 is described below
commit acffec7a92be540aa8531dbe06a3ea7bb813ab93
Author: Sunil G
AuthorDate: Fri Aug 9 12:25:10 2019 +0530
YARN-9715. [UI2] yarn-container-log URI need to be encoded to avoid
potential misuses. Contributed by Akhil PB.
---
.../src/main/webapp/app/controllers/yarn-container-log.js | 5 +++--
.../src/main/webapp/app/controllers/yarn-node-app.js | 5 +
.../src/main/webapp/app/controllers/yarn-node-apps.js | 5 +
.../src/main/webapp/app/controllers/yarn-node-container.js| 5 +
.../src/main/webapp/app/controllers/yarn-node-containers.js | 5 +
.../src/main/webapp/app/routes/yarn-container-log.js | 8 +---
.../hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node-app.js| 8 +---
.../src/main/webapp/app/templates/yarn-node-app.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-apps.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-container.hbs | 2 +-
.../src/main/webapp/app/templates/yarn-node-containers.hbs| 4 ++--
11 files changed, 38 insertions(+), 13 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
index db7470d..d23b2b3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-container-log.js
@@ -23,6 +23,7 @@ export default Ember.Controller.extend({
breadcrumbs: Ember.computed('model.nodeInfo', 'model.containerInfo',
function () {
var nodeInfo = this.get('model.nodeInfo'),
containerInfo = this.get('model.containerInfo');
+var nodeAddr = encodeURIComponent(nodeInfo.addr);
return [{
text: "Home",
routeName: 'application'
@@ -31,10 +32,10 @@ export default Ember.Controller.extend({
routeName: 'yarn-nodes.table'
}, {
text: `Node [ ${nodeInfo.id} ]`,
- href: `#/yarn-node/${nodeInfo.id}/${nodeInfo.addr}`,
+ href: `#/yarn-node/${nodeInfo.id}/${nodeAddr}/info`,
}, {
text: `Container [ ${containerInfo.id} ]`,
- href:
`#/yarn-node-container/${nodeInfo.id}/${nodeInfo.addr}/${containerInfo.id}`,
+ href:
`#/yarn-node-container/${nodeInfo.id}/${nodeAddr}/${containerInfo.id}`,
}, {
text: "Log",
}];
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
index e0d58ec..5ecf706 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-app.js
@@ -35,6 +35,11 @@ export default Ember.Controller.extend({
}, {
text: `Application [ ${nodeInfo.appId} ]`,
}];
+ }),
+
+ encodedNodeAddr: Ember.computed("model.nodeInfo", function() {
+var nodeInfo = this.get("model.nodeInfo");
+return encodeURIComponent(nodeInfo.addr);
})
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
index ddc8256..460f186 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-apps.js
@@ -35,6 +35,11 @@ export default Ember.Controller.extend({
}, {
text: "Applications",
}];
+ }),
+
+ encodedNodeAddr: Ember.computed("model.nodeInfo", function() {
+var nodeInfo = this.get("model.nodeInfo");
+return encodeURIComponent(nodeInfo.addr);
})
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
index b8e7e36..bac26b2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-container.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-node-c
[hadoop] branch branch-3.2 updated: YARN-9644. First RMContext object is always leaked during switch over. Contributed by Bibin A Chundatt.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new d18986e YARN-9644. First RMContext object is always leaked during
switch over. Contributed by Bibin A Chundatt.
d18986e is described below
commit d18986e4e89bd4bb3600e95b4690fd32f54a41e5
Author: Sunil G
AuthorDate: Thu Jul 4 11:05:54 2019 +0530
YARN-9644. First RMContext object is always leaked during switch over.
Contributed by Bibin A Chundatt.
---
.../apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java | 11 ---
.../hadoop/yarn/server/resourcemanager/ResourceManager.java | 11 ---
2 files changed, 16 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index 1b7ddd3..18cf6f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
import javax.management.StandardMBean;
import org.apache.commons.logging.Log;
@@ -42,6 +43,7 @@ public class RMNMInfo implements RMNMInfoBeans {
private static final Log LOG = LogFactory.getLog(RMNMInfo.class);
private RMContext rmContext;
private ResourceScheduler scheduler;
+ private ObjectName mbeanObjectName;
/**
* Constructor for RMNMInfo registers the bean with JMX.
@@ -55,14 +57,17 @@ public class RMNMInfo implements RMNMInfoBeans {
StandardMBean bean;
try {
-bean = new StandardMBean(this,RMNMInfoBeans.class);
-MBeans.register("ResourceManager", "RMNMInfo", bean);
+ bean = new StandardMBean(this, RMNMInfoBeans.class);
+ mbeanObjectName = MBeans.register("ResourceManager", "RMNMInfo", bean);
} catch (NotCompliantMBeanException e) {
-LOG.warn("Error registering RMNMInfo MBean", e);
+ LOG.warn("Error registering RMNMInfo MBean", e);
}
LOG.info("Registered RMNMInfo MBean");
}
+ public void unregister() {
+MBeans.unregister(mbeanObjectName);
+ }
static class InfoMap extends LinkedHashMap {
private static final long serialVersionUID = 1L;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 16f019f..b395d0f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -630,6 +630,7 @@ public class ResourceManager extends CompositeService
private ResourceManager rm;
private boolean fromActive = false;
private StandByTransitionRunnable standByTransitionRunnable;
+private RMNMInfo rmnmInfo;
RMActiveServices(ResourceManager rm) {
super("RMActiveServices");
@@ -830,7 +831,7 @@ public class ResourceManager extends CompositeService
LOG.info("Initialized Federation membership.");
}
- new RMNMInfo(rmContext, scheduler);
+ rmnmInfo = new RMNMInfo(rmContext, scheduler);
if (conf.getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE,
false)) {
@@ -899,6 +900,10 @@ public class ResourceManager extends CompositeService
super.serviceStop();
DefaultMetricsSystem.shutdown();
+ // unregister rmnmInfo bean
+ if (rmnmInfo != null) {
+rmnmInfo.unregister();
+ }
if (rmContext != null) {
RMStateStore store = rmContext.getStateStore();
try {
@@ -1156,9 +1161,9 @@ public class ResourceManager extends CompositeService
params.put("com.sun.jersey.config.property.packages", apiPackages);
}
-Builder builder =
+Builder builder =
WebApps
-.$for("cluster", ApplicationMaste
[hadoop] branch branch-3.1 updated: YARN-9644. First RMContext object is always leaked during switch over. Contributed by Bibin A Chundatt.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 9eb96b0 YARN-9644. First RMContext object is always leaked during
switch over. Contributed by Bibin A Chundatt.
9eb96b0 is described below
commit 9eb96b0fbfae5b7232556fa9a4235fb976ee6d47
Author: Sunil G
AuthorDate: Thu Jul 4 11:05:54 2019 +0530
YARN-9644. First RMContext object is always leaked during switch over.
Contributed by Bibin A Chundatt.
(cherry picked from commit d18986e4e89bd4bb3600e95b4690fd32f54a41e5)
---
.../apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java | 11 ---
.../hadoop/yarn/server/resourcemanager/ResourceManager.java | 11 ---
2 files changed, 16 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index 1b7ddd3..18cf6f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
import javax.management.StandardMBean;
import org.apache.commons.logging.Log;
@@ -42,6 +43,7 @@ public class RMNMInfo implements RMNMInfoBeans {
private static final Log LOG = LogFactory.getLog(RMNMInfo.class);
private RMContext rmContext;
private ResourceScheduler scheduler;
+ private ObjectName mbeanObjectName;
/**
* Constructor for RMNMInfo registers the bean with JMX.
@@ -55,14 +57,17 @@ public class RMNMInfo implements RMNMInfoBeans {
StandardMBean bean;
try {
-bean = new StandardMBean(this,RMNMInfoBeans.class);
-MBeans.register("ResourceManager", "RMNMInfo", bean);
+ bean = new StandardMBean(this, RMNMInfoBeans.class);
+ mbeanObjectName = MBeans.register("ResourceManager", "RMNMInfo", bean);
} catch (NotCompliantMBeanException e) {
-LOG.warn("Error registering RMNMInfo MBean", e);
+ LOG.warn("Error registering RMNMInfo MBean", e);
}
LOG.info("Registered RMNMInfo MBean");
}
+ public void unregister() {
+MBeans.unregister(mbeanObjectName);
+ }
static class InfoMap extends LinkedHashMap {
private static final long serialVersionUID = 1L;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index d459f0e..d1a4e34 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -616,6 +616,7 @@ public class ResourceManager extends CompositeService
private ResourceManager rm;
private boolean fromActive = false;
private StandByTransitionRunnable standByTransitionRunnable;
+private RMNMInfo rmnmInfo;
RMActiveServices(ResourceManager rm) {
super("RMActiveServices");
@@ -806,7 +807,7 @@ public class ResourceManager extends CompositeService
LOG.info("Initialized Federation membership.");
}
- new RMNMInfo(rmContext, scheduler);
+ rmnmInfo = new RMNMInfo(rmContext, scheduler);
if (conf.getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE,
false)) {
@@ -875,6 +876,10 @@ public class ResourceManager extends CompositeService
super.serviceStop();
DefaultMetricsSystem.shutdown();
+ // unregister rmnmInfo bean
+ if (rmnmInfo != null) {
+rmnmInfo.unregister();
+ }
if (rmContext != null) {
RMStateStore store = rmContext.getStateStore();
try {
@@ -1132,9 +1137,9 @@ public class ResourceManager extends CompositeService
params.put("com.sun.jersey.config.property.packages", apiPackages);
}
-Builder builder =
+Builder bu
[hadoop] branch trunk updated: YARN-9644. First RMContext object is always leaked during switch over. Contributed by Bibin A Chundatt.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e966edd YARN-9644. First RMContext object is always leaked during
switch over. Contributed by Bibin A Chundatt.
e966edd is described below
commit e966edd025332394701fe0d2cfa0d76731183aaf
Author: Sunil G
AuthorDate: Tue Jul 2 12:18:16 2019 +0530
YARN-9644. First RMContext object is always leaked during switch over.
Contributed by Bibin A Chundatt.
---
.../apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java | 11 ---
.../hadoop/yarn/server/resourcemanager/ResourceManager.java | 11 ---
2 files changed, 16 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index 4a5da02..8f1cda9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import javax.management.NotCompliantMBeanException;
+import javax.management.ObjectName;
import javax.management.StandardMBean;
import org.slf4j.Logger;
@@ -43,6 +44,7 @@ public class RMNMInfo implements RMNMInfoBeans {
LoggerFactory.getLogger(RMNMInfo.class);
private RMContext rmContext;
private ResourceScheduler scheduler;
+ private ObjectName mbeanObjectName;
/**
* Constructor for RMNMInfo registers the bean with JMX.
@@ -56,14 +58,17 @@ public class RMNMInfo implements RMNMInfoBeans {
StandardMBean bean;
try {
-bean = new StandardMBean(this,RMNMInfoBeans.class);
-MBeans.register("ResourceManager", "RMNMInfo", bean);
+ bean = new StandardMBean(this, RMNMInfoBeans.class);
+ mbeanObjectName = MBeans.register("ResourceManager", "RMNMInfo", bean);
} catch (NotCompliantMBeanException e) {
-LOG.warn("Error registering RMNMInfo MBean", e);
+ LOG.warn("Error registering RMNMInfo MBean", e);
}
LOG.info("Registered RMNMInfo MBean");
}
+ public void unregister() {
+MBeans.unregister(mbeanObjectName);
+ }
static class InfoMap extends LinkedHashMap {
private static final long serialVersionUID = 1L;
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index c795bfa..6244b77 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -641,6 +641,7 @@ public class ResourceManager extends CompositeService
private ResourceManager rm;
private boolean fromActive = false;
private StandByTransitionRunnable standByTransitionRunnable;
+private RMNMInfo rmnmInfo;
RMActiveServices(ResourceManager rm) {
super("RMActiveServices");
@@ -845,7 +846,7 @@ public class ResourceManager extends CompositeService
addService(proxyCAManager);
rmContext.setProxyCAManager(proxyCAManager);
- new RMNMInfo(rmContext, scheduler);
+ rmnmInfo = new RMNMInfo(rmContext, scheduler);
if (conf.getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE,
false)) {
@@ -924,6 +925,10 @@ public class ResourceManager extends CompositeService
super.serviceStop();
DefaultMetricsSystem.shutdown();
+ // unregister rmnmInfo bean
+ if (rmnmInfo != null) {
+rmnmInfo.unregister();
+ }
if (rmContext != null) {
RMStateStore store = rmContext.getStateStore();
try {
@@ -1182,9 +1187,9 @@ public class ResourceManager extends CompositeService
params.put("com.sun.jersey.config.property.packages", apiPackages);
}
-Builder builder =
+Builder builder =
WebApps
-.$for("cluster", ApplicationMasterService.class, masterService,
+
[hadoop] branch branch-3.1 updated: YARN-9327. Improve synchronisation in ProtoUtils#convertToProtoFormat block. Contributed by Bibin A Chundatt.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d2a5749 YARN-9327. Improve synchronisation in
ProtoUtils#convertToProtoFormat block. Contributed by Bibin A Chundatt.
d2a5749 is described below
commit d2a574948262c5cfe438e7d557611e2537f7d8e8
Author: Sunil G
AuthorDate: Tue Jul 2 11:44:11 2019 +0530
YARN-9327. Improve synchronisation in ProtoUtils#convertToProtoFormat
block. Contributed by Bibin A Chundatt.
(cherry picked from commit 0c8813f135f8c17f88660bb92529c15bb3a157ca)
---
.../java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java | 2 +-
.../org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 76e86ad..3568a70 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -400,7 +400,7 @@ public class ProtoUtils {
/*
* Resource
*/
- public static synchronized ResourceProto convertToProtoFormat(Resource r) {
+ public static ResourceProto convertToProtoFormat(Resource r) {
return ResourcePBImpl.getProto(r);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 6ebed6e..7d334bf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -71,14 +71,14 @@ public class ResourcePBImpl extends Resource {
initResources();
}
- public ResourceProto getProto() {
+ synchronized public ResourceProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
- private void maybeInitBuilder() {
+ synchronized private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ResourceProto.newBuilder(proto);
}
@@ -127,7 +127,7 @@ public class ResourcePBImpl extends Resource {
resources[VCORES_INDEX].setValue(vCores);
}
- private void initResources() {
+ synchronized private void initResources() {
if (this.resources != null) {
return;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-9327. Improve synchronisation in ProtoUtils#convertToProtoFormat block. Contributed by Bibin A Chundatt.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new bea79e7 YARN-9327. Improve synchronisation in
ProtoUtils#convertToProtoFormat block. Contributed by Bibin A Chundatt.
bea79e7 is described below
commit bea79e7645b15ccf69b71b457be7e62848f2e37b
Author: Sunil G
AuthorDate: Tue Jul 2 11:44:11 2019 +0530
YARN-9327. Improve synchronisation in ProtoUtils#convertToProtoFormat
block. Contributed by Bibin A Chundatt.
(cherry picked from commit 0c8813f135f8c17f88660bb92529c15bb3a157ca)
---
.../java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java | 2 +-
.../org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 76e86ad..3568a70 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -400,7 +400,7 @@ public class ProtoUtils {
/*
* Resource
*/
- public static synchronized ResourceProto convertToProtoFormat(Resource r) {
+ public static ResourceProto convertToProtoFormat(Resource r) {
return ResourcePBImpl.getProto(r);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 144f48f..b2d63b4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -71,14 +71,14 @@ public class ResourcePBImpl extends Resource {
initResources();
}
- public ResourceProto getProto() {
+ synchronized public ResourceProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
- private void maybeInitBuilder() {
+ synchronized private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ResourceProto.newBuilder(proto);
}
@@ -127,7 +127,7 @@ public class ResourcePBImpl extends Resource {
resources[VCORES_INDEX].setValue(vCores);
}
- private void initResources() {
+ synchronized private void initResources() {
if (this.resources != null) {
return;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-9327. Improve synchronisation in ProtoUtils#convertToProtoFormat block. Contributed by Bibin A Chundatt.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 0c8813f YARN-9327. Improve synchronisation in
ProtoUtils#convertToProtoFormat block. Contributed by Bibin A Chundatt.
0c8813f is described below
commit 0c8813f135f8c17f88660bb92529c15bb3a157ca
Author: Sunil G
AuthorDate: Tue Jul 2 11:44:11 2019 +0530
YARN-9327. Improve synchronisation in ProtoUtils#convertToProtoFormat
block. Contributed by Bibin A Chundatt.
---
.../java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java | 2 +-
.../org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 3f360d7..64b49c8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -412,7 +412,7 @@ public class ProtoUtils {
/*
* Resource
*/
- public static synchronized ResourceProto convertToProtoFormat(Resource r) {
+ public static ResourceProto convertToProtoFormat(Resource r) {
return ResourcePBImpl.getProto(r);
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 36fc959..ed9f3db 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -75,14 +75,14 @@ public class ResourcePBImpl extends Resource {
initResources();
}
- public ResourceProto getProto() {
+ synchronized public ResourceProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
- private void maybeInitBuilder() {
+ synchronized private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ResourceProto.newBuilder(proto);
}
@@ -131,7 +131,7 @@ public class ResourcePBImpl extends Resource {
resources[VCORES_INDEX].setValue(vCores);
}
- private void initResources() {
+ synchronized private void initResources() {
if (this.resources != null) {
return;
}
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: YARN-9543. [UI2] Handle ATSv2 server down or failures cases gracefully in YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new c343554 YARN-9543. [UI2] Handle ATSv2 server down or failures cases
gracefully in YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
c343554 is described below
commit c343554e2ae9c4c44178a17c0843cd83425c8b0f
Author: Sunil G
AuthorDate: Fri May 31 12:29:44 2019 +0530
YARN-9543. [UI2] Handle ATSv2 server down or failures cases gracefully in
YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
(cherry picked from commit 52128e352a30b70b83483f9290d9e94e98929705)
---
.../hadoop-yarn-ui/src/main/webapp/.gitignore | 4
.../timeline-error.js => adapters/timeline-health.js} | 19 ++-
.../src/main/webapp/app/controllers/application.js| 7 +++
.../timeline-error.js => models/timeline-health.js} | 16 +++-
.../src/main/webapp/app/routes/application.js | 4
.../src/main/webapp/app/routes/timeline-error.js | 3 +++
.../timeline-health.js} | 18 ++
.../src/main/webapp/app/templates/application.hbs | 4 ++--
8 files changed, 47 insertions(+), 28 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
new file mode 100644
index 000..338997f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
@@ -0,0 +1,4 @@
+tmp/
+node_modules/
+bower_components/
+dist/
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
similarity index 73%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
index c2e5fc5..8ca2310 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
@@ -16,14 +16,15 @@
* limitations under the License.
*/
-import Ember from 'ember';
+import RESTAbstractAdapter from './restabstract';
-export default Ember.Route.extend({
- afterModel(model/*, transition*/) {
-model.error_id = "error";
-model.isValidErrorCode = false;
-if (model.errorCode && model.errorCode !== "0") {
- model.isValidErrorCode = true;
-}
+export default RESTAbstractAdapter.extend({
+ address: "timelineWebAddress",
+ restNameSpace: "timelineV2",
+ serverName: "ATS",
+
+ urlForQueryRecord(/*query, modelName*/) {
+var url = this.buildURL();
+return url + '/health';
}
-});
+});
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
index 50a2909..34702ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
@@ -73,4 +73,11 @@ export default Ember.Controller.extend({
}
return null;
}.property('model.userInfo'),
+
+ isTimelineUnHealthy: function() {
+if (this.model && this.model.timelineHealth) {
+ return this.model.timelineHealth.get('isTimelineUnHealthy');
+}
+return true;
+ }.property('model.timelineHealth')
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
similarity index 75%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
index c2e5fc5..367ab07 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
@@ -16,14 +16,12 @@
* limitations under the License.
*/
-import Ember from 'ember';
+import DS from 'ember-data';
-export default Ember.Route.extend({
- afterModel(model/*, transition*/) {
-model.error_id = "error";
-model.isValidErrorCode = false;
-if (model.errorCode && model.error
[hadoop] branch branch-3.1 updated: YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new bc028d3 YARN-9545. Create healthcheck REST endpoint for ATSv2.
Contributed by Zoltan Siegl.
bc028d3 is described below
commit bc028d3ebbc62617810d2c358b98bd2c71877dfd
Author: Sunil G
AuthorDate: Wed Jun 12 19:23:40 2019 +0530
YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by
Zoltan Siegl.
(cherry picked from commit 72203f7a12c943ca231fbc40c058a1a094b009cd)
---
.../yarn/api/records/timeline/TimelineHealth.java | 82 ++
.../storage/HBaseTimelineReaderImpl.java | 13
.../reader/TimelineReaderManager.java | 10 +++
.../reader/TimelineReaderWebServices.java | 33 +
.../storage/FileSystemTimelineReaderImpl.java | 23 ++
.../timelineservice/storage/TimelineReader.java| 8 +++
.../reader/TestTimelineReaderWebServices.java | 19 +
7 files changed, 188 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
new file mode 100644
index 000..d592167
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timeline;
+
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class holds health information for ATS.
+ */
+@XmlRootElement(name = "health")
+@XmlAccessorType(XmlAccessType.NONE)
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineHealth {
+
+ /**
+ * Timline health status.
+ *
+ * RUNNING - Service is up and running
+ * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation
+ *reported an error
+ */
+ public enum TimelineHealthStatus {
+RUNNING,
+READER_CONNECTION_FAILURE
+ }
+
+ private TimelineHealthStatus healthStatus;
+ private String diagnosticsInfo;
+
+ public TimelineHealth(TimelineHealthStatus healthy, String diagnosticsInfo) {
+this.healthStatus = healthy;
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+ public TimelineHealth() {
+
+ }
+
+ @XmlElement(name = "healthStatus")
+ public TimelineHealthStatus getHealthStatus() {
+return healthStatus;
+ }
+
+ @XmlElement(name = "diagnosticsInfo")
+ public String getDiagnosticsInfo() {
+return diagnosticsInfo;
+ }
+
+
+ public void setHealthStatus(TimelineHealthStatus healthStatus) {
+this.healthStatus = healthStatus;
+ }
+
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+
+}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index fadfd14..b83d60f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-ya
[hadoop] branch branch-3.1 updated: Revert "YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by Zoltan Siegl."
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 1bb9e9a Revert "YARN-9545. Create healthcheck REST endpoint for
ATSv2. Contributed by Zoltan Siegl."
1bb9e9a is described below
commit 1bb9e9a4f29e074204c4116a004934d0de1ecd25
Author: Sunil G
AuthorDate: Wed Jun 12 19:27:21 2019 +0530
Revert "YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed
by Zoltan Siegl."
This reverts commit d65371c4e8bbb4ae655ccacda389cd37a18fab32.
---
.../yarn/api/records/timeline/TimelineHealth.java | 82
.../node_modules/.bin/apidoc | 1 -
.../node_modules/.bin/markdown-it | 1 -
.../node_modules/.bin/r.js | 1 -
.../node_modules/.bin/r_js | 1 -
.../node_modules/.bin/semver | 1 -
.../node_modules/.bin/shjs | 1 -
.../yarn.lock | 422 -
.../storage/HBaseTimelineReaderImpl.java | 13 -
.../reader/TimelineReaderManager.java | 10 -
.../reader/TimelineReaderWebServices.java | 33 --
.../storage/FileSystemTimelineReaderImpl.java | 23 --
.../timelineservice/storage/TimelineReader.java| 8 -
.../reader/TestTimelineReaderWebServices.java | 19 -
14 files changed, 616 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
deleted file mode 100644
index d592167..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.api.records.timeline;
-
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * This class holds health information for ATS.
- */
-@XmlRootElement(name = "health")
-@XmlAccessorType(XmlAccessType.NONE)
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class TimelineHealth {
-
- /**
- * Timline health status.
- *
- * RUNNING - Service is up and running
- * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation
- *reported an error
- */
- public enum TimelineHealthStatus {
-RUNNING,
-READER_CONNECTION_FAILURE
- }
-
- private TimelineHealthStatus healthStatus;
- private String diagnosticsInfo;
-
- public TimelineHealth(TimelineHealthStatus healthy, String diagnosticsInfo) {
-this.healthStatus = healthy;
-this.diagnosticsInfo = diagnosticsInfo;
- }
-
- public TimelineHealth() {
-
- }
-
- @XmlElement(name = "healthStatus")
- public TimelineHealthStatus getHealthStatus() {
-return healthStatus;
- }
-
- @XmlElement(name = "diagnosticsInfo")
- public String getDiagnosticsInfo() {
-return diagnosticsInfo;
- }
-
-
- public void setHealthStatus(TimelineHealthStatus healthStatus) {
-this.healthStatus = healthStatus;
- }
-
- public void setDiagnosticsInfo(String diagnosticsInfo) {
-this.diagnosticsInfo = diagnosticsInfo;
- }
-
-
-}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
deleted file mode 12
index a588095..0
[hadoop] branch branch-3.2 updated: YARN-9543. [UI2] Handle ATSv2 server down or failures cases gracefully in YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 253dcde YARN-9543. [UI2] Handle ATSv2 server down or failures cases
gracefully in YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
253dcde is described below
commit 253dcde5176a266998aaed0b9f7d10b922e40330
Author: Sunil G
AuthorDate: Fri May 31 12:29:44 2019 +0530
YARN-9543. [UI2] Handle ATSv2 server down or failures cases gracefully in
YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
(cherry picked from commit 52128e352a30b70b83483f9290d9e94e98929705)
---
.../hadoop-yarn-ui/src/main/webapp/.gitignore | 4
.../timeline-error.js => adapters/timeline-health.js} | 19 ++-
.../src/main/webapp/app/controllers/application.js| 7 +++
.../timeline-error.js => models/timeline-health.js} | 16 +++-
.../src/main/webapp/app/routes/application.js | 4
.../src/main/webapp/app/routes/timeline-error.js | 3 +++
.../timeline-health.js} | 18 ++
.../src/main/webapp/app/templates/application.hbs | 4 ++--
8 files changed, 47 insertions(+), 28 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
new file mode 100644
index 000..338997f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
@@ -0,0 +1,4 @@
+tmp/
+node_modules/
+bower_components/
+dist/
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
similarity index 73%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
index c2e5fc5..8ca2310 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
@@ -16,14 +16,15 @@
* limitations under the License.
*/
-import Ember from 'ember';
+import RESTAbstractAdapter from './restabstract';
-export default Ember.Route.extend({
- afterModel(model/*, transition*/) {
-model.error_id = "error";
-model.isValidErrorCode = false;
-if (model.errorCode && model.errorCode !== "0") {
- model.isValidErrorCode = true;
-}
+export default RESTAbstractAdapter.extend({
+ address: "timelineWebAddress",
+ restNameSpace: "timelineV2",
+ serverName: "ATS",
+
+ urlForQueryRecord(/*query, modelName*/) {
+var url = this.buildURL();
+return url + '/health';
}
-});
+});
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
index 50a2909..34702ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
@@ -73,4 +73,11 @@ export default Ember.Controller.extend({
}
return null;
}.property('model.userInfo'),
+
+ isTimelineUnHealthy: function() {
+if (this.model && this.model.timelineHealth) {
+ return this.model.timelineHealth.get('isTimelineUnHealthy');
+}
+return true;
+ }.property('model.timelineHealth')
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
similarity index 75%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
index c2e5fc5..367ab07 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
@@ -16,14 +16,12 @@
* limitations under the License.
*/
-import Ember from 'ember';
+import DS from 'ember-data';
-export default Ember.Route.extend({
- afterModel(model/*, transition*/) {
-model.error_id = "error";
-model.isValidErrorCode = false;
-if (model.errorCode && model.error
[hadoop] branch branch-3.2 updated: YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 72203f7 YARN-9545. Create healthcheck REST endpoint for ATSv2.
Contributed by Zoltan Siegl.
72203f7 is described below
commit 72203f7a12c943ca231fbc40c058a1a094b009cd
Author: Sunil G
AuthorDate: Wed Jun 12 19:23:40 2019 +0530
YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by
Zoltan Siegl.
---
.../yarn/api/records/timeline/TimelineHealth.java | 82 ++
.../storage/HBaseTimelineReaderImpl.java | 13
.../reader/TimelineReaderManager.java | 10 +++
.../reader/TimelineReaderWebServices.java | 33 +
.../storage/FileSystemTimelineReaderImpl.java | 23 ++
.../timelineservice/storage/TimelineReader.java| 8 +++
.../reader/TestTimelineReaderWebServices.java | 19 +
7 files changed, 188 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
new file mode 100644
index 000..d592167
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timeline;
+
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class holds health information for ATS.
+ */
+@XmlRootElement(name = "health")
+@XmlAccessorType(XmlAccessType.NONE)
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineHealth {
+
+ /**
+ * Timline health status.
+ *
+ * RUNNING - Service is up and running
+ * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation
+ *reported an error
+ */
+ public enum TimelineHealthStatus {
+RUNNING,
+READER_CONNECTION_FAILURE
+ }
+
+ private TimelineHealthStatus healthStatus;
+ private String diagnosticsInfo;
+
+ public TimelineHealth(TimelineHealthStatus healthy, String diagnosticsInfo) {
+this.healthStatus = healthy;
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+ public TimelineHealth() {
+
+ }
+
+ @XmlElement(name = "healthStatus")
+ public TimelineHealthStatus getHealthStatus() {
+return healthStatus;
+ }
+
+ @XmlElement(name = "diagnosticsInfo")
+ public String getDiagnosticsInfo() {
+return diagnosticsInfo;
+ }
+
+
+ public void setHealthStatus(TimelineHealthStatus healthStatus) {
+this.healthStatus = healthStatus;
+ }
+
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+
+}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index fadfd14..b83d60f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-tim
[hadoop] branch branch-3.2 updated: Revert "YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by Zoltan Siegl."
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new f1ead03 Revert "YARN-9545. Create healthcheck REST endpoint for
ATSv2. Contributed by Zoltan Siegl."
f1ead03 is described below
commit f1ead036728fe09f27954beff9074a04c27cd1b2
Author: Sunil G
AuthorDate: Wed Jun 12 19:10:23 2019 +0530
Revert "YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed
by Zoltan Siegl."
This reverts commit f1d3a17d3e67ec2acad52227a3f4eb7cca83e468.
---
.../yarn/api/records/timeline/TimelineHealth.java | 82
.../node_modules/.bin/apidoc | 1 -
.../node_modules/.bin/markdown-it | 1 -
.../node_modules/.bin/r.js | 1 -
.../node_modules/.bin/r_js | 1 -
.../node_modules/.bin/semver | 1 -
.../node_modules/.bin/shjs | 1 -
.../yarn.lock | 422 -
.../storage/HBaseTimelineReaderImpl.java | 13 -
.../reader/TimelineReaderManager.java | 10 -
.../reader/TimelineReaderWebServices.java | 33 --
.../storage/FileSystemTimelineReaderImpl.java | 23 --
.../timelineservice/storage/TimelineReader.java| 8 -
.../reader/TestTimelineReaderWebServices.java | 19 -
14 files changed, 616 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
deleted file mode 100644
index d592167..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.api.records.timeline;
-
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * This class holds health information for ATS.
- */
-@XmlRootElement(name = "health")
-@XmlAccessorType(XmlAccessType.NONE)
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class TimelineHealth {
-
- /**
- * Timline health status.
- *
- * RUNNING - Service is up and running
- * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation
- *reported an error
- */
- public enum TimelineHealthStatus {
-RUNNING,
-READER_CONNECTION_FAILURE
- }
-
- private TimelineHealthStatus healthStatus;
- private String diagnosticsInfo;
-
- public TimelineHealth(TimelineHealthStatus healthy, String diagnosticsInfo) {
-this.healthStatus = healthy;
-this.diagnosticsInfo = diagnosticsInfo;
- }
-
- public TimelineHealth() {
-
- }
-
- @XmlElement(name = "healthStatus")
- public TimelineHealthStatus getHealthStatus() {
-return healthStatus;
- }
-
- @XmlElement(name = "diagnosticsInfo")
- public String getDiagnosticsInfo() {
-return diagnosticsInfo;
- }
-
-
- public void setHealthStatus(TimelineHealthStatus healthStatus) {
-this.healthStatus = healthStatus;
- }
-
- public void setDiagnosticsInfo(String diagnosticsInfo) {
-this.diagnosticsInfo = diagnosticsInfo;
- }
-
-
-}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
deleted file mode 12
index a588095..0
[hadoop] branch trunk updated: YARN-9573. DistributedShell cannot specify LogAggregationContext. Contributed by Adam Antal.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 649666e YARN-9573. DistributedShell cannot specify
LogAggregationContext. Contributed by Adam Antal.
649666e is described below
commit 649666e118a7cf92b676eaa56a8be318176c443e
Author: Sunil G
AuthorDate: Thu Jun 6 14:51:55 2019 +0530
YARN-9573. DistributedShell cannot specify LogAggregationContext.
Contributed by Adam Antal.
---
.../yarn/applications/distributedshell/Client.java | 46 --
.../distributedshell/TestDistributedShell.java | 27 +
2 files changed, 60 insertions(+), 13 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 4bd57dd..ff07693 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -68,6 +68,7 @@ import
org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -203,7 +204,9 @@ public class Client {
private String nodeAttributeSpec = "";
// log4j.properties file
// if available, add to local resources and set into classpath
- private String log4jPropFile = "";
+ private String log4jPropFile = "";
+ // rolling
+ private String rollingFilesPattern = "";
// Start time for client
private final long clientStartTime = System.currentTimeMillis();
@@ -280,7 +283,7 @@ public class Client {
}
if (result) {
LOG.info("Application completed successfully");
- System.exit(0);
+ System.exit(0);
}
LOG.error("Application failed to complete successfully");
System.exit(2);
@@ -344,6 +347,8 @@ public class Client {
opts.addOption("enforce_execution_type", false,
"Flag to indicate whether to enforce execution type of containers");
opts.addOption("log_properties", true, "log4j.properties file");
+opts.addOption("rolling_log_pattern", true,
+"pattern for files that should be aggregated in a rolling fashion");
opts.addOption("keep_containers_across_application_attempts", false,
"Flag to indicate whether to keep containers across application "
+ "attempts."
@@ -443,6 +448,10 @@ public class Client {
}
}
+if (cliParser.hasOption("rolling_log_pattern")) {
+ rollingFilesPattern = cliParser.getOptionValue("rolling_log_pattern");
+}
+
if (cliParser.hasOption("help")) {
printUsage();
return false;
@@ -488,7 +497,7 @@ public class Client {
if (!cliParser.hasOption("jar")) {
throw new IllegalArgumentException("No jar file specified for
application master");
-}
+}
appMasterJar = cliParser.getOptionValue("jar");
@@ -689,16 +698,16 @@ public class Client {
+ ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity()
+ ", queueMaxCapacity=" + queueInfo.getMaximumCapacity()
+ ", queueApplicationCount=" + queueInfo.getApplications().size()
-+ ", queueChildQueueCount=" + queueInfo.getChildQueues().size());
++ ", queueChildQueueCount=" + queueInfo.getChildQueues().size());
List listAclInfo = yarnClient.getQueueAclsInfo();
for (QueueUserACLInfo aclInfo : listAclInfo) {
for (QueueACL userAcl : aclInfo.getUserAcls()) {
LOG.info("User ACL Info for Queue"
-+ ", queueName=" + aclInfo.getQueueName()
++ ", queueName=" + aclInfo.getQueueName()
+ ", userAcl=" + userAcl.name());
}
-}
+}
[hadoop] branch branch-3.1 updated: YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d65371c YARN-9545. Create healthcheck REST endpoint for ATSv2.
Contributed by Zoltan Siegl.
d65371c is described below
commit d65371c4e8bbb4ae655ccacda389cd37a18fab32
Author: Sunil G
AuthorDate: Thu Jun 6 06:24:01 2019 +0530
YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by
Zoltan Siegl.
(cherry picked from commit f1d3a17d3e67ec2acad52227a3f4eb7cca83e468)
---
.../yarn/api/records/timeline/TimelineHealth.java | 82
.../node_modules/.bin/apidoc | 1 +
.../node_modules/.bin/markdown-it | 1 +
.../node_modules/.bin/r.js | 1 +
.../node_modules/.bin/r_js | 1 +
.../node_modules/.bin/semver | 1 +
.../node_modules/.bin/shjs | 1 +
.../yarn.lock | 422 +
.../storage/HBaseTimelineReaderImpl.java | 13 +
.../reader/TimelineReaderManager.java | 10 +
.../reader/TimelineReaderWebServices.java | 33 ++
.../storage/FileSystemTimelineReaderImpl.java | 23 ++
.../timelineservice/storage/TimelineReader.java| 8 +
.../reader/TestTimelineReaderWebServices.java | 19 +
14 files changed, 616 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
new file mode 100644
index 000..d592167
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timeline;
+
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class holds health information for ATS.
+ */
+@XmlRootElement(name = "health")
+@XmlAccessorType(XmlAccessType.NONE)
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineHealth {
+
+ /**
+ * Timline health status.
+ *
+ * RUNNING - Service is up and running
+ * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation
+ *reported an error
+ */
+ public enum TimelineHealthStatus {
+RUNNING,
+READER_CONNECTION_FAILURE
+ }
+
+ private TimelineHealthStatus healthStatus;
+ private String diagnosticsInfo;
+
+ public TimelineHealth(TimelineHealthStatus healthy, String diagnosticsInfo) {
+this.healthStatus = healthy;
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+ public TimelineHealth() {
+
+ }
+
+ @XmlElement(name = "healthStatus")
+ public TimelineHealthStatus getHealthStatus() {
+return healthStatus;
+ }
+
+ @XmlElement(name = "diagnosticsInfo")
+ public String getDiagnosticsInfo() {
+return diagnosticsInfo;
+ }
+
+
+ public void setHealthStatus(TimelineHealthStatus healthStatus) {
+this.healthStatus = healthStatus;
+ }
+
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+
+}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
new file mode 12
index 000..a588095
--- /dev/null
+++
b/hadoop-yarn-p
[hadoop] branch branch-3.2 updated: YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new f1d3a17 YARN-9545. Create healthcheck REST endpoint for ATSv2.
Contributed by Zoltan Siegl.
f1d3a17 is described below
commit f1d3a17d3e67ec2acad52227a3f4eb7cca83e468
Author: Sunil G
AuthorDate: Thu Jun 6 06:24:01 2019 +0530
YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by
Zoltan Siegl.
---
.../yarn/api/records/timeline/TimelineHealth.java | 82
.../node_modules/.bin/apidoc | 1 +
.../node_modules/.bin/markdown-it | 1 +
.../node_modules/.bin/r.js | 1 +
.../node_modules/.bin/r_js | 1 +
.../node_modules/.bin/semver | 1 +
.../node_modules/.bin/shjs | 1 +
.../yarn.lock | 422 +
.../storage/HBaseTimelineReaderImpl.java | 13 +
.../reader/TimelineReaderManager.java | 10 +
.../reader/TimelineReaderWebServices.java | 33 ++
.../storage/FileSystemTimelineReaderImpl.java | 23 ++
.../timelineservice/storage/TimelineReader.java| 8 +
.../reader/TestTimelineReaderWebServices.java | 19 +
14 files changed, 616 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
new file mode 100644
index 000..d592167
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timeline;
+
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class holds health information for ATS.
+ */
+@XmlRootElement(name = "health")
+@XmlAccessorType(XmlAccessType.NONE)
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineHealth {
+
+ /**
+ * Timline health status.
+ *
+ * RUNNING - Service is up and running
+ * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation
+ *reported an error
+ */
+ public enum TimelineHealthStatus {
+RUNNING,
+READER_CONNECTION_FAILURE
+ }
+
+ private TimelineHealthStatus healthStatus;
+ private String diagnosticsInfo;
+
+ public TimelineHealth(TimelineHealthStatus healthy, String diagnosticsInfo) {
+this.healthStatus = healthy;
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+ public TimelineHealth() {
+
+ }
+
+ @XmlElement(name = "healthStatus")
+ public TimelineHealthStatus getHealthStatus() {
+return healthStatus;
+ }
+
+ @XmlElement(name = "diagnosticsInfo")
+ public String getDiagnosticsInfo() {
+return diagnosticsInfo;
+ }
+
+
+ public void setHealthStatus(TimelineHealthStatus healthStatus) {
+this.healthStatus = healthStatus;
+ }
+
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+
+}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
new file mode 12
index 000..a588095
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/ha
[hadoop] annotated tag submarine-0.2.0-RC0 created (now 526bbd4)
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a change to annotated tag submarine-0.2.0-RC0 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at 526bbd4 (tag) tagging 4c49cae231beb7859d34c2171eb24d1fd5d51c4e (commit) by Sunil G on Wed Jun 5 08:06:12 2019 + - Log - Submarine 0.2.0 release -BEGIN PGP SIGNATURE- Version: GnuPG v2.0.22 (GNU/Linux) iQIcBAABAgAGBQJc93f0AAoJEPQxMxUmZSbLFLgP+wRIOiMP0qqERwp3H2fEGlr/ u5yWZhDr1BkEJxuC7hXJtNVv87OG/b84EbD1htdoB0i/jG+L9ibQ1n50mBJkCyNQ ssMtC/x17m84BbQE3ZBVK0g0KJiz64wvSFsJqBF0q4M3aR2mdk8C9rtr8dw0wxs8 muy09X5QxjUYcIzqjCq+eooQfQY1cVnblIIfxDpdpUrb+TQoqm2e0N+WTJlNk1RF qt5FsCHH0tgyYdoUbbgxZ/X8syN0W2HGHAduEgl4DQ49tCBeE4rNIOtN6TupGL7P STePfSxz3mGb3xC8UIXGroq0qUKPJCgpCS8een1Z1F+vUCBAjsA10snAuOpPlVEp DSjShKw/qmP9/3ts9IFM4uGkWS/iLOGc3x6ObFjPOtiRyInYvk3+VVqYmZrtgNzc +yoBIuNZzUvn9Y6qi37L0hT1czLxO7FRBJz6P4JM0b2FlsExs2fQm3/l1hoVaYQw hjsCRyMvT7IHLsUv1bZwLEMbk2f0iMacCfQmW53Ua+6yoNqTGaiNv4790I+sIUFt QCWoNCpI8nfquPGTXQzlutExCPa1awI7/5GkPTYgGuBFAfd13zoE8lqcMAeY5YXx 0kpsMBRm1S3ZY4Ns+mNJo5hGlpdUlUT24KVwrSKlWc6JHPC/MSAMbI8p7BPXeKkV G3hzGOp4LqLBcaFhBxeg =4VmC -END PGP SIGNATURE- --- No new revisions were added by this update. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch submarine-0.2.0 updated: SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang.
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch submarine-0.2.0 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/submarine-0.2.0 by this push: new 6735fd0 SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang. 6735fd0 is described below commit 6735fd0ccc136888f3525ac2260a389bf54be0da Author: Sunil G AuthorDate: Wed Jun 5 07:48:24 2019 +0530 SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang. (cherry picked from commit 96d818a2f7a0cd62f252234ba1a27c8c70bfed1f) --- hadoop-submarine/dev-support/checks/rat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-submarine/dev-support/checks/rat.sh b/hadoop-submarine/dev-support/checks/rat.sh index ffead21..0e4a034 100755 --- a/hadoop-submarine/dev-support/checks/rat.sh +++ b/hadoop-submarine/dev-support/checks/rat.sh @@ -17,7 +17,7 @@ mkdir -p target rm target/rat-aggregated.txt mvn apache-rat:check -grep -r --include=rat.txt "\!\?\?\?\?" ./* | tee ./target/rat-aggregated.txt +grep -r --include=rat.txt -E "\!\?\?\?\?" ./* | tee ./target/rat-aggregated.txt if [ "$(cat target/rat-aggregated.txt)" ]; then echo "Failed to pass apache rat check!" exit -1 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch submarine-0.2 updated: SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang.
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch submarine-0.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/submarine-0.2 by this push: new a1a3983 SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang. a1a3983 is described below commit a1a3983d4ad3f0d897f467006aeabe28cb8d8c6d Author: Sunil G AuthorDate: Wed Jun 5 07:48:24 2019 +0530 SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang. (cherry picked from commit 96d818a2f7a0cd62f252234ba1a27c8c70bfed1f) --- hadoop-submarine/dev-support/checks/rat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-submarine/dev-support/checks/rat.sh b/hadoop-submarine/dev-support/checks/rat.sh index ffead21..0e4a034 100755 --- a/hadoop-submarine/dev-support/checks/rat.sh +++ b/hadoop-submarine/dev-support/checks/rat.sh @@ -17,7 +17,7 @@ mkdir -p target rm target/rat-aggregated.txt mvn apache-rat:check -grep -r --include=rat.txt "\!\?\?\?\?" ./* | tee ./target/rat-aggregated.txt +grep -r --include=rat.txt -E "\!\?\?\?\?" ./* | tee ./target/rat-aggregated.txt if [ "$(cat target/rat-aggregated.txt)" ]; then echo "Failed to pass apache rat check!" exit -1 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang.
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 96d818a SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang. 96d818a is described below commit 96d818a2f7a0cd62f252234ba1a27c8c70bfed1f Author: Sunil G AuthorDate: Wed Jun 5 07:48:24 2019 +0530 SUBMARINE-88. rat.sh regex pattern not working issue while using lower version of grep in ubuntu16.04. Contributed by Zhankun Tang. --- hadoop-submarine/dev-support/checks/rat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-submarine/dev-support/checks/rat.sh b/hadoop-submarine/dev-support/checks/rat.sh index ffead21..0e4a034 100755 --- a/hadoop-submarine/dev-support/checks/rat.sh +++ b/hadoop-submarine/dev-support/checks/rat.sh @@ -17,7 +17,7 @@ mkdir -p target rm target/rat-aggregated.txt mvn apache-rat:check -grep -r --include=rat.txt "\!\?\?\?\?" ./* | tee ./target/rat-aggregated.txt +grep -r --include=rat.txt -E "\!\?\?\?\?" ./* | tee ./target/rat-aggregated.txt if [ "$(cat target/rat-aggregated.txt)" ]; then echo "Failed to pass apache rat check!" exit -1 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch submarine-0.2.0 created (now 214c210)
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a change to branch submarine-0.2.0 in repository https://gitbox.apache.org/repos/asf/hadoop.git. at 214c210 Preparing for submarine-0.2.0 release This branch includes the following new commits: new 214c210 Preparing for submarine-0.2.0 release The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/01: Preparing for submarine-0.2.0 release
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch submarine-0.2.0
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 214c2104ed9cc05fbccecb809855e5054675725f
Author: Sunil G
AuthorDate: Tue Jun 4 12:20:34 2019 +0530
Preparing for submarine-0.2.0 release
---
hadoop-submarine/hadoop-submarine-all/pom.xml | 4 ++--
hadoop-submarine/hadoop-submarine-core/pom.xml| 4 ++--
hadoop-submarine/hadoop-submarine-dist/pom.xml| 4 ++--
hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml| 6 +++---
hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml | 8
hadoop-submarine/pom.xml | 2 +-
6 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/hadoop-submarine/hadoop-submarine-all/pom.xml
b/hadoop-submarine/hadoop-submarine-all/pom.xml
index ade3dfd..d2ebf36 100644
--- a/hadoop-submarine/hadoop-submarine-all/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-all/pom.xml
@@ -20,7 +20,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.1-SNAPSHOT
+0.2.0
${project.artifactId}
${project.version}
@@ -30,7 +30,7 @@
${project.parent.parent.basedir}
hadoop-submarine-all
-0.2.1-SNAPSHOT
+0.2.0
diff --git a/hadoop-submarine/hadoop-submarine-core/pom.xml
b/hadoop-submarine/hadoop-submarine-core/pom.xml
index 1383577..e636ec5 100644
--- a/hadoop-submarine/hadoop-submarine-core/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-core/pom.xml
@@ -20,10 +20,10 @@
hadoop-submarine
org.apache.hadoop
-0.2.1-SNAPSHOT
+0.2.0
hadoop-submarine-core
- 0.2.1-SNAPSHOT
+ 0.2.0
Hadoop Submarine Core
diff --git a/hadoop-submarine/hadoop-submarine-dist/pom.xml
b/hadoop-submarine/hadoop-submarine-dist/pom.xml
index e5684f6..76a54a6 100644
--- a/hadoop-submarine/hadoop-submarine-dist/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-dist/pom.xml
@@ -20,7 +20,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.1-SNAPSHOT
+0.2.0
${project.artifactId}
${project.version}
@@ -31,7 +31,7 @@
${project.parent.parent.basedir}
hadoop-submarine-dist
-0.2.1-SNAPSHOT
+0.2.0
diff --git a/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
b/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
index 3e7c4d0..00d1e03 100644
--- a/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
@@ -18,7 +18,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.1-SNAPSHOT
+0.2.0
4.0.0
@@ -28,7 +28,7 @@
org.apache.hadoop
hadoop-submarine-core
-0.2.1-SNAPSHOT
+0.2.0
compile
@@ -59,7 +59,7 @@
hadoop-submarine-core
test-jar
test
-0.2.1-SNAPSHOT
+0.2.0
org.mockito
diff --git a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
index fb2703c..6253b16 100644
--- a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
@@ -20,10 +20,10 @@
hadoop-submarine
org.apache.hadoop
-0.2.1-SNAPSHOT
+0.2.0
hadoop-submarine-yarnservice-runtime
- 0.2.1-SNAPSHOT
+ 0.2.0
Hadoop Submarine YARN Service Runtime
@@ -98,12 +98,12 @@
hadoop-submarine-core
test-jar
test
- 0.2.1-SNAPSHOT
+ 0.2.0
org.apache.hadoop
hadoop-submarine-core
- 0.2.1-SNAPSHOT
+ 0.2.0
org.apache.hadoop
diff --git a/hadoop-submarine/pom.xml b/hadoop-submarine/pom.xml
index f253e21..f997f13 100644
--- a/hadoop-submarine/pom.xml
+++ b/hadoop-submarine/pom.xml
@@ -24,7 +24,7 @@
hadoop-submarine
- 0.2.1-SNAPSHOT
+ 0.2.0
Hadoop Submarine
pom
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch submarine-0.2 updated: SUBMARINE-82. Fix english grammar mistakes in documentation. Contributed by Szilard Nemeth.
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch submarine-0.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/submarine-0.2 by this push: new c177cc9 SUBMARINE-82. Fix english grammar mistakes in documentation. Contributed by Szilard Nemeth. c177cc9 is described below commit c177cc97508743f7e112876a97280554c01813a4 Author: Zhankun Tang AuthorDate: Tue Jun 4 14:44:37 2019 +0800 SUBMARINE-82. Fix english grammar mistakes in documentation. Contributed by Szilard Nemeth. (cherry picked from commit 799115967d6e1a4074d0186b06b4eb97251a19df) --- .../src/site/markdown/Examples.md | 2 +- .../src/site/markdown/HowToInstall.md | 24 +++ .../src/site/markdown/Index.md | 14 ++-- .../src/site/markdown/InstallationGuide.md | 79 +- .../src/site/markdown/QuickStart.md| 29 .../markdown/RunningDistributedCifar10TFJobs.md| 16 ++--- .../src/site/markdown/TestAndTroubleshooting.md| 8 +-- 7 files changed, 95 insertions(+), 77 deletions(-) diff --git a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Examples.md b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Examples.md index b66b32d..fd61e83 100644 --- a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Examples.md +++ b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Examples.md @@ -14,7 +14,7 @@ # Examples -Here're some examples about Submarine usage. +Here are some examples about how to use Submarine: [Running Distributed CIFAR 10 Tensorflow Job](RunningDistributedCifar10TFJobs.html) diff --git a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/HowToInstall.md b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/HowToInstall.md index 65e56ea..af96d6d 100644 --- a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/HowToInstall.md +++ b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/HowToInstall.md @@ -14,23 +14,23 @@ # How to Install Dependencies -Submarine project uses YARN Service, Docker container, and GPU (when GPU hardware available and properly configured). +Submarine project uses YARN Service, Docker container and GPU. +GPU could only be used if a GPU hardware is available and properly configured. -That means as an admin, you have to properly setup YARN Service related dependencies, including: +As an administrator, you have to properly setup YARN Service related dependencies, including: - YARN Registry DNS +- Docker related dependencies, including: + - Docker binary with expected versions + - Docker network that allows Docker containers to talk to each other across different nodes -Docker related dependencies, including: -- Docker binary with expected versions. -- Docker network which allows Docker container can talk to each other across different nodes. +If you would like to use GPU, you need to set up: +- GPU Driver +- Nvidia-docker -And when GPU wanna to be used: -- GPU Driver. -- Nvidia-docker. - -For your convenience, we provided installation documents to help you to setup your environment. You can always choose to have them installed in your own way. +For your convenience, we provided some installation documents to help you setup your environment. You can always choose to have them installed in your own way. Use Submarine installer to install dependencies: [EN](https://github.com/hadoopsubmarine/hadoop-submarine-ecosystem/tree/master/submarine-installer) [CN](https://github.com/hadoopsubmarine/hadoop-submarine-ecosystem/blob/master/submarine-installer/README-CN.md) -Alternatively, you can follow manual install dependencies: [EN](InstallationGuide.html) [CN](InstallationGuideChineseVersion.html) +Alternatively, you can follow this guide to manually install dependencies: [EN](InstallationGuide.html) [CN](InstallationGuideChineseVersion.html) -Once you have installed dependencies, please follow following guide to [TestAndTroubleshooting](TestAndTroubleshooting.html). \ No newline at end of file +Once you have installed all the dependencies, please follow this guide: [TestAndTroubleshooting](TestAndTroubleshooting.html). \ No newline at end of file diff --git a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Index.md b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Index.md index d11fa45..e2c7979 100644 --- a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Index.md +++ b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Index.md @@ -21,20 +21,20 @@ Goals of Submarine: - Can launch services to serve Tensorflow/MXNet models. -- Support run distributed Tensorflow jobs with simple configs. +- Supports running distributed Tensorflow jobs with simple configs. -- Support run standalone PyTorch jobs with simple configs. +- Supports running stand
[hadoop] branch branch-3.2 updated: YARN-8906. [UI2] NM hostnames not displayed correctly in Node Heatmap Chart. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 2f01204 YARN-8906. [UI2] NM hostnames not displayed correctly in Node
Heatmap Chart. Contributed by Akhil PB.
2f01204 is described below
commit 2f012044ff6b8e14a3c2138305ed6e177f3c92dd
Author: Sunil G
AuthorDate: Mon Jun 3 15:53:23 2019 +0530
YARN-8906. [UI2] NM hostnames not displayed correctly in Node Heatmap
Chart. Contributed by Akhil PB.
(cherry picked from commit 59719dc560cf67f485d8e5b4a6f0f38ef97d536b)
---
.../hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js | 6 --
.../src/main/webapp/app/templates/components/nodes-heatmap.hbs | 1 -
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
index 1f772de..7eac266 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
@@ -230,12 +230,14 @@ export default BaseChartComponent.extend({
var node_id = data.get("id"),
node_addr = encodeURIComponent(data.get("nodeHTTPAddress")),
href = `#/yarn-node/${node_id}/${node_addr}/info`;
+var nodeHostName = data.get("nodeHostName");
var a = g.append("a")
.attr("href", href);
a.append("text")
- .text(data.get("nodeHostName"))
+ .text(nodeHostName.length > 30 ? nodeHostName.substr(0, 30) + '...' :
nodeHostName)
.attr("y", yOffset + this.CELL_HEIGHT / 2 + 5)
- .attr("x", xOffset + this.CELL_WIDTH / 2)
+ .attr("x", nodeHostName.length > 30 ? xOffset + 10 : xOffset +
this.CELL_WIDTH / 2)
+ .style("text-anchor", nodeHostName.length > 30 ? "start" : "middle")
.attr("class", this.isNodeSelected(data) ? "heatmap-cell" :
"heatmap-cell-notselected");
if (this.isNodeSelected(data)) {
this.bindTP(a, rect);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
index f68bba6..d1ac8e7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
@@ -25,4 +25,3 @@
-
\ No newline at end of file
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.1 updated: YARN-8906. [UI2] NM hostnames not displayed correctly in Node Heatmap Chart. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 6be665c YARN-8906. [UI2] NM hostnames not displayed correctly in Node
Heatmap Chart. Contributed by Akhil PB.
6be665c is described below
commit 6be665cfc6850aea612f40e26b3b6d5c0a0a3f41
Author: Sunil G
AuthorDate: Mon Jun 3 15:53:23 2019 +0530
YARN-8906. [UI2] NM hostnames not displayed correctly in Node Heatmap
Chart. Contributed by Akhil PB.
(cherry picked from commit 59719dc560cf67f485d8e5b4a6f0f38ef97d536b)
---
.../hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js | 6 --
.../src/main/webapp/app/templates/components/nodes-heatmap.hbs | 1 -
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
index 1f772de..7eac266 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
@@ -230,12 +230,14 @@ export default BaseChartComponent.extend({
var node_id = data.get("id"),
node_addr = encodeURIComponent(data.get("nodeHTTPAddress")),
href = `#/yarn-node/${node_id}/${node_addr}/info`;
+var nodeHostName = data.get("nodeHostName");
var a = g.append("a")
.attr("href", href);
a.append("text")
- .text(data.get("nodeHostName"))
+ .text(nodeHostName.length > 30 ? nodeHostName.substr(0, 30) + '...' :
nodeHostName)
.attr("y", yOffset + this.CELL_HEIGHT / 2 + 5)
- .attr("x", xOffset + this.CELL_WIDTH / 2)
+ .attr("x", nodeHostName.length > 30 ? xOffset + 10 : xOffset +
this.CELL_WIDTH / 2)
+ .style("text-anchor", nodeHostName.length > 30 ? "start" : "middle")
.attr("class", this.isNodeSelected(data) ? "heatmap-cell" :
"heatmap-cell-notselected");
if (this.isNodeSelected(data)) {
this.bindTP(a, rect);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
index f68bba6..d1ac8e7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
@@ -25,4 +25,3 @@
-
\ No newline at end of file
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-8906. [UI2] NM hostnames not displayed correctly in Node Heatmap Chart. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 59719dc YARN-8906. [UI2] NM hostnames not displayed correctly in Node
Heatmap Chart. Contributed by Akhil PB.
59719dc is described below
commit 59719dc560cf67f485d8e5b4a6f0f38ef97d536b
Author: Sunil G
AuthorDate: Mon Jun 3 15:53:23 2019 +0530
YARN-8906. [UI2] NM hostnames not displayed correctly in Node Heatmap
Chart. Contributed by Akhil PB.
---
.../hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js | 6 --
.../src/main/webapp/app/templates/components/nodes-heatmap.hbs | 1 -
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
index 1f772de..7eac266 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
@@ -230,12 +230,14 @@ export default BaseChartComponent.extend({
var node_id = data.get("id"),
node_addr = encodeURIComponent(data.get("nodeHTTPAddress")),
href = `#/yarn-node/${node_id}/${node_addr}/info`;
+var nodeHostName = data.get("nodeHostName");
var a = g.append("a")
.attr("href", href);
a.append("text")
- .text(data.get("nodeHostName"))
+ .text(nodeHostName.length > 30 ? nodeHostName.substr(0, 30) + '...' :
nodeHostName)
.attr("y", yOffset + this.CELL_HEIGHT / 2 + 5)
- .attr("x", xOffset + this.CELL_WIDTH / 2)
+ .attr("x", nodeHostName.length > 30 ? xOffset + 10 : xOffset +
this.CELL_WIDTH / 2)
+ .style("text-anchor", nodeHostName.length > 30 ? "start" : "middle")
.attr("class", this.isNodeSelected(data) ? "heatmap-cell" :
"heatmap-cell-notselected");
if (this.isNodeSelected(data)) {
this.bindTP(a, rect);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
index f68bba6..d1ac8e7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/nodes-heatmap.hbs
@@ -25,4 +25,3 @@
-
\ No newline at end of file
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.2 updated: YARN-8947. [UI2] Active User info missing from UI2. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 58042da YARN-8947. [UI2] Active User info missing from UI2.
Contributed by Akhil PB.
58042da is described below
commit 58042dadc398dc26a74a639a71c7e53931344abf
Author: Sunil G
AuthorDate: Mon Jun 3 12:24:20 2019 +0530
YARN-8947. [UI2] Active User info missing from UI2. Contributed by Akhil PB.
(cherry picked from commit 7f46dda513fb79c349acb73bdb90b689df9cc18d)
---
.../src/main/webapp/app/models/yarn-user.js| 11 -
.../app/serializers/yarn-queue/capacity-queue.js | 11 +
.../src/main/webapp/app/styles/app.scss| 18 +
.../main/webapp/app/templates/yarn-queue/apps.hbs | 47 +-
4 files changed, 84 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
index 84b0fab..9016e92 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
@@ -22,5 +22,14 @@ export default DS.Model.extend({
name: DS.attr('string'),
queueName: DS.attr('string'),
usedMemoryMB: DS.attr('number'),
- usedVCore: DS.attr('number')
+ usedVCore: DS.attr('number'),
+ maxMemoryMB: DS.attr('number'),
+ maxVCore: DS.attr('number'),
+ amUsedMemoryMB: DS.attr('number'),
+ amUsedVCore: DS.attr('number'),
+ maxAMMemoryMB: DS.attr('number'),
+ maxAMVCore: DS.attr('number'),
+ userWeight: DS.attr('string'),
+ activeApps: DS.attr('number'),
+ pendingApps: DS.attr('number')
});
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
index 57714e3..2dec520 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -36,6 +36,8 @@ export default DS.JSONAPISerializer.extend({
// update user models
if (payload.users && payload.users.user) {
payload.users.user.forEach(function(u) {
+ var defaultPartitionResource =
u.resources.resourceUsagesByPartition[0];
+ var maxAMResource = defaultPartitionResource.amLimit;
includedData.push({
type: "YarnUser",
id: u.username + "_" + payload.queueName,
@@ -44,6 +46,15 @@ export default DS.JSONAPISerializer.extend({
queueName: payload.queueName,
usedMemoryMB: u.resourcesUsed.memory || 0,
usedVCore: u.resourcesUsed.vCores || 0,
+ maxMemoryMB: u.userResourceLimit.memory || 0,
+ maxVCore: u.userResourceLimit.vCores || 0,
+ amUsedMemoryMB: u.AMResourceUsed.memory || 0,
+ amUsedVCore: u.AMResourceUsed.vCores || 0,
+ maxAMMemoryMB: maxAMResource.memory || 0,
+ maxAMVCore: maxAMResource.vCores || 0,
+ userWeight: u.userWeight || '',
+ activeApps: u.numActiveApplications || 0,
+ pendingApps: u.numPendingApplications || 0
}
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
index 01dc9f5..59e437a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
@@ -754,4 +754,22 @@ div.loggedin-user {
border: none;
border-radius: 2px;
}
+}
+
+/**
+ * Active User Info table styles
+ */
+.active-user-panel {
+ margin-top: 10px;
+}
+
+table.active-user-table {
+ border: 1px solid #ddd;
+ > thead > tr > th {
+background-color: #f7f7f7;
+ }
+ &.table-bordered > thead > tr > th,
+ &.table-bordered > tbody > tr > td {
+border: 1px solid #dcdcdc !important;
+ }
}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
index 97820c9..dc5413e 100644
---
a/ha
[hadoop] branch branch-3.1 updated: YARN-8947. [UI2] Active User info missing from UI2. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 894ef5c YARN-8947. [UI2] Active User info missing from UI2.
Contributed by Akhil PB.
894ef5c is described below
commit 894ef5c07b962082697db9c9c6bebb3b26903858
Author: Sunil G
AuthorDate: Mon Jun 3 12:24:20 2019 +0530
YARN-8947. [UI2] Active User info missing from UI2. Contributed by Akhil PB.
(cherry picked from commit 7f46dda513fb79c349acb73bdb90b689df9cc18d)
---
.../src/main/webapp/app/models/yarn-user.js| 11 -
.../app/serializers/yarn-queue/capacity-queue.js | 11 +
.../src/main/webapp/app/styles/app.scss| 18 +
.../main/webapp/app/templates/yarn-queue/apps.hbs | 47 +-
4 files changed, 84 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
index 84b0fab..9016e92 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
@@ -22,5 +22,14 @@ export default DS.Model.extend({
name: DS.attr('string'),
queueName: DS.attr('string'),
usedMemoryMB: DS.attr('number'),
- usedVCore: DS.attr('number')
+ usedVCore: DS.attr('number'),
+ maxMemoryMB: DS.attr('number'),
+ maxVCore: DS.attr('number'),
+ amUsedMemoryMB: DS.attr('number'),
+ amUsedVCore: DS.attr('number'),
+ maxAMMemoryMB: DS.attr('number'),
+ maxAMVCore: DS.attr('number'),
+ userWeight: DS.attr('string'),
+ activeApps: DS.attr('number'),
+ pendingApps: DS.attr('number')
});
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
index 57714e3..2dec520 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -36,6 +36,8 @@ export default DS.JSONAPISerializer.extend({
// update user models
if (payload.users && payload.users.user) {
payload.users.user.forEach(function(u) {
+ var defaultPartitionResource =
u.resources.resourceUsagesByPartition[0];
+ var maxAMResource = defaultPartitionResource.amLimit;
includedData.push({
type: "YarnUser",
id: u.username + "_" + payload.queueName,
@@ -44,6 +46,15 @@ export default DS.JSONAPISerializer.extend({
queueName: payload.queueName,
usedMemoryMB: u.resourcesUsed.memory || 0,
usedVCore: u.resourcesUsed.vCores || 0,
+ maxMemoryMB: u.userResourceLimit.memory || 0,
+ maxVCore: u.userResourceLimit.vCores || 0,
+ amUsedMemoryMB: u.AMResourceUsed.memory || 0,
+ amUsedVCore: u.AMResourceUsed.vCores || 0,
+ maxAMMemoryMB: maxAMResource.memory || 0,
+ maxAMVCore: maxAMResource.vCores || 0,
+ userWeight: u.userWeight || '',
+ activeApps: u.numActiveApplications || 0,
+ pendingApps: u.numPendingApplications || 0
}
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
index 01dc9f5..59e437a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
@@ -754,4 +754,22 @@ div.loggedin-user {
border: none;
border-radius: 2px;
}
+}
+
+/**
+ * Active User Info table styles
+ */
+.active-user-panel {
+ margin-top: 10px;
+}
+
+table.active-user-table {
+ border: 1px solid #ddd;
+ > thead > tr > th {
+background-color: #f7f7f7;
+ }
+ &.table-bordered > thead > tr > th,
+ &.table-bordered > tbody > tr > td {
+border: 1px solid #dcdcdc !important;
+ }
}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
index 97820c9..dc5413e 100644
---
a/ha
[hadoop] branch trunk updated: YARN-8947. [UI2] Active User info missing from UI2. Contributed by Akhil PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7f46dda YARN-8947. [UI2] Active User info missing from UI2.
Contributed by Akhil PB.
7f46dda is described below
commit 7f46dda513fb79c349acb73bdb90b689df9cc18d
Author: Sunil G
AuthorDate: Mon Jun 3 12:24:20 2019 +0530
YARN-8947. [UI2] Active User info missing from UI2. Contributed by Akhil PB.
---
.../src/main/webapp/app/models/yarn-user.js| 11 -
.../app/serializers/yarn-queue/capacity-queue.js | 11 +
.../src/main/webapp/app/styles/app.scss| 18 +
.../main/webapp/app/templates/yarn-queue/apps.hbs | 47 +-
4 files changed, 84 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
index 84b0fab..9016e92 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-user.js
@@ -22,5 +22,14 @@ export default DS.Model.extend({
name: DS.attr('string'),
queueName: DS.attr('string'),
usedMemoryMB: DS.attr('number'),
- usedVCore: DS.attr('number')
+ usedVCore: DS.attr('number'),
+ maxMemoryMB: DS.attr('number'),
+ maxVCore: DS.attr('number'),
+ amUsedMemoryMB: DS.attr('number'),
+ amUsedVCore: DS.attr('number'),
+ maxAMMemoryMB: DS.attr('number'),
+ maxAMVCore: DS.attr('number'),
+ userWeight: DS.attr('string'),
+ activeApps: DS.attr('number'),
+ pendingApps: DS.attr('number')
});
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
index 57714e3..2dec520 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -36,6 +36,8 @@ export default DS.JSONAPISerializer.extend({
// update user models
if (payload.users && payload.users.user) {
payload.users.user.forEach(function(u) {
+ var defaultPartitionResource =
u.resources.resourceUsagesByPartition[0];
+ var maxAMResource = defaultPartitionResource.amLimit;
includedData.push({
type: "YarnUser",
id: u.username + "_" + payload.queueName,
@@ -44,6 +46,15 @@ export default DS.JSONAPISerializer.extend({
queueName: payload.queueName,
usedMemoryMB: u.resourcesUsed.memory || 0,
usedVCore: u.resourcesUsed.vCores || 0,
+ maxMemoryMB: u.userResourceLimit.memory || 0,
+ maxVCore: u.userResourceLimit.vCores || 0,
+ amUsedMemoryMB: u.AMResourceUsed.memory || 0,
+ amUsedVCore: u.AMResourceUsed.vCores || 0,
+ maxAMMemoryMB: maxAMResource.memory || 0,
+ maxAMVCore: maxAMResource.vCores || 0,
+ userWeight: u.userWeight || '',
+ activeApps: u.numActiveApplications || 0,
+ pendingApps: u.numPendingApplications || 0
}
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
index 01dc9f5..59e437a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
@@ -754,4 +754,22 @@ div.loggedin-user {
border: none;
border-radius: 2px;
}
+}
+
+/**
+ * Active User Info table styles
+ */
+.active-user-panel {
+ margin-top: 10px;
+}
+
+table.active-user-table {
+ border: 1px solid #ddd;
+ > thead > tr > th {
+background-color: #f7f7f7;
+ }
+ &.table-bordered > thead > tr > th,
+ &.table-bordered > tbody > tr > td {
+border: 1px solid #dcdcdc !important;
+ }
}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
index 97820c9..dc5413e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/apps.hbs
+++
b/
[hadoop] branch trunk updated: Preparing for submarine-0.3.0 development
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2185249 Preparing for submarine-0.3.0 development
2185249 is described below
commit 21852494815e7314e0873c3963a54457ac2aab28
Author: Sunil G
AuthorDate: Mon Jun 3 10:41:25 2019 +0530
Preparing for submarine-0.3.0 development
---
hadoop-submarine/hadoop-submarine-all/pom.xml | 4 ++--
hadoop-submarine/hadoop-submarine-core/pom.xml| 4 ++--
hadoop-submarine/hadoop-submarine-dist/pom.xml| 4 ++--
hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml| 6 +++---
hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml | 8
hadoop-submarine/pom.xml | 2 +-
6 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/hadoop-submarine/hadoop-submarine-all/pom.xml
b/hadoop-submarine/hadoop-submarine-all/pom.xml
index 639a919..c26e0d5 100644
--- a/hadoop-submarine/hadoop-submarine-all/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-all/pom.xml
@@ -20,7 +20,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
${project.artifactId}
${project.version}
@@ -30,7 +30,7 @@
${project.parent.parent.basedir}
hadoop-submarine-all
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
diff --git a/hadoop-submarine/hadoop-submarine-core/pom.xml
b/hadoop-submarine/hadoop-submarine-core/pom.xml
index 332c4db..0ef6f71 100644
--- a/hadoop-submarine/hadoop-submarine-core/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-core/pom.xml
@@ -20,10 +20,10 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
hadoop-submarine-core
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
Hadoop Submarine Core
diff --git a/hadoop-submarine/hadoop-submarine-dist/pom.xml
b/hadoop-submarine/hadoop-submarine-dist/pom.xml
index 7196df1..8aa7db1 100644
--- a/hadoop-submarine/hadoop-submarine-dist/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-dist/pom.xml
@@ -20,7 +20,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
${project.artifactId}
${project.version}
@@ -31,7 +31,7 @@
${project.parent.parent.basedir}
hadoop-submarine-dist
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
diff --git a/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
b/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
index cc2ebfc..7ab2fdd 100644
--- a/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
@@ -18,7 +18,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
4.0.0
@@ -28,7 +28,7 @@
org.apache.hadoop
hadoop-submarine-core
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
compile
@@ -59,7 +59,7 @@
hadoop-submarine-core
test-jar
test
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
org.mockito
diff --git a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
index 034a29e..19e629e 100644
--- a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
@@ -20,10 +20,10 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
hadoop-submarine-yarnservice-runtime
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
Hadoop Submarine YARN Service Runtime
@@ -98,12 +98,12 @@
hadoop-submarine-core
test-jar
test
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
org.apache.hadoop
hadoop-submarine-core
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
org.apache.hadoop
diff --git a/hadoop-submarine/pom.xml b/hadoop-submarine/pom.xml
index 2d33da2..501dfb4 100644
--- a/hadoop-submarine/pom.xml
+++ b/hadoop-submarine/pom.xml
@@ -24,7 +24,7 @@
hadoop-submarine
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
Hadoop Submarine
pom
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch submarine-0.2 updated: SUBMARINE-87. Add an apache rat check script for submarine. Contributed by Zhankun Tang.
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch submarine-0.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/submarine-0.2 by this push: new c7924cc SUBMARINE-87. Add an apache rat check script for submarine. Contributed by Zhankun Tang. c7924cc is described below commit c7924cccb1655aacc663db7ca8c1be83a15da436 Author: Sunil G AuthorDate: Mon Jun 3 09:41:17 2019 +0530 SUBMARINE-87. Add an apache rat check script for submarine. Contributed by Zhankun Tang. (cherry picked from commit 08363db09326d421bd38b72b86a24cb75c339043) --- hadoop-submarine/dev-support/checks/rat.sh | 24 1 file changed, 24 insertions(+) diff --git a/hadoop-submarine/dev-support/checks/rat.sh b/hadoop-submarine/dev-support/checks/rat.sh new file mode 100755 index 000..ffead21 --- /dev/null +++ b/hadoop-submarine/dev-support/checks/rat.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir -p target +rm target/rat-aggregated.txt +mvn apache-rat:check +grep -r --include=rat.txt "\!\?\?\?\?" ./* | tee ./target/rat-aggregated.txt +if [ "$(cat target/rat-aggregated.txt)" ]; then + echo "Failed to pass apache rat check!" + exit -1 +fi - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: SUBMARINE-87. Add an apache rat check script for submarine. Contributed by Zhankun Tang.
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 08363db SUBMARINE-87. Add an apache rat check script for submarine. Contributed by Zhankun Tang. 08363db is described below commit 08363db09326d421bd38b72b86a24cb75c339043 Author: Sunil G AuthorDate: Mon Jun 3 09:41:17 2019 +0530 SUBMARINE-87. Add an apache rat check script for submarine. Contributed by Zhankun Tang. --- hadoop-submarine/dev-support/checks/rat.sh | 24 1 file changed, 24 insertions(+) diff --git a/hadoop-submarine/dev-support/checks/rat.sh b/hadoop-submarine/dev-support/checks/rat.sh new file mode 100755 index 000..ffead21 --- /dev/null +++ b/hadoop-submarine/dev-support/checks/rat.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mkdir -p target +rm target/rat-aggregated.txt +mvn apache-rat:check +grep -r --include=rat.txt "\!\?\?\?\?" ./* | tee ./target/rat-aggregated.txt +if [ "$(cat target/rat-aggregated.txt)" ]; then + echo "Failed to pass apache rat check!" + exit -1 +fi - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch submarine-0.2 updated: Preparing for submarine-0.2.1 development
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch submarine-0.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/submarine-0.2 by this push:
new 86e2d3b Preparing for submarine-0.2.1 development
86e2d3b is described below
commit 86e2d3b79c301e89be55a3ddeff86d3dd7f7cafe
Author: Sunil G
AuthorDate: Fri May 31 12:45:48 2019 +0530
Preparing for submarine-0.2.1 development
---
hadoop-submarine/hadoop-submarine-all/pom.xml | 4 ++--
hadoop-submarine/hadoop-submarine-core/pom.xml| 4 ++--
hadoop-submarine/hadoop-submarine-dist/pom.xml| 4 ++--
hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml| 6 +++---
hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml | 8
hadoop-submarine/pom.xml | 2 +-
6 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/hadoop-submarine/hadoop-submarine-all/pom.xml
b/hadoop-submarine/hadoop-submarine-all/pom.xml
index 922dac9..ade3dfd 100644
--- a/hadoop-submarine/hadoop-submarine-all/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-all/pom.xml
@@ -20,7 +20,7 @@
hadoop-submarine
org.apache.hadoop
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
${project.artifactId}
${project.version}
@@ -30,7 +30,7 @@
${project.parent.parent.basedir}
hadoop-submarine-all
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
diff --git a/hadoop-submarine/hadoop-submarine-core/pom.xml
b/hadoop-submarine/hadoop-submarine-core/pom.xml
index 84ddcb3..1383577 100644
--- a/hadoop-submarine/hadoop-submarine-core/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-core/pom.xml
@@ -20,10 +20,10 @@
hadoop-submarine
org.apache.hadoop
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
hadoop-submarine-core
- 0.3.0-SNAPSHOT
+ 0.2.1-SNAPSHOT
Hadoop Submarine Core
diff --git a/hadoop-submarine/hadoop-submarine-dist/pom.xml
b/hadoop-submarine/hadoop-submarine-dist/pom.xml
index 51cc64b..e5684f6 100644
--- a/hadoop-submarine/hadoop-submarine-dist/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-dist/pom.xml
@@ -20,7 +20,7 @@
hadoop-submarine
org.apache.hadoop
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
${project.artifactId}
${project.version}
@@ -31,7 +31,7 @@
${project.parent.parent.basedir}
hadoop-submarine-dist
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
diff --git a/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
b/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
index 3b90d0a..3e7c4d0 100644
--- a/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
@@ -18,7 +18,7 @@
hadoop-submarine
org.apache.hadoop
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
4.0.0
@@ -28,7 +28,7 @@
org.apache.hadoop
hadoop-submarine-core
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
compile
@@ -59,7 +59,7 @@
hadoop-submarine-core
test-jar
test
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
org.mockito
diff --git a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
index b52b97d..fb2703c 100644
--- a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
@@ -20,10 +20,10 @@
hadoop-submarine
org.apache.hadoop
-0.3.0-SNAPSHOT
+0.2.1-SNAPSHOT
hadoop-submarine-yarnservice-runtime
- 0.3.0-SNAPSHOT
+ 0.2.1-SNAPSHOT
Hadoop Submarine YARN Service Runtime
@@ -98,12 +98,12 @@
hadoop-submarine-core
test-jar
test
- 0.3.0-SNAPSHOT
+ 0.2.1-SNAPSHOT
org.apache.hadoop
hadoop-submarine-core
- 0.3.0-SNAPSHOT
+ 0.2.1-SNAPSHOT
org.apache.hadoop
diff --git a/hadoop-submarine/pom.xml b/hadoop-submarine/pom.xml
index 5972cb6..f253e21 100644
--- a/hadoop-submarine/pom.xml
+++ b/hadoop-submarine/pom.xml
@@ -24,7 +24,7 @@
hadoop-submarine
- 0.3.0-SNAPSHOT
+ 0.2.1-SNAPSHOT
Hadoop Submarine
pom
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch submarine-0.2 updated: Preparing for submarine-0.3.0 development
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch submarine-0.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/submarine-0.2 by this push:
new 2c5e819 Preparing for submarine-0.3.0 development
2c5e819 is described below
commit 2c5e8195acc7045cdac3e8eca77aa8b5e0ca3c56
Author: Sunil G
AuthorDate: Fri May 31 12:43:03 2019 +0530
Preparing for submarine-0.3.0 development
---
hadoop-submarine/hadoop-submarine-all/pom.xml | 4 ++--
hadoop-submarine/hadoop-submarine-core/pom.xml| 4 ++--
hadoop-submarine/hadoop-submarine-dist/pom.xml| 4 ++--
hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml| 6 +++---
hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml | 8
hadoop-submarine/pom.xml | 2 +-
6 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/hadoop-submarine/hadoop-submarine-all/pom.xml
b/hadoop-submarine/hadoop-submarine-all/pom.xml
index e2d2e17..922dac9 100644
--- a/hadoop-submarine/hadoop-submarine-all/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-all/pom.xml
@@ -20,7 +20,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
${project.artifactId}
${project.version}
@@ -30,7 +30,7 @@
${project.parent.parent.basedir}
hadoop-submarine-all
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
diff --git a/hadoop-submarine/hadoop-submarine-core/pom.xml
b/hadoop-submarine/hadoop-submarine-core/pom.xml
index 3b267fa..84ddcb3 100644
--- a/hadoop-submarine/hadoop-submarine-core/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-core/pom.xml
@@ -20,10 +20,10 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
hadoop-submarine-core
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
Hadoop Submarine Core
diff --git a/hadoop-submarine/hadoop-submarine-dist/pom.xml
b/hadoop-submarine/hadoop-submarine-dist/pom.xml
index 423fd0b..51cc64b 100644
--- a/hadoop-submarine/hadoop-submarine-dist/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-dist/pom.xml
@@ -20,7 +20,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
${project.artifactId}
${project.version}
@@ -31,7 +31,7 @@
${project.parent.parent.basedir}
hadoop-submarine-dist
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
diff --git a/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
b/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
index 8a952be..3b90d0a 100644
--- a/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-tony-runtime/pom.xml
@@ -18,7 +18,7 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
4.0.0
@@ -28,7 +28,7 @@
org.apache.hadoop
hadoop-submarine-core
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
compile
@@ -59,7 +59,7 @@
hadoop-submarine-core
test-jar
test
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
org.mockito
diff --git a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
index 3609b5a..b52b97d 100644
--- a/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
+++ b/hadoop-submarine/hadoop-submarine-yarnservice-runtime/pom.xml
@@ -20,10 +20,10 @@
hadoop-submarine
org.apache.hadoop
-0.2.0-SNAPSHOT
+0.3.0-SNAPSHOT
hadoop-submarine-yarnservice-runtime
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
Hadoop Submarine YARN Service Runtime
@@ -98,12 +98,12 @@
hadoop-submarine-core
test-jar
test
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
org.apache.hadoop
hadoop-submarine-core
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
org.apache.hadoop
diff --git a/hadoop-submarine/pom.xml b/hadoop-submarine/pom.xml
index 951e9d3..5972cb6 100644
--- a/hadoop-submarine/pom.xml
+++ b/hadoop-submarine/pom.xml
@@ -24,7 +24,7 @@
hadoop-submarine
- 0.2.0-SNAPSHOT
+ 0.3.0-SNAPSHOT
Hadoop Submarine
pom
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch submarine-0.2 updated: SUBMARINE-86. Bump TonY version to pick up a docker related fix & Update documentation. Contributed by Keqiu Hu.
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch submarine-0.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/submarine-0.2 by this push: new 64d21aa SUBMARINE-86. Bump TonY version to pick up a docker related fix & Update documentation. Contributed by Keqiu Hu. 64d21aa is described below commit 64d21aa21e87fc52f61d15ae42ae99cf498197f7 Author: Zhankun Tang AuthorDate: Thu May 30 16:25:37 2019 +0800 SUBMARINE-86. Bump TonY version to pick up a docker related fix & Update documentation. Contributed by Keqiu Hu. (cherry picked from commit 2b303e9d5f2108fee621dd0c6e5582a861c401e4) --- .../src/site/markdown/TonYRuntimeGuide.md | 48 +- .../hadoop-submarine-tony-runtime/pom.xml | 7 +++- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/TonYRuntimeGuide.md b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/TonYRuntimeGuide.md index 105a724..72e2d0f 100644 --- a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/TonYRuntimeGuide.md +++ b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/TonYRuntimeGuide.md @@ -68,24 +68,31 @@ usage: memory-mb=2048,vcores=2,yarn.io/gpu=2 -localization Specify localization to remote/local file/directory available to all container(Docker). - Argument format is "RemoteUri:LocalFilePath[:rw]" - (ro permission is not supported yet). - The RemoteUri can be a file or directory in local - or HDFS or s3 or abfs or http .etc. - The LocalFilePath can be absolute or relative. - If relative, it'll be under container's implied - working directory. + Argument format is "RemoteUri:LocalFileName" + The LocalFilePath is the local file or folder name. + You should access it with relative path to working directory. This option can be set mutiple times. Examples are - -localization "hdfs:///user/yarn/mydir2:/opt/data" - -localization "s3a:///a/b/myfile1:./" - -localization "https:///a/b/myfile2:./myfile"; - -localization "/user/yarn/mydir3:/opt/mydir3" - -localization "./mydir1:." + -localization "hdfs:///user/yarn/mydir2:data" + -localization "s3a:///a/b/myfile1:file1" + -localization "https:///a/b/myfile2:myfile"; + -localization "/user/yarn/mydir3:mydir3" + -localization "./mydir1:mydir1" -insecureWhether running in an insecure cluster -confOverride configurations via commandline ``` +> Note: all --localization files will be localized to working directory. You should access them use +relative path. Alternatively, you could use `--conf tony.containers.resources +=src_file::dest_file_name,src_file2::dest_file_name2`. It accepts a list of resources to be localized to all containers, +delimited by comma. If a resource has no scheme like `hdfs://` or `s3://`, the file is considered a local file. You +could add #archive annotation, if an entry has `#archive`, the file will be automatically unzipped when localized to the +containers, folder name is the same as the file name. For example: `/user/khu/abc.zip#archive` would be inferred as a +local file and will be unarchived in containers. You would anticipate an abc.zip/ folder in your container's working +directory. Annotation `::` is added since TonY 0.3.3. If you use `PATH/TO/abc.txt::def.txt`, the `abc.txt` file +would be localized as `def.txt` in the container working directory. +Details: [tony configurations](https://github.com/linkedin/TonY/wiki/TonY-Configurations) + ### Submarine Configuration For submarine internal configuration, please create a `submarine.xml` which should be placed under `$HADOOP_CONF_DIR`. @@ -144,7 +151,7 @@ CLASSPATH=$(hadoop classpath --glob): \ ./hadoop-submarine-core/target/hadoop-submarine-core-0.2.0-SNAPSHOT.jar: \ ./hadoop-submarine-yarnservice-runtime/target/hadoop-submarine-score-yarnservice-runtime-0.2.0-SNAPSHOT.jar: \ ./hadoop-submarine-tony-runtime/target/hadoop-submarine-tony-runtime-0.2.0-SNAPSHOT.jar:
[hadoop] branch submarine-0.2 updated: SUBMARINE-83. Refine the documents of submarine targeting 0.2.0 release. Contributed by Zhankun Tang.
This is an automated email from the ASF dual-hosted git repository. sunilg pushed a commit to branch submarine-0.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/submarine-0.2 by this push: new e54baab SUBMARINE-83. Refine the documents of submarine targeting 0.2.0 release. Contributed by Zhankun Tang. e54baab is described below commit e54baab9ffd4872e7652226726fb0df4b5acc608 Author: Zhankun Tang AuthorDate: Thu May 23 10:02:00 2019 +0800 SUBMARINE-83. Refine the documents of submarine targeting 0.2.0 release. Contributed by Zhankun Tang. (cherry picked from commit 03aa70fe19e06a0b3d637acc7b5db4c955f0a693) --- hadoop-submarine/hadoop-submarine-core/README.md | 7 ++-- .../src/site/markdown/DeveloperGuide.md| 24 -- .../src/site/markdown/Examples.md | 4 +-- .../src/site/markdown/Index.md | 5 ++- .../src/site/markdown/QuickStart.md| 19 +-- .../src/site/markdown/RunningZeppelinOnYARN.md | 37 -- .../src/site/markdown/TonYRuntimeGuide.md} | 10 +++--- .../src/site/resources/css/site.css| 29 - .../src/site/site.xml | 28 9 files changed, 28 insertions(+), 135 deletions(-) diff --git a/hadoop-submarine/hadoop-submarine-core/README.md b/hadoop-submarine/hadoop-submarine-core/README.md index cb2e2da..cc137ea 100644 --- a/hadoop-submarine/hadoop-submarine-core/README.md +++ b/hadoop-submarine/hadoop-submarine-core/README.md @@ -37,11 +37,12 @@ \__/ (_) ``` -Submarine is a project which allows infra engineer / data scientist to run *unmodified* Tensorflow programs on YARN. +Submarine is a project which allows infra engineer / data scientist to run +*unmodified* Tensorflow or PyTorch programs on YARN or Kubernetes. Goals of Submarine: - It allows jobs easy access data/models in HDFS and other storages. -- Can launch services to serve Tensorflow/MXNet models. +- Can launch services to serve Tensorflow/PyTorch models. - Support run distributed Tensorflow jobs with simple configs. - Support run user-specified Docker images. - Support specify GPU and other resources. @@ -51,5 +52,3 @@ Goals of Submarine: Please jump to [QuickStart](src/site/markdown/QuickStart.md) guide to quickly understand how to use this framework. Please jump to [Examples](src/site/markdown/Examples.md) to try other examples like running Distributed Tensorflow Training for CIFAR 10. - -If you're a developer, please find [Developer](src/site/markdown/DeveloperGuide.md) guide for more details. diff --git a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/DeveloperGuide.md b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/DeveloperGuide.md deleted file mode 100644 index 9ab0641..000 --- a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/DeveloperGuide.md +++ /dev/null @@ -1,24 +0,0 @@ - - -# Developer Guide - -By default, Submarine uses YARN service framework as runtime. If you want to add your own implementation, you can add a new `RuntimeFactory` implementation and configure following option to `submarine.xml` (which should be placed under same `$HADOOP_CONF_DIR`) - -``` - - submarine.runtime.class - ... full qualified class name for your runtime factory ... - -``` diff --git a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Examples.md b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Examples.md index d878add..b66b32d 100644 --- a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Examples.md +++ b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Examples.md @@ -18,6 +18,4 @@ Here're some examples about Submarine usage. [Running Distributed CIFAR 10 Tensorflow Job](RunningDistributedCifar10TFJobs.html) -[Running Standalone CIFAR 10 PyTorch Job](RunningSingleNodeCifar10PTJobs.html) - -[Running Zeppelin Notebook on YARN](RunningZeppelinOnYARN.html) \ No newline at end of file +[Running Standalone CIFAR 10 PyTorch Job](RunningSingleNodeCifar10PTJobs.html) \ No newline at end of file diff --git a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Index.md b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Index.md index f8556a6..d11fa45 100644 --- a/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Index.md +++ b/hadoop-submarine/hadoop-submarine-core/src/site/markdown/Index.md @@ -12,7 +12,8 @@ limitations under the License. See accompanying LICENSE file. --> -Submarine is a project which allows infra engineer / data scientist to run *unmodified* Tensorflow programs on YARN. +Submarine is a project which allows infra engineer / data scientist to run +*unmodified* Tensorflow or PyTorch programs on YARN or Kubernetes. Goals of Submarine: @@ -43,6 +44,4 @@ Click b
[hadoop] branch trunk updated: YARN-9543. [UI2] Handle ATSv2 server down or failures cases gracefully in YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 52128e3 YARN-9543. [UI2] Handle ATSv2 server down or failures cases
gracefully in YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
52128e3 is described below
commit 52128e352a30b70b83483f9290d9e94e98929705
Author: Sunil G
AuthorDate: Fri May 31 12:29:44 2019 +0530
YARN-9543. [UI2] Handle ATSv2 server down or failures cases gracefully in
YARN UI v2. Contributed by Zoltan Siegl and Akhil P B.
---
.../hadoop-yarn-ui/src/main/webapp/.gitignore | 4
.../timeline-error.js => adapters/timeline-health.js} | 19 ++-
.../src/main/webapp/app/controllers/application.js| 7 +++
.../timeline-error.js => models/timeline-health.js} | 16 +++-
.../src/main/webapp/app/routes/application.js | 4
.../src/main/webapp/app/routes/timeline-error.js | 3 +++
.../timeline-health.js} | 18 ++
.../src/main/webapp/app/templates/application.hbs | 4 ++--
8 files changed, 47 insertions(+), 28 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
new file mode 100644
index 000..338997f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/.gitignore
@@ -0,0 +1,4 @@
+tmp/
+node_modules/
+bower_components/
+dist/
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
similarity index 73%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
index c2e5fc5..8ca2310 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/timeline-health.js
@@ -16,14 +16,15 @@
* limitations under the License.
*/
-import Ember from 'ember';
+import RESTAbstractAdapter from './restabstract';
-export default Ember.Route.extend({
- afterModel(model/*, transition*/) {
-model.error_id = "error";
-model.isValidErrorCode = false;
-if (model.errorCode && model.errorCode !== "0") {
- model.isValidErrorCode = true;
-}
+export default RESTAbstractAdapter.extend({
+ address: "timelineWebAddress",
+ restNameSpace: "timelineV2",
+ serverName: "ATS",
+
+ urlForQueryRecord(/*query, modelName*/) {
+var url = this.buildURL();
+return url + '/health';
}
-});
+});
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
index 50a2909..34702ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/application.js
@@ -73,4 +73,11 @@ export default Ember.Controller.extend({
}
return null;
}.property('model.userInfo'),
+
+ isTimelineUnHealthy: function() {
+if (this.model && this.model.timelineHealth) {
+ return this.model.timelineHealth.get('isTimelineUnHealthy');
+}
+return true;
+ }.property('model.timelineHealth')
});
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
similarity index 75%
copy from
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
copy to
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
index c2e5fc5..367ab07 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/timeline-error.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/timeline-health.js
@@ -16,14 +16,12 @@
* limitations under the License.
*/
-import Ember from 'ember';
+import DS from 'ember-data';
-export default Ember.Route.extend({
- afterModel(model/*, transition*/) {
-model.error_id = "error";
-model.isValidErrorCode = false;
-if (model.errorCode && model.errorCode !== "0") {
- model.isValidErrorCode = true;
-}
- }
+export default DS.Model.e
[hadoop] branch trunk updated: YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by Zoltan Siegl.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new e49162f YARN-9545. Create healthcheck REST endpoint for ATSv2.
Contributed by Zoltan Siegl.
e49162f is described below
commit e49162f4b3791dbf51079e3b19dd0c8bc2a85158
Author: Sunil G
AuthorDate: Fri May 31 10:28:09 2019 +0530
YARN-9545. Create healthcheck REST endpoint for ATSv2. Contributed by
Zoltan Siegl.
---
.../yarn/api/records/timeline/TimelineHealth.java | 82 ++
.../DocumentStoreTimelineReaderImpl.java | 13
.../storage/HBaseTimelineReaderImpl.java | 13
.../reader/TimelineReaderManager.java | 10 +++
.../reader/TimelineReaderWebServices.java | 33 +
.../storage/FileSystemTimelineReaderImpl.java | 15
.../storage/NoOpTimelineReaderImpl.java| 7 ++
.../timelineservice/storage/TimelineReader.java| 8 +++
.../reader/TestTimelineReaderWebServices.java | 19 +
9 files changed, 200 insertions(+)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
new file mode 100644
index 000..d592167
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineHealth.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timeline;
+
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * This class holds health information for ATS.
+ */
+@XmlRootElement(name = "health")
+@XmlAccessorType(XmlAccessType.NONE)
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineHealth {
+
+ /**
+ * Timline health status.
+ *
+ * RUNNING - Service is up and running
+ * READER_CONNECTION_FAULURE - isConnectionAlive() of reader implementation
+ *reported an error
+ */
+ public enum TimelineHealthStatus {
+RUNNING,
+READER_CONNECTION_FAILURE
+ }
+
+ private TimelineHealthStatus healthStatus;
+ private String diagnosticsInfo;
+
+ public TimelineHealth(TimelineHealthStatus healthy, String diagnosticsInfo) {
+this.healthStatus = healthy;
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+ public TimelineHealth() {
+
+ }
+
+ @XmlElement(name = "healthStatus")
+ public TimelineHealthStatus getHealthStatus() {
+return healthStatus;
+ }
+
+ @XmlElement(name = "diagnosticsInfo")
+ public String getDiagnosticsInfo() {
+return diagnosticsInfo;
+ }
+
+
+ public void setHealthStatus(TimelineHealthStatus healthStatus) {
+this.healthStatus = healthStatus;
+ }
+
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+this.diagnosticsInfo = diagnosticsInfo;
+ }
+
+
+}
\ No newline at end of file
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java
index 2159132..8de3b86 100755
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-documentstore/src/main/java/org/apache/hadoop/yarn/server/timelineservice/documentstore/DocumentStoreTimelineReaderImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-
