[hadoop] branch trunk updated: YARN-11020. [UI2] No container is found for an application attempt with a single AM container. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7a8c2e0 YARN-11020. [UI2] No container is found for an application
attempt with a single AM container. Contributed by Andras Gyori
7a8c2e0 is described below
commit 7a8c2e009761b8405406e185ab6302113be1f487
Author: Szilard Nemeth
AuthorDate: Tue Dec 7 08:51:03 2021 +0100
YARN-11020. [UI2] No container is found for an application attempt with a
single AM container. Contributed by Andras Gyori
---
.../src/main/webapp/app/serializers/yarn-jhs-container.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
index 599cf7f..8aebb71 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-jhs-container.js
@@ -45,8 +45,8 @@ export default DS.JSONAPISerializer.extend({
},
normalizeArrayResponse(store, primaryModelClass, payload/*, id,
requestType*/) {
-
-payload = payload["containerLogsInfo"]
+// Handling single container and multiple containers case at the same time
+payload = [].concat(payload["containerLogsInfo"]);
var normalizedArrayResponse = {
data: []
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-11014. YARN incorrectly validates maximum capacity resources on the validation API. Contributed by Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d75f2db YARN-11014. YARN incorrectly validates maximum capacity
resources on the validation API. Contributed by Benjamin Teke
d75f2db is described below
commit d75f2db2f952c5cb785345d912f169d0d45a491b
Author: Szilard Nemeth
AuthorDate: Tue Dec 7 08:42:57 2021 +0100
YARN-11014. YARN incorrectly validates maximum capacity resources on the
validation API. Contributed by Benjamin Teke
---
.../scheduler/capacity/CapacityScheduler.java | 16 ++
.../capacity/CapacitySchedulerConfigValidator.java | 2 +
.../TestCapacitySchedulerConfigValidator.java | 270 -
3 files changed, 284 insertions(+), 4 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index d2c84e7..bd1089b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2239,6 +2239,22 @@ public class CapacityScheduler extends
}
}
+ /**
+ * Add node to nodeTracker. Used when validating CS configuration by
instantiating a new
+ * CS instance.
+ * @param nodesToAdd node to be added
+ */
+ public void addNodes(List nodesToAdd) {
+writeLock.lock();
+try {
+ for (FiCaSchedulerNode node : nodesToAdd) {
+nodeTracker.addNode(node);
+ }
+} finally {
+ writeLock.unlock();
+}
+ }
+
private void addNode(RMNode nodeManager) {
writeLock.lock();
try {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
index ca0d586..fd601ac 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfigValidator.java
@@ -42,6 +42,7 @@ public final class CapacitySchedulerConfigValidator {
public static boolean validateCSConfiguration(
final Configuration oldConf, final Configuration newConf,
final RMContext rmContext) throws IOException {
+CapacityScheduler liveScheduler = (CapacityScheduler)
rmContext.getScheduler();
CapacityScheduler newCs = new CapacityScheduler();
try {
//TODO: extract all the validation steps and replace reinitialize with
@@ -49,6 +50,7 @@ public final class CapacitySchedulerConfigValidator {
newCs.setConf(oldConf);
newCs.setRMContext(rmContext);
newCs.init(oldConf);
+ newCs.addNodes(liveScheduler.getAllNodes());
newCs.reinitialize(newConf, rmContext, true);
return true;
} finally {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
index 04f4349..ad114d9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerConfigValidator.java
@@ -19,13 +19,23 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.conf.Configuration;
+import
[hadoop] branch branch-2.10 updated: YARN-9063. ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing (#3728)
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new 0b9ae87 YARN-9063. ATS 1.5 fails to start if RollingLevelDb files are
corrupt or missing (#3728)
0b9ae87 is described below
commit 0b9ae87ee70d95fe6422c2417a48589a17eddc80
Author: Ashutosh Gupta
AuthorDate: Mon Dec 6 16:15:44 2021 +0530
YARN-9063. ATS 1.5 fails to start if RollingLevelDb files are corrupt or
missing (#3728)
Signed-off-by: Akira Ajisaka
(cherry picked from commit 5a950b8900aa143f6db281eb22928965f3b2754f)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java
---
.../yarn/server/timeline/LeveldbTimelineStore.java | 18 ++
.../timeline/RollingLevelDBTimelineStore.java | 27 +++
.../yarn/server/timeline/util/LeveldbUtils.java| 32 ++
.../timeline/TestRollingLevelDBTimelineStore.java | 38 +-
4 files changed, 91 insertions(+), 24 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
index e3db1dc..ac4d495 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.timeline;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.collections.map.LRUMap;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability;
@@ -32,7 +31,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.timeline.*;
import
org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
import
org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
@@ -41,6 +39,7 @@ import
org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl;
+import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils;
import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyBuilder;
import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyParser;
import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
@@ -48,7 +47,6 @@ import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.*;
import org.slf4j.LoggerFactory;
-import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.*;
@@ -242,19 +240,7 @@ public class LeveldbTimelineStore extends AbstractService
IOUtils.cleanupWithLogger(LOG, localFS);
}
LOG.info("Using leveldb path " + dbPath);
-try {
- db = factory.open(new File(dbPath.toString()), options);
-} catch (IOException ioe) {
- File dbFile = new File(dbPath.toString());
- File backupPath = new File(
- dbPath.toString() + BACKUP_EXT + Time.monotonicNow());
- LOG.warn("Incurred exception while loading LevelDb database. Backing " +
- "up at "+ backupPath, ioe);
- FileUtils.copyDirectory(dbFile, backupPath);
- LOG.warn("Going to try repair");
- factory.repair(dbFile, options);
- db = factory.open(dbFile, options);
-}
+db = LeveldbUtils.loadOrRepairLevelDb(factory, dbPath, options);
checkVersion();
startTimeWriteCache =
Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize(
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
[hadoop] branch branch-3.2 updated (b4fc326 -> c8cf7a9)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a change to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git. from b4fc326 HDFS-16268. Balancer stuck when moving striped blocks due to NPE (#3546) add c8cf7a9 YARN-9063. ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing (#3728) No new revisions were added by this update. Summary of changes: .../yarn/server/timeline/LeveldbTimelineStore.java | 18 ++ .../timeline/RollingLevelDBTimelineStore.java | 27 +++ .../yarn/server/timeline/util/LeveldbUtils.java| 32 ++ .../timeline/TestRollingLevelDBTimelineStore.java | 38 +- 4 files changed, 91 insertions(+), 24 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: YARN-9063. ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing (#3728)
This is an automated email from the ASF dual-hosted git repository.
aajisaka pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 53f76b9 YARN-9063. ATS 1.5 fails to start if RollingLevelDb files are
corrupt or missing (#3728)
53f76b9 is described below
commit 53f76b9b1c4d6a2a1fd50cfae8739b34243441ad
Author: Ashutosh Gupta
AuthorDate: Mon Dec 6 16:15:44 2021 +0530
YARN-9063. ATS 1.5 fails to start if RollingLevelDb files are corrupt or
missing (#3728)
Signed-off-by: Akira Ajisaka
(cherry picked from commit 5a950b8900aa143f6db281eb22928965f3b2754f)
---
.../yarn/server/timeline/LeveldbTimelineStore.java | 18 ++
.../timeline/RollingLevelDBTimelineStore.java | 27 +++
.../yarn/server/timeline/util/LeveldbUtils.java| 32 ++
.../timeline/TestRollingLevelDBTimelineStore.java | 38 +-
4 files changed, 91 insertions(+), 24 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
index 7d34a04..d252ed1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.timeline;
import
org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.apache.commons.collections.map.LRUMap;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability;
@@ -32,7 +31,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.timeline.*;
import
org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
import
org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
@@ -41,6 +39,7 @@ import
org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager.CheckAcl;
+import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils;
import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyBuilder;
import org.apache.hadoop.yarn.server.timeline.util.LeveldbUtils.KeyParser;
import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
@@ -48,7 +47,6 @@ import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.*;
import org.slf4j.LoggerFactory;
-import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.*;
@@ -242,19 +240,7 @@ public class LeveldbTimelineStore extends AbstractService
IOUtils.cleanupWithLogger(LOG, localFS);
}
LOG.info("Using leveldb path " + dbPath);
-try {
- db = factory.open(new File(dbPath.toString()), options);
-} catch (IOException ioe) {
- File dbFile = new File(dbPath.toString());
- File backupPath = new File(
- dbPath.toString() + BACKUP_EXT + Time.monotonicNow());
- LOG.warn("Incurred exception while loading LevelDb database. Backing " +
- "up at "+ backupPath, ioe);
- FileUtils.copyDirectory(dbFile, backupPath);
- LOG.warn("Going to try repair");
- factory.repair(dbFile, options);
- db = factory.open(dbFile, options);
-}
+db = LeveldbUtils.loadOrRepairLevelDb(factory, dbPath, options);
checkVersion();
startTimeWriteCache =
Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize(
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java
index 1d6fa36..9d53f3d 100644
---
[hadoop] branch trunk updated (1509ea5 -> 5a950b8)
This is an automated email from the ASF dual-hosted git repository. aajisaka pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 1509ea5 HDFS-16338. Correct fsimage error configuration message (#3684). Contributed by guophilipse. add 5a950b8 YARN-9063. ATS 1.5 fails to start if RollingLevelDb files are corrupt or missing (#3728) No new revisions were added by this update. Summary of changes: .../yarn/server/timeline/LeveldbTimelineStore.java | 18 ++ .../timeline/RollingLevelDBTimelineStore.java | 27 +++ .../yarn/server/timeline/util/LeveldbUtils.java| 32 ++ .../timeline/TestRollingLevelDBTimelineStore.java | 38 +- 4 files changed, 91 insertions(+), 24 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (e8e69de -> 1509ea5)
This is an automated email from the ASF dual-hosted git repository. ayushsaxena pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from e8e69de HDFS-16293. Client sleeps and holds 'dataQueue' when DataNodes are congested. Contributed by Yuanxin Zhu. add 1509ea5 HDFS-16338. Correct fsimage error configuration message (#3684). Contributed by guophilipse. No new revisions were added by this update. Summary of changes: .../org/apache/hadoop/hdfs/server/namenode/FSImage.java | 4 ++-- .../apache/hadoop/hdfs/server/namenode/TestFSImage.java | 17 + 2 files changed, 19 insertions(+), 2 deletions(-) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
