[2/2] git commit: YARN-90. NodeManager should identify failed disks becoming good again. Contributed by Varun Vasudev (cherry picked from commit 6f2028bd1514d90b831f889fd0ee7f2ba5c15000)
YARN-90. NodeManager should identify failed disks becoming good again.
Contributed by Varun Vasudev
(cherry picked from commit 6f2028bd1514d90b831f889fd0ee7f2ba5c15000)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/413fe4cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/413fe4cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/413fe4cb
Branch: refs/heads/branch-2.6
Commit: 413fe4cb022179c3cec6482bb3ecc7e49116f38b
Parents: fcdeba2
Author: Jason Lowe
Authored: Tue Oct 21 17:43:22 2014 +
Committer: Jason Lowe
Committed: Tue Oct 21 17:43:22 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/nodemanager/DirectoryCollection.java | 219 ---
.../nodemanager/LocalDirsHandlerService.java| 176 +++--
.../nodemanager/NodeHealthCheckerService.java | 4 +-
.../launcher/ContainerLaunch.java | 2 +-
.../localizer/ResourceLocalizationService.java | 270 ++---
.../logaggregation/AppLogAggregatorImpl.java| 35 +-
.../logaggregation/LogAggregationService.java | 13 +-
.../loghandler/NonAggregatingLogHandler.java| 47 ++-
.../nodemanager/TestDirectoryCollection.java| 83 +++-
.../TestLocalDirsHandlerService.java| 43 ++-
.../nodemanager/TestNodeHealthService.java | 2 +-
.../TestResourceLocalizationService.java| 312 ++-
.../TestLogAggregationService.java | 147 ---
.../TestNonAggregatingLogHandler.java | 382 +++
15 files changed, 1445 insertions(+), 293 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/413fe4cb/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ef1e8ad..5c1dabe 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -312,6 +312,9 @@ Release 2.6.0 - UNRELEASED
YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS.
(Xuan
Gong via zjshen)
+YARN-90. NodeManager should identify failed disks becoming good again
+(Varun Vasudev via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/413fe4cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
index f6ee128..279787b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
@@ -21,18 +21,23 @@ package org.apache.hadoop.yarn.server.nodemanager;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.DiskChecker;
-import org.apache.hadoop.util.DiskChecker.DiskErrorException;
/**
* Manages a list of local storage directories.
@@ -40,9 +45,38 @@ import org.apache.hadoop.util.DiskChecker.DiskErrorException;
class DirectoryCollection {
private static final Log LOG = LogFactory.getLog(DirectoryCollection.class);
+ public enum DiskErrorCause {
+DISK_FULL, OTHER
+ }
+
+ static class DiskErrorInformation {
+DiskErrorCause cause;
+String message;
+
+DiskErrorInformation(DiskErrorCause cause, String message) {
+ this.cause = cause;
+ this.message = message
[1/2] YARN-90. NodeManager should identify failed disks becoming good again. Contributed by Varun Vasudev (cherry picked from commit 6f2028bd1514d90b831f889fd0ee7f2ba5c15000)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 fcdeba204 -> 413fe4cb0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/413fe4cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index ab86a18..8a5441a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -25,6 +25,7 @@ import static org.mockito.Matchers.anyMap;
import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.atLeast;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -58,6 +59,8 @@ import org.junit.Assert;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -105,6 +108,7 @@ import
org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerM
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.TestNonAggregatingLogHandler;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
@@ -136,12 +140,19 @@ public class TestLogAggregationService extends
BaseContainerManagerTest {
super();
this.remoteRootLogDir.mkdir();
}
+
+ DrainDispatcher dispatcher;
+ EventHandler appEventHandler;
@Override
+ @SuppressWarnings("unchecked")
public void setup() throws IOException {
super.setup();
NodeId nodeId = NodeId.newInstance("0.0.0.0", );
((NMContext)context).setNodeId(nodeId);
+dispatcher = createDispatcher();
+appEventHandler = mock(EventHandler.class);
+dispatcher.register(ApplicationEventType.class, appEventHandler);
}
@Override
@@ -149,10 +160,12 @@ public class TestLogAggregationService extends
BaseContainerManagerTest {
super.tearDown();
createContainerExecutor().deleteAsUser(user,
new Path(remoteRootLogDir.getAbsolutePath()), new Path[] {});
+dispatcher.await();
+dispatcher.stop();
+dispatcher.close();
}
@Test
- @SuppressWarnings("unchecked")
public void testLocalFileDeletionAfterUpload() throws Exception {
this.delSrvc = new DeletionService(createContainerExecutor());
delSrvc = spy(delSrvc);
@@ -161,10 +174,6 @@ public class TestLogAggregationService extends
BaseContainerManagerTest {
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
this.remoteRootLogDir.getAbsolutePath());
-DrainDispatcher dispatcher = createDispatcher();
-EventHandler appEventHandler = mock(EventHandler.class);
-dispatcher.register(ApplicationEventType.class, appEventHandler);
-
LogAggregationService logAggregationService = spy(
new LogAggregationService(dispatcher, this.context, this.delSrvc,
super.dirsHandler));
@@ -236,16 +245,11 @@ public class TestLogAggregationService extends
BaseContainerManagerTest {
}
@Test
- @SuppressWarnings("unchecked")
public void testNoContainerOnNode() throws Exception {
this.conf.set(YarnConfiguration.NM_LOG_DIRS,
localLogDir.getAbsolutePath());
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
this.remoteRootLogDir.getAbsolutePath());
git commit: HADOOP-9740. Fix FsShell '-text' command to be able to read Avro files stored in HDFS and other filesystems. Contributed by Allan Yan
Repository: hadoop
Updated Branches:
refs/heads/branch-2 112d69408 -> 0c15130db
HADOOP-9740. Fix FsShell '-text' command to be able to read Avro files stored
in HDFS and other filesystems. Contributed by Allan Yan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c15130d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c15130d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c15130d
Branch: refs/heads/branch-2
Commit: 0c15130db2a127ffe2ddc767dc9519567ae8c756
Parents: 112d694
Author: Jason Lowe
Authored: Wed Oct 29 20:19:33 2014 +
Committer: Jason Lowe
Committed: Wed Oct 29 20:23:17 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/fs/shell/Display.java | 5 +-
.../apache/hadoop/fs/shell/TestTextCommand.java | 218 +++
3 files changed, 225 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c15130d/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index acc8d66..da197f5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -683,6 +683,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11068. Match hadoop.auth cookie format to jetty output.
(Gregory Chanan via cnauroth)
+HADOOP-9740. Fix FsShell '-text' command to be able to read Avro
+files stored in HDFS and other filesystems. (Allan Yan via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c15130d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
index 3d607e4..9289d91 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
@@ -36,8 +36,10 @@ import org.apache.avro.io.JsonEncoder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AvroFSInput;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsDirectoryException;
@@ -270,8 +272,9 @@ class Display extends FsCommand {
pos = 0;
buffer = new byte[0];
GenericDatumReader reader = new GenericDatumReader();
+ FileContext fc = FileContext.getFileContext(new Configuration());
fileReader =
-DataFileReader.openReader(new File(status.getPath().toUri()), reader);
+DataFileReader.openReader(new AvroFSInput(fc,
status.getPath()),reader);
Schema schema = fileReader.getSchema();
writer = new GenericDatumWriter(schema);
output = new ByteArrayOutputStream();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c15130d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
new file mode 100644
index 000..49da6e1
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the
git commit: HADOOP-9740. Fix FsShell '-text' command to be able to read Avro files stored in HDFS and other filesystems. Contributed by Allan Yan (cherry picked from commit 0c15130db2a127ffe2ddc767dc9
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 5a0285fa9 -> ba86f06cf
HADOOP-9740. Fix FsShell '-text' command to be able to read Avro files stored
in HDFS and other filesystems. Contributed by Allan Yan
(cherry picked from commit 0c15130db2a127ffe2ddc767dc9519567ae8c756)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba86f06c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba86f06c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba86f06c
Branch: refs/heads/branch-2.6
Commit: ba86f06cf7a7a243fac7c6ded2e3da96fac67862
Parents: 5a0285f
Author: Jason Lowe
Authored: Wed Oct 29 20:19:33 2014 +
Committer: Jason Lowe
Committed: Wed Oct 29 20:25:25 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/fs/shell/Display.java | 5 +-
.../apache/hadoop/fs/shell/TestTextCommand.java | 218 +++
3 files changed, 225 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba86f06c/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f1941e4..a85fb0c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -650,6 +650,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11068. Match hadoop.auth cookie format to jetty output.
(Gregory Chanan via cnauroth)
+HADOOP-9740. Fix FsShell '-text' command to be able to read Avro
+files stored in HDFS and other filesystems. (Allan Yan via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba86f06c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
index 3d607e4..9289d91 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java
@@ -36,8 +36,10 @@ import org.apache.avro.io.JsonEncoder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AvroFSInput;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsDirectoryException;
@@ -270,8 +272,9 @@ class Display extends FsCommand {
pos = 0;
buffer = new byte[0];
GenericDatumReader reader = new GenericDatumReader();
+ FileContext fc = FileContext.getFileContext(new Configuration());
fileReader =
-DataFileReader.openReader(new File(status.getPath().toUri()), reader);
+DataFileReader.openReader(new AvroFSInput(fc,
status.getPath()),reader);
Schema schema = fileReader.getSchema();
writer = new GenericDatumWriter(schema);
output = new ByteArrayOutputStream();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba86f06c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
new file mode 100644
index 000..49da6e1
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDIT
git commit: YARN-2755. NM fails to clean up usercache_DEL_ dirs after YARN-661. Contributed by Siqi Li
Repository: hadoop
Updated Branches:
refs/heads/trunk 179cab81e -> 73e626ad9
YARN-2755. NM fails to clean up usercache_DEL_ dirs after YARN-661.
Contributed by Siqi Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/73e626ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/73e626ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/73e626ad
Branch: refs/heads/trunk
Commit: 73e626ad91cd5c06a005068d8432fd16e06fe6a0
Parents: 179cab8
Author: Jason Lowe
Authored: Thu Oct 30 15:10:27 2014 +
Committer: Jason Lowe
Committed: Thu Oct 30 15:10:27 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../localizer/ResourceLocalizationService.java | 2 +-
.../nodemanager/TestNodeManagerReboot.java | 66 +++-
3 files changed, 54 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73e626ad/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f4e4afa..0e06661 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -801,6 +801,9 @@ Release 2.6.0 - UNRELEASED
YARN-2769. Fixed the problem that timeline domain is not set in
distributed shell
AM when using shell_command on Windows. (Varun Vasudev via zjshen)
+YARN-2755. NM fails to clean up usercache_DEL_ dirs after
+YARN-661 (Siqi Li via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73e626ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 549d8e7..08ed3a1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1324,7 +1324,7 @@ public class ResourceLocalizationService extends
CompositeService
RemoteIterator userDirStatus = lfs.listStatus(userDirPath);
FileDeletionTask dependentDeletionTask =
del.createFileDeletionTask(null, userDirPath, new Path[] {});
-if (userDirStatus != null) {
+if (userDirStatus != null && userDirStatus.hasNext()) {
List deletionTasks = new ArrayList();
while (userDirStatus.hasNext()) {
FileStatus status = userDirStatus.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/73e626ad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
index e69170e..e9aea0e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
@@ -35,7 +35,9 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
@@ -190,17 +192,41 @@ public class TestNodeMa
git commit: YARN-2755. NM fails to clean up usercache_DEL_ dirs after YARN-661. Contributed by Siqi Li (cherry picked from commit 73e626ad91cd5c06a005068d8432fd16e06fe6a0)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d2ba115f0 -> 3e8544c5f
YARN-2755. NM fails to clean up usercache_DEL_ dirs after YARN-661.
Contributed by Siqi Li
(cherry picked from commit 73e626ad91cd5c06a005068d8432fd16e06fe6a0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e8544c5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e8544c5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e8544c5
Branch: refs/heads/branch-2
Commit: 3e8544c5f2d6c6bece2fb89830a75f03560d7dc2
Parents: d2ba115
Author: Jason Lowe
Authored: Thu Oct 30 15:10:27 2014 +
Committer: Jason Lowe
Committed: Thu Oct 30 15:11:57 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../localizer/ResourceLocalizationService.java | 2 +-
.../nodemanager/TestNodeManagerReboot.java | 66 +++-
3 files changed, 54 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e8544c5/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index da29754..ec86979 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -771,6 +771,9 @@ Release 2.6.0 - UNRELEASED
YARN-2769. Fixed the problem that timeline domain is not set in
distributed shell
AM when using shell_command on Windows. (Varun Vasudev via zjshen)
+YARN-2755. NM fails to clean up usercache_DEL_ dirs after
+YARN-661 (Siqi Li via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e8544c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 549d8e7..08ed3a1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1324,7 +1324,7 @@ public class ResourceLocalizationService extends
CompositeService
RemoteIterator userDirStatus = lfs.listStatus(userDirPath);
FileDeletionTask dependentDeletionTask =
del.createFileDeletionTask(null, userDirPath, new Path[] {});
-if (userDirStatus != null) {
+if (userDirStatus != null && userDirStatus.hasNext()) {
List deletionTasks = new ArrayList();
while (userDirStatus.hasNext()) {
FileStatus status = userDirStatus.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e8544c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
index e69170e..e9aea0e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
@@ -35,7 +35,9 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformati
git commit: YARN-2755. NM fails to clean up usercache_DEL_ dirs after YARN-661. Contributed by Siqi Li (cherry picked from commit 73e626ad91cd5c06a005068d8432fd16e06fe6a0)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 2f313cfae -> 773ba8b60
YARN-2755. NM fails to clean up usercache_DEL_ dirs after YARN-661.
Contributed by Siqi Li
(cherry picked from commit 73e626ad91cd5c06a005068d8432fd16e06fe6a0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/773ba8b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/773ba8b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/773ba8b6
Branch: refs/heads/branch-2.6
Commit: 773ba8b6063545f0f225802c363424b9066d25d6
Parents: 2f313cf
Author: Jason Lowe
Authored: Thu Oct 30 15:10:27 2014 +
Committer: Jason Lowe
Committed: Thu Oct 30 15:12:35 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../localizer/ResourceLocalizationService.java | 2 +-
.../nodemanager/TestNodeManagerReboot.java | 66 +++-
3 files changed, 54 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773ba8b6/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2cc7a67..f8b9454 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -729,6 +729,9 @@ Release 2.6.0 - UNRELEASED
YARN-2769. Fixed the problem that timeline domain is not set in
distributed shell
AM when using shell_command on Windows. (Varun Vasudev via zjshen)
+YARN-2755. NM fails to clean up usercache_DEL_ dirs after
+YARN-661 (Siqi Li via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773ba8b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 549d8e7..08ed3a1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1324,7 +1324,7 @@ public class ResourceLocalizationService extends
CompositeService
RemoteIterator userDirStatus = lfs.listStatus(userDirPath);
FileDeletionTask dependentDeletionTask =
del.createFileDeletionTask(null, userDirPath, new Path[] {});
-if (userDirStatus != null) {
+if (userDirStatus != null && userDirStatus.hasNext()) {
List deletionTasks = new ArrayList();
while (userDirStatus.hasNext()) {
FileStatus status = userDirStatus.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/773ba8b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
index e69170e..e9aea0e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
@@ -35,7 +35,9 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformati
git commit: YARN-2707. Potential null dereference in FSDownload. Contributed by Gera Shegalov
Repository: hadoop
Updated Branches:
refs/heads/trunk 6ce32f593 -> 7bc179f9f
YARN-2707. Potential null dereference in FSDownload. Contributed by Gera
Shegalov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bc179f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bc179f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bc179f9
Branch: refs/heads/trunk
Commit: 7bc179f9f5e1e270fd5e6f1ad29b0f32fe371d15
Parents: 6ce32f5
Author: Jason Lowe
Authored: Fri Oct 31 22:53:04 2014 +
Committer: Jason Lowe
Committed: Fri Oct 31 22:53:04 2014 +
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../java/org/apache/hadoop/yarn/util/FSDownload.java | 13 +
.../org/apache/hadoop/yarn/util/TestFSDownload.java| 3 +--
3 files changed, 9 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc179f9/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8816b29..0aab364 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -826,6 +826,9 @@ Release 2.6.0 - UNRELEASED
modified in YARN-2698 so that tests in YARN frameworks don't break. (Wangda
Tan via vinodkv)
+YARN-2707. Potential null dereference in FSDownload (Gera Shegalov via
+ jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc179f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index 8cc5ed3..fb37701 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -266,7 +266,7 @@ public class FSDownload implements Callable {
return dCopy;
}
- private long unpack(File localrsrc, File dst, Pattern pattern) throws
IOException {
+ private long unpack(File localrsrc, File dst) throws IOException {
switch (resource.getType()) {
case ARCHIVE: {
String lowerDst = dst.getName().toLowerCase();
@@ -290,7 +290,9 @@ public class FSDownload implements Callable {
case PATTERN: {
String lowerDst = dst.getName().toLowerCase();
if (lowerDst.endsWith(".jar")) {
-RunJar.unJar(localrsrc, dst, pattern);
+String p = resource.getPattern();
+RunJar.unJar(localrsrc, dst,
+p == null ? RunJar.MATCH_ANY : Pattern.compile(p));
File newDst = new File(dst, dst.getName());
if (!dst.exists() && !dst.mkdir()) {
throw new IOException("Unable to create directory: [" + dst + "]");
@@ -356,12 +358,7 @@ public class FSDownload implements Callable {
return files.makeQualified(copy(sCopy, dst_work));
};
});
- Pattern pattern = null;
- String p = resource.getPattern();
- if (p != null) {
-pattern = Pattern.compile(p);
- }
- unpack(new File(dTmp.toUri()), new File(dFinal.toUri()), pattern);
+ unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
changePermissions(dFinal.getFileSystem(conf), dFinal);
files.rename(dst_work, destDirPath, Rename.OVERWRITE);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bc179f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
index 02ba5fa..07dd630 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
@@ -500,9 +500,8 @@ public class TestFSDownload {
pending.put(rsrc, exec.submit(fsd));
exec.shutdown();
while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS));
-Assert.assertTrue(pending.get(rsrc).isDone());
-
try {
+
git commit: YARN-2707. Potential null dereference in FSDownload. Contributed by Gera Shegalov (cherry picked from commit 7bc179f9f5e1e270fd5e6f1ad29b0f32fe371d15)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6d4a04a03 -> ada40aa93
YARN-2707. Potential null dereference in FSDownload. Contributed by Gera
Shegalov
(cherry picked from commit 7bc179f9f5e1e270fd5e6f1ad29b0f32fe371d15)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ada40aa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ada40aa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ada40aa9
Branch: refs/heads/branch-2
Commit: ada40aa9339acf09864edb1b2669da8f3d033b5f
Parents: 6d4a04a
Author: Jason Lowe
Authored: Fri Oct 31 22:53:04 2014 +
Committer: Jason Lowe
Committed: Fri Oct 31 22:54:49 2014 +
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../java/org/apache/hadoop/yarn/util/FSDownload.java | 13 +
.../org/apache/hadoop/yarn/util/TestFSDownload.java| 3 +--
3 files changed, 9 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada40aa9/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bf42142..96e5b2b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -796,6 +796,9 @@ Release 2.6.0 - UNRELEASED
modified in YARN-2698 so that tests in YARN frameworks don't break. (Wangda
Tan via vinodkv)
+YARN-2707. Potential null dereference in FSDownload (Gera Shegalov via
+ jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada40aa9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index 8cc5ed3..fb37701 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -266,7 +266,7 @@ public class FSDownload implements Callable {
return dCopy;
}
- private long unpack(File localrsrc, File dst, Pattern pattern) throws
IOException {
+ private long unpack(File localrsrc, File dst) throws IOException {
switch (resource.getType()) {
case ARCHIVE: {
String lowerDst = dst.getName().toLowerCase();
@@ -290,7 +290,9 @@ public class FSDownload implements Callable {
case PATTERN: {
String lowerDst = dst.getName().toLowerCase();
if (lowerDst.endsWith(".jar")) {
-RunJar.unJar(localrsrc, dst, pattern);
+String p = resource.getPattern();
+RunJar.unJar(localrsrc, dst,
+p == null ? RunJar.MATCH_ANY : Pattern.compile(p));
File newDst = new File(dst, dst.getName());
if (!dst.exists() && !dst.mkdir()) {
throw new IOException("Unable to create directory: [" + dst + "]");
@@ -356,12 +358,7 @@ public class FSDownload implements Callable {
return files.makeQualified(copy(sCopy, dst_work));
};
});
- Pattern pattern = null;
- String p = resource.getPattern();
- if (p != null) {
-pattern = Pattern.compile(p);
- }
- unpack(new File(dTmp.toUri()), new File(dFinal.toUri()), pattern);
+ unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
changePermissions(dFinal.getFileSystem(conf), dFinal);
files.rename(dst_work, destDirPath, Rename.OVERWRITE);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada40aa9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
index 02ba5fa..07dd630 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
@@ -500,9 +500,8 @@ public class TestFSDownload {
pending.put(rsrc, exec.submit(fsd));
exec.shutdown();
while (!exec.awaitTermination(1000, TimeUnit.MILLISECO
git commit: YARN-2707. Potential null dereference in FSDownload. Contributed by Gera Shegalov (cherry picked from commit 7bc179f9f5e1e270fd5e6f1ad29b0f32fe371d15)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 f71d940e4 -> 239c4cc57
YARN-2707. Potential null dereference in FSDownload. Contributed by Gera
Shegalov
(cherry picked from commit 7bc179f9f5e1e270fd5e6f1ad29b0f32fe371d15)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/239c4cc5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/239c4cc5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/239c4cc5
Branch: refs/heads/branch-2.6
Commit: 239c4cc57286656d28114f320cf4bc172fcef178
Parents: f71d940
Author: Jason Lowe
Authored: Fri Oct 31 22:53:04 2014 +
Committer: Jason Lowe
Committed: Fri Oct 31 22:55:06 2014 +
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../java/org/apache/hadoop/yarn/util/FSDownload.java | 13 +
.../org/apache/hadoop/yarn/util/TestFSDownload.java| 3 +--
3 files changed, 9 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/239c4cc5/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b8e5fe5..1087545 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -751,6 +751,9 @@ Release 2.6.0 - UNRELEASED
modified in YARN-2698 so that tests in YARN frameworks don't break. (Wangda
Tan via vinodkv)
+YARN-2707. Potential null dereference in FSDownload (Gera Shegalov via
+ jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/239c4cc5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
index 8cc5ed3..fb37701 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
@@ -266,7 +266,7 @@ public class FSDownload implements Callable {
return dCopy;
}
- private long unpack(File localrsrc, File dst, Pattern pattern) throws
IOException {
+ private long unpack(File localrsrc, File dst) throws IOException {
switch (resource.getType()) {
case ARCHIVE: {
String lowerDst = dst.getName().toLowerCase();
@@ -290,7 +290,9 @@ public class FSDownload implements Callable {
case PATTERN: {
String lowerDst = dst.getName().toLowerCase();
if (lowerDst.endsWith(".jar")) {
-RunJar.unJar(localrsrc, dst, pattern);
+String p = resource.getPattern();
+RunJar.unJar(localrsrc, dst,
+p == null ? RunJar.MATCH_ANY : Pattern.compile(p));
File newDst = new File(dst, dst.getName());
if (!dst.exists() && !dst.mkdir()) {
throw new IOException("Unable to create directory: [" + dst + "]");
@@ -356,12 +358,7 @@ public class FSDownload implements Callable {
return files.makeQualified(copy(sCopy, dst_work));
};
});
- Pattern pattern = null;
- String p = resource.getPattern();
- if (p != null) {
-pattern = Pattern.compile(p);
- }
- unpack(new File(dTmp.toUri()), new File(dFinal.toUri()), pattern);
+ unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
changePermissions(dFinal.getFileSystem(conf), dFinal);
files.rename(dst_work, destDirPath, Rename.OVERWRITE);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/239c4cc5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
index 02ba5fa..07dd630 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
@@ -500,9 +500,8 @@ public class TestFSDownload {
pending.put(rsrc, exec.submit(fsd));
exec.shutdown();
while (!exec.awaitTermination(1000, TimeUnit.MILL
git commit: YARN-2730. DefaultContainerExecutor runs only one localizer at a time. Contributed by Siqi Li
Repository: hadoop
Updated Branches:
refs/heads/trunk 67f13b58e -> 6157ace54
YARN-2730. DefaultContainerExecutor runs only one localizer at a time.
Contributed by Siqi Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6157ace5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6157ace5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6157ace5
Branch: refs/heads/trunk
Commit: 6157ace5475fff8d2513fd3cd99134b532b0b406
Parents: 67f13b5
Author: Jason Lowe
Authored: Mon Nov 3 20:37:47 2014 +
Committer: Jason Lowe
Committed: Mon Nov 3 20:37:47 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../nodemanager/DefaultContainerExecutor.java| 19 ---
2 files changed, 15 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6157ace5/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6f91bd8..1bb116b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -839,6 +839,9 @@ Release 2.6.0 - UNRELEASED
YARN-2785. Fixed intermittent TestContainerResourceUsage failure. (Varun
Vasudev
via zjshen)
+YARN-2730. DefaultContainerExecutor runs only one localizer at a time
+(Siqi Li via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6157ace5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 834b138..cc2de99 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -94,7 +94,7 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
}
@Override
- public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
+ public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
@@ -102,10 +102,6 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
List localDirs = dirsHandler.getLocalDirs();
List logDirs = dirsHandler.getLogDirs();
-ContainerLocalizer localizer =
-new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
-RecordFactoryProvider.getRecordFactory(getConf()));
-
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
@@ -118,8 +114,17 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
Path tokenDst = new Path(appStorageDir, tokenFn);
copyFile(nmPrivateContainerTokensPath, tokenDst, user);
LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " +
tokenDst);
-lfs.setWorkingDirectory(appStorageDir);
-LOG.info("CWD set to " + appStorageDir + " = " +
lfs.getWorkingDirectory());
+
+
+FileContext localizerFc = FileContext.getFileContext(
+lfs.getDefaultFileSystem(), getConf());
+localizerFc.setUMask(lfs.getUMask());
+localizerFc.setWorkingDirectory(appStorageDir);
+LOG.info("Localizer CWD set to " + appStorageDir + " = "
++ localizerFc.getWorkingDirectory());
+ContainerLocalizer localizer =
+new ContainerLocalizer(localizerFc, user, appId, locId,
+getPaths(localDirs),
RecordFactoryProvider.getRecordFactory(getConf()));
// TODO: DO it over RPC for maintaining similarity?
localizer.runLocalization(nmAddr);
}
git commit: YARN-2730. DefaultContainerExecutor runs only one localizer at a time. Contributed by Siqi Li (cherry picked from commit 6157ace5475fff8d2513fd3cd99134b532b0b406)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c8648bd27 -> 8091ea54d
YARN-2730. DefaultContainerExecutor runs only one localizer at a time.
Contributed by Siqi Li
(cherry picked from commit 6157ace5475fff8d2513fd3cd99134b532b0b406)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8091ea54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8091ea54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8091ea54
Branch: refs/heads/branch-2
Commit: 8091ea54d5e5c47da95ac1b1d921134399a5f066
Parents: c8648bd
Author: Jason Lowe
Authored: Mon Nov 3 20:37:47 2014 +
Committer: Jason Lowe
Committed: Mon Nov 3 20:39:14 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../nodemanager/DefaultContainerExecutor.java| 19 ---
2 files changed, 15 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8091ea54/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9ae57e4..56359e5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -809,6 +809,9 @@ Release 2.6.0 - UNRELEASED
YARN-2785. Fixed intermittent TestContainerResourceUsage failure. (Varun
Vasudev
via zjshen)
+YARN-2730. DefaultContainerExecutor runs only one localizer at a time
+(Siqi Li via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8091ea54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 834b138..cc2de99 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -94,7 +94,7 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
}
@Override
- public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
+ public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
@@ -102,10 +102,6 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
List localDirs = dirsHandler.getLocalDirs();
List logDirs = dirsHandler.getLogDirs();
-ContainerLocalizer localizer =
-new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
-RecordFactoryProvider.getRecordFactory(getConf()));
-
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
@@ -118,8 +114,17 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
Path tokenDst = new Path(appStorageDir, tokenFn);
copyFile(nmPrivateContainerTokensPath, tokenDst, user);
LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " +
tokenDst);
-lfs.setWorkingDirectory(appStorageDir);
-LOG.info("CWD set to " + appStorageDir + " = " +
lfs.getWorkingDirectory());
+
+
+FileContext localizerFc = FileContext.getFileContext(
+lfs.getDefaultFileSystem(), getConf());
+localizerFc.setUMask(lfs.getUMask());
+localizerFc.setWorkingDirectory(appStorageDir);
+LOG.info("Localizer CWD set to " + appStorageDir + " = "
++ localizerFc.getWorkingDirectory());
+ContainerLocalizer localizer =
+new ContainerLocalizer(localizerFc, user, appId, locId,
+getPaths(localDirs),
RecordFactoryProvider.getRecordFactory(getConf()));
// TODO: DO it over RPC for maintaining similarity?
localizer.runLocalization(nmAddr);
}
git commit: YARN-2730. DefaultContainerExecutor runs only one localizer at a time. Contributed by Siqi Li (cherry picked from commit 6157ace5475fff8d2513fd3cd99134b532b0b406)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 3e4182863 -> 2098c68ac
YARN-2730. DefaultContainerExecutor runs only one localizer at a time.
Contributed by Siqi Li
(cherry picked from commit 6157ace5475fff8d2513fd3cd99134b532b0b406)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2098c68a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2098c68a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2098c68a
Branch: refs/heads/branch-2.6
Commit: 2098c68acbcdf8cc412e0c2d43eddbfc1dfe3e79
Parents: 3e41828
Author: Jason Lowe
Authored: Mon Nov 3 20:37:47 2014 +
Committer: Jason Lowe
Committed: Mon Nov 3 20:40:02 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../nodemanager/DefaultContainerExecutor.java| 19 ---
2 files changed, 15 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2098c68a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b84fc1e..cba4b34 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -764,6 +764,9 @@ Release 2.6.0 - UNRELEASED
YARN-2785. Fixed intermittent TestContainerResourceUsage failure. (Varun
Vasudev
via zjshen)
+YARN-2730. DefaultContainerExecutor runs only one localizer at a time
+(Siqi Li via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2098c68a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 3e212bf..5bf8cec 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -94,7 +94,7 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
}
@Override
- public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
+ public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
@@ -102,10 +102,6 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
List localDirs = dirsHandler.getLocalDirs();
List logDirs = dirsHandler.getLogDirs();
-ContainerLocalizer localizer =
-new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
-RecordFactoryProvider.getRecordFactory(getConf()));
-
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
createAppDirs(localDirs, user, appId);
@@ -118,8 +114,17 @@ public class DefaultContainerExecutor extends
ContainerExecutor {
Path tokenDst = new Path(appStorageDir, tokenFn);
copyFile(nmPrivateContainerTokensPath, tokenDst, user);
LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " +
tokenDst);
-lfs.setWorkingDirectory(appStorageDir);
-LOG.info("CWD set to " + appStorageDir + " = " +
lfs.getWorkingDirectory());
+
+
+FileContext localizerFc = FileContext.getFileContext(
+lfs.getDefaultFileSystem(), getConf());
+localizerFc.setUMask(lfs.getUMask());
+localizerFc.setWorkingDirectory(appStorageDir);
+LOG.info("Localizer CWD set to " + appStorageDir + " = "
++ localizerFc.getWorkingDirectory());
+ContainerLocalizer localizer =
+new ContainerLocalizer(localizerFc, user, appId, locId,
+getPaths(localDirs),
RecordFactoryProvider.getRecordFactory(getConf()));
// TODO: DO it over RPC for maintaining similarity?
localizer.runLocalization(nmAddr);
}
git commit: MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect with no authority in job jar path. Contributed by Gera Shegalov
Repository: hadoop
Updated Branches:
refs/heads/trunk 80d7d183c -> 10f9f5101
MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect with
no authority in job jar path. Contributed by Gera Shegalov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10f9f510
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10f9f510
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10f9f510
Branch: refs/heads/trunk
Commit: 10f9f5101c44be7c675a44ded4aad212627ecdee
Parents: 80d7d18
Author: Jason Lowe
Authored: Thu Nov 6 15:10:40 2014 +
Committer: Jason Lowe
Committed: Thu Nov 6 15:10:40 2014 +
--
.../java/org/apache/hadoop/fs/FileContext.java | 3 ++
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../v2/app/job/impl/TaskAttemptImpl.java| 8 ++--
.../apache/hadoop/mapreduce/JobSubmitter.java | 9 ++---
.../org/apache/hadoop/mapred/YARNRunner.java| 5 ++-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 41
.../apache/hadoop/mapreduce/v2/TestUberAM.java | 40 +++
7 files changed, 56 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10f9f510/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 2323650..85f8136 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -430,6 +430,9 @@ public class FileContext {
final Configuration aConf) throws UnsupportedFileSystemException {
UserGroupInformation currentUser = null;
AbstractFileSystem defaultAfs = null;
+if (defaultFsUri.getScheme() == null) {
+ return getFileContext(aConf);
+}
try {
currentUser = UserGroupInformation.getCurrentUser();
defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10f9f510/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index bbe96c2..fd42f82 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -459,6 +459,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6048. Fixed TestJavaSerialization failure. (Varun Vasudev via
jianhe)
+MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect
+with no authority in job jar path. (Gera Shegalov via jlowe)
+
Release 2.5.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10f9f510/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 288e18e..dfc6a3f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -657,9 +657,11 @@ public abstract class TaskAttemptImpl implements
// Set up JobJar to be localized properly on the remote NM.
String jobJar = conf.get(MRJobConfig.JAR);
if (jobJar != null) {
-Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
-.getUri(), remoteFS.getWorkingDirectory());
-LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
+final Path jobJarPath = new Path(jobJar);
+final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf);
+Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(),
+jobJarFs.getWorkingDirectory());
+LocalResource rc = createLocalResource(jobJarFs, remoteJobJar,
LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
String pattern = co
git commit: MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect with no authority in job jar path. Contributed by Gera Shegalov (cherry picked from commit 10f9f5101c44be7c675a44
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5c8b55856 -> c0bbb7291
MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect with
no authority in job jar path. Contributed by Gera Shegalov
(cherry picked from commit 10f9f5101c44be7c675a44ded4aad212627ecdee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0bbb729
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0bbb729
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0bbb729
Branch: refs/heads/branch-2
Commit: c0bbb72912b8e9c1de28bc82ba4b541cd0b222a8
Parents: 5c8b558
Author: Jason Lowe
Authored: Thu Nov 6 15:10:40 2014 +
Committer: Jason Lowe
Committed: Thu Nov 6 15:12:06 2014 +
--
.../java/org/apache/hadoop/fs/FileContext.java | 3 ++
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../v2/app/job/impl/TaskAttemptImpl.java| 8 ++--
.../apache/hadoop/mapreduce/JobSubmitter.java | 9 ++---
.../org/apache/hadoop/mapred/YARNRunner.java| 5 ++-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 41
.../apache/hadoop/mapreduce/v2/TestUberAM.java | 40 +++
7 files changed, 56 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0bbb729/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 57bd3b5..40d271d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -440,6 +440,9 @@ public class FileContext {
final Configuration aConf) throws UnsupportedFileSystemException {
UserGroupInformation currentUser = null;
AbstractFileSystem defaultAfs = null;
+if (defaultFsUri.getScheme() == null) {
+ return getFileContext(aConf);
+}
try {
currentUser = UserGroupInformation.getCurrentUser();
defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0bbb729/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index e51dfb3..c9f9ed2 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -240,6 +240,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6048. Fixed TestJavaSerialization failure. (Varun Vasudev via
jianhe)
+MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect
+with no authority in job jar path. (Gera Shegalov via jlowe)
+
Release 2.5.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0bbb729/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 58a8537..5a8a658 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -653,9 +653,11 @@ public abstract class TaskAttemptImpl implements
// Set up JobJar to be localized properly on the remote NM.
String jobJar = conf.get(MRJobConfig.JAR);
if (jobJar != null) {
-Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
-.getUri(), remoteFS.getWorkingDirectory());
-LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
+final Path jobJarPath = new Path(jobJar);
+final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf);
+Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(),
+jobJarFs.getWorkingDirectory());
+LocalResource rc = createLocalResource(jobJarFs, remoteJobJar,
LocalResourc
git commit: MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect with no authority in job jar path. Contributed by Gera Shegalov (cherry picked from commit 10f9f5101c44be7c675a44
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 43854764a -> 7be74ea09
MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect with
no authority in job jar path. Contributed by Gera Shegalov
(cherry picked from commit 10f9f5101c44be7c675a44ded4aad212627ecdee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7be74ea0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7be74ea0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7be74ea0
Branch: refs/heads/branch-2.6
Commit: 7be74ea09a4721b39f78df47c11b9ceca1b34d1b
Parents: 4385476
Author: Jason Lowe
Authored: Thu Nov 6 15:10:40 2014 +
Committer: Jason Lowe
Committed: Thu Nov 6 15:12:37 2014 +
--
.../java/org/apache/hadoop/fs/FileContext.java | 3 ++
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../v2/app/job/impl/TaskAttemptImpl.java| 8 ++--
.../apache/hadoop/mapreduce/JobSubmitter.java | 9 ++---
.../org/apache/hadoop/mapred/YARNRunner.java| 5 ++-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 41
.../apache/hadoop/mapreduce/v2/TestUberAM.java | 40 +++
7 files changed, 56 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be74ea0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 57bd3b5..40d271d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -440,6 +440,9 @@ public class FileContext {
final Configuration aConf) throws UnsupportedFileSystemException {
UserGroupInformation currentUser = null;
AbstractFileSystem defaultAfs = null;
+if (defaultFsUri.getScheme() == null) {
+ return getFileContext(aConf);
+}
try {
currentUser = UserGroupInformation.getCurrentUser();
defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be74ea0/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index bc70c89..bf758c4 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -225,6 +225,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6048. Fixed TestJavaSerialization failure. (Varun Vasudev via
jianhe)
+MAPREDUCE-5960. JobSubmitter's check whether job.jar is local is incorrect
+with no authority in job jar path. (Gera Shegalov via jlowe)
+
Release 2.5.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7be74ea0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 58a8537..5a8a658 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -653,9 +653,11 @@ public abstract class TaskAttemptImpl implements
// Set up JobJar to be localized properly on the remote NM.
String jobJar = conf.get(MRJobConfig.JAR);
if (jobJar != null) {
-Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
-.getUri(), remoteFS.getWorkingDirectory());
-LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
+final Path jobJarPath = new Path(jobJar);
+final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf);
+Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(),
+jobJarFs.getWorkingDirectory());
+LocalResource rc = createLocalResource(jobJarFs, remoteJobJar,
LocalResourc
hadoop git commit: YARN-2825. Container leak on NM. Contributed by Jian He
Repository: hadoop
Updated Branches:
refs/heads/trunk 68db5b373 -> c3d475070
YARN-2825. Container leak on NM. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3d47507
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3d47507
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3d47507
Branch: refs/heads/trunk
Commit: c3d475070a1ec54c4b05923f4782cef204effd2c
Parents: 68db5b3
Author: Jason Lowe
Authored: Fri Nov 7 23:16:37 2014 +
Committer: Jason Lowe
Committed: Fri Nov 7 23:16:37 2014 +
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../nodemanager/NodeStatusUpdaterImpl.java | 27 ++
.../nodemanager/TestNodeStatusUpdater.java | 54 ++--
3 files changed, 71 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3d47507/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6e5f59a..e4b116d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -906,6 +906,8 @@ Release 2.6.0 - UNRELEASED
YARN-2753. Fixed a bunch of bugs in the NodeLabelsManager classes. (Zhihai
xu
via vinodkv)
+YARN-2825. Container leak on NM (Jian He via jlowe)
+
Release 2.5.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3d47507/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index ebbe503..fcdd2c1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -66,8 +66,8 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
-import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@@ -115,6 +115,7 @@ public class NodeStatusUpdaterImpl extends AbstractService
implements
private Runnable statusUpdaterRunnable;
private Thread statusUpdater;
private long rmIdentifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
+ Set pendingContainersToRemove = new HashSet();
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
@@ -446,19 +447,27 @@ public class NodeStatusUpdaterImpl extends
AbstractService implements
@VisibleForTesting
@Private
- public void removeCompletedContainersFromContext(
+ public void removeOrTrackCompletedContainersFromContext(
List containerIds) throws IOException {
Set removedContainers = new HashSet();
-// If the AM has pulled the completedContainer it can be removed
-for (ContainerId containerId : containerIds) {
- context.getContainers().remove(containerId);
- removedContainers.add(containerId);
+pendingContainersToRemove.addAll(containerIds);
+Iterator iter = pendingContainersToRemove.iterator();
+while (iter.hasNext()) {
+ ContainerId containerId = iter.next();
+ // remove the container only if the container is at DONE state
+ Container nmContainer = context.getContainers().get(containerId);
+ if (nmContainer != null && nmContainer.getContainerState().equals(
+
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.DONE))
{
+context.getContainers().remove(
hadoop git commit: YARN-2825. Container leak on NM. Contributed by Jian He (cherry picked from commit c3d475070a1ec54c4b05923f4782cef204effd2c)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 3ef876712 -> a5764cb78
YARN-2825. Container leak on NM. Contributed by Jian He
(cherry picked from commit c3d475070a1ec54c4b05923f4782cef204effd2c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5764cb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5764cb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5764cb7
Branch: refs/heads/branch-2
Commit: a5764cb7831d20848de444e870a6ed0fd52b3515
Parents: 3ef8767
Author: Jason Lowe
Authored: Fri Nov 7 23:16:37 2014 +
Committer: Jason Lowe
Committed: Fri Nov 7 23:17:34 2014 +
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../nodemanager/NodeStatusUpdaterImpl.java | 27 ++
.../nodemanager/TestNodeStatusUpdater.java | 54 ++--
3 files changed, 71 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5764cb7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 162b2ff..530492f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -879,6 +879,8 @@ Release 2.6.0 - UNRELEASED
YARN-2753. Fixed a bunch of bugs in the NodeLabelsManager classes. (Zhihai
xu
via vinodkv)
+YARN-2825. Container leak on NM (Jian He via jlowe)
+
Release 2.5.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5764cb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index ebbe503..fcdd2c1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -66,8 +66,8 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
-import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@@ -115,6 +115,7 @@ public class NodeStatusUpdaterImpl extends AbstractService
implements
private Runnable statusUpdaterRunnable;
private Thread statusUpdater;
private long rmIdentifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
+ Set pendingContainersToRemove = new HashSet();
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
@@ -446,19 +447,27 @@ public class NodeStatusUpdaterImpl extends
AbstractService implements
@VisibleForTesting
@Private
- public void removeCompletedContainersFromContext(
+ public void removeOrTrackCompletedContainersFromContext(
List containerIds) throws IOException {
Set removedContainers = new HashSet();
-// If the AM has pulled the completedContainer it can be removed
-for (ContainerId containerId : containerIds) {
- context.getContainers().remove(containerId);
- removedContainers.add(containerId);
+pendingContainersToRemove.addAll(containerIds);
+Iterator iter = pendingContainersToRemove.iterator();
+while (iter.hasNext()) {
+ ContainerId containerId = iter.next();
+ // remove the container only if the container is at DONE state
+ Container nmContainer = context.getContainers().get(containerId);
+ if (nmContainer != null && nmContainer.getContainerSta
hadoop git commit: YARN-2825. Container leak on NM. Contributed by Jian He (cherry picked from commit c3d475070a1ec54c4b05923f4782cef204effd2c)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 a07f478b4 -> e5c0fa2c9
YARN-2825. Container leak on NM. Contributed by Jian He
(cherry picked from commit c3d475070a1ec54c4b05923f4782cef204effd2c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5c0fa2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5c0fa2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5c0fa2c
Branch: refs/heads/branch-2.6
Commit: e5c0fa2c956a2db4f93ac8ea9b592e6791665a06
Parents: a07f478
Author: Jason Lowe
Authored: Fri Nov 7 23:16:37 2014 +
Committer: Jason Lowe
Committed: Fri Nov 7 23:17:49 2014 +
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../nodemanager/NodeStatusUpdaterImpl.java | 27 ++
.../nodemanager/TestNodeStatusUpdater.java | 54 ++--
3 files changed, 71 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5c0fa2c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0577123..9e09317 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -834,6 +834,8 @@ Release 2.6.0 - UNRELEASED
YARN-2753. Fixed a bunch of bugs in the NodeLabelsManager classes. (Zhihai
xu
via vinodkv)
+YARN-2825. Container leak on NM (Jian He via jlowe)
+
Release 2.5.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5c0fa2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index ebbe503..fcdd2c1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -66,8 +66,8 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
-import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import
org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@@ -115,6 +115,7 @@ public class NodeStatusUpdaterImpl extends AbstractService
implements
private Runnable statusUpdaterRunnable;
private Thread statusUpdater;
private long rmIdentifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER;
+ Set pendingContainersToRemove = new HashSet();
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
@@ -446,19 +447,27 @@ public class NodeStatusUpdaterImpl extends
AbstractService implements
@VisibleForTesting
@Private
- public void removeCompletedContainersFromContext(
+ public void removeOrTrackCompletedContainersFromContext(
List containerIds) throws IOException {
Set removedContainers = new HashSet();
-// If the AM has pulled the completedContainer it can be removed
-for (ContainerId containerId : containerIds) {
- context.getContainers().remove(containerId);
- removedContainers.add(containerId);
+pendingContainersToRemove.addAll(containerIds);
+Iterator iter = pendingContainersToRemove.iterator();
+while (iter.hasNext()) {
+ ContainerId containerId = iter.next();
+ // remove the container only if the container is at DONE state
+ Container nmContainer = context.getContainers().get(containerId);
+ if (nmContainer != null && nmContainer.getContainerSta
hadoop git commit: YARN-2632. Document NM Restart feature. Contributed by Junping Du and Vinod Kumar Vavilapalli
Repository: hadoop
Updated Branches:
refs/heads/trunk c3d475070 -> 1e215e8ba
YARN-2632. Document NM Restart feature. Contributed by Junping Du and Vinod
Kumar Vavilapalli
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e215e8b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e215e8b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e215e8b
Branch: refs/heads/trunk
Commit: 1e215e8ba2e801eb26f16c307daee756d6b2ca66
Parents: c3d4750
Author: Jason Lowe
Authored: Fri Nov 7 23:40:22 2014 +
Committer: Jason Lowe
Committed: Fri Nov 7 23:40:22 2014 +
--
hadoop-project/src/site/site.xml| 1 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../src/site/apt/NodeManagerRestart.apt.vm | 86
3 files changed, 90 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e215e8b/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index e1d4c92..6a61a83 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -123,6 +123,7 @@
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e215e8b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e4b116d..df7e3ea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -195,6 +195,9 @@ Release 2.6.0 - UNRELEASED
YARN-2647. Added a queue CLI for getting queue information. (Sunil Govind
via
vinodkv)
+YARN-2632. Document NM Restart feature. (Junping Du and Vinod Kumar
+Vavilapalli via jlowe)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e215e8b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
new file mode 100644
index 000..ba03f4e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
@@ -0,0 +1,86 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+ ---
+ NodeManager Restart
+ ---
+ ---
+ ${maven.build.timestamp}
+
+NodeManager Restart
+
+* Introduction
+
+ This document gives an overview of NodeManager (NM) restart, a feature that
+ enables the NodeManager to be restarted without losing
+ the active containers running on the node. At a high level, the NM stores
any
+ necessary state to a local state-store as it processes container-management
+ requests. When the NM restarts, it recovers by first loading state for
+ various subsystems and then letting those subsystems perform recovery using
+ the loaded state.
+
+* Enabling NM Restart
+
+ [[1]] To enable NM Restart functionality, set the following property in
<> to true:
+
+*--+--+
+|| Property|| Value
|
+*--+--+
+| <<>> | |
+| | <<>>, (default value is set to false) |
+*--+--+
+
+ [[2]] Configure a path to the local file-system directory where the
+ NodeManager can save its run state
+
+*--+--+
+|| Property|| Description|
+*--+--+
+| <<>> | |
+| | The local filesystem directory in which the node manager will store state |
+| | when recovery is enabled. |
+| | The def
hadoop git commit: YARN-2632. Document NM Restart feature. Contributed by Junping Du and Vinod Kumar Vavilapalli (cherry picked from commit 1e215e8ba2e801eb26f16c307daee756d6b2ca66)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a5764cb78 -> 944723552
YARN-2632. Document NM Restart feature. Contributed by Junping Du and Vinod
Kumar Vavilapalli
(cherry picked from commit 1e215e8ba2e801eb26f16c307daee756d6b2ca66)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94472355
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94472355
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94472355
Branch: refs/heads/branch-2
Commit: 9447235527b262605e932065a6f6ac891ec9a338
Parents: a5764cb
Author: Jason Lowe
Authored: Fri Nov 7 23:40:22 2014 +
Committer: Jason Lowe
Committed: Fri Nov 7 23:41:29 2014 +
--
hadoop-project/src/site/site.xml| 1 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../src/site/apt/NodeManagerRestart.apt.vm | 86
3 files changed, 90 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94472355/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 2fd1532..4a2c221 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -124,6 +124,7 @@
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94472355/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 530492f..f3c9d4e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -168,6 +168,9 @@ Release 2.6.0 - UNRELEASED
YARN-2647. Added a queue CLI for getting queue information. (Sunil Govind
via
vinodkv)
+YARN-2632. Document NM Restart feature. (Junping Du and Vinod Kumar
+Vavilapalli via jlowe)
+
IMPROVEMENTS
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94472355/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
new file mode 100644
index 000..ba03f4e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
@@ -0,0 +1,86 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+ ---
+ NodeManager Restart
+ ---
+ ---
+ ${maven.build.timestamp}
+
+NodeManager Restart
+
+* Introduction
+
+ This document gives an overview of NodeManager (NM) restart, a feature that
+ enables the NodeManager to be restarted without losing
+ the active containers running on the node. At a high level, the NM stores
any
+ necessary state to a local state-store as it processes container-management
+ requests. When the NM restarts, it recovers by first loading state for
+ various subsystems and then letting those subsystems perform recovery using
+ the loaded state.
+
+* Enabling NM Restart
+
+ [[1]] To enable NM Restart functionality, set the following property in
<> to true:
+
+*--+--+
+|| Property|| Value
|
+*--+--+
+| <<>> | |
+| | <<>>, (default value is set to false) |
+*--+--+
+
+ [[2]] Configure a path to the local file-system directory where the
+ NodeManager can save its run state
+
+*--+--+
+|| Property|| Description|
+*--+--+
+| <<>> | |
+| | The local filesystem directory in which the node manager will store state |
+
hadoop git commit: YARN-2632. Document NM Restart feature. Contributed by Junping Du and Vinod Kumar Vavilapalli (cherry picked from commit 1e215e8ba2e801eb26f16c307daee756d6b2ca66)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 e5c0fa2c9 -> dcc1ab15a
YARN-2632. Document NM Restart feature. Contributed by Junping Du and Vinod
Kumar Vavilapalli
(cherry picked from commit 1e215e8ba2e801eb26f16c307daee756d6b2ca66)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcc1ab15
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcc1ab15
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcc1ab15
Branch: refs/heads/branch-2.6
Commit: dcc1ab15ab4276a36fdb4f5376e65880ed8349ec
Parents: e5c0fa2
Author: Jason Lowe
Authored: Fri Nov 7 23:40:22 2014 +
Committer: Jason Lowe
Committed: Fri Nov 7 23:42:26 2014 +
--
hadoop-project/src/site/site.xml| 1 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../src/site/apt/NodeManagerRestart.apt.vm | 86
3 files changed, 90 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcc1ab15/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 2fd1532..4a2c221 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -124,6 +124,7 @@
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcc1ab15/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9e09317..12c9af9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -123,6 +123,9 @@ Release 2.6.0 - UNRELEASED
YARN-2647. Added a queue CLI for getting queue information. (Sunil Govind
via
vinodkv)
+YARN-2632. Document NM Restart feature. (Junping Du and Vinod Kumar
+Vavilapalli via jlowe)
+
IMPROVEMENTS
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcc1ab15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
new file mode 100644
index 000..ba03f4e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRestart.apt.vm
@@ -0,0 +1,86 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+ ---
+ NodeManager Restart
+ ---
+ ---
+ ${maven.build.timestamp}
+
+NodeManager Restart
+
+* Introduction
+
+ This document gives an overview of NodeManager (NM) restart, a feature that
+ enables the NodeManager to be restarted without losing
+ the active containers running on the node. At a high level, the NM stores
any
+ necessary state to a local state-store as it processes container-management
+ requests. When the NM restarts, it recovers by first loading state for
+ various subsystems and then letting those subsystems perform recovery using
+ the loaded state.
+
+* Enabling NM Restart
+
+ [[1]] To enable NM Restart functionality, set the following property in
<> to true:
+
+*--+--+
+|| Property|| Value
|
+*--+--+
+| <<>> | |
+| | <<>>, (default value is set to false) |
+*--+--+
+
+ [[2]] Configure a path to the local file-system directory where the
+ NodeManager can save its run state
+
+*--+--+
+|| Property|| Description|
+*--+--+
+| <<>> | |
+| | The local filesystem directory in which the node manager will store state
hadoop git commit: YARN-2780. Log aggregated resource allocation in rm-appsummary.log. Contributed by Eric Payne (cherry picked from commit f8aefa5e9c8c6d2817205b5ed8d914db31f56ae7)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d863f54f5 -> 6984d899e
YARN-2780. Log aggregated resource allocation in rm-appsummary.log. Contributed
by Eric Payne
(cherry picked from commit f8aefa5e9c8c6d2817205b5ed8d914db31f56ae7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6984d899
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6984d899
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6984d899
Branch: refs/heads/branch-2
Commit: 6984d899e35ecf2a32f4d50ff2aa5356b043bc02
Parents: d863f54
Author: Jason Lowe
Authored: Wed Nov 12 17:01:15 2014 +
Committer: Jason Lowe
Committed: Wed Nov 12 17:02:30 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/RMAppManager.java | 9 -
.../yarn/server/resourcemanager/TestAppManager.java | 12 +++-
3 files changed, 22 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6984d899/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 51413ca..f9a00fa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -41,6 +41,9 @@ Release 2.7.0 - UNRELEASED
YARN-570. Time strings are formated in different timezone.
(Akira Ajisaka and Peng Zhang via kasha)
+YARN-2780. Log aggregated resource allocation in rm-appsummary.log (Eric
+Payne via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6984d899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 02c6d2f..ab8df62 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -47,6 +47,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRecoverEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -155,6 +156,7 @@ public class RMAppManager implements
EventHandler,
trackingUrl = attempt.getTrackingUrl();
host = attempt.getHost();
}
+ RMAppMetrics metrics = app.getRMAppMetrics();
SummaryBuilder summary = new SummaryBuilder()
.add("appId", app.getApplicationId())
.add("name", app.getName())
@@ -165,7 +167,12 @@ public class RMAppManager implements
EventHandler,
.add("appMasterHost", host)
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
- .add("finalStatus", app.getFinalApplicationStatus());
+ .add("finalStatus", app.getFinalApplicationStatus())
+ .add("memorySeconds", metrics.getMemorySeconds())
+ .add("vcoreSeconds", metrics.getVcoreSeconds())
+ .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
+ .add("preemptedNonAMContainers",
metrics.getNumNonAMContainersPreempted())
+ .add("preemptedResources", metrics.getResourcePreempted());
return summary;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6984d899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-ser
hadoop git commit: YARN-2780. Log aggregated resource allocation in rm-appsummary.log. Contributed by Eric Payne
Repository: hadoop
Updated Branches:
refs/heads/trunk 53f64ee51 -> f8aefa5e9
YARN-2780. Log aggregated resource allocation in rm-appsummary.log. Contributed
by Eric Payne
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8aefa5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8aefa5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8aefa5e
Branch: refs/heads/trunk
Commit: f8aefa5e9c8c6d2817205b5ed8d914db31f56ae7
Parents: 53f64ee
Author: Jason Lowe
Authored: Wed Nov 12 17:01:15 2014 +
Committer: Jason Lowe
Committed: Wed Nov 12 17:01:15 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/RMAppManager.java | 9 -
.../yarn/server/resourcemanager/TestAppManager.java | 12 +++-
3 files changed, 22 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8aefa5e/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c3bdf3d..05bb8f6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -68,6 +68,9 @@ Release 2.7.0 - UNRELEASED
YARN-570. Time strings are formated in different timezone.
(Akira Ajisaka and Peng Zhang via kasha)
+YARN-2780. Log aggregated resource allocation in rm-appsummary.log (Eric
+Payne via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8aefa5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 02c6d2f..ab8df62 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -47,6 +47,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRecoverEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -155,6 +156,7 @@ public class RMAppManager implements
EventHandler,
trackingUrl = attempt.getTrackingUrl();
host = attempt.getHost();
}
+ RMAppMetrics metrics = app.getRMAppMetrics();
SummaryBuilder summary = new SummaryBuilder()
.add("appId", app.getApplicationId())
.add("name", app.getName())
@@ -165,7 +167,12 @@ public class RMAppManager implements
EventHandler,
.add("appMasterHost", host)
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
- .add("finalStatus", app.getFinalApplicationStatus());
+ .add("finalStatus", app.getFinalApplicationStatus())
+ .add("memorySeconds", metrics.getMemorySeconds())
+ .add("vcoreSeconds", metrics.getVcoreSeconds())
+ .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
+ .add("preemptedNonAMContainers",
metrics.getNumNonAMContainersPreempted())
+ .add("preemptedResources", metrics.getResourcePreempted());
return summary;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8aefa5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourceman
hadoop git commit: MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused correctly. Contributed by Junping Du
Repository: hadoop
Updated Branches:
refs/heads/trunk 7dae5b5a8 -> 177e8090f
MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused
correctly. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/177e8090
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/177e8090
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/177e8090
Branch: refs/heads/trunk
Commit: 177e8090f5809beb3ebcb656cd0affbb3f487de8
Parents: 7dae5b5
Author: Jason Lowe
Authored: Thu Nov 13 15:42:25 2014 +
Committer: Jason Lowe
Committed: Thu Nov 13 15:42:25 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../hadoop/mapreduce/task/reduce/Fetcher.java | 39 +++-
2 files changed, 33 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177e8090/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 96bb690..c0105e6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -465,6 +465,9 @@ Release 2.6.0 - 2014-11-15
MAPREDUCE-5958. Wrong reduce task progress if map output is compressed
(Emilio Coppa and jlowe via kihwal)
+MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused
+correctly (Junping Du via jlowe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/177e8090/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index 796394f..3f40853 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -407,7 +407,7 @@ class Fetcher extends Thread {
}
if ((Time.monotonicNow() - startTime) >= this.fetchRetryTimeout) {
LOG.warn("Failed to connect to host: " + url + "after "
- + fetchRetryTimeout + "milliseconds.");
+ + fetchRetryTimeout + " milliseconds.");
throw e;
}
try {
@@ -596,7 +596,7 @@ class Fetcher extends Thread {
} else {
// timeout, prepare to be failed.
LOG.warn("Timeout for copying MapOutput with retry on host " + host
- + "after " + fetchRetryTimeout + "milliseconds.");
+ + "after " + fetchRetryTimeout + " milliseconds.");
}
}
@@ -678,28 +678,49 @@ class Fetcher extends Thread {
} else if (connectionTimeout > 0) {
unit = Math.min(UNIT_CONNECT_TIMEOUT, connectionTimeout);
}
+long startTime = Time.monotonicNow();
+long lastTime = startTime;
+int attempts = 0;
// set the connect timeout to the unit-connect-timeout
connection.setConnectTimeout(unit);
while (true) {
try {
+attempts++;
connection.connect();
break;
} catch (IOException ioe) {
-// update the total remaining connect-timeout
-connectionTimeout -= unit;
-
+long currentTime = Time.monotonicNow();
+long retryTime = currentTime - startTime;
+long leftTime = connectionTimeout - retryTime;
+long timeSinceLastIteration = currentTime - lastTime;
// throw an exception if we have waited for timeout amount of time
// note that the updated value if timeout is used here
-if (connectionTimeout == 0) {
+if (leftTime <= 0) {
+ int retryTimeInSeconds = (int) retryTime/1000;
+ LOG.error("Connection retry failed with " + attempts +
+ " attempts in " + retryTimeInSeconds + " seconds");
throw ioe;
}
-
// reset the connect timeout for the last try
-if (connectionTimeout < unit) {
- unit = connectionTimeout;
+if (leftTime < unit) {
+ unit = (int)leftTime;
// reset the connect time out for the final connect
connection
hadoop git commit: MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused correctly. Contributed by Junping Du (cherry picked from commit 177e8090f5809beb3ebcb656cd0affbb3f487de8)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9c94945fe -> e62a5f9d0
MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused
correctly. Contributed by Junping Du
(cherry picked from commit 177e8090f5809beb3ebcb656cd0affbb3f487de8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e62a5f9d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e62a5f9d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e62a5f9d
Branch: refs/heads/branch-2
Commit: e62a5f9d0af481eb4cba55d39253f64a1abdff1b
Parents: 9c94945
Author: Jason Lowe
Authored: Thu Nov 13 15:42:25 2014 +
Committer: Jason Lowe
Committed: Thu Nov 13 15:43:28 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../hadoop/mapreduce/task/reduce/Fetcher.java | 39 +++-
2 files changed, 33 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e62a5f9d/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 2caa828..e81d24f 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -246,6 +246,9 @@ Release 2.6.0 - 2014-11-15
MAPREDUCE-5958. Wrong reduce task progress if map output is compressed
(Emilio Coppa and jlowe via kihwal)
+MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused
+correctly (Junping Du via jlowe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e62a5f9d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index 796394f..3f40853 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -407,7 +407,7 @@ class Fetcher extends Thread {
}
if ((Time.monotonicNow() - startTime) >= this.fetchRetryTimeout) {
LOG.warn("Failed to connect to host: " + url + "after "
- + fetchRetryTimeout + "milliseconds.");
+ + fetchRetryTimeout + " milliseconds.");
throw e;
}
try {
@@ -596,7 +596,7 @@ class Fetcher extends Thread {
} else {
// timeout, prepare to be failed.
LOG.warn("Timeout for copying MapOutput with retry on host " + host
- + "after " + fetchRetryTimeout + "milliseconds.");
+ + "after " + fetchRetryTimeout + " milliseconds.");
}
}
@@ -678,28 +678,49 @@ class Fetcher extends Thread {
} else if (connectionTimeout > 0) {
unit = Math.min(UNIT_CONNECT_TIMEOUT, connectionTimeout);
}
+long startTime = Time.monotonicNow();
+long lastTime = startTime;
+int attempts = 0;
// set the connect timeout to the unit-connect-timeout
connection.setConnectTimeout(unit);
while (true) {
try {
+attempts++;
connection.connect();
break;
} catch (IOException ioe) {
-// update the total remaining connect-timeout
-connectionTimeout -= unit;
-
+long currentTime = Time.monotonicNow();
+long retryTime = currentTime - startTime;
+long leftTime = connectionTimeout - retryTime;
+long timeSinceLastIteration = currentTime - lastTime;
// throw an exception if we have waited for timeout amount of time
// note that the updated value if timeout is used here
-if (connectionTimeout == 0) {
+if (leftTime <= 0) {
+ int retryTimeInSeconds = (int) retryTime/1000;
+ LOG.error("Connection retry failed with " + attempts +
+ " attempts in " + retryTimeInSeconds + " seconds");
throw ioe;
}
-
// reset the connect timeout for the last try
-if (connectionTimeout < unit) {
- unit = connectionTimeout;
+if (leftTime < unit) {
+ unit = (int)left
hadoop git commit: MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused correctly. Contributed by Junping Du (cherry picked from commit 177e8090f5809beb3ebcb656cd0affbb3f487de8)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 59b2a74ea -> 9b383f681
MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused
correctly. Contributed by Junping Du
(cherry picked from commit 177e8090f5809beb3ebcb656cd0affbb3f487de8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b383f68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b383f68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b383f68
Branch: refs/heads/branch-2.6
Commit: 9b383f6813d2b455f1f2f3fb56373164a38acf5b
Parents: 59b2a74
Author: Jason Lowe
Authored: Thu Nov 13 15:42:25 2014 +
Committer: Jason Lowe
Committed: Thu Nov 13 15:44:00 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../hadoop/mapreduce/task/reduce/Fetcher.java | 39 +++-
2 files changed, 33 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b383f68/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index a4e88fa..40d5167 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -231,6 +231,9 @@ Release 2.6.0 - 2014-11-15
MAPREDUCE-5958. Wrong reduce task progress if map output is compressed
(Emilio Coppa and jlowe via kihwal)
+MAPREDUCE-6156. Fetcher - connect() doesn't handle connection refused
+correctly (Junping Du via jlowe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b383f68/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index 796394f..3f40853 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -407,7 +407,7 @@ class Fetcher extends Thread {
}
if ((Time.monotonicNow() - startTime) >= this.fetchRetryTimeout) {
LOG.warn("Failed to connect to host: " + url + "after "
- + fetchRetryTimeout + "milliseconds.");
+ + fetchRetryTimeout + " milliseconds.");
throw e;
}
try {
@@ -596,7 +596,7 @@ class Fetcher extends Thread {
} else {
// timeout, prepare to be failed.
LOG.warn("Timeout for copying MapOutput with retry on host " + host
- + "after " + fetchRetryTimeout + "milliseconds.");
+ + "after " + fetchRetryTimeout + " milliseconds.");
}
}
@@ -678,28 +678,49 @@ class Fetcher extends Thread {
} else if (connectionTimeout > 0) {
unit = Math.min(UNIT_CONNECT_TIMEOUT, connectionTimeout);
}
+long startTime = Time.monotonicNow();
+long lastTime = startTime;
+int attempts = 0;
// set the connect timeout to the unit-connect-timeout
connection.setConnectTimeout(unit);
while (true) {
try {
+attempts++;
connection.connect();
break;
} catch (IOException ioe) {
-// update the total remaining connect-timeout
-connectionTimeout -= unit;
-
+long currentTime = Time.monotonicNow();
+long retryTime = currentTime - startTime;
+long leftTime = connectionTimeout - retryTime;
+long timeSinceLastIteration = currentTime - lastTime;
// throw an exception if we have waited for timeout amount of time
// note that the updated value if timeout is used here
-if (connectionTimeout == 0) {
+if (leftTime <= 0) {
+ int retryTimeInSeconds = (int) retryTime/1000;
+ LOG.error("Connection retry failed with " + attempts +
+ " attempts in " + retryTimeInSeconds + " seconds");
throw ioe;
}
-
// reset the connect timeout for the last try
-if (connectionTimeout < unit) {
- unit = connectionTimeout;
+if (leftTime < unit) {
+ unit = (int)left
hadoop git commit: YARN-2846. Incorrect persist exit code for running containers in reacquireContainer() that interrupted by NodeManager restart. Contributed by Junping Du
Repository: hadoop
Updated Branches:
refs/heads/trunk 177e8090f -> 33ea5ae92
YARN-2846. Incorrect persist exit code for running containers in
reacquireContainer() that interrupted by NodeManager restart. Contributed by
Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33ea5ae9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33ea5ae9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33ea5ae9
Branch: refs/heads/trunk
Commit: 33ea5ae92b9dd3abace104903d9a94d17dd75af5
Parents: 177e809
Author: Jason Lowe
Authored: Thu Nov 13 16:11:04 2014 +
Committer: Jason Lowe
Committed: Thu Nov 13 16:11:04 2014 +
--
hadoop-yarn-project/CHANGES.txt | 4
.../server/nodemanager/ContainerExecutor.java | 21 +++-
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../launcher/RecoveredContainerLaunch.java | 20 ---
4 files changed, 25 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33ea5ae9/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 01dc246..326d33c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -955,6 +955,10 @@ Release 2.6.0 - 2014-11-15
YARN-2794. Fixed log messages about distributing system-credentials. (Jian
He via
zjshen)
+YARN-2846. Incorrect persist exit code for running containers in
+reacquireContainer() that interrupted by NodeManager restart. (Junping Du
+via jlowe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33ea5ae9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 8133413..327f882 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -159,9 +159,10 @@ public abstract class ContainerExecutor implements
Configurable {
* @param containerId The ID of the container to reacquire
* @return The exit code of the pre-existing container
* @throws IOException
+ * @throws InterruptedException
*/
public int reacquireContainer(String user, ContainerId containerId)
- throws IOException {
+ throws IOException, InterruptedException {
Path pidPath = getPidFilePath(containerId);
if (pidPath == null) {
LOG.warn(containerId + " is not active, returning terminated error");
@@ -175,13 +176,8 @@ public abstract class ContainerExecutor implements
Configurable {
}
LOG.info("Reacquiring " + containerId + " with pid " + pid);
-try {
- while(isContainerProcessAlive(user, pid)) {
-Thread.sleep(1000);
- }
-} catch (InterruptedException e) {
- throw new IOException("Interrupted while waiting for process " + pid
- + " to exit", e);
+while(isContainerProcessAlive(user, pid)) {
+ Thread.sleep(1000);
}
// wait for exit code file to appear
@@ -194,12 +190,9 @@ public abstract class ContainerExecutor implements
Configurable {
LOG.info(containerId + " was deactivated");
return ExitCode.TERMINATED.getExitCode();
}
- try {
-Thread.sleep(sleepMsec);
- } catch (InterruptedException e) {
-throw new IOException(
-"Interrupted while waiting for exit code from " + containerId, e);
- }
+
+ Thread.sleep(sleepMsec);
+
msecLeft -= sleepMsec;
}
if (msecLeft < 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33ea5ae9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yar
hadoop git commit: YARN-2846. Incorrect persist exit code for running containers in reacquireContainer() that interrupted by NodeManager restart. Contributed by Junping Du (cherry picked from commit 3
Repository: hadoop
Updated Branches:
refs/heads/branch-2 ed3e5cb16 -> 6fd547da8
YARN-2846. Incorrect persist exit code for running containers in
reacquireContainer() that interrupted by NodeManager restart. Contributed by
Junping Du
(cherry picked from commit 33ea5ae92b9dd3abace104903d9a94d17dd75af5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fd547da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fd547da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fd547da
Branch: refs/heads/branch-2
Commit: 6fd547da8feb71058f701eb94296cf6cace357b2
Parents: ed3e5cb
Author: Jason Lowe
Authored: Thu Nov 13 16:11:04 2014 +
Committer: Jason Lowe
Committed: Thu Nov 13 16:17:08 2014 +
--
hadoop-yarn-project/CHANGES.txt | 4
.../server/nodemanager/ContainerExecutor.java | 21 +++-
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../launcher/RecoveredContainerLaunch.java | 20 ---
4 files changed, 25 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fd547da/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b080aeb..df04fcc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -931,6 +931,10 @@ Release 2.6.0 - 2014-11-15
YARN-2794. Fixed log messages about distributing system-credentials. (Jian
He via
zjshen)
+YARN-2846. Incorrect persist exit code for running containers in
+reacquireContainer() that interrupted by NodeManager restart. (Junping Du
+via jlowe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fd547da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 8133413..327f882 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -159,9 +159,10 @@ public abstract class ContainerExecutor implements
Configurable {
* @param containerId The ID of the container to reacquire
* @return The exit code of the pre-existing container
* @throws IOException
+ * @throws InterruptedException
*/
public int reacquireContainer(String user, ContainerId containerId)
- throws IOException {
+ throws IOException, InterruptedException {
Path pidPath = getPidFilePath(containerId);
if (pidPath == null) {
LOG.warn(containerId + " is not active, returning terminated error");
@@ -175,13 +176,8 @@ public abstract class ContainerExecutor implements
Configurable {
}
LOG.info("Reacquiring " + containerId + " with pid " + pid);
-try {
- while(isContainerProcessAlive(user, pid)) {
-Thread.sleep(1000);
- }
-} catch (InterruptedException e) {
- throw new IOException("Interrupted while waiting for process " + pid
- + " to exit", e);
+while(isContainerProcessAlive(user, pid)) {
+ Thread.sleep(1000);
}
// wait for exit code file to appear
@@ -194,12 +190,9 @@ public abstract class ContainerExecutor implements
Configurable {
LOG.info(containerId + " was deactivated");
return ExitCode.TERMINATED.getExitCode();
}
- try {
-Thread.sleep(sleepMsec);
- } catch (InterruptedException e) {
-throw new IOException(
-"Interrupted while waiting for exit code from " + containerId, e);
- }
+
+ Thread.sleep(sleepMsec);
+
msecLeft -= sleepMsec;
}
if (msecLeft < 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fd547da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
--
diff --
hadoop git commit: YARN-2846. Incorrect persist exit code for running containers in reacquireContainer() that interrupted by NodeManager restart. Contributed by Junping Du (cherry picked from commit 3
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 9b383f681 -> 685790e02
YARN-2846. Incorrect persist exit code for running containers in
reacquireContainer() that interrupted by NodeManager restart. Contributed by
Junping Du
(cherry picked from commit 33ea5ae92b9dd3abace104903d9a94d17dd75af5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/685790e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/685790e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/685790e0
Branch: refs/heads/branch-2.6
Commit: 685790e027ed24f2752cfa6b8db95b7516b9817c
Parents: 9b383f6
Author: Jason Lowe
Authored: Thu Nov 13 16:11:04 2014 +
Committer: Jason Lowe
Committed: Thu Nov 13 16:12:30 2014 +
--
hadoop-yarn-project/CHANGES.txt | 4
.../server/nodemanager/ContainerExecutor.java | 21 +++-
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../launcher/RecoveredContainerLaunch.java | 20 ---
4 files changed, 25 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/685790e0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c809fd0..6cdd648 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -868,6 +868,10 @@ Release 2.6.0 - 2014-11-15
YARN-2794. Fixed log messages about distributing system-credentials. (Jian
He via
zjshen)
+YARN-2846. Incorrect persist exit code for running containers in
+reacquireContainer() that interrupted by NodeManager restart. (Junping Du
+via jlowe)
+
Release 2.5.2 - 2014-11-10
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/685790e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 8133413..327f882 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -159,9 +159,10 @@ public abstract class ContainerExecutor implements
Configurable {
* @param containerId The ID of the container to reacquire
* @return The exit code of the pre-existing container
* @throws IOException
+ * @throws InterruptedException
*/
public int reacquireContainer(String user, ContainerId containerId)
- throws IOException {
+ throws IOException, InterruptedException {
Path pidPath = getPidFilePath(containerId);
if (pidPath == null) {
LOG.warn(containerId + " is not active, returning terminated error");
@@ -175,13 +176,8 @@ public abstract class ContainerExecutor implements
Configurable {
}
LOG.info("Reacquiring " + containerId + " with pid " + pid);
-try {
- while(isContainerProcessAlive(user, pid)) {
-Thread.sleep(1000);
- }
-} catch (InterruptedException e) {
- throw new IOException("Interrupted while waiting for process " + pid
- + " to exit", e);
+while(isContainerProcessAlive(user, pid)) {
+ Thread.sleep(1000);
}
// wait for exit code file to appear
@@ -194,12 +190,9 @@ public abstract class ContainerExecutor implements
Configurable {
LOG.info(containerId + " was deactivated");
return ExitCode.TERMINATED.getExitCode();
}
- try {
-Thread.sleep(sleepMsec);
- } catch (InterruptedException e) {
-throw new IOException(
-"Interrupted while waiting for exit code from " + containerId, e);
- }
+
+ Thread.sleep(sleepMsec);
+
msecLeft -= sleepMsec;
}
if (msecLeft < 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/685790e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
--
diff --
hadoop git commit: YARN-2857. ConcurrentModificationException in ContainerLogAppender. Contributed by Mohammad Kamrul Islam
Repository: hadoop
Updated Branches:
refs/heads/trunk 1a1dcce82 -> f2fe8a800
YARN-2857. ConcurrentModificationException in ContainerLogAppender. Contributed
by Mohammad Kamrul Islam
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2fe8a80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2fe8a80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2fe8a80
Branch: refs/heads/trunk
Commit: f2fe8a800e5b0f3875931adba9ae20c6a95aa4ff
Parents: 1a1dcce
Author: Jason Lowe
Authored: Fri Nov 14 20:47:26 2014 +
Committer: Jason Lowe
Committed: Fri Nov 14 20:47:26 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/ContainerLogAppender.java | 7 ++-
.../hadoop/yarn/TestContainerLogAppender.java | 46
3 files changed, 55 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2fe8a80/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4373813..34568ff 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -87,6 +87,9 @@ Release 2.7.0 - UNRELEASED
YARN-2856. Fixed RMAppImpl to handle ATTEMPT_KILLED event at ACCEPTED state
on app recovery. (Rohith Sharmaks via jianhe)
+YARN-2857. ConcurrentModificationException in ContainerLogAppender
+(Mohammad Kamrul Islam via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2fe8a80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
index 95876ef..0aba8b5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
@@ -41,6 +41,7 @@ public class ContainerLogAppender extends FileAppender
//so that log4j can configure it from the configuration(log4j.properties).
private int maxEvents;
private Queue tail = null;
+ private boolean closing = false;
@Override
public void activateOptions() {
@@ -57,6 +58,9 @@ public class ContainerLogAppender extends FileAppender
@Override
public void append(LoggingEvent event) {
synchronized (this) {
+ if (closing) { // When closing drop any new/transitive CLA appending
+return;
+ }
if (tail == null) {
super.append(event);
} else {
@@ -77,8 +81,9 @@ public class ContainerLogAppender extends FileAppender
@Override
public synchronized void close() {
+closing = true;
if (tail != null) {
- for(LoggingEvent event: tail) {
+ for (LoggingEvent event : tail) {
super.append(event);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2fe8a80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
new file mode 100644
index 000..3b20d18
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package
hadoop git commit: YARN-2857. ConcurrentModificationException in ContainerLogAppender. Contributed by Mohammad Kamrul Islam (cherry picked from commit f2fe8a800e5b0f3875931adba9ae20c6a95aa4ff)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 45692effe -> 00d135d75
YARN-2857. ConcurrentModificationException in ContainerLogAppender. Contributed
by Mohammad Kamrul Islam
(cherry picked from commit f2fe8a800e5b0f3875931adba9ae20c6a95aa4ff)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00d135d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00d135d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00d135d7
Branch: refs/heads/branch-2
Commit: 00d135d75a8e89fa00fc4f58217ed3b90959c697
Parents: 45692ef
Author: Jason Lowe
Authored: Fri Nov 14 20:47:26 2014 +
Committer: Jason Lowe
Committed: Fri Nov 14 20:48:51 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/ContainerLogAppender.java | 7 ++-
.../hadoop/yarn/TestContainerLogAppender.java | 46
3 files changed, 55 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00d135d7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cd096f2..2d83526 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -60,6 +60,9 @@ Release 2.7.0 - UNRELEASED
YARN-2856. Fixed RMAppImpl to handle ATTEMPT_KILLED event at ACCEPTED state
on app recovery. (Rohith Sharmaks via jianhe)
+YARN-2857. ConcurrentModificationException in ContainerLogAppender
+(Mohammad Kamrul Islam via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00d135d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
index 95876ef..0aba8b5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
@@ -41,6 +41,7 @@ public class ContainerLogAppender extends FileAppender
//so that log4j can configure it from the configuration(log4j.properties).
private int maxEvents;
private Queue tail = null;
+ private boolean closing = false;
@Override
public void activateOptions() {
@@ -57,6 +58,9 @@ public class ContainerLogAppender extends FileAppender
@Override
public void append(LoggingEvent event) {
synchronized (this) {
+ if (closing) { // When closing drop any new/transitive CLA appending
+return;
+ }
if (tail == null) {
super.append(event);
} else {
@@ -77,8 +81,9 @@ public class ContainerLogAppender extends FileAppender
@Override
public synchronized void close() {
+closing = true;
if (tail != null) {
- for(LoggingEvent event: tail) {
+ for (LoggingEvent event : tail) {
super.append(event);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00d135d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
new file mode 100644
index 000..3b20d18
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLogAppender.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language go
hadoop git commit: MAPREDUCE-6161. mapred hsadmin command missing from trunk. Contributed by Allen Wittenauer
Repository: hadoop
Updated Branches:
refs/heads/trunk f95017b3e -> 3baaa4294
MAPREDUCE-6161. mapred hsadmin command missing from trunk. Contributed by Allen
Wittenauer
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3baaa429
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3baaa429
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3baaa429
Branch: refs/heads/trunk
Commit: 3baaa42945fd432e10c547d3ead57e220f64cd57
Parents: f95017b
Author: Jason Lowe
Authored: Fri Nov 14 21:10:06 2014 +
Committer: Jason Lowe
Committed: Fri Nov 14 21:10:06 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
hadoop-mapreduce-project/bin/mapred | 10 +-
2 files changed, 12 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3baaa429/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 28595ca..653ad44 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -160,6 +160,9 @@ Trunk (Unreleased)
MAPREDUCE-5972. Fix typo 'programatically' in job.xml (and a few other
places) (Akira AJISAKA via aw)
+MAPREDUCE-6161. mapred hsadmin command missing from trunk (Allen Wittenauer
+via jlowe)
+
BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3baaa429/hadoop-mapreduce-project/bin/mapred
--
diff --git a/hadoop-mapreduce-project/bin/mapred
b/hadoop-mapreduce-project/bin/mapred
index 1fa5d44..9131ba7 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -25,8 +25,9 @@ function hadoop_usage
echo " classpathprints the class path needed for running"
echo " mapreduce subcommands"
echo " distcp copy file or directories recursively"
- echo " job manipulate MapReduce jobs"
echo " historyserverrun job history servers as a standalone daemon"
+ echo " hsadmin job history server admin interface"
+ echo " job manipulate MapReduce jobs"
echo " pipesrun a Pipes job"
echo " queueget information regarding JobQueues"
echo " sampler sampler"
@@ -98,8 +99,15 @@ case ${COMMAND} in
fi
HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_JHS_LOGGER:-$HADOOP_DAEMON_ROOT_LOGGER}
;;
+ hsadmin)
+CLASS=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin
+hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
job)
CLASS=org.apache.hadoop.mapred.JobClient
+hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
pipes)
CLASS=org.apache.hadoop.mapred.pipes.Submitter
hadoop git commit: YARN-2816. NM fail to start with NPE during container recovery. Contributed by Zhihai Xu
Repository: hadoop
Updated Branches:
refs/heads/trunk 3baaa4294 -> 49c38898b
YARN-2816. NM fail to start with NPE during container recovery. Contributed by
Zhihai Xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49c38898
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49c38898
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49c38898
Branch: refs/heads/trunk
Commit: 49c38898b0be64fc686d039ed2fb2dea1378df02
Parents: 3baaa42
Author: Jason Lowe
Authored: Fri Nov 14 21:25:59 2014 +
Committer: Jason Lowe
Committed: Fri Nov 14 21:25:59 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../recovery/NMLeveldbStateStoreService.java| 24 +++-
.../TestNMLeveldbStateStoreService.java | 7 ++
3 files changed, 33 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49c38898/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 34568ff..1f8af55 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -90,6 +90,9 @@ Release 2.7.0 - UNRELEASED
YARN-2857. ConcurrentModificationException in ContainerLogAppender
(Mohammad Kamrul Islam via jlowe)
+YARN-2816. NM fail to start with NPE during container recovery (Zhihai Xu
+via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49c38898/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 7cf4921..9d54688 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -146,6 +146,8 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
throws IOException {
ArrayList containers =
new ArrayList();
+ArrayList containersToRemove =
+ new ArrayList();
LeveldbIterator iter = null;
try {
iter = new LeveldbIterator(db);
@@ -165,7 +167,14 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
ContainerId containerId = ConverterUtils.toContainerId(
key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos));
String keyPrefix = key.substring(0, idEndPos+1);
-containers.add(loadContainerState(containerId, iter, keyPrefix));
+RecoveredContainerState rcs = loadContainerState(containerId,
+iter, keyPrefix);
+// Don't load container without StartContainerRequest
+if (rcs.startRequest != null) {
+ containers.add(rcs);
+} else {
+ containersToRemove.add(containerId);
+}
}
} catch (DBException e) {
throw new IOException(e);
@@ -175,6 +184,19 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
}
}
+// remove container without StartContainerRequest
+for (ContainerId containerId : containersToRemove) {
+ LOG.warn("Remove container " + containerId +
+ " with incomplete records");
+ try {
+removeContainer(containerId);
+// TODO: kill and cleanup the leaked container
+ } catch (IOException e) {
+LOG.error("Unable to remove container " + containerId +
+" in store", e);
+ }
+}
+
return containers;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/49c38898/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLevel
hadoop git commit: YARN-2816. NM fail to start with NPE during container recovery. Contributed by Zhihai Xu (cherry picked from commit 49c38898b0be64fc686d039ed2fb2dea1378df02)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5b90e428a -> ad140d1fc
YARN-2816. NM fail to start with NPE during container recovery. Contributed by
Zhihai Xu
(cherry picked from commit 49c38898b0be64fc686d039ed2fb2dea1378df02)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad140d1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad140d1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad140d1f
Branch: refs/heads/branch-2
Commit: ad140d1fc831735fb9335e27b38d2fc040847af1
Parents: 5b90e42
Author: Jason Lowe
Authored: Fri Nov 14 21:25:59 2014 +
Committer: Jason Lowe
Committed: Fri Nov 14 21:27:16 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../recovery/NMLeveldbStateStoreService.java| 24 +++-
.../TestNMLeveldbStateStoreService.java | 7 ++
3 files changed, 33 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad140d1f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2d83526..0f3d31e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -63,6 +63,9 @@ Release 2.7.0 - UNRELEASED
YARN-2857. ConcurrentModificationException in ContainerLogAppender
(Mohammad Kamrul Islam via jlowe)
+YARN-2816. NM fail to start with NPE during container recovery (Zhihai Xu
+via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad140d1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 7cf4921..9d54688 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -146,6 +146,8 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
throws IOException {
ArrayList containers =
new ArrayList();
+ArrayList containersToRemove =
+ new ArrayList();
LeveldbIterator iter = null;
try {
iter = new LeveldbIterator(db);
@@ -165,7 +167,14 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
ContainerId containerId = ConverterUtils.toContainerId(
key.substring(CONTAINERS_KEY_PREFIX.length(), idEndPos));
String keyPrefix = key.substring(0, idEndPos+1);
-containers.add(loadContainerState(containerId, iter, keyPrefix));
+RecoveredContainerState rcs = loadContainerState(containerId,
+iter, keyPrefix);
+// Don't load container without StartContainerRequest
+if (rcs.startRequest != null) {
+ containers.add(rcs);
+} else {
+ containersToRemove.add(containerId);
+}
}
} catch (DBException e) {
throw new IOException(e);
@@ -175,6 +184,19 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
}
}
+// remove container without StartContainerRequest
+for (ContainerId containerId : containersToRemove) {
+ LOG.warn("Remove container " + containerId +
+ " with incomplete records");
+ try {
+removeContainer(containerId);
+// TODO: kill and cleanup the leaked container
+ } catch (IOException e) {
+LOG.error("Unable to remove container " + containerId +
+" in store", e);
+ }
+}
+
return containers;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad140d1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodeman
hadoop git commit: YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created. Contributed by Wangda Tan
Repository: hadoop
Updated Branches:
refs/heads/trunk bd8196e85 -> 81c9d17af
YARN-2414. RM web UI: app page will crash if app is failed before any attempt
has been created. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81c9d17a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81c9d17a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81c9d17a
Branch: refs/heads/trunk
Commit: 81c9d17af84ed87b9ded7057cb726a3855ddd32d
Parents: bd8196e
Author: Jason Lowe
Authored: Mon Nov 17 21:13:22 2014 +
Committer: Jason Lowe
Committed: Mon Nov 17 21:15:48 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/security/ApplicationACLsManager.java | 7 ++
.../security/QueueACLsManager.java | 7 ++
.../server/resourcemanager/webapp/AppBlock.java | 30 +--
.../resourcemanager/webapp/TestAppPage.java | 90
5 files changed, 130 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81c9d17a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b303e53..0f526dd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -99,6 +99,9 @@ Release 2.7.0 - UNRELEASED
YARN-1703. Fixed ResourceManager web-proxy to close connections correctly.
(Rohith Sharma via vinodkv)
+YARN-2414. RM web UI: app page will crash if app is failed before any
+attempt has been created (Wangda Tan via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81c9d17a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
index e8e3cb5..4cacfca 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
@@ -35,6 +35,8 @@ import
org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager;
+import com.google.common.annotations.VisibleForTesting;
+
@InterfaceAudience.Private
public class ApplicationACLsManager {
@@ -48,6 +50,11 @@ public class ApplicationACLsManager {
private final ConcurrentMap> applicationACLS
= new ConcurrentHashMap>();
+ @VisibleForTesting
+ public ApplicationACLsManager() {
+this(new Configuration());
+ }
+
public ApplicationACLsManager(Configuration conf) {
this.conf = conf;
this.adminAclsManager = new AdminACLsManager(this.conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81c9d17a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
index eb5037a..fb8279d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
@@ -24,9 +24,16 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import com.google.common.annotations.VisibleForTesting;
+
public class QueueACLsManager {
private ResourceScheduler scheduler;
private boolean isACLsEnable;
+
+ @VisibleForTesting
+ public QueueACLsManager() {
+this(null, new Conf
hadoop git commit: YARN-2414. RM web UI: app page will crash if app is failed before any attempt has been created. Contributed by Wangda Tan (cherry picked from commit 81c9d17af84ed87b9ded7057cb726a38
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d35eba7b1 -> 242fd0e39
YARN-2414. RM web UI: app page will crash if app is failed before any attempt
has been created. Contributed by Wangda Tan
(cherry picked from commit 81c9d17af84ed87b9ded7057cb726a3855ddd32d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/242fd0e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/242fd0e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/242fd0e3
Branch: refs/heads/branch-2
Commit: 242fd0e39ad1c5d51719cd0f6c197166066e3288
Parents: d35eba7
Author: Jason Lowe
Authored: Mon Nov 17 21:13:22 2014 +
Committer: Jason Lowe
Committed: Mon Nov 17 21:16:54 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/security/ApplicationACLsManager.java | 7 ++
.../security/QueueACLsManager.java | 7 ++
.../server/resourcemanager/webapp/AppBlock.java | 30 +--
.../resourcemanager/webapp/TestAppPage.java | 90
5 files changed, 130 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/242fd0e3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 208c66b..8e9db91 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -72,6 +72,9 @@ Release 2.7.0 - UNRELEASED
YARN-1703. Fixed ResourceManager web-proxy to close connections correctly.
(Rohith Sharma via vinodkv)
+YARN-2414. RM web UI: app page will crash if app is failed before any
+attempt has been created (Wangda Tan via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/242fd0e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
index e8e3cb5..4cacfca 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
@@ -35,6 +35,8 @@ import
org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager;
+import com.google.common.annotations.VisibleForTesting;
+
@InterfaceAudience.Private
public class ApplicationACLsManager {
@@ -48,6 +50,11 @@ public class ApplicationACLsManager {
private final ConcurrentMap> applicationACLS
= new ConcurrentHashMap>();
+ @VisibleForTesting
+ public ApplicationACLsManager() {
+this(new Configuration());
+ }
+
public ApplicationACLsManager(Configuration conf) {
this.conf = conf;
this.adminAclsManager = new AdminACLsManager(this.conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/242fd0e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
index eb5037a..fb8279d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/QueueACLsManager.java
@@ -24,9 +24,16 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import com.google.common.annotations.VisibleForTesting;
+
public class QueueACLsManager {
private ResourceScheduler scheduler;
private boolean isA
hadoop git commit: MAPREDUCE-6162. mapred hsadmin fails on a secure cluster. Contributed by Jason Lowe
Repository: hadoop
Updated Branches:
refs/heads/trunk 48d62fad8 -> 09b3dee12
MAPREDUCE-6162. mapred hsadmin fails on a secure cluster. Contributed by Jason
Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09b3dee1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09b3dee1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09b3dee1
Branch: refs/heads/trunk
Commit: 09b3dee122ad20fae22ae8de74886dbf11c39343
Parents: 48d62fa
Author: Jason Lowe
Authored: Mon Nov 17 22:59:59 2014 +
Committer: Jason Lowe
Committed: Mon Nov 17 22:59:59 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 6 +-
.../authorize/ClientHSPolicyProvider.java | 17 +++-
.../hadoop-mapreduce-client-common/pom.xml | 1 +
.../mapreduce/v2/api/HSAdminProtocol.java | 34 +++
.../v2/api/HSAdminRefreshProtocol.java | 59 +++
.../v2/api/HSAdminRefreshProtocolPB.java| 33 ++
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 4 +-
.../src/main/proto/HSAdminRefreshProtocol.proto | 100 +++
.../hadoop-mapreduce-client-hs/pom.xml | 30 --
.../hadoop/mapreduce/v2/hs/HSProxies.java | 4 +-
.../hadoop/mapreduce/v2/hs/client/HSAdmin.java | 18 +++-
.../v2/hs/protocol/HSAdminProtocol.java | 38 ---
.../v2/hs/protocol/HSAdminRefreshProtocol.java | 63
...inRefreshProtocolClientSideTranslatorPB.java | 3 +-
.../hs/protocolPB/HSAdminRefreshProtocolPB.java | 33 --
...inRefreshProtocolServerSideTranslatorPB.java | 3 +-
.../mapreduce/v2/hs/server/HSAdminServer.java | 14 ++-
.../src/main/proto/HSAdminRefreshProtocol.proto | 100 ---
.../v2/hs/server/TestHSAdminServer.java | 20
19 files changed, 303 insertions(+), 277 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b3dee1/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 653ad44..cd5f965 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -239,9 +239,11 @@ Release 2.7.0 - UNRELEASED
BUG FIXES
- MAPREDUCE-5918. LineRecordReader can return the same decompressor to
CodecPool
- multiple times (Sergey Murylev via raviprak)
+MAPREDUCE-5918. LineRecordReader can return the same decompressor to
+CodecPool multiple times (Sergey Murylev via raviprak)
+MAPREDUCE-6162. mapred hsadmin fails on a secure cluster (jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b3dee1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
index dfb7469..a3bf8ae 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
@@ -19,13 +19,17 @@ package
org.apache.hadoop.mapreduce.v2.app.security.authorize;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocol;
import org.apache.hadoop.mapreduce.v2.api.HSClientProtocolPB;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.Service;
+import org.apache.hadoop.tools.GetUserMappingsProtocol;
/**
- * {@link PolicyProvider} for YARN MapReduce protocols.
+ * {@link PolicyProvider} for MapReduce history server protocols.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -35,7 +39,16 @@ public class ClientHSPolicyProvider extends PolicyProvider {
new Service[] {
new Service(
JHAdminConfig.MR_HS_SECURITY_SERVICE_AUTHORIZAT
hadoop git commit: MAPREDUCE-6162. mapred hsadmin fails on a secure cluster. Contributed by Jason Lowe (cherry picked from commit 09b3dee122ad20fae22ae8de74886dbf11c39343)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1deefa6cf -> 0b9a1a3e0
MAPREDUCE-6162. mapred hsadmin fails on a secure cluster. Contributed by Jason
Lowe
(cherry picked from commit 09b3dee122ad20fae22ae8de74886dbf11c39343)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b9a1a3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b9a1a3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b9a1a3e
Branch: refs/heads/branch-2
Commit: 0b9a1a3e033fa2d0fec2bf0c4903cb8eacd25d4e
Parents: 1deefa6
Author: Jason Lowe
Authored: Mon Nov 17 22:59:59 2014 +
Committer: Jason Lowe
Committed: Mon Nov 17 23:01:16 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 6 +-
.../authorize/ClientHSPolicyProvider.java | 17 +++-
.../hadoop-mapreduce-client-common/pom.xml | 1 +
.../mapreduce/v2/api/HSAdminProtocol.java | 34 +++
.../v2/api/HSAdminRefreshProtocol.java | 59 +++
.../v2/api/HSAdminRefreshProtocolPB.java| 33 ++
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 4 +-
.../src/main/proto/HSAdminRefreshProtocol.proto | 100 +++
.../hadoop-mapreduce-client-hs/pom.xml | 30 --
.../hadoop/mapreduce/v2/hs/HSProxies.java | 4 +-
.../hadoop/mapreduce/v2/hs/client/HSAdmin.java | 18 +++-
.../v2/hs/protocol/HSAdminProtocol.java | 38 ---
.../v2/hs/protocol/HSAdminRefreshProtocol.java | 63
...inRefreshProtocolClientSideTranslatorPB.java | 3 +-
.../hs/protocolPB/HSAdminRefreshProtocolPB.java | 33 --
...inRefreshProtocolServerSideTranslatorPB.java | 3 +-
.../mapreduce/v2/hs/server/HSAdminServer.java | 14 ++-
.../src/main/proto/HSAdminRefreshProtocol.proto | 100 ---
.../v2/hs/server/TestHSAdminServer.java | 20
19 files changed, 303 insertions(+), 277 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b9a1a3e/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 64ac66e..6402bbe 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,9 +12,11 @@ Release 2.7.0 - UNRELEASED
BUG FIXES
- MAPREDUCE-5918. LineRecordReader can return the same decompressor to
CodecPool
- multiple times (Sergey Murylev via raviprak)
+MAPREDUCE-5918. LineRecordReader can return the same decompressor to
+CodecPool multiple times (Sergey Murylev via raviprak)
+MAPREDUCE-6162. mapred hsadmin fails on a secure cluster (jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b9a1a3e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
index dfb7469..a3bf8ae 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/security/authorize/ClientHSPolicyProvider.java
@@ -19,13 +19,17 @@ package
org.apache.hadoop.mapreduce.v2.app.security.authorize;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.mapreduce.v2.api.HSAdminRefreshProtocol;
import org.apache.hadoop.mapreduce.v2.api.HSClientProtocolPB;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.security.RefreshUserMappingsProtocol;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.Service;
+import org.apache.hadoop.tools.GetUserMappingsProtocol;
/**
- * {@link PolicyProvider} for YARN MapReduce protocols.
+ * {@link PolicyProvider} for MapReduce history server protocols.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -35,7 +39,16 @@ public class ClientHSPolicyProvider extends PolicyProvider {
new Service[] {
hadoop git commit: HADOOP-11309. System class pattern package.Foo should match package.Foo$Bar, too. Contributed by Gera Shegalov
Repository: hadoop
Updated Branches:
refs/heads/trunk 7250b0bf9 -> b4ca72769
HADOOP-11309. System class pattern package.Foo should match package.Foo$Bar,
too. Contributed by Gera Shegalov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4ca7276
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4ca7276
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4ca7276
Branch: refs/heads/trunk
Commit: b4ca7276902ad362f746ea997f7e977a7a6abd0e
Parents: 7250b0b
Author: Jason Lowe
Authored: Tue Nov 18 17:22:21 2014 +
Committer: Jason Lowe
Committed: Tue Nov 18 17:22:21 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/util/ApplicationClassLoader.java | 11 ++---
.../hadoop/util/TestApplicationClassLoader.java | 26 ++--
3 files changed, 29 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4ca7276/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index e4cc8e7..7fc29a6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -447,6 +447,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11311. Restrict uppercase key names from being created with JCEKS.
(wang)
+HADOOP-11309. System class pattern package.Foo should match
+package.Foo$Bar, too (Gera Shegalov via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4ca7276/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index b18997c..d2ab015 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -228,10 +228,13 @@ public class ApplicationClassLoader extends
URLClassLoader {
c = c.substring(1);
result = false;
}
-if (c.endsWith(".") && canonicalName.startsWith(c)) {
- return result;
-} else if (canonicalName.equals(c)) {
- return result;
+if (canonicalName.startsWith(c)) {
+ if ( c.endsWith(".") // package
+ || canonicalName.length() == c.length() // class
+ ||canonicalName.length() > c.length()// nested
+ && canonicalName.charAt(c.length()) == '$' ) {
+return result;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4ca7276/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
index 5d0e131..cc16493 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
@@ -90,15 +90,27 @@ public class TestApplicationClassLoader {
@Test
public void testIsSystemClass() {
-assertFalse(isSystemClass("org.example.Foo", null));
-assertTrue(isSystemClass("org.example.Foo", classes("org.example.Foo")));
-assertTrue(isSystemClass("/org.example.Foo", classes("org.example.Foo")));
-assertTrue(isSystemClass("org.example.Foo", classes("org.example.")));
-assertTrue(isSystemClass("net.example.Foo",
+testIsSystemClassInternal("");
+ }
+
+ @Test
+ public void testIsSystemNestedClass() {
+testIsSystemClassInternal("$Klass");
+ }
+
+ private void testIsSystemClassInternal(String nestedClass) {
+assertFalse(isSystemClass("org.example.Foo" + nestedClass, null));
+assertTrue(isSystemClass("org.example.Foo" + nestedClass,
+classes("org.example.Foo")));
+assertTr
hadoop git commit: HADOOP-11309. System class pattern package.Foo should match package.Foo$Bar, too. Contributed by Gera Shegalov (cherry picked from commit b4ca7276902ad362f746ea997f7e977a7a6abd0e)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 09df35e51 -> 5125a8e59
HADOOP-11309. System class pattern package.Foo should match package.Foo$Bar,
too. Contributed by Gera Shegalov
(cherry picked from commit b4ca7276902ad362f746ea997f7e977a7a6abd0e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5125a8e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5125a8e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5125a8e5
Branch: refs/heads/branch-2
Commit: 5125a8e59ea7fa0680cab5cda60ddcc731f74f39
Parents: 09df35e
Author: Jason Lowe
Authored: Tue Nov 18 17:22:21 2014 +
Committer: Jason Lowe
Committed: Tue Nov 18 17:23:37 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/util/ApplicationClassLoader.java | 11 ++---
.../hadoop/util/TestApplicationClassLoader.java | 26 ++--
3 files changed, 29 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5125a8e5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 875fd99..4935850 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -91,6 +91,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11311. Restrict uppercase key names from being created with JCEKS.
(wang)
+HADOOP-11309. System class pattern package.Foo should match
+package.Foo$Bar, too (Gera Shegalov via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5125a8e5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index b18997c..d2ab015 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -228,10 +228,13 @@ public class ApplicationClassLoader extends
URLClassLoader {
c = c.substring(1);
result = false;
}
-if (c.endsWith(".") && canonicalName.startsWith(c)) {
- return result;
-} else if (canonicalName.equals(c)) {
- return result;
+if (canonicalName.startsWith(c)) {
+ if ( c.endsWith(".") // package
+ || canonicalName.length() == c.length() // class
+ ||canonicalName.length() > c.length()// nested
+ && canonicalName.charAt(c.length()) == '$' ) {
+return result;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5125a8e5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
index 5d0e131..cc16493 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java
@@ -90,15 +90,27 @@ public class TestApplicationClassLoader {
@Test
public void testIsSystemClass() {
-assertFalse(isSystemClass("org.example.Foo", null));
-assertTrue(isSystemClass("org.example.Foo", classes("org.example.Foo")));
-assertTrue(isSystemClass("/org.example.Foo", classes("org.example.Foo")));
-assertTrue(isSystemClass("org.example.Foo", classes("org.example.")));
-assertTrue(isSystemClass("net.example.Foo",
+testIsSystemClassInternal("");
+ }
+
+ @Test
+ public void testIsSystemNestedClass() {
+testIsSystemClassInternal("$Klass");
+ }
+
+ private void testIsSystemClassInternal(String nestedClass) {
+assertFalse(isSystemClass("org.example.Foo" + nestedClass, null));
+assertTrue(isSystemClass("org.example.Foo" + nestedClass,
git commit: MAPREDUCE-6104. TestJobHistoryParsing.testPartialJob fails in branch-2. Contributed by Mit Desai
Repository: hadoop
Updated Branches:
refs/heads/trunk 034df0e2e -> 7af4c3888
MAPREDUCE-6104. TestJobHistoryParsing.testPartialJob fails in branch-2.
Contributed by Mit Desai
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7af4c388
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7af4c388
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7af4c388
Branch: refs/heads/trunk
Commit: 7af4c3888bf7be0822dc880170e5e3d09e9280f1
Parents: 034df0e
Author: Jason Lowe
Authored: Wed Sep 24 16:09:33 2014 +
Committer: Jason Lowe
Committed: Wed Sep 24 16:09:33 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af4c388/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 859c899..a71e3ef 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -367,6 +367,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs (Gera Shegalov
via jlowe)
+MAPREDUCE-6104. TestJobHistoryParsing.testPartialJob fails in branch-2
+(Mit Desai via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af4c388/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
index d2f5e70..fe5706a 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
@@ -810,7 +810,7 @@ public class TestJobHistoryParsing {
/**
* Simple test PartialJob
*/
- @Test(timeout = 1000)
+ @Test(timeout = 3000)
public void testPartialJob() throws Exception {
JobId jobId = new JobIdPBImpl();
jobId.setId(0);
git commit: MAPREDUCE-6104. TestJobHistoryParsing.testPartialJob fails in branch-2. Contributed by Mit Desai (cherry picked from commit 7af4c3888bf7be0822dc880170e5e3d09e9280f1)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 e5a531a8a -> d5d9fd325
MAPREDUCE-6104. TestJobHistoryParsing.testPartialJob fails in branch-2.
Contributed by Mit Desai
(cherry picked from commit 7af4c3888bf7be0822dc880170e5e3d09e9280f1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5d9fd32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5d9fd32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5d9fd32
Branch: refs/heads/branch-2
Commit: d5d9fd32553aa09534f336b4976434a76126de0a
Parents: e5a531a
Author: Jason Lowe
Authored: Wed Sep 24 16:09:33 2014 +
Committer: Jason Lowe
Committed: Wed Sep 24 16:10:28 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d9fd32/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 952295a..d163417 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -150,6 +150,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs (Gera Shegalov
via jlowe)
+MAPREDUCE-6104. TestJobHistoryParsing.testPartialJob fails in branch-2
+(Mit Desai via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5d9fd32/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
index c2246b3..93538dc 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
@@ -806,7 +806,7 @@ public class TestJobHistoryParsing {
/**
* Simple test PartialJob
*/
- @Test(timeout = 1000)
+ @Test(timeout = 3000)
public void testPartialJob() throws Exception {
JobId jobId = new JobIdPBImpl();
jobId.setId(0);
git commit: YARN-2523. ResourceManager UI showing negative value for "Decommissioned Nodes" field. Contributed by Rohith
Repository: hadoop
Updated Branches:
refs/heads/trunk 9f9aa -> 8269bfa61
YARN-2523. ResourceManager UI showing negative value for "Decommissioned Nodes"
field. Contributed by Rohith
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8269bfa6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8269bfa6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8269bfa6
Branch: refs/heads/trunk
Commit: 8269bfa613999f71767de3c0369817b58cfe1416
Parents: 9f9a222
Author: Jason Lowe
Authored: Thu Sep 25 22:37:05 2014 +
Committer: Jason Lowe
Committed: Thu Sep 25 22:37:05 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/NodesListManager.java | 1 -
.../resourcemanager/rmnode/RMNodeImpl.java | 13 --
.../server/resourcemanager/TestRMRestart.java | 26 +---
.../TestResourceTrackerService.java | 25 +--
5 files changed, 48 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8269bfa6/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index aaac7b7..bbda48d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -447,6 +447,9 @@ Release 2.6.0 - UNRELEASED
YARN-2546. Made REST API for application creation/submission use numeric
and
boolean types instead of the string of them. (Varun Vasudev via zjshen)
+YARN-2523. ResourceManager UI showing negative value for "Decommissioned
+Nodes" field (Rohith via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8269bfa6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index d7797cc..90d7b51 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -121,7 +121,6 @@ public class NodesListManager extends AbstractService
implements
this.conf, includesFile), excludesFile.isEmpty() ? null
: this.rmContext.getConfigurationProvider()
.getConfigurationInputStream(this.conf, excludesFile));
- setDecomissionedNMsMetrics();
printConfiguredHosts();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8269bfa6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index f0ae826..1123a98 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -460,22 +460,9 @@ public class RMNodeImpl implements RMNode,
EventHandler {
break;
}
-// Decomissioned NMs equals to the nodes missing in include list (if
-// include list not empty) or the nodes listed in excluded list.
-// DecomissionedNMs as per exclude list is set upfront when the
-// exclude list is read so that RM restart can also reflect the
-// decomissionedNMs. Note that RM is still not able to know
decomissionedNMs
-// as per include list after it restarts as they are known when those nodes
-// come for registration.
-// Decomiss
git commit: YARN-2523. ResourceManager UI showing negative value for "Decommissioned Nodes" field. Contributed by Rohith (cherry picked from commit 8269bfa613999f71767de3c0369817b58cfe1416)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 87f07e67b -> e4d46e5ff
YARN-2523. ResourceManager UI showing negative value for "Decommissioned Nodes"
field. Contributed by Rohith
(cherry picked from commit 8269bfa613999f71767de3c0369817b58cfe1416)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4d46e5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4d46e5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4d46e5f
Branch: refs/heads/branch-2
Commit: e4d46e5ff609858e7c76b0eda7fab3882092396d
Parents: 87f07e6
Author: Jason Lowe
Authored: Thu Sep 25 22:39:57 2014 +
Committer: Jason Lowe
Committed: Thu Sep 25 22:39:57 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/NodesListManager.java | 1 -
.../resourcemanager/rmnode/RMNodeImpl.java | 13 --
.../server/resourcemanager/TestRMRestart.java | 26 +---
.../TestResourceTrackerService.java | 25 +--
5 files changed, 48 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d46e5f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6608015..421fac6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -417,6 +417,9 @@ Release 2.6.0 - UNRELEASED
YARN-2546. Made REST API for application creation/submission use numeric
and
boolean types instead of the string of them. (Varun Vasudev via zjshen)
+YARN-2523. ResourceManager UI showing negative value for "Decommissioned
+Nodes" field (Rohith via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d46e5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index d7797cc..90d7b51 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -121,7 +121,6 @@ public class NodesListManager extends AbstractService
implements
this.conf, includesFile), excludesFile.isEmpty() ? null
: this.rmContext.getConfigurationProvider()
.getConfigurationInputStream(this.conf, excludesFile));
- setDecomissionedNMsMetrics();
printConfiguredHosts();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4d46e5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index f0ae826..1123a98 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -460,22 +460,9 @@ public class RMNodeImpl implements RMNode,
EventHandler {
break;
}
-// Decomissioned NMs equals to the nodes missing in include list (if
-// include list not empty) or the nodes listed in excluded list.
-// DecomissionedNMs as per exclude list is set upfront when the
-// exclude list is read so that RM restart can also reflect the
-// decomissionedNMs. Note that RM is still not able to know
decomissionedNMs
-// as per include list after it restarts as th
git commit: HADOOP-11048. user/custom LogManager fails to load if the client classloader is enabled. Contributed by Sangjin Lee
Repository: hadoop
Updated Branches:
refs/heads/trunk 32870db0f -> f154ebe8c
HADOOP-11048. user/custom LogManager fails to load if the client classloader is
enabled. Contributed by Sangjin Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f154ebe8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f154ebe8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f154ebe8
Branch: refs/heads/trunk
Commit: f154ebe8c44e41edc443198a14e0491604cc613f
Parents: 32870db
Author: Jason Lowe
Authored: Fri Sep 26 19:49:23 2014 +
Committer: Jason Lowe
Committed: Fri Sep 26 19:49:23 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/util/RunJar.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f154ebe8/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d359698..a8c34e8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -873,6 +873,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11064. UnsatisifedLinkError with hadoop 2.4 JARs on hadoop-2.6 due
to
NativeCRC32 method changes. (cnauroth)
+HADOOP-11048. user/custom LogManager fails to load if the client
+classloader is enabled (Sangjin Lee via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f154ebe8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 75b43b6..4b26b76 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -176,7 +176,7 @@ public class RunJar {
}
mainClassName = mainClassName.replaceAll("/", ".");
-File tmpDir = new File(new Configuration().get("hadoop.tmp.dir"));
+File tmpDir = new File(System.getProperty("java.io.tmpdir"));
ensureDirectory(tmpDir);
final File workDir;
@@ -185,7 +185,7 @@ public class RunJar {
} catch (IOException ioe) {
// If user has insufficient perms to write to tmpDir, default
// "Permission denied" message doesn't specify a filename.
- System.err.println("Error creating temp dir in hadoop.tmp.dir "
+ System.err.println("Error creating temp dir in java.io.tmpdir "
+ tmpDir + " due to " + ioe.getMessage());
System.exit(-1);
return;
git commit: HADOOP-11048. user/custom LogManager fails to load if the client classloader is enabled. Contributed by Sangjin Lee (cherry picked from commit f154ebe8c44e41edc443198a14e0491604cc613f)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6f5d555ad -> 50a5227da
HADOOP-11048. user/custom LogManager fails to load if the client classloader is
enabled. Contributed by Sangjin Lee
(cherry picked from commit f154ebe8c44e41edc443198a14e0491604cc613f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50a5227d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50a5227d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50a5227d
Branch: refs/heads/branch-2
Commit: 50a5227dad4afb16c7bc7f92dc52732155ed0093
Parents: 6f5d555
Author: Jason Lowe
Authored: Fri Sep 26 19:49:23 2014 +
Committer: Jason Lowe
Committed: Fri Sep 26 19:51:18 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/util/RunJar.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50a5227d/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8c6cf16..9f3cb19 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -535,6 +535,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11064. UnsatisifedLinkError with hadoop 2.4 JARs on hadoop-2.6 due
to
NativeCRC32 method changes. (cnauroth)
+HADOOP-11048. user/custom LogManager fails to load if the client
+classloader is enabled (Sangjin Lee via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50a5227d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
index 75b43b6..4b26b76 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/RunJar.java
@@ -176,7 +176,7 @@ public class RunJar {
}
mainClassName = mainClassName.replaceAll("/", ".");
-File tmpDir = new File(new Configuration().get("hadoop.tmp.dir"));
+File tmpDir = new File(System.getProperty("java.io.tmpdir"));
ensureDirectory(tmpDir);
final File workDir;
@@ -185,7 +185,7 @@ public class RunJar {
} catch (IOException ioe) {
// If user has insufficient perms to write to tmpDir, default
// "Permission denied" message doesn't specify a filename.
- System.err.println("Error creating temp dir in hadoop.tmp.dir "
+ System.err.println("Error creating temp dir in java.io.tmpdir "
+ tmpDir + " due to " + ioe.getMessage());
System.exit(-1);
return;
[2/3] YARN-1769. CapacityScheduler: Improve reservations. Contributed by Thomas Graves
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c220651/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 083cb71..092ff83 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
@@ -252,7 +253,7 @@ public class TestLeafQueue {
doNothing().when(parent).completedContainer(
any(Resource.class), any(FiCaSchedulerApp.class),
any(FiCaSchedulerNode.class),
any(RMContainer.class), any(ContainerStatus.class),
-any(RMContainerEventType.class), any(CSQueue.class));
+any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());
return queue;
}
@@ -325,7 +326,7 @@ public class TestLeafQueue {
// Start testing...
// Only 1 container
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(
(int)(node_0.getTotalResource().getMemory() * a.getCapacity()) -
(1*GB),
a.getMetrics().getAvailableMB());
@@ -460,7 +461,7 @@ public class TestLeafQueue {
// Start testing...
// Only 1 container
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
@@ -470,7 +471,7 @@ public class TestLeafQueue {
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
// you can get one container more than user-limit
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
@@ -478,7 +479,7 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Can't allocate 3rd due to user-limit
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
@@ -487,7 +488,7 @@ public class TestLeafQueue {
// Bump up user-limit-factor, now allocate should work
a.setUserLimitFactor(10);
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(3*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
@@ -495,7 +496,7 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getMetrics().getAllocatedMB());
// One more should work, for app_1, due to user-limit-factor
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
@@ -505,7 +506,7 @@ public class TestLeafQueue {
// Test max-capacity
// Now - no more allocs since we are at max-cap
a.setMaxCapacity(0.5f);
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
@@ -
[1/3] YARN-1769. CapacityScheduler: Improve reservations. Contributed by Thomas Graves
Repository: hadoop
Updated Branches:
refs/heads/trunk b38e52b5e -> 9c2206510
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c220651/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
new file mode 100644
index 000..0f8290e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -0,0 +1,1184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import
org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
+import
org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import
org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class TestReservations {
+
+ private static final Log LOG = LogFactory.getLog(TestReservations.class);
+
+ private final RecordFactory recordFactory = RecordFactoryProvider
+ .getRecordFactory(null);
+
+ RMContext rmContext;
+ CapacityScheduler cs;
+ // CapacitySchedulerConfiguration csConf;
+ CapacitySchedulerContex
[3/3] git commit: YARN-1769. CapacityScheduler: Improve reservations. Contributed by Thomas Graves
YARN-1769. CapacityScheduler: Improve reservations. Contributed by Thomas Graves
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c220651
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c220651
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c220651
Branch: refs/heads/trunk
Commit: 9c22065109a77681bc2534063eabe8692fbcb3cd
Parents: b38e52b
Author: Jason Lowe
Authored: Mon Sep 29 14:12:18 2014 +
Committer: Jason Lowe
Committed: Mon Sep 29 14:12:18 2014 +
--
hadoop-yarn-project/CHANGES.txt |3 +
.../dev-support/findbugs-exclude.xml| 12 +
.../scheduler/capacity/CSQueue.java |7 +-
.../scheduler/capacity/CapacityScheduler.java | 15 +-
.../CapacitySchedulerConfiguration.java | 18 +
.../capacity/CapacitySchedulerContext.java |6 +-
.../scheduler/capacity/LeafQueue.java | 349 +-
.../scheduler/capacity/ParentQueue.java | 99 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 27 +
.../capacity/TestApplicationLimits.java |8 +-
.../scheduler/capacity/TestChildQueueOrder.java | 32 +-
.../scheduler/capacity/TestLeafQueue.java | 148 +--
.../scheduler/capacity/TestParentQueue.java | 81 +-
.../scheduler/capacity/TestReservations.java| 1184 ++
14 files changed, 1760 insertions(+), 229 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c220651/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0a6500b..7fa83be 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -265,6 +265,9 @@ Release 2.6.0 - UNRELEASED
YARN-668. Changed
NMTokenIdentifier/AMRMTokenIdentifier/ContainerTokenIdentifier
to use protobuf object as the payload. (Junping Du via jianhe)
+YARN-1769. CapacityScheduler: Improve reservations (Thomas Graves via
+jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c220651/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index b1dfb1e..0e6207b 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -351,4 +351,16 @@
+
+
+
+
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c220651/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 04c2fd5..db893dc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -184,10 +184,11 @@ extends
org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
* Assign containers to applications in the queue or it's children (if any).
* @param clusterResource the resource of the cluster.
* @param node node on which resources are available
+ * @param needToUnreserve assign container only if it can unreserve one first
* @return the assignment
*/
public CSAssignment assignContainers(
- Resource clusterResource, FiCaSchedulerNode node);
+ Resource clusterResource, FiCaSchedulerNode node, boolean
needToUnreserve);
/**
* A container assigned to the queue has completed.
@@ -200,11 +201,13 @@ extends
org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
*container
* @param childQueue CSQueue to reinsert in childQueues
* @param event event to be sent to the container
+ * @param sortQueues indicates whether it should re-sort the queues
*/
public void completedCont
[3/3] git commit: YARN-1769. CapacityScheduler: Improve reservations. Contributed by Thomas Graves (cherry picked from commit 9c22065109a77681bc2534063eabe8692fbcb3cd)
YARN-1769. CapacityScheduler: Improve reservations. Contributed by Thomas Graves
(cherry picked from commit 9c22065109a77681bc2534063eabe8692fbcb3cd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f6a3671
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f6a3671
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f6a3671
Branch: refs/heads/branch-2
Commit: 4f6a3671665f3dda5c6213d5deb2495db981cbd1
Parents: 8ebea28
Author: Jason Lowe
Authored: Mon Sep 29 14:12:18 2014 +
Committer: Jason Lowe
Committed: Mon Sep 29 14:13:21 2014 +
--
hadoop-yarn-project/CHANGES.txt |3 +
.../dev-support/findbugs-exclude.xml| 12 +
.../scheduler/capacity/CSQueue.java |7 +-
.../scheduler/capacity/CapacityScheduler.java | 15 +-
.../CapacitySchedulerConfiguration.java | 18 +
.../capacity/CapacitySchedulerContext.java |6 +-
.../scheduler/capacity/LeafQueue.java | 349 +-
.../scheduler/capacity/ParentQueue.java | 99 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 27 +
.../capacity/TestApplicationLimits.java |8 +-
.../scheduler/capacity/TestChildQueueOrder.java | 32 +-
.../scheduler/capacity/TestLeafQueue.java | 148 +--
.../scheduler/capacity/TestParentQueue.java | 81 +-
.../scheduler/capacity/TestReservations.java| 1184 ++
14 files changed, 1760 insertions(+), 229 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6a3671/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 925cefb..f55f2f6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -235,6 +235,9 @@ Release 2.6.0 - UNRELEASED
YARN-668. Changed
NMTokenIdentifier/AMRMTokenIdentifier/ContainerTokenIdentifier
to use protobuf object as the payload. (Junping Du via jianhe)
+YARN-1769. CapacityScheduler: Improve reservations (Thomas Graves via
+jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6a3671/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index b1dfb1e..0e6207b 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -351,4 +351,16 @@
+
+
+
+
+
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6a3671/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 04c2fd5..db893dc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -184,10 +184,11 @@ extends
org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
* Assign containers to applications in the queue or it's children (if any).
* @param clusterResource the resource of the cluster.
* @param node node on which resources are available
+ * @param needToUnreserve assign container only if it can unreserve one first
* @return the assignment
*/
public CSAssignment assignContainers(
- Resource clusterResource, FiCaSchedulerNode node);
+ Resource clusterResource, FiCaSchedulerNode node, boolean
needToUnreserve);
/**
* A container assigned to the queue has completed.
@@ -200,11 +201,13 @@ extends
org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
*container
* @param childQueue CSQueue to reinsert in childQueues
* @param event event to be sent to the container
+ * @param sortQueues indicates wheth
[1/3] YARN-1769. CapacityScheduler: Improve reservations. Contributed by Thomas Graves (cherry picked from commit 9c22065109a77681bc2534063eabe8692fbcb3cd)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 8ebea28b4 -> 4f6a36716
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6a3671/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
new file mode 100644
index 000..0f8290e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -0,0 +1,1184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import
org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
+import
org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import
org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+public class TestReservations {
+
+ private static final Log LOG = LogFactory.getLog(TestReservations.class);
+
+ private final RecordFactory recordFactory = RecordFactoryProvider
+ .getRecordFactory(null);
+
+ RMContext rmContext;
+ CapacityScheduler cs;
+ // CapacitySchedulerConfiguration csConf;
+ CapacitySchedulerCon
[2/3] YARN-1769. CapacityScheduler: Improve reservations. Contributed by Thomas Graves (cherry picked from commit 9c22065109a77681bc2534063eabe8692fbcb3cd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f6a3671/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 083cb71..092ff83 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doNothing;
@@ -252,7 +253,7 @@ public class TestLeafQueue {
doNothing().when(parent).completedContainer(
any(Resource.class), any(FiCaSchedulerApp.class),
any(FiCaSchedulerNode.class),
any(RMContainer.class), any(ContainerStatus.class),
-any(RMContainerEventType.class), any(CSQueue.class));
+any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());
return queue;
}
@@ -325,7 +326,7 @@ public class TestLeafQueue {
// Start testing...
// Only 1 container
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(
(int)(node_0.getTotalResource().getMemory() * a.getCapacity()) -
(1*GB),
a.getMetrics().getAvailableMB());
@@ -460,7 +461,7 @@ public class TestLeafQueue {
// Start testing...
// Only 1 container
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
@@ -470,7 +471,7 @@ public class TestLeafQueue {
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
// you can get one container more than user-limit
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
@@ -478,7 +479,7 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Can't allocate 3rd due to user-limit
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
@@ -487,7 +488,7 @@ public class TestLeafQueue {
// Bump up user-limit-factor, now allocate should work
a.setUserLimitFactor(10);
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(3*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
@@ -495,7 +496,7 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getMetrics().getAllocatedMB());
// One more should work, for app_1, due to user-limit-factor
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
@@ -505,7 +506,7 @@ public class TestLeafQueue {
// Test max-capacity
// Now - no more allocs since we are at max-cap
a.setMaxCapacity(0.5f);
-a.assignContainers(clusterResource, node_0);
+a.assignContainers(clusterResource, node_0, false);
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
@@ -
git commit: HADOOP-11049. javax package system class default is too broad. Contributed by Sangjin Lee
Repository: hadoop
Updated Branches:
refs/heads/trunk 9c2206510 -> cf4631d7c
HADOOP-11049. javax package system class default is too broad. Contributed by
Sangjin Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf4631d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf4631d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf4631d7
Branch: refs/heads/trunk
Commit: cf4631d7c9448205f4952341c071e1650e7922dc
Parents: 9c22065
Author: Jason Lowe
Authored: Mon Sep 29 14:36:29 2014 +
Committer: Jason Lowe
Committed: Mon Sep 29 14:36:29 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/util/ApplicationClassLoader.java | 47 +++-
...he.hadoop.application-classloader.properties | 57
.../java/org/apache/hadoop/util/TestRunJar.java | 2 +-
.../hadoop/mapreduce/v2/util/TestMRApps.java| 4 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 2 +-
6 files changed, 98 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4631d7/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 14c21fa..d6be9f0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -885,6 +885,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11143 NetUtils.wrapException loses inner stack trace on
BindException
(stevel)
+HADOOP-11049. javax package system class default is too broad (Sangjin Lee
+via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4631d7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index 5dda10f..b18997c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -20,12 +20,15 @@ package org.apache.hadoop.util;
import java.io.File;
import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -45,18 +48,12 @@ public class ApplicationClassLoader extends URLClassLoader {
* classes are considered system classes, and are not loaded by the
* application classloader.
*/
- public static final String DEFAULT_SYSTEM_CLASSES =
-"java.," +
-"javax.," +
-"org.w3c.dom.," +
-"org.xml.sax.," +
-"org.apache.commons.logging.," +
-"org.apache.log4j.," +
-"org.apache.hadoop.," +
-"core-default.xml," +
-"hdfs-default.xml," +
-"mapred-default.xml," +
-"yarn-default.xml";
+ public static final String SYSTEM_CLASSES_DEFAULT;
+
+ private static final String PROPERTIES_FILE =
+ "org.apache.hadoop.application-classloader.properties";
+ private static final String SYSTEM_CLASSES_DEFAULT_KEY =
+ "system.classes.default";
private static final Log LOG =
LogFactory.getLog(ApplicationClassLoader.class.getName());
@@ -69,6 +66,30 @@ public class ApplicationClassLoader extends URLClassLoader {
}
};
+ static {
+InputStream is = null;
+try {
+ is = ApplicationClassLoader.class.getClassLoader().
+ getResourceAsStream(PROPERTIES_FILE);
+ if (is == null) {
+throw new ExceptionInInitializerError("properties file " +
+PROPERTIES_FILE + " is not found");
+ }
+ Properties props = new Properties();
+ props.load(is);
+ // get the system classes default
+ String systemClassesDefault =
+ props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY);
+ if (systemClassesDefault == null) {
+throw new ExceptionInInitializerError("property " +
+SYSTEM_CLASSES_DEFAULT_KEY + " is not found");
git commit: HADOOP-11049. javax package system class default is too broad. Contributed by Sangjin Lee (cherry picked from commit cf4631d7c9448205f4952341c071e1650e7922dc)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 4f6a36716 -> 0cfff28f5
HADOOP-11049. javax package system class default is too broad. Contributed by
Sangjin Lee
(cherry picked from commit cf4631d7c9448205f4952341c071e1650e7922dc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cfff28f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cfff28f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cfff28f
Branch: refs/heads/branch-2
Commit: 0cfff28f57811fe3dbac52a3286086b11fabfa24
Parents: 4f6a367
Author: Jason Lowe
Authored: Mon Sep 29 14:36:29 2014 +
Committer: Jason Lowe
Committed: Mon Sep 29 14:37:28 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../hadoop/util/ApplicationClassLoader.java | 47 +++-
...he.hadoop.application-classloader.properties | 57
.../java/org/apache/hadoop/util/TestRunJar.java | 2 +-
.../hadoop/mapreduce/v2/util/TestMRApps.java| 4 +-
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 2 +-
6 files changed, 98 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cfff28f/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 28bd6e0..2d1f400 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -547,6 +547,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11143 NetUtils.wrapException loses inner stack trace on
BindException
(stevel)
+HADOOP-11049. javax package system class default is too broad (Sangjin Lee
+via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cfff28f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
index 5dda10f..b18997c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ApplicationClassLoader.java
@@ -20,12 +20,15 @@ package org.apache.hadoop.util;
import java.io.File;
import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -45,18 +48,12 @@ public class ApplicationClassLoader extends URLClassLoader {
* classes are considered system classes, and are not loaded by the
* application classloader.
*/
- public static final String DEFAULT_SYSTEM_CLASSES =
-"java.," +
-"javax.," +
-"org.w3c.dom.," +
-"org.xml.sax.," +
-"org.apache.commons.logging.," +
-"org.apache.log4j.," +
-"org.apache.hadoop.," +
-"core-default.xml," +
-"hdfs-default.xml," +
-"mapred-default.xml," +
-"yarn-default.xml";
+ public static final String SYSTEM_CLASSES_DEFAULT;
+
+ private static final String PROPERTIES_FILE =
+ "org.apache.hadoop.application-classloader.properties";
+ private static final String SYSTEM_CLASSES_DEFAULT_KEY =
+ "system.classes.default";
private static final Log LOG =
LogFactory.getLog(ApplicationClassLoader.class.getName());
@@ -69,6 +66,30 @@ public class ApplicationClassLoader extends URLClassLoader {
}
};
+ static {
+InputStream is = null;
+try {
+ is = ApplicationClassLoader.class.getClassLoader().
+ getResourceAsStream(PROPERTIES_FILE);
+ if (is == null) {
+throw new ExceptionInInitializerError("properties file " +
+PROPERTIES_FILE + " is not found");
+ }
+ Properties props = new Properties();
+ props.load(is);
+ // get the system classes default
+ String systemClassesDefault =
+ props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY);
+ if (systemClassesDefault == null) {
+throw new ExceptionInInitializerError("property " +
+
git commit: MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk. Contributed by Akira AJISAKA
Repository: hadoop
Updated Branches:
refs/heads/trunk 371ef4cef -> 23fbfe36c
MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk.
Contributed by Akira AJISAKA
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23fbfe36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23fbfe36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23fbfe36
Branch: refs/heads/trunk
Commit: 23fbfe36c31674907a26c8ecc6fd47888dfcc1ea
Parents: 371ef4c
Author: Jason Lowe
Authored: Mon Sep 29 18:11:16 2014 +
Committer: Jason Lowe
Committed: Mon Sep 29 18:12:59 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java | 3 +--
2 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23fbfe36/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index b5c7a77..2dc7b6d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -396,6 +396,9 @@ Release 2.6.0 - UNRELEASED
MRJobConfig#MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS. Contributed by
Akira AJISAKA. (Akira AJISAKA via jianhe)
+MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk
+(Akira AJISAKA via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23fbfe36/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
index aeba171..b806630 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
@@ -47,9 +47,8 @@ public class TestMRCJCFileInputFormat {
@Test
public void testAddInputPath() throws IOException {
final Configuration conf = new Configuration();
-conf.set("fs.defaultFS", "s3://abc:xyz@hostname/");
+conf.set("fs.defaultFS", "file:///abc/");
final Job j = Job.getInstance(conf);
-j.getConfiguration().set("fs.defaultFS", "s3://abc:xyz@hostname/");
//setup default fs
final FileSystem defaultfs = FileSystem.get(conf);
git commit: MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk. Contributed by Akira AJISAKA (cherry picked from commit 23fbfe36c31674907a26c8ecc6fd47888dfcc1ea)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c7cf212b2 -> 7b8f2f5b1
MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk.
Contributed by Akira AJISAKA
(cherry picked from commit 23fbfe36c31674907a26c8ecc6fd47888dfcc1ea)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b8f2f5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b8f2f5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b8f2f5b
Branch: refs/heads/branch-2
Commit: 7b8f2f5b1dbc0c3df2dd9e06371057395b34d6e9
Parents: c7cf212
Author: Jason Lowe
Authored: Mon Sep 29 18:11:16 2014 +
Committer: Jason Lowe
Committed: Mon Sep 29 18:14:11 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java | 3 +--
2 files changed, 4 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b8f2f5b/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 23d991f..e499d2a 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -177,6 +177,9 @@ Release 2.6.0 - UNRELEASED
MRJobConfig#MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS. Contributed by
Akira AJISAKA. (Akira AJISAKA via jianhe)
+MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk
+(Akira AJISAKA via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b8f2f5b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
index aeba171..b806630 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestMRCJCFileInputFormat.java
@@ -47,9 +47,8 @@ public class TestMRCJCFileInputFormat {
@Test
public void testAddInputPath() throws IOException {
final Configuration conf = new Configuration();
-conf.set("fs.defaultFS", "s3://abc:xyz@hostname/");
+conf.set("fs.defaultFS", "file:///abc/");
final Job j = Job.getInstance(conf);
-j.getConfiguration().set("fs.defaultFS", "s3://abc:xyz@hostname/");
//setup default fs
final FileSystem defaultfs = FileSystem.get(conf);
git commit: YARN-2387. Resource Manager crashes with NPE due to lack of synchronization. Contributed by Mit Desai
Repository: hadoop
Updated Branches:
refs/heads/trunk 9582a5017 -> feaf139b4
YARN-2387. Resource Manager crashes with NPE due to lack of synchronization.
Contributed by Mit Desai
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/feaf139b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/feaf139b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/feaf139b
Branch: refs/heads/trunk
Commit: feaf139b4f327d33011e5a4424c06fb44c630955
Parents: 9582a50
Author: Jason Lowe
Authored: Tue Sep 30 22:37:28 2014 +
Committer: Jason Lowe
Committed: Tue Sep 30 22:37:28 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../records/impl/pb/ContainerStatusPBImpl.java | 22 ++--
2 files changed, 14 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/feaf139b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6771839..be61dfe 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -485,6 +485,9 @@ Release 2.6.0 - UNRELEASED
YARN-2610. Hamlet should close table tags. (Ray Chiang via kasha)
+YARN-2387. Resource Manager crashes with NPE due to lack of
+synchronization (Mit Desai via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/feaf139b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index 8e3e2cc..86f2af9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -50,7 +50,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
viaProto = true;
}
- public ContainerStatusProto getProto() {
+ public synchronized ContainerStatusProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@@ -90,7 +90,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
}
- private void mergeLocalToProto() {
+ private synchronized void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
@@ -98,7 +98,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
viaProto = true;
}
- private void maybeInitBuilder() {
+ private synchronized void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ContainerStatusProto.newBuilder(proto);
}
@@ -107,7 +107,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
@Override
- public ContainerState getState() {
+ public synchronized ContainerState getState() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasState()) {
return null;
@@ -116,7 +116,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
@Override
- public void setState(ContainerState state) {
+ public synchronized void setState(ContainerState state) {
maybeInitBuilder();
if (state == null) {
builder.clearState();
@@ -125,7 +125,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
builder.setState(convertToProtoFormat(state));
}
@Override
- public ContainerId getContainerId() {
+ public synchronized ContainerId getContainerId() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
if (this.containerId != null) {
return this.containerId;
@@ -138,32 +138,32 @@ public class ContainerStatusPBImpl extends
ContainerStatus {
}
@Override
- public void setContainerId(ContainerId containerId) {
+ public synchronized void setContainerId(ContainerId containerId) {
maybeInitBuilder();
if (containerId == null)
builder.clearContainerId();
this.containerId = containerId;
}
@Override
- public int getExitStatus() {
+ public synchronized int getExitStatus() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
return p.getExitStatus();
}
@Override
- public void setExitStatus(int exitStatus) {
+ public synchronized v
git commit: YARN-2387. Resource Manager crashes with NPE due to lack of synchronization. Contributed by Mit Desai (cherry picked from commit feaf139b4f327d33011e5a4424c06fb44c630955)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d977205b9 -> a0305ba3f
YARN-2387. Resource Manager crashes with NPE due to lack of synchronization.
Contributed by Mit Desai
(cherry picked from commit feaf139b4f327d33011e5a4424c06fb44c630955)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0305ba3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0305ba3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0305ba3
Branch: refs/heads/branch-2
Commit: a0305ba3fcf3c7fd5285aac4117ca80538988f89
Parents: d977205
Author: Jason Lowe
Authored: Tue Sep 30 22:37:28 2014 +
Committer: Jason Lowe
Committed: Tue Sep 30 22:38:27 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../records/impl/pb/ContainerStatusPBImpl.java | 22 ++--
2 files changed, 14 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0305ba3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c773145..017ce10 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -455,6 +455,9 @@ Release 2.6.0 - UNRELEASED
YARN-2610. Hamlet should close table tags. (Ray Chiang via kasha)
+YARN-2387. Resource Manager crashes with NPE due to lack of
+synchronization (Mit Desai via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0305ba3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index 8e3e2cc..86f2af9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -50,7 +50,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
viaProto = true;
}
- public ContainerStatusProto getProto() {
+ public synchronized ContainerStatusProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@@ -90,7 +90,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
}
- private void mergeLocalToProto() {
+ private synchronized void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
@@ -98,7 +98,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
viaProto = true;
}
- private void maybeInitBuilder() {
+ private synchronized void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ContainerStatusProto.newBuilder(proto);
}
@@ -107,7 +107,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
@Override
- public ContainerState getState() {
+ public synchronized ContainerState getState() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasState()) {
return null;
@@ -116,7 +116,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
@Override
- public void setState(ContainerState state) {
+ public synchronized void setState(ContainerState state) {
maybeInitBuilder();
if (state == null) {
builder.clearState();
@@ -125,7 +125,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
builder.setState(convertToProtoFormat(state));
}
@Override
- public ContainerId getContainerId() {
+ public synchronized ContainerId getContainerId() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
if (this.containerId != null) {
return this.containerId;
@@ -138,32 +138,32 @@ public class ContainerStatusPBImpl extends
ContainerStatus {
}
@Override
- public void setContainerId(ContainerId containerId) {
+ public synchronized void setContainerId(ContainerId containerId) {
maybeInitBuilder();
if (containerId == null)
builder.clearContainerId();
this.containerId = containerId;
}
@Override
- public int getExitStatus() {
+ public synchronized int getExitStatus() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
return p.getExitStatus();
}
@Overr
git commit: YARN-2387. Resource Manager crashes with NPE due to lack of synchronization. Contributed by Mit Desai (cherry picked from commit feaf139b4f327d33011e5a4424c06fb44c630955)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 b19f668f5 -> ab3cab0b5
YARN-2387. Resource Manager crashes with NPE due to lack of synchronization.
Contributed by Mit Desai
(cherry picked from commit feaf139b4f327d33011e5a4424c06fb44c630955)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab3cab0b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab3cab0b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab3cab0b
Branch: refs/heads/branch-2.6
Commit: ab3cab0b536661ac6fc3e272ec18bf5735bd6339
Parents: b19f668
Author: Jason Lowe
Authored: Tue Sep 30 22:42:53 2014 +
Committer: Jason Lowe
Committed: Tue Sep 30 22:42:53 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../records/impl/pb/ContainerStatusPBImpl.java | 22 ++--
2 files changed, 14 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3cab0b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 628f64f..aaeaf85 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -441,6 +441,9 @@ Release 2.6.0 - UNRELEASED
YARN-2606. Application History Server tries to access hdfs before doing
secure login (Mit Desai via jeagles)
+YARN-2387. Resource Manager crashes with NPE due to lack of
+synchronization (Mit Desai via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3cab0b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index 8e3e2cc..86f2af9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -50,7 +50,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
viaProto = true;
}
- public ContainerStatusProto getProto() {
+ public synchronized ContainerStatusProto getProto() {
mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
@@ -90,7 +90,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
}
- private void mergeLocalToProto() {
+ private synchronized void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
mergeLocalToBuilder();
@@ -98,7 +98,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
viaProto = true;
}
- private void maybeInitBuilder() {
+ private synchronized void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = ContainerStatusProto.newBuilder(proto);
}
@@ -107,7 +107,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
@Override
- public ContainerState getState() {
+ public synchronized ContainerState getState() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasState()) {
return null;
@@ -116,7 +116,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
}
@Override
- public void setState(ContainerState state) {
+ public synchronized void setState(ContainerState state) {
maybeInitBuilder();
if (state == null) {
builder.clearState();
@@ -125,7 +125,7 @@ public class ContainerStatusPBImpl extends ContainerStatus {
builder.setState(convertToProtoFormat(state));
}
@Override
- public ContainerId getContainerId() {
+ public synchronized ContainerId getContainerId() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
if (this.containerId != null) {
return this.containerId;
@@ -138,32 +138,32 @@ public class ContainerStatusPBImpl extends
ContainerStatus {
}
@Override
- public void setContainerId(ContainerId containerId) {
+ public synchronized void setContainerId(ContainerId containerId) {
maybeInitBuilder();
if (containerId == null)
builder.clearContainerId();
this.containerId = containerId;
}
@Override
- public int getExitStatus() {
+ public synchronized int getExitStatus() {
ContainerStatusProtoOrBuilder p = viaProto ? proto : buil
git commit: YARN-2624. Resource Localization fails on a cluster due to existing cache directories. Contributed by Anubhav Dhoot
Repository: hadoop
Updated Branches:
refs/heads/trunk 5e0b49da9 -> 29f520052
YARN-2624. Resource Localization fails on a cluster due to existing cache
directories. Contributed by Anubhav Dhoot
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29f52005
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29f52005
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29f52005
Branch: refs/heads/trunk
Commit: 29f520052e2b02f44979980e446acc0dccd96d54
Parents: 5e0b49d
Author: Jason Lowe
Authored: Thu Oct 2 17:39:34 2014 +
Committer: Jason Lowe
Committed: Thu Oct 2 17:39:34 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../localizer/ResourceLocalizationService.java | 2 +-
.../recovery/NMLeveldbStateStoreService.java| 7 ++
.../recovery/NMStateStoreService.java | 3 +
.../TestResourceLocalizationService.java| 76
.../TestNMLeveldbStateStoreService.java | 8 +++
6 files changed, 98 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29f52005/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a67ff27..deedb83 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -521,6 +521,9 @@ Release 2.6.0 - UNRELEASED
YARN-2617. Fixed NM to not send duplicate container status whose app is not
running. (Jun Gong via jianhe)
+YARN-2624. Resource Localization fails on a cluster due to existing cache
+directories (Anubhav Dhoot via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29f52005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index a092b59..c2dcebf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -222,7 +222,7 @@ public class ResourceLocalizationService extends
CompositeService
FileContext lfs = getLocalFileContext(conf);
lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
- if (!stateStore.canRecover()) {
+ if (!stateStore.canRecover() || stateStore.isNewlyCreated()) {
cleanUpLocalDir(lfs,delService);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29f52005/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 7c95fff..7cf4921 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -118,6 +118,7 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
private static final byte[] EMPTY_VALUE = new byte[0];
private DB db;
+ private boolean isNewlyCreated;
public NMLeveldbStateStoreService() {
super(NMLeveldbStateStoreService.class.getName());
@@ -134,6 +135,11 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
}
}
+ @Overr
git commit: YARN-2624. Resource Localization fails on a cluster due to existing cache directories. Contributed by Anubhav Dhoot (cherry picked from commit 29f520052e2b02f44979980e446acc0dccd96d54)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b6f4d1ef4 -> 531c1fd00
YARN-2624. Resource Localization fails on a cluster due to existing cache
directories. Contributed by Anubhav Dhoot
(cherry picked from commit 29f520052e2b02f44979980e446acc0dccd96d54)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/531c1fd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/531c1fd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/531c1fd0
Branch: refs/heads/branch-2
Commit: 531c1fd00add260825e00284e6868c4614f8e19f
Parents: b6f4d1e
Author: Jason Lowe
Authored: Thu Oct 2 17:39:34 2014 +
Committer: Jason Lowe
Committed: Thu Oct 2 17:40:44 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../localizer/ResourceLocalizationService.java | 2 +-
.../recovery/NMLeveldbStateStoreService.java| 7 ++
.../recovery/NMStateStoreService.java | 3 +
.../TestResourceLocalizationService.java| 76
.../TestNMLeveldbStateStoreService.java | 8 +++
6 files changed, 98 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/531c1fd0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a6585a6..b4b0b75 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -491,6 +491,9 @@ Release 2.6.0 - UNRELEASED
YARN-2617. Fixed NM to not send duplicate container status whose app is not
running. (Jun Gong via jianhe)
+YARN-2624. Resource Localization fails on a cluster due to existing cache
+directories (Anubhav Dhoot via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/531c1fd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index a092b59..c2dcebf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -222,7 +222,7 @@ public class ResourceLocalizationService extends
CompositeService
FileContext lfs = getLocalFileContext(conf);
lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
- if (!stateStore.canRecover()) {
+ if (!stateStore.canRecover() || stateStore.isNewlyCreated()) {
cleanUpLocalDir(lfs,delService);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/531c1fd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 7c95fff..7cf4921 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -118,6 +118,7 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
private static final byte[] EMPTY_VALUE = new byte[0];
private DB db;
+ private boolean isNewlyCreated;
public NMLeveldbStateStoreService() {
super(NMLeveldbStateStoreService.class.getName());
@@ -134,6 +135,11 @@ public cl
git commit: YARN-2624. Resource Localization fails on a cluster due to existing cache directories. Contributed by Anubhav Dhoot (cherry picked from commit 29f520052e2b02f44979980e446acc0dccd96d54)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 116eb0bdc -> 95757899d
YARN-2624. Resource Localization fails on a cluster due to existing cache
directories. Contributed by Anubhav Dhoot
(cherry picked from commit 29f520052e2b02f44979980e446acc0dccd96d54)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95757899
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95757899
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95757899
Branch: refs/heads/branch-2.6
Commit: 95757899d4f2526129dcc7bd4adef89da77055ab
Parents: 116eb0b
Author: Jason Lowe
Authored: Thu Oct 2 17:39:34 2014 +
Committer: Jason Lowe
Committed: Thu Oct 2 17:45:12 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../localizer/ResourceLocalizationService.java | 2 +-
.../recovery/NMLeveldbStateStoreService.java| 7 ++
.../recovery/NMStateStoreService.java | 3 +
.../TestResourceLocalizationService.java| 76
.../TestNMLeveldbStateStoreService.java | 8 +++
6 files changed, 98 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95757899/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2719dfe..ffc9d79 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -464,6 +464,9 @@ Release 2.6.0 - UNRELEASED
YARN-2617. Fixed NM to not send duplicate container status whose app is not
running. (Jun Gong via jianhe)
+YARN-2624. Resource Localization fails on a cluster due to existing cache
+directories (Anubhav Dhoot via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95757899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index a092b59..c2dcebf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -222,7 +222,7 @@ public class ResourceLocalizationService extends
CompositeService
FileContext lfs = getLocalFileContext(conf);
lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
- if (!stateStore.canRecover()) {
+ if (!stateStore.canRecover() || stateStore.isNewlyCreated()) {
cleanUpLocalDir(lfs,delService);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/95757899/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 7c95fff..7cf4921 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -118,6 +118,7 @@ public class NMLeveldbStateStoreService extends
NMStateStoreService {
private static final byte[] EMPTY_VALUE = new byte[0];
private DB db;
+ private boolean isNewlyCreated;
public NMLeveldbStateStoreService() {
super(NMLeveldbStateStoreService.class.getName());
@@ -134,6 +135,11 @@ public cl
git commit: MAPREDUCE-6029. TestCommitterEventHandler fails in trunk. Contributed by Mit Desai
Repository: hadoop
Updated Branches:
refs/heads/trunk 0fb273570 -> 2e789eb22
MAPREDUCE-6029. TestCommitterEventHandler fails in trunk. Contributed by Mit
Desai
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e789eb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e789eb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e789eb2
Branch: refs/heads/trunk
Commit: 2e789eb2261fa607489d8ae630fb77e8147d0266
Parents: 0fb2735
Author: Jason Lowe
Authored: Tue Oct 7 14:10:39 2014 +
Committer: Jason Lowe
Committed: Tue Oct 7 14:10:39 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../mapreduce/v2/app/commit/TestCommitterEventHandler.java | 9 +++--
2 files changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e789eb2/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 8bc1e53..1adf7cf 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -411,6 +411,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk
(Akira AJISAKA via jlowe)
+MAPREDUCE-6029. TestCommitterEventHandler fails in trunk (Mit Desai via
+ jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e789eb2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
index 83e43f5..a553bbd 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
@@ -58,6 +58,7 @@ import
org.apache.hadoop.mapreduce.v2.app.job.event.JobCommitCompletedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobCommitFailedEvent;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Event;
@@ -84,8 +85,12 @@ public class TestCommitterEventHandler {
public synchronized Event getAndClearEvent() throws InterruptedException {
if (event == null) {
-//Wait for at most 10 ms
-wait(100);
+final long waitTime = 5000;
+long waitStartTime = Time.monotonicNow();
+while(event == null && Time.monotonicNow() - waitStartTime < waitTime)
{
+ //Wait for at most 5 sec
+ wait(waitTime);
+}
}
Event e = event;
event = null;
git commit: MAPREDUCE-6029. TestCommitterEventHandler fails in trunk. Contributed by Mit Desai (cherry picked from commit 2e789eb2261fa607489d8ae630fb77e8147d0266)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 70c8cb561 -> 979b915ec
MAPREDUCE-6029. TestCommitterEventHandler fails in trunk. Contributed by Mit
Desai
(cherry picked from commit 2e789eb2261fa607489d8ae630fb77e8147d0266)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/979b915e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/979b915e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/979b915e
Branch: refs/heads/branch-2
Commit: 979b915ecc3b74d796d91cb8594180d68cf127e2
Parents: 70c8cb5
Author: Jason Lowe
Authored: Tue Oct 7 14:10:39 2014 +
Committer: Jason Lowe
Committed: Tue Oct 7 14:11:48 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../mapreduce/v2/app/commit/TestCommitterEventHandler.java | 9 +++--
2 files changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979b915e/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 32b8f15..81633f3 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -192,6 +192,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk
(Akira AJISAKA via jlowe)
+MAPREDUCE-6029. TestCommitterEventHandler fails in trunk (Mit Desai via
+ jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/979b915e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
index 83e43f5..a553bbd 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
@@ -58,6 +58,7 @@ import
org.apache.hadoop.mapreduce.v2.app.job.event.JobCommitCompletedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobCommitFailedEvent;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Event;
@@ -84,8 +85,12 @@ public class TestCommitterEventHandler {
public synchronized Event getAndClearEvent() throws InterruptedException {
if (event == null) {
-//Wait for at most 10 ms
-wait(100);
+final long waitTime = 5000;
+long waitStartTime = Time.monotonicNow();
+while(event == null && Time.monotonicNow() - waitStartTime < waitTime)
{
+ //Wait for at most 5 sec
+ wait(waitTime);
+}
}
Event e = event;
event = null;
git commit: MAPREDUCE-6029. TestCommitterEventHandler fails in trunk. Contributed by Mit Desai (cherry picked from commit 2e789eb2261fa607489d8ae630fb77e8147d0266)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 1bd45fa16 -> e835cb5c8
MAPREDUCE-6029. TestCommitterEventHandler fails in trunk. Contributed by Mit
Desai
(cherry picked from commit 2e789eb2261fa607489d8ae630fb77e8147d0266)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e835cb5c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e835cb5c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e835cb5c
Branch: refs/heads/branch-2.6
Commit: e835cb5c86afeb3ca052621874452c31af9735b7
Parents: 1bd45fa
Author: Jason Lowe
Authored: Tue Oct 7 14:10:39 2014 +
Committer: Jason Lowe
Committed: Tue Oct 7 14:13:12 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../mapreduce/v2/app/commit/TestCommitterEventHandler.java | 9 +++--
2 files changed, 10 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e835cb5c/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index e499d2a..1399f4a 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -180,6 +180,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6094. TestMRCJCFileInputFormat.testAddInputPath() fails on trunk
(Akira AJISAKA via jlowe)
+MAPREDUCE-6029. TestCommitterEventHandler fails in trunk (Mit Desai via
+ jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e835cb5c/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
index 83e43f5..a553bbd 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/commit/TestCommitterEventHandler.java
@@ -58,6 +58,7 @@ import
org.apache.hadoop.mapreduce.v2.app.job.event.JobCommitCompletedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobCommitFailedEvent;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Event;
@@ -84,8 +85,12 @@ public class TestCommitterEventHandler {
public synchronized Event getAndClearEvent() throws InterruptedException {
if (event == null) {
-//Wait for at most 10 ms
-wait(100);
+final long waitTime = 5000;
+long waitStartTime = Time.monotonicNow();
+while(event == null && Time.monotonicNow() - waitStartTime < waitTime)
{
+ //Wait for at most 5 sec
+ wait(waitTime);
+}
}
Event e = event;
event = null;
[2/2] git commit: YARN-2431. NM restart: cgroup is not removed for reacquired containers. Contributed by Jason Lowe (cherry picked from commit 3fa5f728c4d6944302621965d6a0376827af5b51)
YARN-2431. NM restart: cgroup is not removed for reacquired containers.
Contributed by Jason Lowe
(cherry picked from commit 3fa5f728c4d6944302621965d6a0376827af5b51)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/04d325af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/04d325af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/04d325af
Branch: refs/heads/branch-2
Commit: 04d325afffd3696fd1d932b1aee47856164b916f
Parents: c035365
Author: Jason Lowe
Authored: Thu Sep 4 21:11:27 2014 +
Committer: Jason Lowe
Committed: Thu Sep 4 21:14:20 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodemanager/LinuxContainerExecutor.java | 11
.../nodemanager/TestLinuxContainerExecutor.java | 60
3 files changed, 74 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04d325af/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index eb38da5..7f7f367 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -255,6 +255,9 @@ Release 2.6.0 - UNRELEASED
YARN-2462.
TestNodeManagerResync#testBlockNewContainerRequestsOnStartAndResync
should have a test timeout (Eric Payne via jlowe)
+YARN-2431. NM restart: cgroup is not removed for reacquired containers
+(jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04d325af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index abfa417..b182420 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.base.Optional;
+
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -340,6 +341,16 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
}
@Override
+ public int reacquireContainer(String user, ContainerId containerId)
+ throws IOException {
+try {
+ return super.reacquireContainer(user, containerId);
+} finally {
+ resourcesHandler.postExecute(containerId);
+}
+ }
+
+ @Override
public boolean signalContainer(String user, String pid, Signal signal)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04d325af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index 3ac132f..cea2e6b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -30,6 +30,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -42,11 +44,15 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import
[1/2] git commit: YARN-2431. NM restart: cgroup is not removed for reacquired containers. Contributed by Jason Lowe
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c03536533 -> 04d325aff
refs/heads/trunk b44b2ee4a -> 3fa5f728c
YARN-2431. NM restart: cgroup is not removed for reacquired containers.
Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fa5f728
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fa5f728
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fa5f728
Branch: refs/heads/trunk
Commit: 3fa5f728c4d6944302621965d6a0376827af5b51
Parents: b44b2ee
Author: Jason Lowe
Authored: Thu Sep 4 21:11:27 2014 +
Committer: Jason Lowe
Committed: Thu Sep 4 21:11:27 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodemanager/LinuxContainerExecutor.java | 11
.../nodemanager/TestLinuxContainerExecutor.java | 60
3 files changed, 74 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fa5f728/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b0e4a01..f6f5c4b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -278,6 +278,9 @@ Release 2.6.0 - UNRELEASED
YARN-2462.
TestNodeManagerResync#testBlockNewContainerRequestsOnStartAndResync
should have a test timeout (Eric Payne via jlowe)
+YARN-2431. NM restart: cgroup is not removed for reacquired containers
+(jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fa5f728/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 804864e..6b8dd28 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.base.Optional;
+
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -342,6 +343,16 @@ public class LinuxContainerExecutor extends
ContainerExecutor {
}
@Override
+ public int reacquireContainer(String user, ContainerId containerId)
+ throws IOException {
+try {
+ return super.reacquireContainer(user, containerId);
+} finally {
+ resourcesHandler.postExecute(containerId);
+}
+ }
+
+ @Override
public boolean signalContainer(String user, String pid, Signal signal)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fa5f728/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index a5ec43b..c02212e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -30,6 +30,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -42,11 +44,15 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
+
[1/2] git commit: MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is disabled because of too much CPUs. Contributed by Tsuyoshi OZAWA
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0bd0423ff -> 0b3f918b3
refs/heads/trunk 772d1fb01 -> 45efc966e
MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is disabled
because of too much CPUs. Contributed by Tsuyoshi OZAWA
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45efc966
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45efc966
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45efc966
Branch: refs/heads/trunk
Commit: 45efc966ee1a1bda0271464dcee2564862d53b1f
Parents: 772d1fb
Author: Jason Lowe
Authored: Fri Sep 5 13:42:00 2014 +
Committer: Jason Lowe
Committed: Fri Sep 5 13:42:00 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java | 2 ++
2 files changed, 5 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45efc966/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 0ee0731..73dd8d8 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -270,6 +270,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6063. Correct spill size calculation for spills wrapping the
circular buffer. (zhihai xu via cdouglas)
+MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is
+disabled because of too much CPUs (Tsuyoshi OZAWA via jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45efc966/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index c1bc17d..dff5ef4 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -1285,6 +1285,8 @@ public class JobImpl implements
org.apache.hadoop.mapreduce.v2.app.job.Job,
msg.append(" too many reduces;");
if (!smallInput)
msg.append(" too much input;");
+ if (!smallCpu)
+msg.append(" too much CPU;");
if (!smallMemory)
msg.append(" too much RAM;");
if (!notChainJob)
[2/2] git commit: MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is disabled because of too much CPUs. Contributed by Tsuyoshi OZAWA (cherry picked from commit 45efc966ee1a1bda027
MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is disabled
because of too much CPUs. Contributed by Tsuyoshi OZAWA
(cherry picked from commit 45efc966ee1a1bda0271464dcee2564862d53b1f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b3f918b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b3f918b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b3f918b
Branch: refs/heads/branch-2
Commit: 0b3f918b3b365406a47cf799f0db2aac48011f01
Parents: 0bd0423
Author: Jason Lowe
Authored: Fri Sep 5 13:42:00 2014 +
Committer: Jason Lowe
Committed: Fri Sep 5 13:48:16 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java | 2 ++
2 files changed, 5 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b3f918b/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 0e91326..e50c4b8 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -121,6 +121,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6063. Correct spill size calculation for spills wrapping the
circular buffer. (zhihai xu via cdouglas)
+MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is
+disabled because of too much CPUs (Tsuyoshi OZAWA via jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b3f918b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index c1bc17d..dff5ef4 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -1285,6 +1285,8 @@ public class JobImpl implements
org.apache.hadoop.mapreduce.v2.app.job.Job,
msg.append(" too many reduces;");
if (!smallInput)
msg.append(" too much input;");
+ if (!smallCpu)
+msg.append(" too much CPU;");
if (!smallMemory)
msg.append(" too much RAM;");
if (!notChainJob)
git commit: MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length files. Contributed by Jason Lowe (cherry picked from commit 17a025d9dda9949f191b650e3ef96090880f568a)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 88e5549d9 -> e6f108c99
MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length files.
Contributed by Jason Lowe
(cherry picked from commit 17a025d9dda9949f191b650e3ef96090880f568a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6f108c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6f108c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6f108c9
Branch: refs/heads/branch-2
Commit: e6f108c99ba1e650c450b693fa696852bd8c0891
Parents: 88e5549
Author: Jason Lowe
Authored: Wed Sep 10 22:10:36 2014 +
Committer: Jason Lowe
Committed: Wed Sep 10 22:12:24 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../v2/hs/HistoryServerFileSystemStateStoreService.java| 6 ++
2 files changed, 9 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6f108c9/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index e50c4b8..d591105 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -124,6 +124,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is
disabled because of too much CPUs (Tsuyoshi OZAWA via jlowe)
+MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length
+ files (jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6f108c9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
index dcea333..9902f5e 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
@@ -189,6 +189,8 @@ public class HistoryServerFileSystemStateStoreService
DataOutputStream dataStream = new DataOutputStream(memStream);
try {
key.write(dataStream);
+ dataStream.close();
+ dataStream = null;
} finally {
IOUtils.cleanup(LOG, dataStream);
}
@@ -260,6 +262,8 @@ public class HistoryServerFileSystemStateStoreService
try {
try {
out.write(data);
+out.close();
+out = null;
} finally {
IOUtils.cleanup(LOG, out);
}
@@ -299,6 +303,8 @@ public class HistoryServerFileSystemStateStoreService
try {
tokenId.write(dataStream);
dataStream.writeLong(renewDate);
+ dataStream.close();
+ dataStream = null;
} finally {
IOUtils.cleanup(LOG, dataStream);
}
git commit: MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length files. Contributed by Jason Lowe
Repository: hadoop
Updated Branches:
refs/heads/trunk 7d38ffc8d -> 7f80e1420
MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length files.
Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f80e142
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f80e142
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f80e142
Branch: refs/heads/trunk
Commit: 7f80e142096ab21b4852635a127589e34a747bc4
Parents: 7d38ffc
Author: Jason Lowe
Authored: Wed Sep 10 22:10:36 2014 +
Committer: Jason Lowe
Committed: Wed Sep 10 22:22:39 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../v2/hs/HistoryServerFileSystemStateStoreService.java| 6 ++
2 files changed, 9 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f80e142/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index a5e5920..490c0a1 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -276,6 +276,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6071. JobImpl#makeUberDecision doesn't log that Uber mode is
disabled because of too much CPUs (Tsuyoshi OZAWA via jlowe)
+MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length
+ files (jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f80e142/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
index dcea333..9902f5e 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryServerFileSystemStateStoreService.java
@@ -189,6 +189,8 @@ public class HistoryServerFileSystemStateStoreService
DataOutputStream dataStream = new DataOutputStream(memStream);
try {
key.write(dataStream);
+ dataStream.close();
+ dataStream = null;
} finally {
IOUtils.cleanup(LOG, dataStream);
}
@@ -260,6 +262,8 @@ public class HistoryServerFileSystemStateStoreService
try {
try {
out.write(data);
+out.close();
+out = null;
} finally {
IOUtils.cleanup(LOG, out);
}
@@ -299,6 +303,8 @@ public class HistoryServerFileSystemStateStoreService
try {
tokenId.write(dataStream);
dataStream.writeLong(renewDate);
+ dataStream.close();
+ dataStream = null;
} finally {
IOUtils.cleanup(LOG, dataStream);
}
git commit: MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented. Contributed by Tsuyoshi OZAWA
Repository: hadoop Updated Branches: refs/heads/trunk 4be95175c -> 9e28e089c MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented. Contributed by Tsuyoshi OZAWA Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e28e089 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e28e089 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e28e089 Branch: refs/heads/trunk Commit: 9e28e089c6bcb0b264751bdfa1f994ce002ef2de Parents: 4be9517 Author: Jason Lowe Authored: Thu Sep 11 16:19:23 2014 + Committer: Jason Lowe Committed: Thu Sep 11 16:19:23 2014 + -- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/resources/mapred-default.xml | 7 ++- 2 files changed, 9 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e28e089/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 490c0a1..0c065c5 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -279,6 +279,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length files (jlowe) +MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but +is not documented (Tsuyoshi OZAWA via jlowe) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e28e089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 703a103..6cefdc9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -508,7 +508,12 @@ Whether to enable the small-jobs "ubertask" optimization, which runs "sufficiently small" jobs sequentially within a single JVM. "Small" is defined by the following maxmaps, maxreduces, and maxbytes - settings. Users may override this value. + settings. Note that configurations for application masters also affect + the "Small" definition - yarn.app.mapreduce.am.resource.mb must be + larger than both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb, + and yarn.app.mapreduce.am.resource.cpu-vcores must be larger than + both mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores to enable + ubertask. Users may override this value.
git commit: MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented. Contributed by Tsuyoshi OZAWA
Repository: hadoop Updated Branches: refs/heads/branch-2 a2a61eec6 -> 6f8abe7ef MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented. Contributed by Tsuyoshi OZAWA (cherry picked from commit 9e28e089c6bcb0b264751bdfa1f994ce002ef2de) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f8abe7e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f8abe7e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f8abe7e Branch: refs/heads/branch-2 Commit: 6f8abe7efee00576e84427b9bd4a28bf250ae520 Parents: a2a61ee Author: Jason Lowe Authored: Thu Sep 11 16:20:51 2014 + Committer: Jason Lowe Committed: Thu Sep 11 16:20:51 2014 + -- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/resources/mapred-default.xml | 7 ++- 2 files changed, 9 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f8abe7e/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d591105..a5b2443 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -127,6 +127,9 @@ Release 2.6.0 - UNRELEASED MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length files (jlowe) +MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but +is not documented (Tsuyoshi OZAWA via jlowe) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f8abe7e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 430c1f9..72d21a8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -773,7 +773,12 @@ Whether to enable the small-jobs "ubertask" optimization, which runs "sufficiently small" jobs sequentially within a single JVM. "Small" is defined by the following maxmaps, maxreduces, and maxbytes - settings. Users may override this value. + settings. Note that configurations for application masters also affect + the "Small" definition - yarn.app.mapreduce.am.resource.mb must be + larger than both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb, + and yarn.app.mapreduce.am.resource.cpu-vcores must be larger than + both mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores to enable + ubertask. Users may override this value.
git commit: HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when value is NaN. Contributed by Mit Desai
Repository: hadoop
Updated Branches:
refs/heads/trunk 80ac6aabc -> 581176cdc
HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when value
is NaN. Contributed by Mit Desai
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/581176cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/581176cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/581176cd
Branch: refs/heads/trunk
Commit: 581176cdc868b8701a761b5a0d52f771cbba709a
Parents: 80ac6aa
Author: Jason Lowe
Authored: Thu Sep 11 19:53:25 2014 +
Committer: Jason Lowe
Committed: Thu Sep 11 19:53:25 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/util/Progress.java| 10 +-
2 files changed, 8 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/581176cd/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ffb8ede..3bf92ec 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -784,6 +784,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-9989. Bug introduced in HADOOP-9374, which parses the
-tokenCacheFile
as binary file but set it to the configuration as JSON file. (zxu via tucu)
+HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when
+value is NaN (Mit Desai via jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/581176cd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
index e3445b5..3fbc935 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
@@ -162,27 +162,27 @@ public class Progress {
public synchronized void set(float progress) {
if (Float.isNaN(progress)) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is Float.NaN. " +
+ LOG.debug("Illegal progress value found, progress is Float.NaN. " +
"Progress will be changed to 0");
}
else if (progress == Float.NEGATIVE_INFINITY) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is " +
+ LOG.debug("Illegal progress value found, progress is " +
"Float.NEGATIVE_INFINITY. Progress will be changed to 0");
}
else if (progress < 0) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is less than 0." +
+ LOG.debug("Illegal progress value found, progress is less than 0." +
" Progress will be changed to 0");
}
else if (progress > 1) {
progress = 1;
- LOG.warn("Illegal progress value found, progress is larger than 1." +
+ LOG.debug("Illegal progress value found, progress is larger than 1." +
" Progress will be changed to 1");
}
else if (progress == Float.POSITIVE_INFINITY) {
progress = 1;
- LOG.warn("Illegal progress value found, progress is " +
+ LOG.debug("Illegal progress value found, progress is " +
"Float.POSITIVE_INFINITY. Progress will be changed to 1");
}
this.progress = progress;
git commit: HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when value is NaN. Contributed by Mit Desai (cherry picked from commit 581176cdc868b8701a761b5a0d52f771cbba709a)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 ac296f952 -> b4ab7aa11
HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when value
is NaN. Contributed by Mit Desai
(cherry picked from commit 581176cdc868b8701a761b5a0d52f771cbba709a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4ab7aa1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4ab7aa1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4ab7aa1
Branch: refs/heads/branch-2
Commit: b4ab7aa110d04934cc63ae98a4db8b295804ba81
Parents: ac296f9
Author: Jason Lowe
Authored: Thu Sep 11 19:53:25 2014 +
Committer: Jason Lowe
Committed: Thu Sep 11 19:54:05 2014 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/util/Progress.java| 10 +-
2 files changed, 8 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4ab7aa1/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2d1375f..8d03b01 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -454,6 +454,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-9989. Bug introduced in HADOOP-9374, which parses the
-tokenCacheFile
as binary file but set it to the configuration as JSON file. (zxu via tucu)
+HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when
+value is NaN (Mit Desai via jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4ab7aa1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
index e3445b5..3fbc935 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
@@ -162,27 +162,27 @@ public class Progress {
public synchronized void set(float progress) {
if (Float.isNaN(progress)) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is Float.NaN. " +
+ LOG.debug("Illegal progress value found, progress is Float.NaN. " +
"Progress will be changed to 0");
}
else if (progress == Float.NEGATIVE_INFINITY) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is " +
+ LOG.debug("Illegal progress value found, progress is " +
"Float.NEGATIVE_INFINITY. Progress will be changed to 0");
}
else if (progress < 0) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is less than 0." +
+ LOG.debug("Illegal progress value found, progress is less than 0." +
" Progress will be changed to 0");
}
else if (progress > 1) {
progress = 1;
- LOG.warn("Illegal progress value found, progress is larger than 1." +
+ LOG.debug("Illegal progress value found, progress is larger than 1." +
" Progress will be changed to 1");
}
else if (progress == Float.POSITIVE_INFINITY) {
progress = 1;
- LOG.warn("Illegal progress value found, progress is " +
+ LOG.debug("Illegal progress value found, progress is " +
"Float.POSITIVE_INFINITY. Progress will be changed to 1");
}
this.progress = progress;
git commit: YARN-2484. FileSystemRMStateStore#readFile/writeFile should close FSData(In|Out)putStream in final block. Contributed by Tsuyoshi OZAWA
Repository: hadoop
Updated Branches:
refs/heads/trunk 469ea3dce -> 78b048393
YARN-2484. FileSystemRMStateStore#readFile/writeFile should close
FSData(In|Out)putStream in final block. Contributed by Tsuyoshi OZAWA
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78b04839
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78b04839
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78b04839
Branch: refs/heads/trunk
Commit: 78b048393a80a9bd1399d08525590bb211a32d8c
Parents: 469ea3d
Author: Jason Lowe
Authored: Fri Sep 12 14:09:14 2014 +
Committer: Jason Lowe
Committed: Fri Sep 12 14:09:14 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../recovery/FileSystemRMStateStore.java| 30 +---
2 files changed, 23 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78b04839/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c5f5cb0..59b8788 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -334,6 +334,9 @@ Release 2.6.0 - UNRELEASED
YARN-2541. Fixed ResourceManagerRest.apt.vm table syntax error. (jianhe)
+YARN-2484. FileSystemRMStateStore#readFile/writeFile should close
+FSData(In|Out)putStream in final block (Tsuyoshi OZAWA via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78b04839/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 4c01a61..4d91831 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -573,12 +574,16 @@ public class FileSystemRMStateStore extends RMStateStore {
}
private byte[] readFile(Path inputPath, long len) throws Exception {
-FSDataInputStream fsIn = fs.open(inputPath);
-// state data will not be that "long"
-byte[] data = new byte[(int)len];
-fsIn.readFully(data);
-fsIn.close();
-return data;
+FSDataInputStream fsIn = null;
+try {
+ fsIn = fs.open(inputPath);
+ // state data will not be that "long"
+ byte[] data = new byte[(int) len];
+ fsIn.readFully(data);
+ return data;
+} finally {
+ IOUtils.cleanup(LOG, fsIn);
+}
}
/*
@@ -592,10 +597,15 @@ public class FileSystemRMStateStore extends RMStateStore {
FSDataOutputStream fsOut = null;
// This file will be overwritten when app/attempt finishes for saving the
// final status.
-fsOut = fs.create(tempPath, true);
-fsOut.write(data);
-fsOut.close();
-fs.rename(tempPath, outputPath);
+try {
+ fsOut = fs.create(tempPath, true);
+ fsOut.write(data);
+ fsOut.close();
+ fsOut = null;
+ fs.rename(tempPath, outputPath);
+} finally {
+ IOUtils.cleanup(LOG, fsOut);
+}
}
/*
git commit: YARN-2484. FileSystemRMStateStore#readFile/writeFile should close FSData(In|Out)putStream in final block. Contributed by Tsuyoshi OZAWA (cherry picked from commit 78b048393a80a9bd1399d0852
Repository: hadoop
Updated Branches:
refs/heads/branch-2 466bdf79c -> 7e69dc877
YARN-2484. FileSystemRMStateStore#readFile/writeFile should close
FSData(In|Out)putStream in final block. Contributed by Tsuyoshi OZAWA
(cherry picked from commit 78b048393a80a9bd1399d08525590bb211a32d8c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e69dc87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e69dc87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e69dc87
Branch: refs/heads/branch-2
Commit: 7e69dc8771726b347bc84640f7b311be2d67c3de
Parents: 466bdf7
Author: Jason Lowe
Authored: Fri Sep 12 14:09:14 2014 +
Committer: Jason Lowe
Committed: Fri Sep 12 14:10:51 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../recovery/FileSystemRMStateStore.java| 30 +---
2 files changed, 23 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e69dc87/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b40db52..57ff8df 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -308,6 +308,9 @@ Release 2.6.0 - UNRELEASED
YARN-2541. Fixed ResourceManagerRest.apt.vm table syntax error. (jianhe)
+YARN-2484. FileSystemRMStateStore#readFile/writeFile should close
+FSData(In|Out)putStream in final block (Tsuyoshi OZAWA via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e69dc87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 4c01a61..4d91831 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -573,12 +574,16 @@ public class FileSystemRMStateStore extends RMStateStore {
}
private byte[] readFile(Path inputPath, long len) throws Exception {
-FSDataInputStream fsIn = fs.open(inputPath);
-// state data will not be that "long"
-byte[] data = new byte[(int)len];
-fsIn.readFully(data);
-fsIn.close();
-return data;
+FSDataInputStream fsIn = null;
+try {
+ fsIn = fs.open(inputPath);
+ // state data will not be that "long"
+ byte[] data = new byte[(int) len];
+ fsIn.readFully(data);
+ return data;
+} finally {
+ IOUtils.cleanup(LOG, fsIn);
+}
}
/*
@@ -592,10 +597,15 @@ public class FileSystemRMStateStore extends RMStateStore {
FSDataOutputStream fsOut = null;
// This file will be overwritten when app/attempt finishes for saving the
// final status.
-fsOut = fs.create(tempPath, true);
-fsOut.write(data);
-fsOut.close();
-fs.rename(tempPath, outputPath);
+try {
+ fsOut = fs.create(tempPath, true);
+ fsOut.write(data);
+ fsOut.close();
+ fsOut = null;
+ fs.rename(tempPath, outputPath);
+} finally {
+ IOUtils.cleanup(LOG, fsOut);
+}
}
/*
git commit: YARN-2363. Submitted applications occasionally lack a tracking URL. Contributed by Jason Lowe
Repository: hadoop
Updated Branches:
refs/heads/trunk 570b8b468 -> 9ea7b6c06
YARN-2363. Submitted applications occasionally lack a tracking URL. Contributed
by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ea7b6c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ea7b6c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ea7b6c0
Branch: refs/heads/trunk
Commit: 9ea7b6c063c0bdd4551962e21d0173f671e9df03
Parents: 570b8b4
Author: Jason Lowe
Authored: Thu Sep 18 20:13:16 2014 +
Committer: Jason Lowe
Committed: Thu Sep 18 20:13:16 2014 +
--
hadoop-yarn-project/CHANGES.txt | 2 ++
.../server/resourcemanager/rmapp/RMAppImpl.java | 19 +++
.../rmapp/TestRMAppTransitions.java | 6 --
3 files changed, 25 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea7b6c0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c179c7f..fbbef7f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -389,6 +389,8 @@ Release 2.6.0 - UNRELEASED
YARN-1779. Fixed AMRMClient to handle AMRMTokens correctly across
ResourceManager work-preserving-restart or failover. (Jian He via vinodkv)
+YARN-2363. Submitted applications occasionally lack a tracking URL (jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ea7b6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 815b86a..ff520be 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
@@ -76,6 +78,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
@@ -84,6 +87,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -542,6 +546,7 @@ public class RMAppImpl implements RMApp, Recoverable {
float progress = 0.0f;
org.apache.hadoop.yarn.api.records.Token amrmToken = null;
if (allowAccess) {
+trackingUrl = getDefaultProxyTrackingUrl();
if (this.currentAttempt != null) {
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
trackingUrl = this.currentAttempt.getTrackingUrl();
@@ -602,6 +607,20 @@ public class RMAppImpl implements RMApp, Recoverable {
}
}
+ private String getDefaultProxyTrackingUrl() {
+try {
+ final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
+ String proxy = WebAppUtils.getProxyHostAndPort(conf);
+ URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
+ URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
+ return result.toASCIIString();
+} catch (URISyntaxException e) {
+ LOG.warn("Could not generate default proxy tracking URL for "
+ + applicationId);
+ retur
git commit: YARN-2363. Submitted applications occasionally lack a tracking URL. Contributed by Jason Lowe (cherry picked from commit 9ea7b6c063c0bdd4551962e21d0173f671e9df03)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2b1c9c621 -> 44c22c3d8
YARN-2363. Submitted applications occasionally lack a tracking URL. Contributed
by Jason Lowe
(cherry picked from commit 9ea7b6c063c0bdd4551962e21d0173f671e9df03)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44c22c3d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44c22c3d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44c22c3d
Branch: refs/heads/branch-2
Commit: 44c22c3d8d9966e72e5f350c933c5d3ea9eb9465
Parents: 2b1c9c6
Author: Jason Lowe
Authored: Thu Sep 18 20:13:16 2014 +
Committer: Jason Lowe
Committed: Thu Sep 18 20:15:46 2014 +
--
hadoop-yarn-project/CHANGES.txt | 2 ++
.../server/resourcemanager/rmapp/RMAppImpl.java | 19 +++
.../rmapp/TestRMAppTransitions.java | 6 --
3 files changed, 25 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44c22c3d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3bdd793..a537749 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -359,6 +359,8 @@ Release 2.6.0 - UNRELEASED
YARN-1779. Fixed AMRMClient to handle AMRMTokens correctly across
ResourceManager work-preserving-restart or failover. (Jian He via vinodkv)
+YARN-2363. Submitted applications occasionally lack a tracking URL (jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44c22c3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 815b86a..ff520be 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
@@ -76,6 +78,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition;
@@ -84,6 +87,7 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -542,6 +546,7 @@ public class RMAppImpl implements RMApp, Recoverable {
float progress = 0.0f;
org.apache.hadoop.yarn.api.records.Token amrmToken = null;
if (allowAccess) {
+trackingUrl = getDefaultProxyTrackingUrl();
if (this.currentAttempt != null) {
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
trackingUrl = this.currentAttempt.getTrackingUrl();
@@ -602,6 +607,20 @@ public class RMAppImpl implements RMApp, Recoverable {
}
}
+ private String getDefaultProxyTrackingUrl() {
+try {
+ final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
+ String proxy = WebAppUtils.getProxyHostAndPort(conf);
+ URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
+ URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
+ return result.toASCIIString();
+} catch (URISyntaxException e) {
+ LOG.warn("Could not generate defau
git commit: MAPREDUCE-6090. mapred hsadmin getGroups fails to connect in some cases. Contributed by Robert Kanter
Repository: hadoop
Updated Branches:
refs/heads/trunk 474f116f5 -> 1cf319804
MAPREDUCE-6090. mapred hsadmin getGroups fails to connect in some cases.
Contributed by Robert Kanter
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cf31980
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cf31980
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cf31980
Branch: refs/heads/trunk
Commit: 1cf31980474bf9ac28e51b6f014eae09eda9e15f
Parents: 474f116
Author: Jason Lowe
Authored: Thu Sep 18 21:00:53 2014 +
Committer: Jason Lowe
Committed: Thu Sep 18 21:00:53 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java | 6 --
.../hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java | 5 +++--
3 files changed, 10 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cf31980/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 5d1e5f5..0be938b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -347,6 +347,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but
is not documented (Tsuyoshi OZAWA via jlowe)
+MAPREDUCE-6090. mapred hsadmin getGroups fails to connect in some cases
+(Robert Kanter via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cf31980/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
index be6ca13..000ea54 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.v2.hs.HSProxies;
import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
@@ -41,7 +42,7 @@ public class HSAdmin extends Configured implements Tool {
super();
}
- public HSAdmin(Configuration conf) {
+ public HSAdmin(JobConf conf) {
super(conf);
}
@@ -331,7 +332,8 @@ public class HSAdmin extends Configured implements Tool {
}
public static void main(String[] args) throws Exception {
-int result = ToolRunner.run(new HSAdmin(), args);
+JobConf conf = new JobConf();
+int result = ToolRunner.run(new HSAdmin(conf), args);
System.exit(result);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cf31980/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
index 277a195..2c239ec 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
@@ -28,6 +28,7 @@ import java.util.List;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.v2.hs.JobHistory;
import org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin;
imp
git commit: MAPREDUCE-6090. mapred hsadmin getGroups fails to connect in some cases. Contributed by Robert Kanter (cherry picked from commit 1cf31980474bf9ac28e51b6f014eae09eda9e15f)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 44c22c3d8 -> 4b4e44a8a
MAPREDUCE-6090. mapred hsadmin getGroups fails to connect in some cases.
Contributed by Robert Kanter
(cherry picked from commit 1cf31980474bf9ac28e51b6f014eae09eda9e15f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b4e44a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b4e44a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b4e44a8
Branch: refs/heads/branch-2
Commit: 4b4e44a8aade35df406fa0ded020a357c92f1e73
Parents: 44c22c3
Author: Jason Lowe
Authored: Thu Sep 18 21:00:53 2014 +
Committer: Jason Lowe
Committed: Thu Sep 18 21:02:13 2014 +
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java | 6 --
.../hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java | 5 +++--
3 files changed, 10 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4e44a8/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 420890b..c4f3d90 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -130,6 +130,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but
is not documented (Tsuyoshi OZAWA via jlowe)
+MAPREDUCE-6090. mapred hsadmin getGroups fails to connect in some cases
+(Robert Kanter via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4e44a8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
index be6ca13..000ea54 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/client/HSAdmin.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.v2.hs.HSProxies;
import org.apache.hadoop.mapreduce.v2.hs.protocol.HSAdminRefreshProtocol;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
@@ -41,7 +42,7 @@ public class HSAdmin extends Configured implements Tool {
super();
}
- public HSAdmin(Configuration conf) {
+ public HSAdmin(JobConf conf) {
super(conf);
}
@@ -331,7 +332,8 @@ public class HSAdmin extends Configured implements Tool {
}
public static void main(String[] args) throws Exception {
-int result = ToolRunner.run(new HSAdmin(), args);
+JobConf conf = new JobConf();
+int result = ToolRunner.run(new HSAdmin(conf), args);
System.exit(result);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b4e44a8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
index 277a195..2c239ec 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/server/TestHSAdminServer.java
@@ -28,6 +28,7 @@ import java.util.List;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.v2.hs.JobHist
git commit: YARN-2561. MR job client cannot reconnect to AM after NM restart. Contributed by Junping Du
Repository: hadoop
Updated Branches:
refs/heads/trunk 1cf319804 -> a337f0e35
YARN-2561. MR job client cannot reconnect to AM after NM restart. Contributed
by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a337f0e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a337f0e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a337f0e3
Branch: refs/heads/trunk
Commit: a337f0e3549351344bce70cb23ddc0a256c894b0
Parents: 1cf3198
Author: Jason Lowe
Authored: Thu Sep 18 21:34:40 2014 +
Committer: Jason Lowe
Committed: Thu Sep 18 21:34:40 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../resourcemanager/rmnode/RMNodeImpl.java | 47 +---
.../TestResourceTrackerService.java | 11 +
3 files changed, 55 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a337f0e3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fbbef7f..836248b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -391,6 +391,9 @@ Release 2.6.0 - UNRELEASED
YARN-2363. Submitted applications occasionally lack a tracking URL (jlowe)
+YARN-2561. MR job client cannot reconnect to AM after NM restart. (Junping
+Du via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a337f0e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 3ce6416..1265958 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -544,12 +544,47 @@ public class RMNodeImpl implements RMNode,
EventHandler {
RMNodeReconnectEvent reconnectEvent = (RMNodeReconnectEvent) event;
RMNode newNode = reconnectEvent.getReconnectedNode();
rmNode.nodeManagerVersion = newNode.getNodeManagerVersion();
- rmNode.httpPort = newNode.getHttpPort();
- rmNode.httpAddress = newNode.getHttpAddress();
- rmNode.totalCapability = newNode.getTotalCapability();
+ List runningApps =
reconnectEvent.getRunningApplications();
+ boolean noRunningApps =
+ (runningApps == null) || (runningApps.size() == 0);
- // Reset heartbeat ID since node just restarted.
- rmNode.getLastNodeHeartBeatResponse().setResponseId(0);
+ // No application running on the node, so send node-removal event with
+ // cleaning up old container info.
+ if (noRunningApps) {
+rmNode.nodeUpdateQueue.clear();
+rmNode.context.getDispatcher().getEventHandler().handle(
+new NodeRemovedSchedulerEvent(rmNode));
+
+if (rmNode.getHttpPort() == newNode.getHttpPort()) {
+ // Reset heartbeat ID since node just restarted.
+ rmNode.getLastNodeHeartBeatResponse().setResponseId(0);
+ if (rmNode.getState() != NodeState.UNHEALTHY) {
+// Only add new node if old state is not UNHEALTHY
+rmNode.context.getDispatcher().getEventHandler().handle(
+new NodeAddedSchedulerEvent(newNode));
+ }
+} else {
+ // Reconnected node differs, so replace old node and start new node
+ switch (rmNode.getState()) {
+case RUNNING:
+ ClusterMetrics.getMetrics().decrNumActiveNodes();
+ break;
+case UNHEALTHY:
+ ClusterMetrics.getMetrics().decrNumUnhealthyNMs();
+ break;
+}
+rmNode.context.getRMNodes().put(newNode.getNodeID(), newNode);
+rmNode.context.getDispatcher().getEventHandler().handle(
+new RMNodeStartedEvent(newNode.getNodeID(), null, null));
+}
+ } else {
+rmNode.httpPort = newNode.getHttpPort();
+rmNode.httpAddress = newNode.getHttpAddress();
+rmNode.totalCapabil
git commit: YARN-2561. MR job client cannot reconnect to AM after NM restart. Contributed by Junping Du (cherry picked from commit a337f0e3549351344bce70cb23ddc0a256c894b0)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 4b4e44a8a -> d9273a954
YARN-2561. MR job client cannot reconnect to AM after NM restart. Contributed
by Junping Du
(cherry picked from commit a337f0e3549351344bce70cb23ddc0a256c894b0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9273a95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9273a95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9273a95
Branch: refs/heads/branch-2
Commit: d9273a95476371b01cc366251a6c67ec2c07f4c4
Parents: 4b4e44a
Author: Jason Lowe
Authored: Thu Sep 18 21:34:40 2014 +
Committer: Jason Lowe
Committed: Thu Sep 18 21:36:20 2014 +
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../resourcemanager/rmnode/RMNodeImpl.java | 47 +---
.../TestResourceTrackerService.java | 11 +
3 files changed, 55 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9273a95/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a537749..0baf910 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -361,6 +361,9 @@ Release 2.6.0 - UNRELEASED
YARN-2363. Submitted applications occasionally lack a tracking URL (jlowe)
+YARN-2561. MR job client cannot reconnect to AM after NM restart. (Junping
+Du via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9273a95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 3ce6416..1265958 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -544,12 +544,47 @@ public class RMNodeImpl implements RMNode,
EventHandler {
RMNodeReconnectEvent reconnectEvent = (RMNodeReconnectEvent) event;
RMNode newNode = reconnectEvent.getReconnectedNode();
rmNode.nodeManagerVersion = newNode.getNodeManagerVersion();
- rmNode.httpPort = newNode.getHttpPort();
- rmNode.httpAddress = newNode.getHttpAddress();
- rmNode.totalCapability = newNode.getTotalCapability();
+ List runningApps =
reconnectEvent.getRunningApplications();
+ boolean noRunningApps =
+ (runningApps == null) || (runningApps.size() == 0);
- // Reset heartbeat ID since node just restarted.
- rmNode.getLastNodeHeartBeatResponse().setResponseId(0);
+ // No application running on the node, so send node-removal event with
+ // cleaning up old container info.
+ if (noRunningApps) {
+rmNode.nodeUpdateQueue.clear();
+rmNode.context.getDispatcher().getEventHandler().handle(
+new NodeRemovedSchedulerEvent(rmNode));
+
+if (rmNode.getHttpPort() == newNode.getHttpPort()) {
+ // Reset heartbeat ID since node just restarted.
+ rmNode.getLastNodeHeartBeatResponse().setResponseId(0);
+ if (rmNode.getState() != NodeState.UNHEALTHY) {
+// Only add new node if old state is not UNHEALTHY
+rmNode.context.getDispatcher().getEventHandler().handle(
+new NodeAddedSchedulerEvent(newNode));
+ }
+} else {
+ // Reconnected node differs, so replace old node and start new node
+ switch (rmNode.getState()) {
+case RUNNING:
+ ClusterMetrics.getMetrics().decrNumActiveNodes();
+ break;
+case UNHEALTHY:
+ ClusterMetrics.getMetrics().decrNumUnhealthyNMs();
+ break;
+}
+rmNode.context.getRMNodes().put(newNode.getNodeID(), newNode);
+rmNode.context.getDispatcher().getEventHandler().handle(
+new RMNodeStartedEvent(newNode.getNodeID(), null, null));
+}
+ } else {
+rmNode.httpPort = newNode.getHttpP
git commit: MAPREDUCE-5891. Improved shuffle error handling across NM restarts. Contributed by Junping Du
Repository: hadoop
Updated Branches:
refs/heads/trunk 70be56d09 -> 2c3da25fd
MAPREDUCE-5891. Improved shuffle error handling across NM restarts. Contributed
by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c3da25f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c3da25f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c3da25f
Branch: refs/heads/trunk
Commit: 2c3da25fd718b3a9c1ed67f05b577975ae613f4e
Parents: 70be56d
Author: Jason Lowe
Authored: Thu Sep 18 21:56:46 2014 +
Committer: Jason Lowe
Committed: Thu Sep 18 22:00:11 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../apache/hadoop/mapreduce/MRJobConfig.java| 8 +
.../hadoop/mapreduce/task/reduce/Fetcher.java | 231 ++-
.../task/reduce/ShuffleSchedulerImpl.java | 17 +-
.../src/main/resources/mapred-default.xml | 21 ++
.../mapreduce/task/reduce/TestFetcher.java | 96 +++-
6 files changed, 312 insertions(+), 64 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c3da25f/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 0be938b..a661266 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -251,6 +251,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-5130. Add missing job config options to mapred-default.xml
(Ray Chiang via Sandy Ryza)
+MAPREDUCE-5891. Improved shuffle error handling across NM restarts
+(Junping Du via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c3da25f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 4c48cf5..e39dd6a 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -298,6 +298,14 @@ public interface MRJobConfig {
public static final String MAX_FETCH_FAILURES_NOTIFICATIONS =
"mapreduce.reduce.shuffle.max-fetch-failures-notifications";
public static final int DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS = 3;
+
+ public static final String SHUFFLE_FETCH_RETRY_INTERVAL_MS =
"mapreduce.reduce.shuffle.fetch.retry.interval-ms";
+ /** Default interval that fetcher retry to fetch during NM restart.*/
+ public final static int DEFAULT_SHUFFLE_FETCH_RETRY_INTERVAL_MS = 1000;
+
+ public static final String SHUFFLE_FETCH_RETRY_TIMEOUT_MS =
"mapreduce.reduce.shuffle.fetch.retry.timeout-ms";
+
+ public static final String SHUFFLE_FETCH_RETRY_ENABLED =
"mapreduce.reduce.shuffle.fetch.retry.enabled";
public static final String SHUFFLE_NOTIFY_READERROR =
"mapreduce.reduce.shuffle.notify.readerror";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c3da25f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index e1e1663..a416200 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -27,6 +27,7 @@ import java.net.URL;
import java.net.URLConnection;
import java.security.GeneralSecurityException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -46,6 +47,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
im
git commit: MAPREDUCE-5891. Improved shuffle error handling across NM restarts. Contributed by Junping Du (cherry picked from commit 2c3da25fd718b3a9c1ed67f05b577975ae613f4e)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 47c9dcfe8 -> 44c711313
MAPREDUCE-5891. Improved shuffle error handling across NM restarts. Contributed
by Junping Du
(cherry picked from commit 2c3da25fd718b3a9c1ed67f05b577975ae613f4e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44c71131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44c71131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44c71131
Branch: refs/heads/branch-2
Commit: 44c711313311bc2c38adc66bca11a100bd7ce4d0
Parents: 47c9dcf
Author: Jason Lowe
Authored: Thu Sep 18 21:56:46 2014 +
Committer: Jason Lowe
Committed: Thu Sep 18 22:01:42 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../apache/hadoop/mapreduce/MRJobConfig.java| 8 +
.../hadoop/mapreduce/task/reduce/Fetcher.java | 231 ++-
.../task/reduce/ShuffleSchedulerImpl.java | 17 +-
.../src/main/resources/mapred-default.xml | 21 ++
.../mapreduce/task/reduce/TestFetcher.java | 96 +++-
6 files changed, 312 insertions(+), 64 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44c71131/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index c4f3d90..2e79e4d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -34,6 +34,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-5130. Add missing job config options to mapred-default.xml
(Ray Chiang via Sandy Ryza)
+MAPREDUCE-5891. Improved shuffle error handling across NM restarts
+(Junping Du via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44c71131/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index fdc1c59..d2199c3 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -293,6 +293,14 @@ public interface MRJobConfig {
public static final String MAX_FETCH_FAILURES_NOTIFICATIONS =
"mapreduce.reduce.shuffle.max-fetch-failures-notifications";
public static final int DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS = 3;
+
+ public static final String SHUFFLE_FETCH_RETRY_INTERVAL_MS =
"mapreduce.reduce.shuffle.fetch.retry.interval-ms";
+ /** Default interval that fetcher retry to fetch during NM restart.*/
+ public final static int DEFAULT_SHUFFLE_FETCH_RETRY_INTERVAL_MS = 1000;
+
+ public static final String SHUFFLE_FETCH_RETRY_TIMEOUT_MS =
"mapreduce.reduce.shuffle.fetch.retry.timeout-ms";
+
+ public static final String SHUFFLE_FETCH_RETRY_ENABLED =
"mapreduce.reduce.shuffle.fetch.retry.enabled";
public static final String SHUFFLE_NOTIFY_READERROR =
"mapreduce.reduce.shuffle.notify.readerror";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44c71131/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index e1e1663..a416200 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -27,6 +27,7 @@ import java.net.URL;
import java.net.URLConnection;
import java.security.GeneralSecurityException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -46,6 +47,8 @@ impo
git commit: MAPREDUCE-6091. YARNRunner.getJobStatus() fails with ApplicationNotFoundException if the job rolled off the RM view. Contributed by Sangjin Lee
Repository: hadoop
Updated Branches:
refs/heads/trunk 9f03a7c01 -> 951847ba9
MAPREDUCE-6091. YARNRunner.getJobStatus() fails with
ApplicationNotFoundException if the job rolled off the RM view. Contributed by
Sangjin Lee
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/951847ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/951847ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/951847ba
Branch: refs/heads/trunk
Commit: 951847ba94442b8d928435decfbea9ea20882912
Parents: 9f03a7c
Author: Jason Lowe
Authored: Fri Sep 19 20:15:54 2014 +
Committer: Jason Lowe
Committed: Fri Sep 19 20:15:54 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 4
.../org/apache/hadoop/mapred/ClientServiceDelegate.java | 3 +++
.../org/apache/hadoop/mapred/TestClientServiceDelegate.java | 8 +---
.../org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java | 9 ++---
4 files changed, 14 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951847ba/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 0c6bbf7..eb4c251 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -356,6 +356,10 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6086. mapreduce.job.credentials.binary should allow all URIs.
(Zhihai Xu via kasha)
+MAPREDUCE-6091. YARNRunner.getJobStatus() fails with
+ApplicationNotFoundException if the job rolled off the RM view (Sangjin
+Lee via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951847ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index c3eee2c..686fa0c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -150,6 +151,8 @@ public class ClientServiceDelegate {
ApplicationReport application = null;
try {
application = rm.getApplicationReport(appId);
+} catch (ApplicationNotFoundException e) {
+ application = null;
} catch (YarnException e2) {
throw new IOException(e2);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/951847ba/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
index 5639e5d..7d6b2f3 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
@@ -31,8 +31,6 @@ import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobStatus;
@@ -56,8 +54,10 @@ imp
git commit: MAPREDUCE-6091. YARNRunner.getJobStatus() fails with ApplicationNotFoundException if the job rolled off the RM view. Contributed by Sangjin Lee (cherry picked from commit 951847ba94442b8d9
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5f6129feb -> bdb864e5e
MAPREDUCE-6091. YARNRunner.getJobStatus() fails with
ApplicationNotFoundException if the job rolled off the RM view. Contributed by
Sangjin Lee
(cherry picked from commit 951847ba94442b8d928435decfbea9ea20882912)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdb864e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdb864e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdb864e5
Branch: refs/heads/branch-2
Commit: bdb864e5e183b47d7cb14bcc0b5fc23312dbb5f9
Parents: 5f6129f
Author: Jason Lowe
Authored: Fri Sep 19 20:15:54 2014 +
Committer: Jason Lowe
Committed: Fri Sep 19 20:17:44 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 4
.../org/apache/hadoop/mapred/ClientServiceDelegate.java | 3 +++
.../org/apache/hadoop/mapred/TestClientServiceDelegate.java | 8 +---
.../org/apache/hadoop/mapreduce/v2/TestNonExistentJob.java | 9 ++---
4 files changed, 14 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb864e5/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 0e6a6cd..7d1768f 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -139,6 +139,10 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6086. mapreduce.job.credentials.binary should allow all URIs.
(Zhihai Xu via kasha)
+MAPREDUCE-6091. YARNRunner.getJobStatus() fails with
+ApplicationNotFoundException if the job rolled off the RM view (Sangjin
+Lee via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb864e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
index c3eee2c..686fa0c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -150,6 +151,8 @@ public class ClientServiceDelegate {
ApplicationReport application = null;
try {
application = rm.getApplicationReport(appId);
+} catch (ApplicationNotFoundException e) {
+ application = null;
} catch (YarnException e2) {
throw new IOException(e2);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb864e5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
index 5639e5d..7d6b2f3 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java
@@ -31,8 +31,6 @@ import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.JobSta
git commit: MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs. Contributed by Gera Shegalov
Repository: hadoop
Updated Branches:
refs/heads/trunk 9721e2c1f -> 7039b98e1
MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs. Contributed by Gera
Shegalov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7039b98e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7039b98e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7039b98e
Branch: refs/heads/trunk
Commit: 7039b98e1c459e9e0d8caa28cdaa2868e2bde2eb
Parents: 9721e2c
Author: Jason Lowe
Authored: Mon Sep 22 15:20:59 2014 +
Committer: Jason Lowe
Committed: Mon Sep 22 15:20:59 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../org/apache/hadoop/mapred/YarnChild.java | 63 +---
.../hadoop/mapreduce/v2/app/MRAppMaster.java| 1 +
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 61 +--
.../apache/hadoop/mapreduce/v2/TestUberAM.java | 7 ---
5 files changed, 62 insertions(+), 73 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7039b98e/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index eb4c251..815b3fb 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -360,6 +360,9 @@ Release 2.6.0 - UNRELEASED
ApplicationNotFoundException if the job rolled off the RM view (Sangjin
Lee via jlowe)
+MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs (Gera Shegalov
+via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7039b98e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index 4ba1991..92bbc4a 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -23,15 +23,11 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
-import java.net.URI;
import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
@@ -43,7 +39,6 @@ import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.counters.Limits;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
@@ -307,7 +302,7 @@ class YarnChild {
task.localizeConfiguration(job);
// Set up the DistributedCache related configs
-setupDistributedCacheConfig(job);
+MRApps.setupDistributedCacheLocal(job);
// Overwrite the localized task jobconf which is linked to in the current
// work-dir.
@@ -317,62 +312,6 @@ class YarnChild {
task.setConf(job);
}
- /**
- * Set up the DistributedCache related configs to make
- * {@link DistributedCache#getLocalCacheFiles(Configuration)}
- * and
- * {@link DistributedCache#getLocalCacheArchives(Configuration)}
- * working.
- * @param job
- * @throws IOException
- */
- private static void setupDistributedCacheConfig(final JobConf job)
- throws IOException {
-
-String localWorkDir = System.getenv("PWD");
-//^ ^ all symlinks are created in the current work-dir
-
-// Update the configuration object with localized archives.
-URI[] cacheArchives = DistributedCache.getCacheArchives(job);
-if (cacheArchives != null) {
- List localArchives = new ArrayList();
- for (int i = 0; i < cacheArchives.length; ++i) {
-URI u = cacheArchives[i];
-Pat
git commit: MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs. Contributed by Gera Shegalov (cherry picked from commit 7039b98e1c459e9e0d8caa28cdaa2868e2bde2eb)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c1a3819a4 -> f4534746f
MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs. Contributed by Gera
Shegalov
(cherry picked from commit 7039b98e1c459e9e0d8caa28cdaa2868e2bde2eb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4534746
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4534746
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4534746
Branch: refs/heads/branch-2
Commit: f4534746fbc2e4c216fbae63cacc4430607dc8ca
Parents: c1a3819
Author: Jason Lowe
Authored: Mon Sep 22 15:20:59 2014 +
Committer: Jason Lowe
Committed: Mon Sep 22 15:22:58 2014 +
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../org/apache/hadoop/mapred/YarnChild.java | 63 +---
.../hadoop/mapreduce/v2/app/MRAppMaster.java| 1 +
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 61 +--
.../apache/hadoop/mapreduce/v2/TestUberAM.java | 7 ---
5 files changed, 62 insertions(+), 73 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4534746/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 7d1768f..9d86721 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -143,6 +143,9 @@ Release 2.6.0 - UNRELEASED
ApplicationNotFoundException if the job rolled off the RM view (Sangjin
Lee via jlowe)
+MAPREDUCE-6095. Enable DistributedCache for uber-mode Jobs (Gera Shegalov
+via jlowe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4534746/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
index 061114e..37dbad3 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java
@@ -23,15 +23,11 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetSocketAddress;
-import java.net.URI;
import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
@@ -43,7 +39,6 @@ import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.counters.Limits;
-import org.apache.hadoop.mapreduce.filecache.DistributedCache;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
@@ -301,7 +296,7 @@ class YarnChild {
task.localizeConfiguration(job);
// Set up the DistributedCache related configs
-setupDistributedCacheConfig(job);
+MRApps.setupDistributedCacheLocal(job);
// Overwrite the localized task jobconf which is linked to in the current
// work-dir.
@@ -311,62 +306,6 @@ class YarnChild {
task.setConf(job);
}
- /**
- * Set up the DistributedCache related configs to make
- * {@link DistributedCache#getLocalCacheFiles(Configuration)}
- * and
- * {@link DistributedCache#getLocalCacheArchives(Configuration)}
- * working.
- * @param job
- * @throws IOException
- */
- private static void setupDistributedCacheConfig(final JobConf job)
- throws IOException {
-
-String localWorkDir = System.getenv("PWD");
-//^ ^ all symlinks are created in the current work-dir
-
-// Update the configuration object with localized archives.
-URI[] cacheArchives = DistributedCache.getCacheArchives(job);
-if (cacheArchives != null) {
- List localArchives = new ArrayList();
- for (int i = 0; i < cacheArch
svn commit: r1588470 - /hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
Author: jlowe
Date: Fri Apr 18 13:40:19 2014
New Revision: 1588470
URL: http://svn.apache.org/r1588470
Log:
svn merge -c 1451291 FIXES: HADOOP-9230. TestUniformSizeInputFormat fails
intermittently. Contributed by Karthik Kambatla
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java?rev=1588470&r1=1588469&r2=1588470&view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java
Fri Apr 18 13:40:19 2014
@@ -33,8 +33,6 @@ import org.apache.hadoop.tools.CopyListi
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.hadoop.tools.StubContext;
import org.apache.hadoop.security.Credentials;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -48,9 +46,6 @@ import java.util.Random;
public class TestUniformSizeInputFormat {
- private static final Log LOG
-= LogFactory.getLog(TestUniformSizeInputFormat.class);
-
private static MiniDFSCluster cluster;
private static final int N_FILES = 20;
private static final int SIZEOF_EACH_FILE=1024;
@@ -118,12 +113,9 @@ public class TestUniformSizeInputFormat
List splits
= uniformSizeInputFormat.getSplits(jobContext);
-List legacySplits = legacyGetSplits(listFile, nMaps);
-
int sizePerMap = totalFileSize/nMaps;
checkSplits(listFile, splits);
-checkAgainstLegacy(splits, legacySplits);
int doubleCheckedTotalSize = 0;
int previousSplitSize = -1;
@@ -155,57 +147,6 @@ public class TestUniformSizeInputFormat
Assert.assertEquals(totalFileSize, doubleCheckedTotalSize);
}
- // From
- //
http://svn.apache.org/repos/asf/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
- private List legacyGetSplits(Path listFile, int numSplits)
- throws IOException {
-
-FileSystem fs = cluster.getFileSystem();
-FileStatus srcst = fs.getFileStatus(listFile);
-Configuration conf = fs.getConf();
-
-ArrayList splits = new ArrayList(numSplits);
-FileStatus value = new FileStatus();
-Text key = new Text();
-final long targetsize = totalFileSize / numSplits;
-long pos = 0L;
-long last = 0L;
-long acc = 0L;
-long cbrem = srcst.getLen();
-SequenceFile.Reader sl = null;
-
-LOG.info("Average bytes per map: " + targetsize +
-", Number of maps: " + numSplits + ", total size: " + totalFileSize);
-
-try {
- sl = new SequenceFile.Reader(conf, SequenceFile.Reader.file(listFile));
- for (; sl.next(key, value); last = sl.getPosition()) {
-// if adding this split would put this split past the target size,
-// cut the last split and put this next file in the next split.
-if (acc + value.getLen() > targetsize && acc != 0) {
- long splitsize = last - pos;
- FileSplit fileSplit = new FileSplit(listFile, pos, splitsize, null);
- LOG.info ("Creating split : " + fileSplit + ", bytes in split: " +
splitsize);
- splits.add(fileSplit);
- cbrem -= splitsize;
- pos = last;
- acc = 0L;
-}
-acc += value.getLen();
- }
-}
-finally {
- IOUtils.closeStream(sl);
-}
-if (cbrem != 0) {
- FileSplit fileSplit = new FileSplit(listFile, pos, cbrem, null);
- LOG.info ("Creating split : " + fileSplit + ", bytes in split: " +
cbrem);
- splits.add(fileSplit);
-}
-
-return splits;
- }
-
private void checkSplits(Path listFile, List splits) throws
IOException {
long lastEnd = 0;
@@ -233,18 +174,6 @@ public class TestUniformSizeInputFormat
}
}
- private void checkAgainstLegacy(List splits,
- List legacySplits)
- throws IOException, InterruptedException {
-
-Assert.assertEquals(legacySplits.size(), splits.size());
-for (int index = 0; index < splits.size(); index++) {
- FileSplit fileSplit = (FileSplit) splits.get(index);
- FileSplit legacyFileSplit = (FileSplit) legacySplits.get(index);
- Assert.assertEquals(fileSplit.getStart(), legacyFileSplit.getStart());
-}
- }
-
@Test
public void testGetSplits() throws Exception {
testGetSplits(9);
svn commit: r1588470 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: jlowe Date: Fri Apr 18 13:40:19 2014 New Revision: 1588470 URL: http://svn.apache.org/r1588470 Log: svn merge -c 1451291 FIXES: HADOOP-9230. TestUniformSizeInputFormat fails intermittently. Contributed by Karthik Kambatla Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1588470&r1=1588469&r2=1588470&view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Fri Apr 18 13:40:19 2014 @@ -35,6 +35,9 @@ Release 0.23.11 - UNRELEASED HADOOP-10332. HttpServer's jetty audit log always logs 200 OK (jeagles) +HADOOP-9230. TestUniformSizeInputFormat fails intermittently. + (kkambatl via jlowe) + Release 0.23.10 - 2013-12-09 INCOMPATIBLE CHANGES
svn commit: r1596041 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: jlowe Date: Mon May 19 20:13:14 2014 New Revision: 1596041 URL: http://svn.apache.org/r1596041 Log: Reverting HADOOP-10485 from branch-2. Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1596041&r1=1596040&r2=1596041&view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Mon May 19 20:13:14 2014 @@ -23,8 +23,6 @@ Release 2.5.0 - UNRELEASED HADOOP-10104. Update jackson to 1.9.13 (Akira Ajisaka via stevel) -HADOOP-10485. Remove dead classes in hadoop-streaming. (wheat9) - HADOOP-10503. Move junit up to v 4.11. (cnauroth) HADOOP-10535. Make the retry numbers in ActiveStandbyElector configurable.
svn commit: r1596041 - in /hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming: ./ dev-support/ src/main/java/org/apache/hadoop/record/ src/main/java/org/apache/hadoop/record/compiler/ src/m
Author: jlowe Date: Mon May 19 20:13:14 2014 New Revision: 1596041 URL: http://svn.apache.org/r1596041 Log: Reverting HADOOP-10485 from branch-2. Added: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/CsvRecordInput.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/CsvRecordInput.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/XmlRecordInput.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/XmlRecordInput.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/XmlRecordOutput.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/XmlRecordOutput.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/ - copied from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/CGenerator.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/CGenerator.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/CodeBuffer.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/CodeBuffer.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/CodeGenerator.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/CodeGenerator.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/Consts.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/Consts.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/CppGenerator.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/CppGenerator.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JBoolean.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JBoolean.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JBuffer.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JBuffer.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JByte.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JByte.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JCompType.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JCompType.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JDouble.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JDouble.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JField.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JField.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JFile.java - copied unchanged from r1586059, hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JFile.java hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/compiler/JFloat.java - copied unchanged from r1586059, hadoop/common
svn commit: r1596048 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ dev-support/ src/main/java/org/apache/hadoop/record/ src/main/java/org/apache/hadoop/record/compiler/
Author: jlowe Date: Mon May 19 20:31:40 2014 New Revision: 1596048 URL: http://svn.apache.org/r1596048 Log: Reverted HADOOP-10474 from branch-2. Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/BinaryRecordInput.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/BinaryRecordInput.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/BinaryRecordOutput.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/BinaryRecordOutput.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/CsvRecordInput.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/CsvRecordInput.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/CsvRecordOutput.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/CsvRecordOutput.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Index.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Index.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Record.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Record.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordComparator.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordComparator.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordInput.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordInput.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Utils.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Utils.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/XmlRecordInput.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/XmlRecordInput.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/XmlRecordOutput.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/XmlRecordOutput.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CGenerator.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CGenerator.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CodeBuffer.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CodeBuffer.java hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CodeGenerator.java - copied unchanged from r1585886, hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/compiler/CodeGenerator.java hadoop/common/branches/branch-2/hadoop-common-project
svn commit: r1596048 - in /hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming: dev-support/findbugs-exclude.xml pom.xml src/main/java/org/apache/hadoop/record/
Author: jlowe Date: Mon May 19 20:31:40 2014 New Revision: 1596048 URL: http://svn.apache.org/r1596048 Log: Reverted HADOOP-10474 from branch-2. Removed: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/record/ Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml?rev=1596048&r1=1596047&r2=1596048&view=diff == --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/dev-support/findbugs-exclude.xml Mon May 19 20:31:40 2014 @@ -16,19 +16,6 @@ limitations under the License. --> - - - - - - - - - - - - - @@ -40,32 +27,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml?rev=1596048&r1=1596047&r2=1596048&view=diff == --- hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml (original) +++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/pom.xml Mon May 19 20:31:40 2014 @@ -42,11 +42,6 @@ test - org.apache.ant - ant - provided - - org.apache.hadoop hadoop-annotations provided
svn commit: r1596049 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: jlowe Date: Mon May 19 20:37:49 2014 New Revision: 1596049 URL: http://svn.apache.org/r1596049 Log: Updated CHANGES.txt to reflect revert of HADOOP 10474 and 10485 from branch-2 Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1596049&r1=1596048&r2=1596049&view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Mon May 19 20:37:49 2014 @@ -7,6 +7,8 @@ Trunk (Unreleased) HADOOP-8124. Remove the deprecated FSDataOutputStream constructor, FSDataOutputStream.sync() and Syncable.sync(). (szetszwo) +HADOOP-10474 Move o.a.h.record to hadoop-streaming. (wheat9) + NEW FEATURES HADOOP-10433. Key Management Server based on KeyProvider API. (tucu) @@ -148,6 +150,8 @@ Trunk (Unreleased) HADOOP-10563. Remove the dependency of jsp in trunk. (wheat9) +HADOOP-10485. Remove dead classes in hadoop-streaming. (wheat9) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. @@ -340,8 +344,6 @@ Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES -HADOOP-10474 Move o.a.h.record to hadoop-streaming. (wheat9) - NEW FEATURES HADOOP-10498. Add support for proxy server. (daryn) @@ -359,8 +361,6 @@ Release 2.5.0 - UNRELEASED HADOOP-10104. Update jackson to 1.9.13 (Akira Ajisaka via stevel) -HADOOP-10485. Remove dead classes in hadoop-streaming. (wheat9) - HADOOP-10503. Move junit up to v 4.11. (cnauroth) HADOOP-10535. Make the retry numbers in ActiveStandbyElector configurable.
svn commit: r1602033 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/util/Shell.java
Author: jlowe
Date: Wed Jun 11 22:05:04 2014
New Revision: 1602033
URL: http://svn.apache.org/r1602033
Log:
HADOOP-10622. Shell.runCommand can deadlock. Contributed by Gera Shegalov
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1602033&r1=1602032&r2=1602033&view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Jun
11 22:05:04 2014
@@ -532,6 +532,8 @@ Release 2.5.0 - UNRELEASED
HADOOP-10656. The password keystore file is not picked by LDAP group
mapping
(brandonli)
+HADOOP-10622. Shell.runCommand can deadlock (Gera Shegalov via jlowe)
+
BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
HADOOP-10520. Extended attributes definition and FileSystem APIs for
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1602033&r1=1602032&r2=1602033&view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Wed Jun 11 22:05:04 2014
@@ -526,12 +526,8 @@ abstract public class Shell {
}
// wait for the process to finish and check the exit code
exitCode = process.waitFor();
- try {
-// make sure that the error thread exits
-errThread.join();
- } catch (InterruptedException ie) {
-LOG.warn("Interrupted while reading the error stream", ie);
- }
+ // make sure that the error thread exits
+ joinThread(errThread);
completed.set(true);
//the timeout thread handling
//taken care in finally block
@@ -560,13 +556,9 @@ abstract public class Shell {
} catch (IOException ioe) {
LOG.warn("Error while closing the input stream", ioe);
}
- try {
-if (!completed.get()) {
- errThread.interrupt();
- errThread.join();
-}
- } catch (InterruptedException ie) {
-LOG.warn("Interrupted while joining errThread");
+ if (!completed.get()) {
+errThread.interrupt();
+joinThread(errThread);
}
try {
InputStream stderr = process.getErrorStream();
@@ -581,6 +573,19 @@ abstract public class Shell {
}
}
+ private static void joinThread(Thread t) {
+while (t.isAlive()) {
+ try {
+t.join();
+ } catch (InterruptedException ie) {
+if (LOG.isWarnEnabled()) {
+ LOG.warn("Interrupted while joining on: " + t, ie);
+}
+t.interrupt(); // propagate interrupt
+ }
+}
+ }
+
/** return an array containing the command name & its parameters */
protected abstract String[] getExecString();
svn commit: r1602034 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/util/Shell.java
Author: jlowe
Date: Wed Jun 11 22:06:12 2014
New Revision: 1602034
URL: http://svn.apache.org/r1602034
Log:
svn merge -c 1602033 FIXES: HADOOP-10622. Shell.runCommand can deadlock.
Contributed by Gera Shegalov
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1602034&r1=1602033&r2=1602034&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Jun 11 22:06:12 2014
@@ -195,6 +195,8 @@ Release 2.5.0 - UNRELEASED
HADOOP-10656. The password keystore file is not picked by LDAP group
mapping
(brandonli)
+HADOOP-10622. Shell.runCommand can deadlock (Gera Shegalov via jlowe)
+
BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
HADOOP-10520. Extended attributes definition and FileSystem APIs for
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java?rev=1602034&r1=1602033&r2=1602034&view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
Wed Jun 11 22:06:12 2014
@@ -526,12 +526,8 @@ abstract public class Shell {
}
// wait for the process to finish and check the exit code
exitCode = process.waitFor();
- try {
-// make sure that the error thread exits
-errThread.join();
- } catch (InterruptedException ie) {
-LOG.warn("Interrupted while reading the error stream", ie);
- }
+ // make sure that the error thread exits
+ joinThread(errThread);
completed.set(true);
//the timeout thread handling
//taken care in finally block
@@ -560,13 +556,9 @@ abstract public class Shell {
} catch (IOException ioe) {
LOG.warn("Error while closing the input stream", ioe);
}
- try {
-if (!completed.get()) {
- errThread.interrupt();
- errThread.join();
-}
- } catch (InterruptedException ie) {
-LOG.warn("Interrupted while joining errThread");
+ if (!completed.get()) {
+errThread.interrupt();
+joinThread(errThread);
}
try {
InputStream stderr = process.getErrorStream();
@@ -581,6 +573,19 @@ abstract public class Shell {
}
}
+ private static void joinThread(Thread t) {
+while (t.isAlive()) {
+ try {
+t.join();
+ } catch (InterruptedException ie) {
+if (LOG.isWarnEnabled()) {
+ LOG.warn("Interrupted while joining on: " + t, ie);
+}
+t.interrupt(); // propagate interrupt
+ }
+}
+ }
+
/** return an array containing the command name & its parameters */
protected abstract String[] getExecString();
svn commit: r1603042 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
Author: jlowe
Date: Tue Jun 17 02:23:58 2014
New Revision: 1603042
URL: http://svn.apache.org/r1603042
Log:
HADOOP-10699. Fix build native library on mac osx. Contributed by Binglin Chang
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1603042&r1=1603041&r2=1603042&view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jun
17 02:23:58 2014
@@ -562,6 +562,9 @@ Release 2.5.0 - UNRELEASED
HADOOP-10702. KerberosAuthenticationHandler does not log the principal
names
correctly. (Benoy Antony via cnauroth)
+HADOOP-10699. Fix build native library on mac osx (Binglin Chang via
+jlowe)
+
BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
HADOOP-10520. Extended attributes definition and FileSystem APIs for
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c?rev=1603042&r1=1603041&r2=1603042&view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c
Tue Jun 17 02:23:58 2014
@@ -73,7 +73,10 @@ Java_org_apache_hadoop_security_JniBased
// was successful or not (as long as it was called we need to call
// endnetgrent)
setnetgrentCalledFlag = 1;
-#ifndef __FreeBSD__
+#if defined(__FreeBSD__) || defined(__MACH__)
+ setnetgrent(cgroup);
+ {
+#else
if(setnetgrent(cgroup) == 1) {
#endif
current = NULL;
@@ -90,9 +93,7 @@ Java_org_apache_hadoop_security_JniBased
userListSize++;
}
}
-#ifndef __FreeBSD__
}
-#endif
//--
// build return data (java array)
