hadoop git commit: YARN-3980. Plumb resource-utilization info in node heartbeat through to the scheduler. (Inigo Goiri via kasha)
Repository: hadoop Updated Branches: refs/heads/trunk f80dc6f49 -> 52948bb20 YARN-3980. Plumb resource-utilization info in node heartbeat through to the scheduler. (Inigo Goiri via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52948bb2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52948bb2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52948bb2 Branch: refs/heads/trunk Commit: 52948bb20bd1446164df1d3920c46c96dad750ae Parents: f80dc6f Author: Karthik Kambatla Authored: Tue Nov 24 10:05:12 2015 +0530 Committer: Karthik Kambatla Committed: Tue Nov 24 13:47:17 2015 +0530 -- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 11 + .../yarn/sls/scheduler/RMNodeWrapper.java | 11 + hadoop-yarn-project/CHANGES.txt | 3 + .../impl/pb/ResourceUtilizationPBImpl.java | 2 +- .../nodemanager/NodeStatusUpdaterImpl.java | 3 +- .../resourcemanager/ResourceTrackerService.java | 5 +- .../server/resourcemanager/rmnode/RMNode.java | 15 +- .../resourcemanager/rmnode/RMNodeImpl.java | 55 + .../rmnode/RMNodeStatusEvent.java | 53 ++-- .../scheduler/SchedulerNode.java| 38 +++ .../scheduler/capacity/CapacityScheduler.java | 5 + .../scheduler/fair/FairScheduler.java | 5 + .../scheduler/fifo/FifoScheduler.java | 4 + .../yarn/server/resourcemanager/MockNodes.java | 28 ++- .../resourcemanager/TestRMNodeTransitions.java | 16 +- .../TestRMAppLogAggregationStatus.java | 32 +-- .../webapp/TestRMWebServicesNodes.java | 6 +- .../hadoop/yarn/server/MiniYARNCluster.java | 54 +++- .../TestMiniYarnClusterNodeUtilization.java | 245 +++ 19 files changed, 522 insertions(+), 69 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/52948bb2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index dae2ce7..f5943a8 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -188,6 +189,16 @@ public class NodeInfo { // TODO Auto-generated method stub return null; } + +@Override +public ResourceUtilization getAggregatedContainersUtilization() { + return null; +} + +@Override +public ResourceUtilization getNodeUtilization() { + return null; +} } public static RMNode newNodeInfo(String rackName, String hostName, http://git-wip-us.apache.org/repos/asf/hadoop/blob/52948bb2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index 8c65ccc..e778188 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -176,4 +177,14 @@ public class RMNodeWrapper implements RMNode { // TODO Auto-generated method stub return null; } + + @Override + public ResourceUtilization getAggreg
hadoop git commit: YARN-3980. Plumb resource-utilization info in node heartbeat through to the scheduler. (Inigo Goiri via kasha)
Repository: hadoop Updated Branches: refs/heads/branch-2 52c889ac1 -> 49ed955c9 YARN-3980. Plumb resource-utilization info in node heartbeat through to the scheduler. (Inigo Goiri via kasha) (cherry picked from commit 52948bb20bd1446164df1d3920c46c96dad750ae) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/49ed955c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/49ed955c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/49ed955c Branch: refs/heads/branch-2 Commit: 49ed955c91a3c495f40a59b8b13371bcebce473d Parents: 52c889a Author: Karthik Kambatla Authored: Tue Nov 24 10:05:12 2015 +0530 Committer: Karthik Kambatla Committed: Tue Nov 24 13:52:29 2015 +0530 -- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 11 + .../yarn/sls/scheduler/RMNodeWrapper.java | 11 + hadoop-yarn-project/CHANGES.txt | 3 + .../impl/pb/ResourceUtilizationPBImpl.java | 2 +- .../nodemanager/NodeStatusUpdaterImpl.java | 3 +- .../resourcemanager/ResourceTrackerService.java | 5 +- .../server/resourcemanager/rmnode/RMNode.java | 15 +- .../resourcemanager/rmnode/RMNodeImpl.java | 55 + .../rmnode/RMNodeStatusEvent.java | 53 ++-- .../scheduler/SchedulerNode.java| 38 +++ .../scheduler/capacity/CapacityScheduler.java | 5 + .../scheduler/fair/FairScheduler.java | 5 + .../scheduler/fifo/FifoScheduler.java | 4 + .../yarn/server/resourcemanager/MockNodes.java | 28 ++- .../resourcemanager/TestRMNodeTransitions.java | 16 +- .../TestRMAppLogAggregationStatus.java | 32 +-- .../webapp/TestRMWebServicesNodes.java | 6 +- .../hadoop/yarn/server/MiniYARNCluster.java | 54 +++- .../TestMiniYarnClusterNodeUtilization.java | 245 +++ 19 files changed, 522 insertions(+), 69 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/49ed955c/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index dae2ce7..f5943a8 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -188,6 +189,16 @@ public class NodeInfo { // TODO Auto-generated method stub return null; } + +@Override +public ResourceUtilization getAggregatedContainersUtilization() { + return null; +} + +@Override +public ResourceUtilization getNodeUtilization() { + return null; +} } public static RMNode newNodeInfo(String rackName, String hostName, http://git-wip-us.apache.org/repos/asf/hadoop/blob/49ed955c/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index 8c65ccc..e778188 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.records.ResourceUtilization; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -176,4 +177,14 @@ public class RMNodeWrapper implements RMNode { // TODO Auto-generated method stub
hadoop git commit: CHANGES.txt: add YARN-4367 to 2.8.0.
Repository: hadoop Updated Branches: refs/heads/trunk 52948bb20 -> fb0f09e46 CHANGES.txt: add YARN-4367 to 2.8.0. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb0f09e4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb0f09e4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb0f09e4 Branch: refs/heads/trunk Commit: fb0f09e46b456789ec1c7470873b6de231430773 Parents: 52948bb Author: Tsuyoshi Ozawa Authored: Tue Nov 24 17:39:34 2015 +0900 Committer: Tsuyoshi Ozawa Committed: Tue Nov 24 17:39:34 2015 +0900 -- hadoop-yarn-project/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb0f09e4/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0532e1d..24807d7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1059,6 +1059,8 @@ Release 2.8.0 - UNRELEASED YARN-4345. yarn rmadmin -updateNodeResource doesn't work (Junping Du via jlowe) +YARN-4367. SLS webapp doesn't load. (kasha). + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES
hadoop git commit: YARN-4298. Fix findbugs warnings in hadoop-yarn-common. Contributed by Sunil G.
Repository: hadoop Updated Branches: refs/heads/trunk fb0f09e46 -> 8c0133f3d YARN-4298. Fix findbugs warnings in hadoop-yarn-common. Contributed by Sunil G. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c0133f3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c0133f3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c0133f3 Branch: refs/heads/trunk Commit: 8c0133f3d4c85733c1e0389d1ee39f1eab58c0b9 Parents: fb0f09e Author: Akira Ajisaka Authored: Tue Nov 24 18:40:33 2015 +0900 Committer: Akira Ajisaka Committed: Tue Nov 24 18:40:33 2015 +0900 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c0133f3/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 24807d7..6558e40 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1061,6 +1061,9 @@ Release 2.8.0 - UNRELEASED YARN-4367. SLS webapp doesn't load. (kasha). +YARN-4298. Fix findbugs warnings in hadoop-yarn-common. +(Sunil G via aajisaka) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c0133f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java index bd460f6..da87465 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java @@ -386,7 +386,7 @@ public class AllocateResponsePBImpl extends AllocateResponse { } @Override - public Priority getApplicationPriority() { + public synchronized Priority getApplicationPriority() { AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; if (this.appPriority != null) { return this.appPriority; @@ -399,7 +399,7 @@ public class AllocateResponsePBImpl extends AllocateResponse { } @Override - public void setApplicationPriority(Priority priority) { + public synchronized void setApplicationPriority(Priority priority) { maybeInitBuilder(); if (priority == null) builder.clearApplicationPriority();
hadoop git commit: YARN-4298. Fix findbugs warnings in hadoop-yarn-common. Contributed by Sunil G.
Repository: hadoop Updated Branches: refs/heads/branch-2 49ed955c9 -> ac0ddc4fe YARN-4298. Fix findbugs warnings in hadoop-yarn-common. Contributed by Sunil G. (cherry picked from commit 8c0133f3d4c85733c1e0389d1ee39f1eab58c0b9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac0ddc4f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac0ddc4f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac0ddc4f Branch: refs/heads/branch-2 Commit: ac0ddc4fec907d19a95c5e5305bb27a82d0fa085 Parents: 49ed955 Author: Akira Ajisaka Authored: Tue Nov 24 18:40:33 2015 +0900 Committer: Akira Ajisaka Committed: Tue Nov 24 18:40:56 2015 +0900 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0ddc4f/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e17883d..d2ce060 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1009,6 +1009,9 @@ Release 2.8.0 - UNRELEASED YARN-4367. SLS webapp doesn't load. (kasha) +YARN-4298. Fix findbugs warnings in hadoop-yarn-common. +(Sunil G via aajisaka) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac0ddc4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java index bd460f6..da87465 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java @@ -386,7 +386,7 @@ public class AllocateResponsePBImpl extends AllocateResponse { } @Override - public Priority getApplicationPriority() { + public synchronized Priority getApplicationPriority() { AllocateResponseProtoOrBuilder p = viaProto ? proto : builder; if (this.appPriority != null) { return this.appPriority; @@ -399,7 +399,7 @@ public class AllocateResponsePBImpl extends AllocateResponse { } @Override - public void setApplicationPriority(Priority priority) { + public synchronized void setApplicationPriority(Priority priority) { maybeInitBuilder(); if (priority == null) builder.clearApplicationPriority();
hadoop git commit: YARN-4387. Fix typo in FairScheduler log message. Contributed by Xin Wang.
Repository: hadoop Updated Branches: refs/heads/trunk 8c0133f3d -> 28dfe721b YARN-4387. Fix typo in FairScheduler log message. Contributed by Xin Wang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28dfe721 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28dfe721 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28dfe721 Branch: refs/heads/trunk Commit: 28dfe721b86ccbaf2ddcfb7e709b226ac766803a Parents: 8c0133f Author: Tsuyoshi Ozawa Authored: Tue Nov 24 17:35:38 2015 +0900 Committer: Tsuyoshi Ozawa Committed: Tue Nov 24 19:24:01 2015 +0900 -- hadoop-yarn-project/CHANGES.txt| 2 ++ .../yarn/server/resourcemanager/scheduler/fair/FairScheduler.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28dfe721/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6558e40..204c338 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1064,6 +1064,8 @@ Release 2.8.0 - UNRELEASED YARN-4298. Fix findbugs warnings in hadoop-yarn-common. (Sunil G via aajisaka) +YARN-4387. Fix typo in FairScheduler log message. (Xin Wang via ozawa) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/28dfe721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index f1839f5..04977a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -500,7 +500,7 @@ public class FairScheduler extends // containers on the RMNode (see SchedulerNode.releaseContainer()). completedContainer(container, status, RMContainerEventType.KILL); LOG.info("Killing container" + container + -" (after waiting for premption for " + +" (after waiting for preemption for " + (getClock().getTime() - time) + "ms)"); } } else {
hadoop git commit: YARN-4387. Fix typo in FairScheduler log message. Contributed by Xin Wang.
Repository: hadoop Updated Branches: refs/heads/branch-2 ac0ddc4fe -> 9e458c3e9 YARN-4387. Fix typo in FairScheduler log message. Contributed by Xin Wang. (cherry picked from commit 28dfe721b86ccbaf2ddcfb7e709b226ac766803a) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e458c3e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e458c3e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e458c3e Branch: refs/heads/branch-2 Commit: 9e458c3e9e5d6ae7850b3d56309157e2213a6603 Parents: ac0ddc4 Author: Tsuyoshi Ozawa Authored: Tue Nov 24 17:35:38 2015 +0900 Committer: Tsuyoshi Ozawa Committed: Tue Nov 24 22:10:31 2015 +0900 -- hadoop-yarn-project/CHANGES.txt| 2 ++ .../yarn/server/resourcemanager/scheduler/fair/FairScheduler.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e458c3e/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d2ce060..fa2049a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1012,6 +1012,8 @@ Release 2.8.0 - UNRELEASED YARN-4298. Fix findbugs warnings in hadoop-yarn-common. (Sunil G via aajisaka) +YARN-4387. Fix typo in FairScheduler log message. (Xin Wang via ozawa) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e458c3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index f1839f5..04977a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -500,7 +500,7 @@ public class FairScheduler extends // containers on the RMNode (see SchedulerNode.releaseContainer()). completedContainer(container, status, RMContainerEventType.KILL); LOG.info("Killing container" + container + -" (after waiting for premption for " + +" (after waiting for preemption for " + (getClock().getTime() - time) + "ms)"); } } else {
hadoop git commit: HDFS-9289. Make DataStreamer#block thread safe and verify genStamp in commitBlock. Contributed by Chang Li.
Repository: hadoop Updated Branches: refs/heads/branch-2.6 b9a6f9aa1 -> 238458b25 HDFS-9289. Make DataStreamer#block thread safe and verify genStamp in commitBlock. Contributed by Chang Li. Conflicts: hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java Change-Id: Ibd44ff1bf92bad7262db724990a6a64c1975ffb6 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/238458b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/238458b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/238458b2 Branch: refs/heads/branch-2.6 Commit: 238458b25921a652eefead2cebd797c1b9de0343 Parents: b9a6f9a Author: Kihwal Lee Authored: Wed Nov 4 12:10:59 2015 -0600 Committer: Zhe Zhang Committed: Tue Nov 24 09:44:50 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +- .../BlockInfoUnderConstruction.java | 2 +- .../server/blockmanagement/BlockManager.java| 4 + .../org/apache/hadoop/hdfs/DFSTestUtil.java | 67 + .../TestCommitBlockWithInvalidGenStamp.java | 98 6 files changed, 174 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/238458b2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index cc7bae8..5e683e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -20,6 +20,9 @@ Release 2.6.3 - UNRELEASED HDFS-9083. Replication violates block placement policy (Rushabh Shah) +HDFS-9289. Make DataStreamer#block thread safe and verify genStamp in +commitBlock. (Chang Li via zhz) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/238458b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 92dbc8e..21e4d4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -363,7 +363,7 @@ public class DFSOutputStream extends FSOutputSummer // class DataStreamer extends Daemon { private volatile boolean streamerClosed = false; -private ExtendedBlock block; // its length is number of bytes acked +private volatile ExtendedBlock block; // its length is number of bytes acked private Token accessToken; private DataOutputStream blockStream; private DataInputStream blockReplyStream; http://git-wip-us.apache.org/repos/asf/hadoop/blob/238458b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java index dd3593f..703373e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java @@ -262,7 +262,7 @@ public class BlockInfoUnderConstruction extends BlockInfo { throw new IOException("Trying to commit inconsistent block: id = " + block.getBlockId() + ", expected id = " + getBlockId()); blockUCState = BlockUCState.COMMITTED; -this.set(getBlockId(), block.getNumBytes(), block.getGenerationStamp()); +this.setNumBytes(block.getNumBytes()); // Sort out invalid replicas. setGenerationStampAndVerifyReplicas(block.getGenerationStamp()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/238458b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoo
[2/2] hadoop git commit: HDFS-6101. TestReplaceDatanodeOnFailure fails occasionally. Contributed by Wei-Chiu Chuang.
HDFS-6101. TestReplaceDatanodeOnFailure fails occasionally. Contributed by Wei-Chiu Chuang. (cherry picked from commit 1777608fa075a807c645619fda87cb8de1b0350c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c79f0177 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c79f0177 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c79f0177 Branch: refs/heads/branch-2 Commit: c79f01775269594f8624264623c8b9a747edbb60 Parents: 9e458c3 Author: cnauroth Authored: Tue Nov 24 09:39:21 2015 -0800 Committer: cnauroth Committed: Tue Nov 24 09:39:37 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/TestReplaceDatanodeOnFailure.java | 64 +++- 2 files changed, 53 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c79f0177/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6955092..421240b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1510,6 +1510,9 @@ Release 2.8.0 - UNRELEASED HDFS-9433. DFS getEZForPath API on a non-existent file should throw FileNotFoundException (Rakesh R via umamahesh) +HDFS-6101. TestReplaceDatanodeOnFailure fails occasionally. +(Wei-Chiu Chuang via cnauroth) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c79f0177/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java index 76d592c..45acd12 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.hdfs; +import com.google.common.base.Supplier; + import java.io.IOException; import java.util.Arrays; +import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -42,7 +46,7 @@ import org.junit.Test; * data can be read by another client. */ public class TestReplaceDatanodeOnFailure { - static final Log LOG = AppendTestUtil.LOG; + static final Log LOG = LogFactory.getLog(TestReplaceDatanodeOnFailure.class); static final String DIR = "/" + TestReplaceDatanodeOnFailure.class.getSimpleName() + "/"; static final short REPLICATION = 3; @@ -114,7 +118,8 @@ public class TestReplaceDatanodeOnFailure { @Test public void testReplaceDatanodeOnFailure() throws Exception { final Configuration conf = new HdfsConfiguration(); - +// do not consider load factor when selecting a data node +conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); //always replace a datanode ReplaceDatanodeOnFailure.write(Policy.ALWAYS, true, conf); @@ -124,31 +129,40 @@ public class TestReplaceDatanodeOnFailure { ).racks(racks).numDataNodes(REPLICATION).build(); try { + cluster.waitActive(); final DistributedFileSystem fs = cluster.getFileSystem(); final Path dir = new Path(DIR); - - final SlowWriter[] slowwriters = new SlowWriter[10]; + final int NUM_WRITERS = 10; + final int FIRST_BATCH = 5; + final SlowWriter[] slowwriters = new SlowWriter[NUM_WRITERS]; for(int i = 1; i <= slowwriters.length; i++) { //create slow writers in different speed slowwriters[i - 1] = new SlowWriter(fs, new Path(dir, "file" + i), i*200L); } - for(SlowWriter s : slowwriters) { -s.start(); + for(int i = 0; i < FIRST_BATCH; i++) { +slowwriters[i].start(); } // Let slow writers write something. - // Some of them are too slow and will be not yet started. - sleepSeconds(1); + // Some of them are too slow and will be not yet started. + sleepSeconds(3); //start new datanodes cluster.startDataNodes(conf, 2, true, null, new String[]{RACK1, RACK1}); + cluster.waitActive(); + // wait for first block reports for up to 10 seconds + cluster.waitFirstBRCompleted(0, 1000
[1/2] hadoop git commit: HDFS-6101. TestReplaceDatanodeOnFailure fails occasionally. Contributed by Wei-Chiu Chuang.
Repository: hadoop Updated Branches: refs/heads/branch-2 9e458c3e9 -> c79f01775 refs/heads/trunk 28dfe721b -> 1777608fa HDFS-6101. TestReplaceDatanodeOnFailure fails occasionally. Contributed by Wei-Chiu Chuang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1777608f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1777608f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1777608f Branch: refs/heads/trunk Commit: 1777608fa075a807c645619fda87cb8de1b0350c Parents: 28dfe72 Author: cnauroth Authored: Tue Nov 24 09:39:21 2015 -0800 Committer: cnauroth Committed: Tue Nov 24 09:39:21 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/TestReplaceDatanodeOnFailure.java | 64 +++- 2 files changed, 53 insertions(+), 14 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1777608f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ca2ed15..d39ed3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2368,6 +2368,9 @@ Release 2.8.0 - UNRELEASED HDFS-9433. DFS getEZForPath API on a non-existent file should throw FileNotFoundException (Rakesh R via umamahesh) +HDFS-6101. TestReplaceDatanodeOnFailure fails occasionally. +(Wei-Chiu Chuang via cnauroth) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/1777608f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java index d351020..bbc447c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplaceDatanodeOnFailure.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.hdfs; +import com.google.common.base.Supplier; + import java.io.IOException; import java.util.Arrays; +import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -41,7 +45,7 @@ import org.junit.Test; * This class tests that data nodes are correctly replaced on failure. */ public class TestReplaceDatanodeOnFailure { - static final Log LOG = AppendTestUtil.LOG; + static final Log LOG = LogFactory.getLog(TestReplaceDatanodeOnFailure.class); static final String DIR = "/" + TestReplaceDatanodeOnFailure.class.getSimpleName() + "/"; static final short REPLICATION = 3; @@ -113,7 +117,8 @@ public class TestReplaceDatanodeOnFailure { @Test public void testReplaceDatanodeOnFailure() throws Exception { final Configuration conf = new HdfsConfiguration(); - +// do not consider load factor when selecting a data node +conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false); //always replace a datanode ReplaceDatanodeOnFailure.write(Policy.ALWAYS, true, conf); @@ -123,31 +128,40 @@ public class TestReplaceDatanodeOnFailure { ).racks(racks).numDataNodes(REPLICATION).build(); try { + cluster.waitActive(); final DistributedFileSystem fs = cluster.getFileSystem(); final Path dir = new Path(DIR); - - final SlowWriter[] slowwriters = new SlowWriter[10]; + final int NUM_WRITERS = 10; + final int FIRST_BATCH = 5; + final SlowWriter[] slowwriters = new SlowWriter[NUM_WRITERS]; for(int i = 1; i <= slowwriters.length; i++) { //create slow writers in different speed slowwriters[i - 1] = new SlowWriter(fs, new Path(dir, "file" + i), i*200L); } - for(SlowWriter s : slowwriters) { -s.start(); + for(int i = 0; i < FIRST_BATCH; i++) { +slowwriters[i].start(); } // Let slow writers write something. - // Some of them are too slow and will be not yet started. - sleepSeconds(1); + // Some of them are too slow and will be not yet started. + sleepSeconds(3); //start new datanodes cluster.startDataNodes(conf, 2, true, null, new String[]{RACK1, RACK1}); + cluster.waitActive(); + // wait for
hadoop git commit: HDFS-9314. Improve BlockPlacementPolicyDefault's picking of excess replicas. (Xiao Chen via mingma)
Repository: hadoop Updated Branches: refs/heads/trunk 1777608fa -> 0e54b164a HDFS-9314. Improve BlockPlacementPolicyDefault's picking of excess replicas. (Xiao Chen via mingma) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e54b164 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e54b164 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e54b164 Branch: refs/heads/trunk Commit: 0e54b164a8d8acf09aca8712116bf7a554cb4846 Parents: 1777608 Author: Ming Ma Authored: Tue Nov 24 10:30:24 2015 -0800 Committer: Ming Ma Committed: Tue Nov 24 10:30:24 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../BlockPlacementPolicyDefault.java| 32 ++-- .../BlockPlacementPolicyRackFaultTolerant.java | 8 ++ .../BlockPlacementPolicyWithNodeGroup.java | 3 +- .../BlockPlacementPolicyWithUpgradeDomain.java | 19 ++--- .../blockmanagement/TestReplicationPolicy.java | 82 .../TestReplicationPolicyWithNodeGroup.java | 6 +- .../TestReplicationPolicyWithUpgradeDomain.java | 32 .../hdfs/server/namenode/ha/TestDNFencing.java | 4 +- 9 files changed, 153 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e54b164/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d39ed3f..b441b35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1676,6 +1676,9 @@ Release 2.8.0 - UNRELEASED HDFS-7988. Replace usage of ExactSizeInputStream with LimitInputStream. (Walter Su via wheat9) +HDFS-9314. Improve BlockPlacementPolicyDefault's picking of excess +replicas. (Xiao Chen via mingma) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e54b164/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 13b17e3..08e7851 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -916,7 +916,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { public DatanodeStorageInfo chooseReplicaToDelete( Collection moreThanOne, Collection exactlyOne, - final List excessTypes) { + final List excessTypes, + Map> rackMap) { long oldestHeartbeat = monotonicNow() - heartbeatInterval * tolerateHeartbeatMultiplier; DatanodeStorageInfo oldestHeartbeatStorage = null; @@ -926,7 +927,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { // Pick the node with the oldest heartbeat or with the least free space, // if all hearbeats are within the tolerable heartbeat interval for(DatanodeStorageInfo storage : pickupReplicaSet(moreThanOne, -exactlyOne)) { +exactlyOne, rackMap)) { if (!excessTypes.contains(storage.getStorageType())) { continue; } @@ -991,7 +992,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { moreThanOne, exactlyOne, excessTypes)) { cur = delNodeHintStorage; } else { // regular excessive replica removal -cur = chooseReplicaToDelete(moreThanOne, exactlyOne, excessTypes); +cur = chooseReplicaToDelete(moreThanOne, exactlyOne, excessTypes, +rackMap); } firstOne = false; if (cur == null) { @@ -1044,16 +1046,34 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { splitNodesWithRack(locs, rackMap, moreThanOne, exactlyOne); return notReduceNumOfGroups(moreThanOne, source, target); } + /** * Pick up replica node set for deleting replica as over-replicated. * First set contains replica nodes on rack with more than one * replica while second set contains remaining replica nodes. - * So pick up first set if not empty. If first is empty, then pick second. + * If only 1 rack, pick all. If 2 racks, pick all that have more than + * 1 replicas on the same rack; if no such
hadoop git commit: HDFS-9314. Improve BlockPlacementPolicyDefault's picking of excess replicas. (Xiao Chen via mingma)
Repository: hadoop Updated Branches: refs/heads/branch-2 c79f01775 -> 85d04dc46 HDFS-9314. Improve BlockPlacementPolicyDefault's picking of excess replicas. (Xiao Chen via mingma) (cherry picked from commit 0e54b164a8d8acf09aca8712116bf7a554cb4846) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85d04dc4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85d04dc4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85d04dc4 Branch: refs/heads/branch-2 Commit: 85d04dc46494c5b627920bbc021f0515af8f753e Parents: c79f017 Author: Ming Ma Authored: Tue Nov 24 10:30:24 2015 -0800 Committer: Ming Ma Committed: Tue Nov 24 10:31:23 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../BlockPlacementPolicyDefault.java| 32 ++-- .../BlockPlacementPolicyRackFaultTolerant.java | 8 ++ .../BlockPlacementPolicyWithNodeGroup.java | 3 +- .../BlockPlacementPolicyWithUpgradeDomain.java | 19 ++--- .../blockmanagement/TestReplicationPolicy.java | 82 .../TestReplicationPolicyWithNodeGroup.java | 6 +- .../TestReplicationPolicyWithUpgradeDomain.java | 32 .../hdfs/server/namenode/ha/TestDNFencing.java | 4 +- 9 files changed, 153 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d04dc4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 421240b..5cacca3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -811,6 +811,9 @@ Release 2.8.0 - UNRELEASED HDFS-7988. Replace usage of ExactSizeInputStream with LimitInputStream. (Walter Su via wheat9) +HDFS-9314. Improve BlockPlacementPolicyDefault's picking of excess +replicas. (Xiao Chen via mingma) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d04dc4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java index 13b17e3..08e7851 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java @@ -916,7 +916,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { public DatanodeStorageInfo chooseReplicaToDelete( Collection moreThanOne, Collection exactlyOne, - final List excessTypes) { + final List excessTypes, + Map> rackMap) { long oldestHeartbeat = monotonicNow() - heartbeatInterval * tolerateHeartbeatMultiplier; DatanodeStorageInfo oldestHeartbeatStorage = null; @@ -926,7 +927,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { // Pick the node with the oldest heartbeat or with the least free space, // if all hearbeats are within the tolerable heartbeat interval for(DatanodeStorageInfo storage : pickupReplicaSet(moreThanOne, -exactlyOne)) { +exactlyOne, rackMap)) { if (!excessTypes.contains(storage.getStorageType())) { continue; } @@ -991,7 +992,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { moreThanOne, exactlyOne, excessTypes)) { cur = delNodeHintStorage; } else { // regular excessive replica removal -cur = chooseReplicaToDelete(moreThanOne, exactlyOne, excessTypes); +cur = chooseReplicaToDelete(moreThanOne, exactlyOne, excessTypes, +rackMap); } firstOne = false; if (cur == null) { @@ -1044,16 +1046,34 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy { splitNodesWithRack(locs, rackMap, moreThanOne, exactlyOne); return notReduceNumOfGroups(moreThanOne, source, target); } + /** * Pick up replica node set for deleting replica as over-replicated. * First set contains replica nodes on rack with more than one * replica while second set contains remaining replica nodes. - * So pick up first set if not empty. If first is empty, then pick second. + * If only 1 rack, pick all. If 2 racks, p
hadoop git commit: HADOOP-12294. Correct CHANGES.txt description.
Repository: hadoop Updated Branches: refs/heads/trunk 0e54b164a -> 0e1c12c17 HADOOP-12294. Correct CHANGES.txt description. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e1c12c1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e1c12c1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e1c12c1 Branch: refs/heads/trunk Commit: 0e1c12c1740132778fb29e41fc47d374fb87021e Parents: 0e54b16 Author: cnauroth Authored: Tue Nov 24 10:51:56 2015 -0800 Committer: cnauroth Committed: Tue Nov 24 10:51:56 2015 -0800 -- hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e1c12c1/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 873cd64..1f384f2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -35,8 +35,8 @@ Trunk (Unreleased) HADOOP-10787 Rename/remove non-HADOOP_*, etc from the shell scripts. (aw via vvasudev) -HADOOP-12294. Remove the support of the deprecated dfs.umask. -(Chang Li vha wheat9) +HADOOP-12294. Throw an Exception when fs.permissions.umask-mode is +misconfigured (Chang Li vha wheat9) NEW FEATURES
hadoop git commit: Revert "HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. Contributed by Brahma Reddy Battula."
Repository: hadoop Updated Branches: refs/heads/trunk 0e1c12c17 -> 17b1a5482 Revert "HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. Contributed by Brahma Reddy Battula." This reverts commit 4356e8a5ef0ac6d11a34704b80ef360a710e623a. Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17b1a548 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17b1a548 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17b1a548 Branch: refs/heads/trunk Commit: 17b1a5482b32dab82225e8233648990bc77674ba Parents: 0e1c12c Author: Jason Lowe Authored: Tue Nov 24 19:12:04 2015 + Committer: Jason Lowe Committed: Tue Nov 24 19:12:04 2015 + -- hadoop-common-project/hadoop-common/CHANGES.txt| 3 --- .../hadoop/metrics2/impl/MetricsSourceAdapter.java | 17 ++--- 2 files changed, 10 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/17b1a548/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 1f384f2..cc1827b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1492,9 +1492,6 @@ Release 2.7.3 - UNRELEASED HADOOP-12374. Updated expunge command description. (WeiWei Yang via eyang) -HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. -(Brahma Reddy Battula via ozawa) - HADOOP-12348. MetricsSystemImpl creates MetricsSourceAdapter with wrong time unit parameter. (zxu via rkanter) http://git-wip-us.apache.org/repos/asf/hadoop/blob/17b1a548/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java index a3665b3..cbba014 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java @@ -158,7 +158,7 @@ class MetricsSourceAdapter implements DynamicMBean { private void updateJmxCache() { boolean getAllMetrics = false; -synchronized (this) { +synchronized(this) { if (Time.now() - jmxCacheTS >= jmxCacheTTL) { // temporarilly advance the expiry while updating the cache jmxCacheTS = Time.now() + jmxCacheTTL; @@ -169,21 +169,24 @@ class MetricsSourceAdapter implements DynamicMBean { getAllMetrics = true; lastRecsCleared = false; } - } else { + } + else { return; } +} - if (getAllMetrics) { -MetricsCollectorImpl builder = new MetricsCollectorImpl(); -getMetrics(builder, true); - } +if (getAllMetrics) { + MetricsCollectorImpl builder = new MetricsCollectorImpl(); + getMetrics(builder, true); +} +synchronized(this) { updateAttrCache(); if (getAllMetrics) { updateInfoCache(); } jmxCacheTS = Time.now(); - lastRecs = null; // in case regular interval update is not running + lastRecs = null; // in case regular interval update is not running lastRecsCleared = true; } }
hadoop git commit: Revert "HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. Contributed by Brahma Reddy Battula."
Repository: hadoop Updated Branches: refs/heads/branch-2 85d04dc46 -> 792313011 Revert "HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. Contributed by Brahma Reddy Battula." This reverts commit 4356e8a5ef0ac6d11a34704b80ef360a710e623a. Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java (cherry picked from commit 17b1a5482b32dab82225e8233648990bc77674ba) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79231301 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79231301 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79231301 Branch: refs/heads/branch-2 Commit: 7923130114c1b9a8e8086809065b9f6cc9860df3 Parents: 85d04dc Author: Jason Lowe Authored: Tue Nov 24 19:12:04 2015 + Committer: Jason Lowe Committed: Tue Nov 24 19:13:26 2015 + -- hadoop-common-project/hadoop-common/CHANGES.txt| 3 --- .../hadoop/metrics2/impl/MetricsSourceAdapter.java | 17 ++--- 2 files changed, 10 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/79231301/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a300c50..66b8870 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -871,9 +871,6 @@ Release 2.7.3 - UNRELEASED HADOOP-12374. Updated expunge command description. (WeiWei Yang via eyang) -HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. -(Brahma Reddy Battula via ozawa) - HADOOP-12348. MetricsSystemImpl creates MetricsSourceAdapter with wrong time unit parameter. (zxu via rkanter) http://git-wip-us.apache.org/repos/asf/hadoop/blob/79231301/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java index 706ef7e..d56ee53 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java @@ -158,7 +158,7 @@ class MetricsSourceAdapter implements DynamicMBean { private void updateJmxCache() { boolean getAllMetrics = false; -synchronized (this) { +synchronized(this) { if (Time.now() - jmxCacheTS >= jmxCacheTTL) { // temporarilly advance the expiry while updating the cache jmxCacheTS = Time.now() + jmxCacheTTL; @@ -169,21 +169,24 @@ class MetricsSourceAdapter implements DynamicMBean { getAllMetrics = true; lastRecsCleared = false; } - } else { + } + else { return; } +} - if (getAllMetrics) { -MetricsCollectorImpl builder = new MetricsCollectorImpl(); -getMetrics(builder, true); - } +if (getAllMetrics) { + MetricsCollectorImpl builder = new MetricsCollectorImpl(); + getMetrics(builder, true); +} +synchronized(this) { updateAttrCache(); if (getAllMetrics) { updateInfoCache(); } jmxCacheTS = Time.now(); - lastRecs = null; // in case regular interval update is not running + lastRecs = null; // in case regular interval update is not running lastRecsCleared = true; } }
hadoop git commit: Revert "HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. Contributed by Brahma Reddy Battula."
Repository: hadoop Updated Branches: refs/heads/branch-2.7 9bd9544bd -> 2946621a5 Revert "HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. Contributed by Brahma Reddy Battula." This reverts commit 4356e8a5ef0ac6d11a34704b80ef360a710e623a. Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java (cherry picked from commit 17b1a5482b32dab82225e8233648990bc77674ba) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2946621a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2946621a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2946621a Branch: refs/heads/branch-2.7 Commit: 2946621a531bd71e2408951ab72ecaf5f9fea3f0 Parents: 9bd9544 Author: Jason Lowe Authored: Tue Nov 24 19:12:04 2015 + Committer: Jason Lowe Committed: Tue Nov 24 19:18:37 2015 + -- hadoop-common-project/hadoop-common/CHANGES.txt| 3 --- .../hadoop/metrics2/impl/MetricsSourceAdapter.java | 17 ++--- 2 files changed, 10 insertions(+), 10 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2946621a/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7065abc..3ea617b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -25,9 +25,6 @@ Release 2.7.3 - UNRELEASED HADOOP-12374. Updated expunge command description. (WeiWei Yang via eyang) -HADOOP-11361. Fix a race condition in MetricsSourceAdapter.updateJmxCache. -(Brahma Reddy Battula via ozawa) - HADOOP-12348. MetricsSystemImpl creates MetricsSourceAdapter with wrong time unit parameter. (zxu via rkanter) http://git-wip-us.apache.org/repos/asf/hadoop/blob/2946621a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java index 706ef7e..d56ee53 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MetricsSourceAdapter.java @@ -158,7 +158,7 @@ class MetricsSourceAdapter implements DynamicMBean { private void updateJmxCache() { boolean getAllMetrics = false; -synchronized (this) { +synchronized(this) { if (Time.now() - jmxCacheTS >= jmxCacheTTL) { // temporarilly advance the expiry while updating the cache jmxCacheTS = Time.now() + jmxCacheTTL; @@ -169,21 +169,24 @@ class MetricsSourceAdapter implements DynamicMBean { getAllMetrics = true; lastRecsCleared = false; } - } else { + } + else { return; } +} - if (getAllMetrics) { -MetricsCollectorImpl builder = new MetricsCollectorImpl(); -getMetrics(builder, true); - } +if (getAllMetrics) { + MetricsCollectorImpl builder = new MetricsCollectorImpl(); + getMetrics(builder, true); +} +synchronized(this) { updateAttrCache(); if (getAllMetrics) { updateInfoCache(); } jmxCacheTS = Time.now(); - lastRecs = null; // in case regular interval update is not running + lastRecs = null; // in case regular interval update is not running lastRecsCleared = true; } }
hadoop git commit: HDFS-5165. Remove the TotalFiles metrics. Contributed by Akira AJISAKA.
Repository: hadoop Updated Branches: refs/heads/trunk 17b1a5482 -> db4cab21f HDFS-5165. Remove the TotalFiles metrics. Contributed by Akira AJISAKA. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db4cab21 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db4cab21 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db4cab21 Branch: refs/heads/trunk Commit: db4cab21f4c661f68d6a6dec50aae00d75168486 Parents: 17b1a54 Author: Haohui Mai Authored: Tue Nov 24 11:41:57 2015 -0800 Committer: Haohui Mai Committed: Tue Nov 24 11:41:57 2015 -0800 -- .../hadoop-common/src/site/markdown/Metrics.md| 1 - hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 8 .../hadoop/hdfs/server/namenode/NameNodeMXBean.java | 10 -- 4 files changed, 2 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db4cab21/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md -- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md index 9e02ffa..a91bbad 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md @@ -231,7 +231,6 @@ Each metrics record contains tags such as HAState and Hostname as additional inf | `MillisSinceLastLoadedEdits` | (HA-only) Time in milliseconds since the last time standby NameNode load edit log. In active NameNode, set to 0 | | `BlockCapacity` | Current number of block capacity | | `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat | -| `TotalFiles` | Deprecated: Use FilesTotal instead | | `MissingReplOneBlocks` | Current number of missing blocks with replication factor 1 | | `NumFilesUnderConstruction` | Current number of files under construction | | `NumActiveClients` | Current number of active clients holding lease | http://git-wip-us.apache.org/repos/asf/hadoop/blob/db4cab21/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b441b35..639b8f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -44,6 +44,8 @@ Trunk (Unreleased) HDFS-9278. Fix preferredBlockSize typo in OIV XML output. (Nicole Pazmany via wang) +HDFS-5165. Remove the TotalFiles metrics. (Akira Ajisaka via wheat9) + NEW FEATURES HDFS-3125. Add JournalService to enable Journal Daemon. (suresh) http://git-wip-us.apache.org/repos/asf/hadoop/blob/db4cab21/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 8d77630..0559288 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6001,14 +6001,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return getBlocksTotal(); } - /** @deprecated Use {@link #getFilesTotal()} instead. */ - @Deprecated - @Override // NameNodeMXBean - @Metric - public long getTotalFiles() { -return getFilesTotal(); - } - @Override // NameNodeMXBean public long getNumberOfMissingBlocks() { return getMissingBlocksCount(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/db4cab21/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java index 3f78155..c0d256a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java @@ -139,16 +139,6 @@ public interface NameNodeMXBean { public long getTotalBlocks(); /** - * Gets the total
hadoop git commit: HDFS-9368. Implement reads with implicit offset state in libhdfs++. Contributed by James Clampffer.
Repository: hadoop Updated Branches: refs/heads/HDFS-8707 68f436689 -> ebfb48b88 HDFS-9368. Implement reads with implicit offset state in libhdfs++. Contributed by James Clampffer. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebfb48b8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebfb48b8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebfb48b8 Branch: refs/heads/HDFS-8707 Commit: ebfb48b88bac2fd56571dd51fcc195db5598f1b1 Parents: 68f4366 Author: James Authored: Tue Nov 24 15:30:08 2015 -0500 Committer: James Committed: Tue Nov 24 15:30:08 2015 -0500 -- .../native/libhdfspp/lib/bindings/c/hdfs.cc | 97 +--- .../native/libhdfspp/lib/bindings/c/hdfs_cpp.cc | 62 - .../native/libhdfspp/lib/bindings/c/hdfs_cpp.h | 22 - .../main/native/libhdfspp/lib/fs/filesystem.h | 2 +- .../main/native/libhdfspp/lib/fs/inputstream.cc | 2 + 5 files changed, 167 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebfb48b8/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc index 9b985a9..853e9d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc @@ -58,6 +58,43 @@ static void ReportError(int errnum, std::string msg) { #endif } +/* Convert Status wrapped error into appropriate errno and return code */ +static int Error(const Status &stat) { + int code = stat.code(); + switch (code) { +case Status::Code::kOk: + return 0; +case Status::Code::kInvalidArgument: + ReportError(EINVAL, "Invalid argument"); + break; +case Status::Code::kResourceUnavailable: + ReportError(EAGAIN, "Resource temporarily unavailable"); + break; +case Status::Code::kUnimplemented: + ReportError(ENOSYS, "Function not implemented"); + break; +case Status::Code::kException: + ReportError(EINTR, "Exception raised"); + break; +default: + ReportError(ENOSYS, "Error: unrecognised code"); + } + return -1; +} + +/* return false on failure */ +bool CheckSystemAndHandle(hdfsFS fs, hdfsFile file) { + if (!fs) { +ReportError(ENODEV, "Cannot perform FS operations with null FS handle."); +return false; + } + if (!file) { +ReportError(EBADF, "Cannot perform FS operations with null File handle."); +return false; + } + return true; +} + /** * C API implementations **/ @@ -110,28 +147,66 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, int bufferSize, } int hdfsCloseFile(hdfsFS fs, hdfsFile file) { - if (!fs) { -ReportError(ENODEV, "Cannot perform FS operations with null FS handle."); -return -1; - } - if (!file) { -ReportError(EBADF, "Cannot perform FS operations with null File handle."); + if (!CheckSystemAndHandle(fs, file)) { return -1; } + delete file; return 0; } tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, void *buffer, tSize length) { - if (!fs) { -ReportError(ENODEV, "Cannot perform FS operations with null FS handle."); + if (!CheckSystemAndHandle(fs, file)) { return -1; } - if (!file) { -ReportError(EBADF, "Cannot perform FS operations with null File handle."); + + size_t len = length; + Status stat = file->get_impl()->Pread(buffer, &len, position); + if (!stat.ok()) { +return Error(stat); + } + return (tSize)len; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile file, void *buffer, tSize length) { + if (!CheckSystemAndHandle(fs, file)) { +return -1; + } + + size_t len = length; + Status stat = file->get_impl()->Read(buffer, &len); + if (!stat.ok()) { +return Error(stat); + } + + return (tSize)len; +} + +int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos) { + if (!CheckSystemAndHandle(fs, file)) { return -1; } - return file->get_impl()->Pread(buffer, length, position); + off_t desired = desiredPos; + Status stat = file->get_impl()->Seek(&desired, std::ios_base::beg); + if (!stat.ok()) { +return Error(stat); + } + + return (int)desired; +} + +tOffset hdfsTell(hdfsFS fs, hdfsFile file) { + if (!CheckSystemAndHandle(fs, file)) { +return -1; + } + + ssize_t offset = 0; + Status stat = file->get_impl()->Seek(&offset, std::ios_base::cur); + if (!stat.ok()) { +return Error(stat); + } + + return offset; } http://git-
hadoop git commit: HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 seconds for printing info log messags.
Repository: hadoop Updated Branches: refs/heads/branch-2.6 238458b25 -> 09dbb1156 HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 seconds for printing info log messags. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09dbb115 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09dbb115 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09dbb115 Branch: refs/heads/branch-2.6 Commit: 09dbb11566410f2e9101555637ad81c7acc7edfd Parents: 238458b Author: Tsz-Wo Nicholas Sze Authored: Thu Nov 19 14:54:01 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 12:40:24 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 8 +++- 2 files changed, 6 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/09dbb115/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5e683e3..fb6df6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -23,6 +23,9 @@ Release 2.6.3 - UNRELEASED HDFS-9289. Make DataStreamer#block thread safe and verify genStamp in commitBlock. (Chang Li via zhz) +HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 +seconds for printing info log messags. (szetszwo) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/09dbb115/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index feaf843..8dbee81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2853,11 +2853,9 @@ public class BlockManager { for(DatanodeStorageInfo storage : blocksMap.getStorages(block, State.NORMAL)) { final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); if (storage.areBlockContentsStale()) { -LOG.info("BLOCK* processOverReplicatedBlock: " + -"Postponing processing of over-replicated " + -block + " since storage + " + storage -+ "datanode " + cur + " does not yet have up-to-date " + -"block information."); +LOG.trace("BLOCK* processOverReplicatedBlock: Postponing {}" ++ " since storage {} does not yet have up-to-date information.", +block, storage); postponeBlock(block); return; }
hadoop git commit: HADOOP-11954. Solaris does not support RLIMIT_MEMLOCK as in Linux (Alan Burlison via aw)
Repository: hadoop Updated Branches: refs/heads/trunk 56493cda0 -> e8a87d739 HADOOP-11954. Solaris does not support RLIMIT_MEMLOCK as in Linux (Alan Burlison via aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8a87d73 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8a87d73 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8a87d73 Branch: refs/heads/trunk Commit: e8a87d739f0ffa98fdf87d15440d1e2528080d0c Parents: 56493cd Author: Allen Wittenauer Authored: Tue Nov 24 12:44:58 2015 -0800 Committer: Allen Wittenauer Committed: Tue Nov 24 12:45:08 2015 -0800 -- hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++ .../main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8a87d73/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index cc1827b..70295be 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1359,6 +1359,9 @@ Release 2.8.0 - UNRELEASED HADOOP-9822. Create constant MAX_CAPACITY in RetryCache rather than hard-coding 16 in RetryCache constructor. (Tsuyoshi Ozawa via wheat9) +HADOOP-11954. Solaris does not support RLIMIT_MEMLOCK as in Linux +(Alan Burlison via aw) + OPTIMIZATIONS HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString() http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8a87d73/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c -- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index bd7784e..a7d4b55 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -1255,9 +1255,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_getMemlockLimit0( JNIEnv *env, jclass clazz) { -#ifdef WINDOWS - return 0; -#else +#ifdef RLIMIT_MEMLOCK struct rlimit rlim; int rc = getrlimit(RLIMIT_MEMLOCK, &rlim); if (rc != 0) { @@ -1266,6 +1264,8 @@ JNIEnv *env, jclass clazz) } return (rlim.rlim_cur == RLIM_INFINITY) ? INT64_MAX : rlim.rlim_cur; +#else + return 0; #endif }
hadoop git commit: Move HDFS-9434 to 2.6.3 in CHANGES.txt.
Repository: hadoop Updated Branches: refs/heads/branch-2 792313011 -> 15ecbc95b Move HDFS-9434 to 2.6.3 in CHANGES.txt. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15ecbc95 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15ecbc95 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15ecbc95 Branch: refs/heads/branch-2 Commit: 15ecbc95bff4c88e5382772dd9204f720c11990f Parents: 7923130 Author: Tsz-Wo Nicholas Sze Authored: Tue Nov 24 12:42:57 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 12:44:37 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/15ecbc95/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5cacca3..9b64fae 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1484,9 +1484,6 @@ Release 2.8.0 - UNRELEASED HDFS-9400. TestRollingUpgradeRollback fails on branch-2. (Brahma Reddy Battula via cnauroth) -HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 -seconds for printing info log messags. (szetszwo) - HDFS-9443. Disabling HDFS client socket cache causes logging message printed to console for CLI commands. (Chris Nauroth via wheat9) @@ -2672,6 +2669,9 @@ Release 2.6.3 - UNRELEASED HDFS-9431. DistributedFileSystem#concat fails if the target path is relative. (Kazuho Fujii via aajisaka) +HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 +seconds for printing info log messags. (szetszwo) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES
hadoop git commit: HDFS-8855. Webhdfs client leaks active NameNode connections. Contributed by Xiaobing Zhou.
Repository: hadoop Updated Branches: refs/heads/trunk e8a87d739 -> fe5624b85 HDFS-8855. Webhdfs client leaks active NameNode connections. Contributed by Xiaobing Zhou. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe5624b8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe5624b8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe5624b8 Branch: refs/heads/trunk Commit: fe5624b85d71720ae9da90a01cad9a3d1ea41160 Parents: e8a87d7 Author: Xiaoyu Yao Authored: Tue Nov 24 12:41:08 2015 -0800 Committer: Xiaoyu Yao Committed: Tue Nov 24 12:47:57 2015 -0800 -- .../org/apache/hadoop/security/token/Token.java | 10 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 + .../server/datanode/web/DatanodeHttpServer.java | 4 +- .../web/webhdfs/DataNodeUGIProvider.java| 106 +++-- .../src/main/resources/hdfs-default.xml | 8 + .../web/webhdfs/TestDataNodeUGIProvider.java| 235 +++ 7 files changed, 350 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe5624b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java -- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java index 2420155..f8b7355 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java @@ -19,6 +19,8 @@ package org.apache.hadoop.security.token; import com.google.common.collect.Maps; +import com.google.common.primitives.Bytes; + import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,6 +34,7 @@ import java.io.*; import java.util.Arrays; import java.util.Map; import java.util.ServiceLoader; +import java.util.UUID; /** * The client-side form of the token. @@ -337,7 +340,12 @@ public class Token implements Writable { identifierToString(buffer); return buffer.toString(); } - + + public String buildCacheKey() { +return UUID.nameUUIDFromBytes( +Bytes.concat(kind.getBytes(), identifier, password)).toString(); + } + private static ServiceLoader renewers = ServiceLoader.load(TokenRenewer.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe5624b8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 95dfbcf..7d9df2e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2373,6 +2373,9 @@ Release 2.8.0 - UNRELEASED HDFS-6101. TestReplaceDatanodeOnFailure fails occasionally. (Wei-Chiu Chuang via cnauroth) +HDFS-8855. Webhdfs client leaks active NameNode connections. +(Xiaobing Zhou via xyao) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe5624b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 54e0d10..6986896 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -70,6 +70,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_WEBHDFS_NETTY_HIGH_WATERMARK = "dfs.webhdfs.netty.high.watermark"; public static final int DFS_WEBHDFS_NETTY_HIGH_WATERMARK_DEFAULT = 65535; + public static final String DFS_WEBHDFS_UGI_EXPIRE_AFTER_ACCESS_KEY = + "dfs.webhdfs.ugi.expire.after.access"; + public static final int DFS_WEBHDFS_UGI_EXPIRE_AFTER_ACCESS_DEFAULT = + 10*60*1000; //10 minutes // HA related configuration public static final String DFS_DATANODE_RESTART_REPLICA_EXPIRY_KEY = "dfs.datanode.restart.replica.expiration"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe5624b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java ---
hadoop git commit: HADOOP-11954. Solaris does not support RLIMIT_MEMLOCK as in Linux (Alan Burlison via aw)
Repository: hadoop Updated Branches: refs/heads/branch-2 15ecbc95b -> 38f8952c4 HADOOP-11954. Solaris does not support RLIMIT_MEMLOCK as in Linux (Alan Burlison via aw) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38f8952c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38f8952c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38f8952c Branch: refs/heads/branch-2 Commit: 38f8952c4da851d4d914e7594afd2287635a9876 Parents: 15ecbc9 Author: Allen Wittenauer Authored: Tue Nov 24 12:44:58 2015 -0800 Committer: Allen Wittenauer Committed: Tue Nov 24 12:48:02 2015 -0800 -- hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++ .../main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/38f8952c/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 66b8870..2720943 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -735,6 +735,9 @@ Release 2.8.0 - UNRELEASED HADOOP-9822. Create constant MAX_CAPACITY in RetryCache rather than hard-coding 16 in RetryCache constructor. (Tsuyoshi Ozawa via wheat9) +HADOOP-11954. Solaris does not support RLIMIT_MEMLOCK as in Linux +(Alan Burlison via aw) + OPTIMIZATIONS HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString() http://git-wip-us.apache.org/repos/asf/hadoop/blob/38f8952c/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c -- diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index bd7784e..a7d4b55 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -1255,9 +1255,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_getMemlockLimit0( JNIEnv *env, jclass clazz) { -#ifdef WINDOWS - return 0; -#else +#ifdef RLIMIT_MEMLOCK struct rlimit rlim; int rc = getrlimit(RLIMIT_MEMLOCK, &rlim); if (rc != 0) { @@ -1266,6 +1264,8 @@ JNIEnv *env, jclass clazz) } return (rlim.rlim_cur == RLIM_INFINITY) ? INT64_MAX : rlim.rlim_cur; +#else + return 0; #endif }
[2/2] hadoop git commit: Move HDFS-9434 to 2.6.3 in CHANGES.txt.
Move HDFS-9434 to 2.6.3 in CHANGES.txt. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af811030 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af811030 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af811030 Branch: refs/heads/branch-2.7 Commit: af811030e5a7294c5b7811a0f40ff78ae1529af4 Parents: 09f4a4e Author: Tsz-Wo Nicholas Sze Authored: Tue Nov 24 12:42:57 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 12:49:34 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/af811030/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a5391de..89aede2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1,8 +1,5 @@ Hadoop HDFS Change Log -HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 -seconds for printing info log messags. (szetszwo) - Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES @@ -1169,6 +1166,9 @@ Release 2.6.3 - UNRELEASED HDFS-9431. DistributedFileSystem#concat fails if the target path is relative. (Kazuho Fujii via aajisaka) +HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 +seconds for printing info log messags. (szetszwo) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES
[1/2] hadoop git commit: HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 seconds for printing info log messags.
Repository: hadoop Updated Branches: refs/heads/branch-2.7 2946621a5 -> af811030e HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 seconds for printing info log messags. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09f4a4ee Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09f4a4ee Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09f4a4ee Branch: refs/heads/branch-2.7 Commit: 09f4a4eefd461bc418addd3862aec83dbf7165d3 Parents: 2946621 Author: Tsz-Wo Nicholas Sze Authored: Thu Nov 19 14:54:01 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 12:47:50 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 8 +++- 2 files changed, 6 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/09f4a4ee/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5e11af9..a5391de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1,5 +1,8 @@ Hadoop HDFS Change Log +HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 +seconds for printing info log messags. (szetszwo) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/09f4a4ee/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 63a7aed..81367ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2884,11 +2884,9 @@ public class BlockManager { for(DatanodeStorageInfo storage : blocksMap.getStorages(block, State.NORMAL)) { final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); if (storage.areBlockContentsStale()) { -LOG.info("BLOCK* processOverReplicatedBlock: " + -"Postponing processing of over-replicated " + -block + " since storage + " + storage -+ "datanode " + cur + " does not yet have up-to-date " + -"block information."); +LOG.trace("BLOCK* processOverReplicatedBlock: Postponing {}" ++ " since storage {} does not yet have up-to-date information.", +block, storage); postponeBlock(block); return; }
hadoop git commit: Move HDFS-9434 to 2.6.3 in CHANGES.txt.
Repository: hadoop Updated Branches: refs/heads/trunk db4cab21f -> 56493cda0 Move HDFS-9434 to 2.6.3 in CHANGES.txt. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56493cda Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56493cda Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56493cda Branch: refs/heads/trunk Commit: 56493cda04e30ab737fc6cecc8c43a87d5b006b7 Parents: db4cab2 Author: Tsz-Wo Nicholas Sze Authored: Tue Nov 24 12:42:57 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 12:42:57 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/56493cda/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 639b8f3..95dfbcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2344,9 +2344,6 @@ Release 2.8.0 - UNRELEASED HDFS-9400. TestRollingUpgradeRollback fails on branch-2. (Brahma Reddy Battula via cnauroth) -HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 -seconds for printing info log messags. (szetszwo) - HDFS-9443. Disabling HDFS client socket cache causes logging message printed to console for CLI commands. (Chris Nauroth via wheat9) @@ -3515,6 +3512,9 @@ Release 2.6.3 - UNRELEASED HDFS-9431. DistributedFileSystem#concat fails if the target path is relative. (Kazuho Fujii via aajisaka) +HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 +seconds for printing info log messags. (szetszwo) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES
hadoop git commit: Move HDFS-9434 to 2.6.3 in CHANGES.txt.
Repository: hadoop Updated Branches: refs/heads/branch-2.7.2 1752fece9 -> 4e8799fff Move HDFS-9434 to 2.6.3 in CHANGES.txt. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e8799ff Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e8799ff Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e8799ff Branch: refs/heads/branch-2.7.2 Commit: 4e8799fff4543868d7ebf218ae73f5ae1f807c38 Parents: 1752fec Author: Tsz-Wo Nicholas Sze Authored: Tue Nov 24 12:42:57 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 12:56:58 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e8799ff/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f55504b..d2e1b7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -95,9 +95,6 @@ Release 2.7.2 - 2015-11-11 HDFS-9413. getContentSummary() on standby should throw StandbyException. (Brahma Reddy Battula via mingma) -HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 -seconds for printing info log messags. (szetszwo) - Release 2.7.1 - 2015-07-06 INCOMPATIBLE CHANGES @@ -1138,6 +1135,9 @@ Release 2.6.3 - UNRELEASED BUG FIXES +HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 +seconds for printing info log messags. (szetszwo) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES
hadoop git commit: HDFS-8335. FSNamesystem should construct FSPermissionChecker only if permission is enabled. Contributed by Gabor Liptak.
Repository: hadoop Updated Branches: refs/heads/branch-2 38f8952c4 -> b07c91a16 HDFS-8335. FSNamesystem should construct FSPermissionChecker only if permission is enabled. Contributed by Gabor Liptak. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b07c91a1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b07c91a1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b07c91a1 Branch: refs/heads/branch-2 Commit: b07c91a16b8c95f55b73cd59352428c9ffe6e990 Parents: 38f8952 Author: Haohui Mai Authored: Tue Nov 24 13:07:26 2015 -0800 Committer: Haohui Mai Committed: Tue Nov 24 13:07:45 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 + .../server/namenode/FSDirStatAndListingOp.java | 25 +--- 2 files changed, 23 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b07c91a1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9b64fae..097c8e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -867,6 +867,9 @@ Release 2.8.0 - UNRELEASED HDFS-9318. considerLoad factor can be improved. (Kuhu Shukla via kihwal) +HDFS-8335. FSNamesystem should construct FSPermissionChecker only if +permission is enabled. (Gabor Liptak via wheat9) + BUG FIXES HDFS-8091: ACLStatus and XAttributes should be presented to @@ -1513,6 +1516,9 @@ Release 2.8.0 - UNRELEASED HDFS-6101. TestReplaceDatanodeOnFailure fails occasionally. (Wei-Chiu Chuang via cnauroth) +HDFS-8335. FSNamesystem should construct FSPermissionChecker only if +permission is enabled. (Gabor Liptak via wheat9) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b07c91a1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index 171e2f1..c5c2fb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -50,12 +50,17 @@ import static org.apache.hadoop.util.Time.now; class FSDirStatAndListingOp { static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg, byte[] startAfter, boolean needLocation) throws IOException { -FSPermissionChecker pc = fsd.getPermissionChecker(); byte[][] pathComponents = FSDirectory .getPathComponentsForReservedPath(srcArg); final String startAfterString = new String(startAfter, Charsets.UTF_8); -final String src = fsd.resolvePath(pc, srcArg, pathComponents); -final INodesInPath iip = fsd.getINodesInPath(src, true); +String src = null; + +if (fsd.isPermissionEnabled()) { + FSPermissionChecker pc = fsd.getPermissionChecker(); + src = fsd.resolvePath(pc, srcArg, pathComponents); +} else { + src = FSDirectory.resolvePath(srcArg, pathComponents, fsd); +} // Get file name when startAfter is an INodePath if (FSDirectory.isReservedName(startAfterString)) { @@ -72,8 +77,10 @@ class FSDirStatAndListingOp { } } +final INodesInPath iip = fsd.getINodesInPath(src, true); boolean isSuperUser = true; if (fsd.isPermissionEnabled()) { + FSPermissionChecker pc = fsd.getPermissionChecker(); if (iip.getLastINode() != null && iip.getLastINode().isDirectory()) { fsd.checkPathAccess(pc, iip, FsAction.READ_EXECUTE); } else { @@ -101,15 +108,17 @@ class FSDirStatAndListingOp { if (!DFSUtil.isValidName(src)) { throw new InvalidPathException("Invalid file name: " + src); } -FSPermissionChecker pc = fsd.getPermissionChecker(); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); -src = fsd.resolvePath(pc, src, pathComponents); -final INodesInPath iip = fsd.getINodesInPath(src, resolveLink); if (fsd.isPermissionEnabled()) { + FSPermissionChecker pc = fsd.getPermissionChecker(); + src = fsd.resolvePath(pc, src, pathComponents); + final INodesInPath iip = fsd.getINodesInPath(src, resolveLink); fsd.checkPermission(pc, iip, false, null, null, null, null, false); +}
hadoop git commit: HDFS-8335. FSNamesystem should construct FSPermissionChecker only if permission is enabled. Contributed by Gabor Liptak.
Repository: hadoop Updated Branches: refs/heads/trunk fe5624b85 -> 977e0b3c4 HDFS-8335. FSNamesystem should construct FSPermissionChecker only if permission is enabled. Contributed by Gabor Liptak. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/977e0b3c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/977e0b3c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/977e0b3c Branch: refs/heads/trunk Commit: 977e0b3c4ce76746a3d8590d2d790fdc96c86ca5 Parents: fe5624b Author: Haohui Mai Authored: Tue Nov 24 13:07:26 2015 -0800 Committer: Haohui Mai Committed: Tue Nov 24 13:14:49 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 + .../server/namenode/FSDirStatAndListingOp.java | 25 +--- 2 files changed, 23 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/977e0b3c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7d9df2e..92897b9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1734,6 +1734,9 @@ Release 2.8.0 - UNRELEASED HDFS-9318. considerLoad factor can be improved. (Kuhu Shukla via kihwal) +HDFS-8335. FSNamesystem should construct FSPermissionChecker only if +permission is enabled. (Gabor Liptak via wheat9) + BUG FIXES HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs. @@ -2376,6 +2379,9 @@ Release 2.8.0 - UNRELEASED HDFS-8855. Webhdfs client leaks active NameNode connections. (Xiaobing Zhou via xyao) +HDFS-8335. FSNamesystem should construct FSPermissionChecker only if +permission is enabled. (Gabor Liptak via wheat9) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/977e0b3c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index a1ac1a7..d8baa6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -52,12 +52,17 @@ import static org.apache.hadoop.util.Time.now; class FSDirStatAndListingOp { static DirectoryListing getListingInt(FSDirectory fsd, final String srcArg, byte[] startAfter, boolean needLocation) throws IOException { -FSPermissionChecker pc = fsd.getPermissionChecker(); byte[][] pathComponents = FSDirectory .getPathComponentsForReservedPath(srcArg); final String startAfterString = new String(startAfter, Charsets.UTF_8); -final String src = fsd.resolvePath(pc, srcArg, pathComponents); -final INodesInPath iip = fsd.getINodesInPath(src, true); +String src = null; + +if (fsd.isPermissionEnabled()) { + FSPermissionChecker pc = fsd.getPermissionChecker(); + src = fsd.resolvePath(pc, srcArg, pathComponents); +} else { + src = FSDirectory.resolvePath(srcArg, pathComponents, fsd); +} // Get file name when startAfter is an INodePath if (FSDirectory.isReservedName(startAfterString)) { @@ -74,8 +79,10 @@ class FSDirStatAndListingOp { } } +final INodesInPath iip = fsd.getINodesInPath(src, true); boolean isSuperUser = true; if (fsd.isPermissionEnabled()) { + FSPermissionChecker pc = fsd.getPermissionChecker(); if (iip.getLastINode() != null && iip.getLastINode().isDirectory()) { fsd.checkPathAccess(pc, iip, FsAction.READ_EXECUTE); } else { @@ -103,15 +110,17 @@ class FSDirStatAndListingOp { if (!DFSUtil.isValidName(src)) { throw new InvalidPathException("Invalid file name: " + src); } -FSPermissionChecker pc = fsd.getPermissionChecker(); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); -src = fsd.resolvePath(pc, src, pathComponents); -final INodesInPath iip = fsd.getINodesInPath(src, resolveLink); if (fsd.isPermissionEnabled()) { + FSPermissionChecker pc = fsd.getPermissionChecker(); + src = fsd.resolvePath(pc, src, pathComponents); + final INodesInPath iip = fsd.getINodesInPath(src, resolveLink); fsd.checkPermission(pc, iip, false, null, null, null, null, false); +} e
hadoop git commit: MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly misleading. Contributed by Nathan Roberts
Repository: hadoop Updated Branches: refs/heads/trunk 977e0b3c4 -> cab3c7c88 MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly misleading. Contributed by Nathan Roberts Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cab3c7c8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cab3c7c8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cab3c7c8 Branch: refs/heads/trunk Commit: cab3c7c8892ad33a7eb0955b01e99872ab95e192 Parents: 977e0b3 Author: Jason Lowe Authored: Tue Nov 24 22:01:03 2015 + Committer: Jason Lowe Committed: Tue Nov 24 22:01:03 2015 + -- hadoop-mapreduce-project/CHANGES.txt | 6 ++ .../org/apache/hadoop/mapreduce/JobCounter.properties| 8 2 files changed, 10 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cab3c7c8/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d22edd0..3cbacc4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -654,6 +654,9 @@ Release 2.7.3 - UNRELEASED MAPREDUCE-6377. JHS sorting on state column not working in webUi. (zhihai xu via devaraj) +MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly +misleading (Nathan Roberts via jlowe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -959,6 +962,9 @@ Release 2.6.3 - UNRELEASED position/key information for uncompressed input sometimes. (Zhihai Xu via jlowe) +MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly +misleading (Nathan Roberts via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/cab3c7c8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties index 1154784..857f31e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties @@ -27,10 +27,10 @@ SLOTS_MILLIS_MAPS.name=Total time spent by all maps in occupied slot SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces in occupied slots (ms) MILLIS_MAPS.name= Total time spent by all map tasks (ms) MILLIS_REDUCES.name= Total time spent by all reduce tasks (ms) -MB_MILLIS_MAPS.name= Total megabyte-seconds taken by all map tasks -MB_MILLIS_REDUCES.name=Total megabyte-seconds taken by all reduce tasks -VCORES_MILLIS_MAPS.name= Total vcore-seconds taken by all map tasks -VCORES_MILLIS_REDUCES.name=Total vcore-seconds taken by all reduce tasks +MB_MILLIS_MAPS.name= Total megabyte-milliseconds taken by all map tasks +MB_MILLIS_REDUCES.name=Total megabyte-milliseconds taken by all reduce tasks +VCORES_MILLIS_MAPS.name= Total vcore-milliseconds taken by all map tasks +VCORES_MILLIS_REDUCES.name=Total vcore-milliseconds taken by all reduce tasks FALLOW_SLOTS_MILLIS_MAPS.name= Total time spent by all maps waiting after reserving slots (ms) FALLOW_SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces waiting after reserving slots (ms) TASKS_REQ_PREEMPT.name=Tasks that have been asked to preempt
hadoop git commit: MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly misleading. Contributed by Nathan Roberts (cherry picked from commit cab3c7c8892ad33a7eb0955b01e99872ab95e192)
Repository: hadoop Updated Branches: refs/heads/branch-2 b07c91a16 -> ef3e01a1e MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly misleading. Contributed by Nathan Roberts (cherry picked from commit cab3c7c8892ad33a7eb0955b01e99872ab95e192) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef3e01a1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef3e01a1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef3e01a1 Branch: refs/heads/branch-2 Commit: ef3e01a1e40d4e1d234b67e7ff5995e73145cff5 Parents: b07c91a Author: Jason Lowe Authored: Tue Nov 24 22:01:03 2015 + Committer: Jason Lowe Committed: Tue Nov 24 22:01:53 2015 + -- hadoop-mapreduce-project/CHANGES.txt | 6 ++ .../org/apache/hadoop/mapreduce/JobCounter.properties| 8 2 files changed, 10 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3e01a1/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a668b12..e84be3a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -364,6 +364,9 @@ Release 2.7.3 - UNRELEASED MAPREDUCE-6377. JHS sorting on state column not working in webUi. (zhihai xu via devaraj) +MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly +misleading (Nathan Roberts via jlowe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -666,6 +669,9 @@ Release 2.6.3 - UNRELEASED position/key information for uncompressed input sometimes. (Zhihai Xu via jlowe) +MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly +misleading (Nathan Roberts via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3e01a1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties index 7a493a8..774002b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties @@ -27,9 +27,9 @@ SLOTS_MILLIS_MAPS.name=Total time spent by all maps in occupied slot SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces in occupied slots (ms) MILLIS_MAPS.name= Total time spent by all map tasks (ms) MILLIS_REDUCES.name= Total time spent by all reduce tasks (ms) -MB_MILLIS_MAPS.name= Total megabyte-seconds taken by all map tasks -MB_MILLIS_REDUCES.name=Total megabyte-seconds taken by all reduce tasks -VCORES_MILLIS_MAPS.name= Total vcore-seconds taken by all map tasks -VCORES_MILLIS_REDUCES.name=Total vcore-seconds taken by all reduce tasks +MB_MILLIS_MAPS.name= Total megabyte-milliseconds taken by all map tasks +MB_MILLIS_REDUCES.name=Total megabyte-milliseconds taken by all reduce tasks +VCORES_MILLIS_MAPS.name= Total vcore-milliseconds taken by all map tasks +VCORES_MILLIS_REDUCES.name=Total vcore-milliseconds taken by all reduce tasks FALLOW_SLOTS_MILLIS_MAPS.name= Total time spent by all maps waiting after reserving slots (ms) FALLOW_SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces waiting after reserving slots (ms)
hadoop git commit: MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly misleading. Contributed by Nathan Roberts (cherry picked from commit cab3c7c8892ad33a7eb0955b01e99872ab95e192)
Repository: hadoop Updated Branches: refs/heads/branch-2.7 af811030e -> 4ba5667ae MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly misleading. Contributed by Nathan Roberts (cherry picked from commit cab3c7c8892ad33a7eb0955b01e99872ab95e192) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ba5667a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ba5667a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ba5667a Branch: refs/heads/branch-2.7 Commit: 4ba5667aed5b5c677eca5b0ed7405fb63452315f Parents: af81103 Author: Jason Lowe Authored: Tue Nov 24 22:01:03 2015 + Committer: Jason Lowe Committed: Tue Nov 24 22:02:21 2015 + -- hadoop-mapreduce-project/CHANGES.txt | 6 ++ .../org/apache/hadoop/mapreduce/JobCounter.properties| 8 2 files changed, 10 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ba5667a/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d9aab2a..7220b3a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -20,6 +20,9 @@ Release 2.7.3 - UNRELEASED MAPREDUCE-6377. JHS sorting on state column not working in webUi. (zhihai xu via devaraj) +MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly +misleading (Nathan Roberts via jlowe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -322,6 +325,9 @@ Release 2.6.3 - UNRELEASED position/key information for uncompressed input sometimes. (Zhihai Xu via jlowe) +MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly +misleading (Nathan Roberts via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ba5667a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties index 7a493a8..774002b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties @@ -27,9 +27,9 @@ SLOTS_MILLIS_MAPS.name=Total time spent by all maps in occupied slot SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces in occupied slots (ms) MILLIS_MAPS.name= Total time spent by all map tasks (ms) MILLIS_REDUCES.name= Total time spent by all reduce tasks (ms) -MB_MILLIS_MAPS.name= Total megabyte-seconds taken by all map tasks -MB_MILLIS_REDUCES.name=Total megabyte-seconds taken by all reduce tasks -VCORES_MILLIS_MAPS.name= Total vcore-seconds taken by all map tasks -VCORES_MILLIS_REDUCES.name=Total vcore-seconds taken by all reduce tasks +MB_MILLIS_MAPS.name= Total megabyte-milliseconds taken by all map tasks +MB_MILLIS_REDUCES.name=Total megabyte-milliseconds taken by all reduce tasks +VCORES_MILLIS_MAPS.name= Total vcore-milliseconds taken by all map tasks +VCORES_MILLIS_REDUCES.name=Total vcore-milliseconds taken by all reduce tasks FALLOW_SLOTS_MILLIS_MAPS.name= Total time spent by all maps waiting after reserving slots (ms) FALLOW_SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces waiting after reserving slots (ms)
hadoop git commit: MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly misleading. Contributed by Nathan Roberts (cherry picked from commit cab3c7c8892ad33a7eb0955b01e99872ab95e192)
Repository: hadoop Updated Branches: refs/heads/branch-2.6 09dbb1156 -> 5074d836b MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly misleading. Contributed by Nathan Roberts (cherry picked from commit cab3c7c8892ad33a7eb0955b01e99872ab95e192) Conflicts: hadoop-mapreduce-project/CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5074d836 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5074d836 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5074d836 Branch: refs/heads/branch-2.6 Commit: 5074d836b08a99361ed8fa348dadfc6746f4b57c Parents: 09dbb11 Author: Jason Lowe Authored: Tue Nov 24 22:03:39 2015 + Committer: Jason Lowe Committed: Tue Nov 24 22:03:39 2015 + -- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapreduce/JobCounter.properties| 8 2 files changed, 7 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5074d836/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b737ecb..a395c8a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -32,6 +32,9 @@ Release 2.6.3 - UNRELEASED position/key information for uncompressed input sometimes. (Zhihai Xu via jlowe) +MAPREDUCE-5883. "Total megabyte-seconds" in job counters is slightly +misleading (Nathan Roberts via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5074d836/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties index 7a493a8..774002b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties @@ -27,9 +27,9 @@ SLOTS_MILLIS_MAPS.name=Total time spent by all maps in occupied slot SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces in occupied slots (ms) MILLIS_MAPS.name= Total time spent by all map tasks (ms) MILLIS_REDUCES.name= Total time spent by all reduce tasks (ms) -MB_MILLIS_MAPS.name= Total megabyte-seconds taken by all map tasks -MB_MILLIS_REDUCES.name=Total megabyte-seconds taken by all reduce tasks -VCORES_MILLIS_MAPS.name= Total vcore-seconds taken by all map tasks -VCORES_MILLIS_REDUCES.name=Total vcore-seconds taken by all reduce tasks +MB_MILLIS_MAPS.name= Total megabyte-milliseconds taken by all map tasks +MB_MILLIS_REDUCES.name=Total megabyte-milliseconds taken by all reduce tasks +VCORES_MILLIS_MAPS.name= Total vcore-milliseconds taken by all map tasks +VCORES_MILLIS_REDUCES.name=Total vcore-milliseconds taken by all reduce tasks FALLOW_SLOTS_MILLIS_MAPS.name= Total time spent by all maps waiting after reserving slots (ms) FALLOW_SLOTS_MILLIS_REDUCES.name= Total time spent by all reduces waiting after reserving slots (ms)
hadoop git commit: MAPREDUCE-5870. Support for passing Job priority through Application Submission Context in Mapreduce Side. Contributed by Sunil G
Repository: hadoop Updated Branches: refs/heads/trunk cab3c7c88 -> f634505d4 MAPREDUCE-5870. Support for passing Job priority through Application Submission Context in Mapreduce Side. Contributed by Sunil G Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f634505d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f634505d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f634505d Branch: refs/heads/trunk Commit: f634505d48d97e4d461980d68a0cbdf87223646d Parents: cab3c7c Author: Jason Lowe Authored: Tue Nov 24 22:07:26 2015 + Committer: Jason Lowe Committed: Tue Nov 24 22:07:26 2015 + -- hadoop-mapreduce-project/CHANGES.txt| 3 + .../v2/app/local/LocalContainerAllocator.java | 6 ++ .../mapreduce/v2/app/job/impl/TestJobImpl.java | 2 +- .../app/local/TestLocalContainerAllocator.java | 5 +- .../apache/hadoop/mapreduce/TypeConverter.java | 48 +- .../hadoop/mapreduce/TestTypeConverter.java | 4 + .../java/org/apache/hadoop/mapred/JobConf.java | 92 ++-- .../org/apache/hadoop/mapred/JobPriority.java | 6 ++ .../org/apache/hadoop/mapred/JobStatus.java | 2 +- .../java/org/apache/hadoop/mapreduce/Job.java | 67 -- .../apache/hadoop/mapreduce/JobPriority.java| 7 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 29 -- .../org/apache/hadoop/mapred/TestJobConf.java | 47 +- .../org/apache/hadoop/mapreduce/TestJob.java| 2 +- .../org/apache/hadoop/mapred/YARNRunner.java| 19 +++- .../apache/hadoop/mapred/TestYARNRunner.java| 25 ++ .../hadoop/mapreduce/TestMRJobClient.java | 5 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 63 ++ 18 files changed, 401 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f634505d/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 3cbacc4..ef4bb18 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -434,6 +434,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6499. Add elapsed time for retired job in JobHistoryServer WebUI. (Lin Yiqun via aajisaka) +MAPREDUCE-5870. Support for passing Job priority through Application +Submission Context in Mapreduce Side (Sunil G via jlowe) + OPTIMIZATIONS MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via http://git-wip-us.apache.org/repos/asf/hadoop/blob/f634505d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java index aed1023..7437357 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.client.ClientRMProxy; @@ -146,6 +147,11 @@ public class LocalContainerAllocator extends RMCommunicator if (token != null) { updateAMRMToken(token); } + Priority priorityFromResponse = Priority.newInstance(allocateResponse + .getApplicationPriority().getPriority()); + + // Update the job priority to Job directly. + getJob().setJobPriority(priorityFromResponse); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f634505d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/te
hadoop git commit: Revert "HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 seconds for printing info log messags."
Repository: hadoop Updated Branches: refs/heads/branch-2.6 5074d836b -> 83fcbd49a Revert "HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 seconds for printing info log messags." This reverts commit 09dbb11566410f2e9101555637ad81c7acc7edfd. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83fcbd49 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83fcbd49 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83fcbd49 Branch: refs/heads/branch-2.6 Commit: 83fcbd49a7508b015a9f91a8f94215b92db9ea2f Parents: 5074d83 Author: Tsz-Wo Nicholas Sze Authored: Tue Nov 24 14:08:45 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 14:08:45 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 --- .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 8 +--- 2 files changed, 5 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fcbd49/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index fb6df6e..5e683e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -23,9 +23,6 @@ Release 2.6.3 - UNRELEASED HDFS-9289. Make DataStreamer#block thread safe and verify genStamp in commitBlock. (Chang Li via zhz) -HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 -seconds for printing info log messags. (szetszwo) - Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fcbd49/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 8dbee81..feaf843 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2853,9 +2853,11 @@ public class BlockManager { for(DatanodeStorageInfo storage : blocksMap.getStorages(block, State.NORMAL)) { final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); if (storage.areBlockContentsStale()) { -LOG.trace("BLOCK* processOverReplicatedBlock: Postponing {}" -+ " since storage {} does not yet have up-to-date information.", -block, storage); +LOG.info("BLOCK* processOverReplicatedBlock: " + +"Postponing processing of over-replicated " + +block + " since storage + " + storage ++ "datanode " + cur + " does not yet have up-to-date " + +"block information."); postponeBlock(block); return; }
hadoop git commit: MAPREDUCE-5870. Support for passing Job priority through Application Submission Context in Mapreduce Side. Contributed by Sunil G (cherry picked from commit f634505d48d97e4d461980d6
Repository: hadoop Updated Branches: refs/heads/branch-2 ef3e01a1e -> 2f1821850 MAPREDUCE-5870. Support for passing Job priority through Application Submission Context in Mapreduce Side. Contributed by Sunil G (cherry picked from commit f634505d48d97e4d461980d68a0cbdf87223646d) Conflicts: hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f182185 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f182185 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f182185 Branch: refs/heads/branch-2 Commit: 2f18218508524b01a3e19b90950f63920f350c6f Parents: ef3e01a Author: Jason Lowe Authored: Tue Nov 24 22:15:37 2015 + Committer: Jason Lowe Committed: Tue Nov 24 22:15:37 2015 + -- hadoop-mapreduce-project/CHANGES.txt| 3 + .../v2/app/local/LocalContainerAllocator.java | 6 ++ .../mapreduce/v2/app/job/impl/TestJobImpl.java | 2 +- .../app/local/TestLocalContainerAllocator.java | 5 +- .../apache/hadoop/mapreduce/TypeConverter.java | 48 +- .../hadoop/mapreduce/TestTypeConverter.java | 4 + .../java/org/apache/hadoop/mapred/JobConf.java | 92 ++-- .../org/apache/hadoop/mapred/JobPriority.java | 6 ++ .../org/apache/hadoop/mapred/JobStatus.java | 2 +- .../java/org/apache/hadoop/mapreduce/Job.java | 67 -- .../apache/hadoop/mapreduce/JobPriority.java| 7 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 29 -- .../org/apache/hadoop/mapred/TestJobConf.java | 47 +- .../org/apache/hadoop/mapreduce/TestJob.java| 2 +- .../org/apache/hadoop/mapred/YARNRunner.java| 19 +++- .../apache/hadoop/mapred/TestYARNRunner.java| 25 ++ .../hadoop/mapreduce/TestMRJobClient.java | 5 +- .../apache/hadoop/mapreduce/v2/TestMRJobs.java | 63 ++ 18 files changed, 401 insertions(+), 31 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f182185/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e84be3a..324f3a4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -139,6 +139,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6499. Add elapsed time for retired job in JobHistoryServer WebUI. (Lin Yiqun via aajisaka) +MAPREDUCE-5870. Support for passing Job priority through Application +Submission Context in Mapreduce Side (Sunil G via jlowe) + OPTIMIZATIONS MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f182185/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java index aed1023..7437357 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.client.ClientRMProxy; @@ -146,6 +147,11 @@ public class LocalContainerAllocator extends RMCommunicator if (token != null) { updateAMRMToken(token); } + Priority priorityFromResponse = Priority.newInstance(allocateResponse + .getApplicationPriority().getPriority()); + + // Update the job priority to Job directly. + getJob().setJobPriority(priorityFromResponse); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f182185/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/jav
hadoop git commit: YARN-4132. Separate configs for nodemanager to resourcemanager connection timeout and retries. Contributed by Chang Li (cherry picked from commit 4ac6799d4a8b071e0d367c2d709e84d8ea0
Repository: hadoop Updated Branches: refs/heads/branch-2 2f1821850 -> 9e54433c6 YARN-4132. Separate configs for nodemanager to resourcemanager connection timeout and retries. Contributed by Chang Li (cherry picked from commit 4ac6799d4a8b071e0d367c2d709e84d8ea06942d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e54433c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e54433c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e54433c Branch: refs/heads/branch-2 Commit: 9e54433c6c0911f7f61f35c6bff75868bf029b1a Parents: 2f18218 Author: Jason Lowe Authored: Tue Nov 24 22:35:37 2015 + Committer: Jason Lowe Committed: Tue Nov 24 22:36:19 2015 + -- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 - .../org/apache/hadoop/yarn/client/RMProxy.java | 39 - .../src/main/resources/yarn-default.xml | 20 + .../hadoop/yarn/server/api/ServerRMProxy.java | 22 +- .../nodemanager/TestNodeStatusUpdater.java | 83 6 files changed, 181 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e54433c/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fa2049a..fee03c5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -519,6 +519,9 @@ Release 2.8.0 - UNRELEASED YARN-3980. Plumb resource-utilization info in node heartbeat through to the scheduler. (Inigo Goiri via kasha) +YARN-4132. Separate configs for nodemanager to resourcemanager connection +timeout and retries (Chang Li via jlowe) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e54433c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 14c1ffc..5a36bd1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2050,7 +2050,23 @@ public class YarnConfiguration extends Configuration { public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy"; public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY .name(); - + + /** + * Max time to wait for NM to connection to RM. + * When not set, proxy will fall back to use value of + * RESOURCEMANAGER_CONNECT_MAX_WAIT_MS. + */ + public static final String NM_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = + YARN_PREFIX + "nodemanager.resourcemanager.connect.max-wait.ms"; + + /** + * Time interval between each NM attempt to connection to RM. + * When not set, proxy will fall back to use value of + * RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS. + */ + public static final String NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = + YARN_PREFIX + "nodemanager.resourcemanager.connect.retry-interval.ms"; + /** * Node-labels configurations */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e54433c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java index 23e1691..3779ce5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java @@ -88,7 +88,32 @@ public class RMProxy { YarnConfiguration conf = (configuration instanceof YarnConfiguration) ? (YarnConfiguration) configuration : new YarnConfiguration(configuration); -RetryPolicy retryPolicy = createRetryPolicy(conf); +RetryPolicy retryPolicy = +createRetryPolicy(conf); +return createRMProxy(conf, protocol, instance, retryPolicy); + } + + /** + * Create a proxy for the specified protocol. F
hadoop git commit: YARN-4132. Separate configs for nodemanager to resourcemanager connection timeout and retries. Contributed by Chang Li
Repository: hadoop Updated Branches: refs/heads/trunk f634505d4 -> 4ac6799d4 YARN-4132. Separate configs for nodemanager to resourcemanager connection timeout and retries. Contributed by Chang Li Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ac6799d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ac6799d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ac6799d Branch: refs/heads/trunk Commit: 4ac6799d4a8b071e0d367c2d709e84d8ea06942d Parents: f634505 Author: Jason Lowe Authored: Tue Nov 24 22:35:37 2015 + Committer: Jason Lowe Committed: Tue Nov 24 22:35:37 2015 + -- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/conf/YarnConfiguration.java | 18 - .../org/apache/hadoop/yarn/client/RMProxy.java | 39 - .../src/main/resources/yarn-default.xml | 20 + .../hadoop/yarn/server/api/ServerRMProxy.java | 22 +- .../nodemanager/TestNodeStatusUpdater.java | 83 6 files changed, 181 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac6799d/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 204c338..4483589 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -571,6 +571,9 @@ Release 2.8.0 - UNRELEASED YARN-3980. Plumb resource-utilization info in node heartbeat through to the scheduler. (Inigo Goiri via kasha) +YARN-4132. Separate configs for nodemanager to resourcemanager connection +timeout and retries (Chang Li via jlowe) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac6799d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 18e6082..f493fd3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2050,7 +2050,23 @@ public class YarnConfiguration extends Configuration { public static final String YARN_HTTP_POLICY_KEY = YARN_PREFIX + "http.policy"; public static final String YARN_HTTP_POLICY_DEFAULT = HttpConfig.Policy.HTTP_ONLY .name(); - + + /** + * Max time to wait for NM to connection to RM. + * When not set, proxy will fall back to use value of + * RESOURCEMANAGER_CONNECT_MAX_WAIT_MS. + */ + public static final String NM_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = + YARN_PREFIX + "nodemanager.resourcemanager.connect.max-wait.ms"; + + /** + * Time interval between each NM attempt to connection to RM. + * When not set, proxy will fall back to use value of + * RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS. + */ + public static final String NM_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = + YARN_PREFIX + "nodemanager.resourcemanager.connect.retry-interval.ms"; + /** * Node-labels configurations */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ac6799d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java index 23e1691..3779ce5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java @@ -88,7 +88,32 @@ public class RMProxy { YarnConfiguration conf = (configuration instanceof YarnConfiguration) ? (YarnConfiguration) configuration : new YarnConfiguration(configuration); -RetryPolicy retryPolicy = createRetryPolicy(conf); +RetryPolicy retryPolicy = +createRetryPolicy(conf); +return createRMProxy(conf, protocol, instance, retryPolicy); + } + + /** + * Create a proxy for the specified protocol. For non-HA, + * this is a direct connection to the ResourceManager address
hadoop git commit: HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 seconds for printing info log messags.
Repository: hadoop Updated Branches: refs/heads/branch-2.6 83fcbd49a -> d2518fd56 HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 seconds for printing info log messags. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2518fd5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2518fd5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2518fd5 Branch: refs/heads/branch-2.6 Commit: d2518fd5627220736e691b4a55638fc1f57d612e Parents: 83fcbd4 Author: Tsz-Wo Nicholas Sze Authored: Tue Nov 24 14:34:51 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 14:34:51 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/blockmanagement/BlockManager.java | 10 +- 2 files changed, 8 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2518fd5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5e683e3..fb6df6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -23,6 +23,9 @@ Release 2.6.3 - UNRELEASED HDFS-9289. Make DataStreamer#block thread safe and verify genStamp in commitBlock. (Chang Li via zhz) +HDFS-9434. Recommission a datanode with 500k blocks may pause NN for 30 +seconds for printing info log messags. (szetszwo) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2518fd5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index feaf843..123a706 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -2853,11 +2853,11 @@ public class BlockManager { for(DatanodeStorageInfo storage : blocksMap.getStorages(block, State.NORMAL)) { final DatanodeDescriptor cur = storage.getDatanodeDescriptor(); if (storage.areBlockContentsStale()) { -LOG.info("BLOCK* processOverReplicatedBlock: " + -"Postponing processing of over-replicated " + -block + " since storage + " + storage -+ "datanode " + cur + " does not yet have up-to-date " + -"block information."); +if (LOG.isTraceEnabled()) { + LOG.trace("BLOCK* processOverReplicatedBlock: Postponing " + block + + " since storage " + storage + + " does not yet have up-to-date information."); +} postponeBlock(block); return; }
hadoop git commit: HDFS-9359. Test libhdfs++ with existing libhdfs tests. Contributed by Stephen Walkauskas.
Repository: hadoop Updated Branches: refs/heads/HDFS-8707 ebfb48b88 -> 8b7769557 HDFS-9359. Test libhdfs++ with existing libhdfs tests. Contributed by Stephen Walkauskas. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b776955 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b776955 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b776955 Branch: refs/heads/HDFS-8707 Commit: 8b77695572d490c548782c798b2cb2fc72bd11f2 Parents: ebfb48b Author: James Authored: Tue Nov 24 17:56:26 2015 -0500 Committer: James Committed: Tue Nov 24 17:56:26 2015 -0500 -- .../main/native/libhdfspp/tests/CMakeLists.txt | 22 ++ .../src/main/native/libhdfspp/tests/hdfs_shim.c | 381 +++ .../native/libhdfspp/tests/libhdfs_wrapper.c| 22 ++ .../native/libhdfspp/tests/libhdfs_wrapper.h| 28 ++ .../libhdfspp/tests/libhdfs_wrapper_defines.h | 92 + .../libhdfspp/tests/libhdfs_wrapper_undefs.h| 92 + .../native/libhdfspp/tests/libhdfspp_wrapper.cc | 22 ++ .../native/libhdfspp/tests/libhdfspp_wrapper.h | 28 ++ .../libhdfspp/tests/libhdfspp_wrapper_defines.h | 92 + 9 files changed, 779 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b776955/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 51b6bfe..abd3858 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -16,6 +16,24 @@ # limitations under the License. # +# Delegate some functionality to libhdfs, until libhdfspp is complete. +set (LIBHDFS_SRC_DIR ../../libhdfs) +set (LIBHDFS_TESTS_DIR ../../libhdfs-tests) +set (LIBHDFSPP_SRC_DIR ..) +set (LIBHDFSPP_LIB_DIR ${LIBHDFSPP_SRC_DIR}/lib) +set (LIBHDFSPP_BINDING_C ${LIBHDFSPP_LIB_DIR}/bindings/c) +include_directories( +${GENERATED_JAVAH} +${CMAKE_CURRENT_SOURCE_DIR} +${CMAKE_BINARY_DIR} +${JNI_INCLUDE_DIRS} +${LIBHDFS_SRC_DIR}/include +${LIBHDFS_SRC_DIR}/.. +${LIBHDFS_SRC_DIR} +${OS_DIR} +) +add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs_cpp.cc) + add_library(test_common OBJECT mock_connection.cc) set(PROTOBUF_IMPORT_DIRS ${PROTO_HADOOP_TEST_DIR}) @@ -49,3 +67,7 @@ add_test(bad_datanode bad_datanode_test) add_executable(node_exclusion_test node_exclusion_test.cc) target_link_libraries(node_exclusion_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) add_test(node_exclusion node_exclusion_test) + +build_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static expect.c test_libhdfs_threaded.c ${OS_DIR}/thread.c) +link_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static fs reader rpc proto common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY}) +add_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static) http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b776955/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c new file mode 100644 index 000..3954440 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c @@ -0,0 +1,381 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "libhdfs_wrapper.h" +#include "libhdfspp_wrapper.h" +#include "h
hadoop git commit: YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla
Repository: hadoop Updated Branches: refs/heads/trunk 4ac6799d4 -> f5acf94ec YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5acf94e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5acf94e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5acf94e Branch: refs/heads/trunk Commit: f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701 Parents: 4ac6799 Author: Jason Lowe Authored: Tue Nov 24 23:47:10 2015 + Committer: Jason Lowe Committed: Tue Nov 24 23:47:10 2015 + -- hadoop-yarn-project/CHANGES.txt | 6 .../nodelabels/FileSystemNodeLabelsStore.java | 6 ++-- .../TestFileSystemNodeLabelsStore.java | 29 3 files changed, 39 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4483589..155d5c9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1100,6 +1100,9 @@ Release 2.7.3 - UNRELEASED YARN-4344. NMs reconnecting with changed capabilities can lead to wrong cluster resource calculations (Varun Vasudev via jlowe) +YARN-4365. FileSystemNodeLabelStore should check for root dir existence on +startup (Kuhu Shukla via jlowe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -1960,6 +1963,9 @@ Release 2.6.3 - UNRELEASED YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. (zhihai xu via jlowe) +YARN-4365. FileSystemNodeLabelStore should check for root dir existence on +startup (Kuhu Shukla via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index 20dc67c..c9727a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -88,7 +88,9 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { setFileSystem(conf); // mkdir of root dir path -fs.mkdirs(fsWorkingPath); +if (!fs.exists(fsWorkingPath)) { + fs.mkdirs(fsWorkingPath); +} } @Override @@ -96,7 +98,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { IOUtils.cleanup(LOG, fs, editlogOs); } - private void setFileSystem(Configuration conf) throws IOException { + void setFileSystem(Configuration conf) throws IOException { Configuration confCopy = new Configuration(conf); confCopy.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5acf94e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 4b052c9..4929f95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -24,6 +24,8 @@ import java.util.Arrays; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -33,6 +35,7 @@ import org.junit.Before; import org.junit.Test; import com.google.common.collect.ImmutableMap; +i
hadoop git commit: YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla (cherry picked from commit f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701)
Repository: hadoop Updated Branches: refs/heads/branch-2 9e54433c6 -> 3a2f5d632 YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla (cherry picked from commit f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a2f5d63 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a2f5d63 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a2f5d63 Branch: refs/heads/branch-2 Commit: 3a2f5d6329e80884b795899f0c42c0d61d729e15 Parents: 9e54433 Author: Jason Lowe Authored: Tue Nov 24 23:47:10 2015 + Committer: Jason Lowe Committed: Tue Nov 24 23:47:54 2015 + -- hadoop-yarn-project/CHANGES.txt | 6 .../nodelabels/FileSystemNodeLabelsStore.java | 6 ++-- .../TestFileSystemNodeLabelsStore.java | 29 3 files changed, 39 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a2f5d63/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fee03c5..fe1d3f0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1048,6 +1048,9 @@ Release 2.7.3 - UNRELEASED YARN-4344. NMs reconnecting with changed capabilities can lead to wrong cluster resource calculations (Varun Vasudev via jlowe) +YARN-4365. FileSystemNodeLabelStore should check for root dir existence on +startup (Kuhu Shukla via jlowe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -1914,6 +1917,9 @@ Release 2.6.3 - UNRELEASED YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. (zhihai xu via jlowe) +YARN-4365. FileSystemNodeLabelStore should check for root dir existence on +startup (Kuhu Shukla via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a2f5d63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index 20dc67c..c9727a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -88,7 +88,9 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { setFileSystem(conf); // mkdir of root dir path -fs.mkdirs(fsWorkingPath); +if (!fs.exists(fsWorkingPath)) { + fs.mkdirs(fsWorkingPath); +} } @Override @@ -96,7 +98,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { IOUtils.cleanup(LOG, fs, editlogOs); } - private void setFileSystem(Configuration conf) throws IOException { + void setFileSystem(Configuration conf) throws IOException { Configuration confCopy = new Configuration(conf); confCopy.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a2f5d63/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 4b052c9..4929f95 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -24,6 +24,8 @@ import java.util.Arrays; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -33,6 +35,7 @@ import org.junit.Before;
hadoop git commit: YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla (cherry picked from commit f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701)
Repository: hadoop Updated Branches: refs/heads/branch-2.7 4ba5667ae -> b68f527b9 YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla (cherry picked from commit f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701) Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b68f527b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b68f527b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b68f527b Branch: refs/heads/branch-2.7 Commit: b68f527b9fee64e3c92766befe9544f780f1347e Parents: 4ba5667 Author: Jason Lowe Authored: Tue Nov 24 23:52:46 2015 + Committer: Jason Lowe Committed: Tue Nov 24 23:52:46 2015 + -- hadoop-yarn-project/CHANGES.txt | 6 .../nodelabels/FileSystemNodeLabelsStore.java | 6 ++-- .../TestFileSystemNodeLabelsStore.java | 29 3 files changed, 39 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68f527b/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a2573fb..a2e9798 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -31,6 +31,9 @@ Release 2.7.3 - UNRELEASED YARN-4344. NMs reconnecting with changed capabilities can lead to wrong cluster resource calculations (Varun Vasudev via jlowe) +YARN-4365. FileSystemNodeLabelStore should check for root dir existence on +startup (Kuhu Shukla via jlowe) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -917,6 +920,9 @@ Release 2.6.3 - UNRELEASED YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. (zhihai xu via jlowe) +YARN-4365. FileSystemNodeLabelStore should check for root dir existence on +startup (Kuhu Shukla via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68f527b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index 6e685ee..7648690 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -84,7 +84,9 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { setFileSystem(conf); // mkdir of root dir path -fs.mkdirs(fsWorkingPath); +if (!fs.exists(fsWorkingPath)) { + fs.mkdirs(fsWorkingPath); +} } @Override @@ -97,7 +99,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { } } - private void setFileSystem(Configuration conf) throws IOException { + void setFileSystem(Configuration conf) throws IOException { Configuration confCopy = new Configuration(conf); confCopy.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68f527b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 56da848..be4e70d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -24,6 +24,8 @@ import java.util.Arrays; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.eve
hadoop git commit: YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla (cherry picked from commit f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701)
Repository: hadoop Updated Branches: refs/heads/branch-2.6 d2518fd56 -> f61e3320c YARN-4365. FileSystemNodeLabelStore should check for root dir existence on startup. Contributed by Kuhu Shukla (cherry picked from commit f5acf94ecafb301a0cc8e8f91f19c8bcbc8da701) Conflicts: hadoop-yarn-project/CHANGES.txt hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f61e3320 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f61e3320 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f61e3320 Branch: refs/heads/branch-2.6 Commit: f61e3320cbc50870c603671bf4b18d28eee95270 Parents: d2518fd Author: Jason Lowe Authored: Tue Nov 24 23:59:40 2015 + Committer: Jason Lowe Committed: Tue Nov 24 23:59:40 2015 + -- hadoop-yarn-project/CHANGES.txt | 3 ++ .../nodelabels/FileSystemNodeLabelsStore.java | 6 ++-- .../TestFileSystemNodeLabelsStore.java | 29 3 files changed, 36 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f61e3320/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fb29938..26c3a19 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -36,6 +36,9 @@ Release 2.6.3 - UNRELEASED YARN-3878. AsyncDispatcher can hang while stopping if it is configured for draining events on stop. (Varun Saxena via kasha) +YARN-4365. FileSystemNodeLabelStore should check for root dir existence on +startup (Kuhu Shukla via jlowe) + Release 2.6.2 - 2015-10-28 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/f61e3320/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index 6e685ee..7648690 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -84,7 +84,9 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { setFileSystem(conf); // mkdir of root dir path -fs.mkdirs(fsWorkingPath); +if (!fs.exists(fsWorkingPath)) { + fs.mkdirs(fsWorkingPath); +} } @Override @@ -97,7 +99,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore { } } - private void setFileSystem(Configuration conf) throws IOException { + void setFileSystem(Configuration conf) throws IOException { Configuration confCopy = new Configuration(conf); confCopy.setBoolean("dfs.client.retry.policy.enabled", true); String retryPolicy = http://git-wip-us.apache.org/repos/asf/hadoop/blob/f61e3320/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 45a2d8d..fd57865 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -24,6 +24,8 @@ import java.util.Arrays; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.InlineDispatcher; import org.junit.After; @@ -32,6 +34,7 @@ import org.junit.Before; import org.junit.Test; import com.google.common.collect.ImmutableMap; +import org.mockito.Mockito; public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { MockNodeLabelManager mgr = null; @@ -249,4 +252,30
hadoop git commit: HDFS-8807. dfs.datanode.data.dir does not handle spaces between storageType and URI correctly. Contributed by Anu Engineer
Repository: hadoop Updated Branches: refs/heads/trunk f5acf94ec -> 78ec38b2e HDFS-8807. dfs.datanode.data.dir does not handle spaces between storageType and URI correctly. Contributed by Anu Engineer Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78ec38b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78ec38b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78ec38b2 Branch: refs/heads/trunk Commit: 78ec38b2ede8bdf3874b2ae051af9580007a9ba1 Parents: f5acf94 Author: Tsz-Wo Nicholas Sze Authored: Tue Nov 24 16:01:55 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 16:01:55 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/datanode/StorageLocation.java | 2 +- .../hdfs/server/datanode/TestDataDirs.java | 29 ++-- 3 files changed, 25 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ec38b2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 92897b9..db49e54 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1681,6 +1681,9 @@ Release 2.8.0 - UNRELEASED HDFS-9314. Improve BlockPlacementPolicyDefault's picking of excess replicas. (Xiao Chen via mingma) +HDFS-8807. dfs.datanode.data.dir does not handle spaces between +storageType and URI correctly. (Anu Engineer via szetszwo) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ec38b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java index 7873459..46e8e8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java @@ -87,7 +87,7 @@ public class StorageLocation { if (matcher.matches()) { String classString = matcher.group(1); - location = matcher.group(2); + location = matcher.group(2).trim(); if (!classString.isEmpty()) { storageType = StorageType.valueOf(StringUtils.toUpperCase(classString)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ec38b2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java index 396945e..d41c13e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode.DataNodeDiskChecker; public class TestDataDirs { - @Test (timeout = 3) + @Test(timeout = 3) public void testDataDirParsing() throws Throwable { Configuration conf = new Configuration(); List locations; @@ -46,12 +46,16 @@ public class TestDataDirs { File dir3 = new File("/dir3"); File dir4 = new File("/dir4"); +File dir5 = new File("/dir5"); +File dir6 = new File("/dir6"); // Verify that a valid string is correctly parsed, and that storage -// type is not case-sensitive -String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3,[ram_disk]/dir4"; +// type is not case-sensitive and we are able to handle white-space between +// storage type and URI. +String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3," + +"[ram_disk]/dir4,[disk]/dir5, [disk] /dir6, [disk] "; conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1); locations = DataNode.getStorageLocations(conf); -assertThat(locations.size(), is(5)); +assertThat(locations.size(), is(8)); assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(0).getUri(), is(dir0.toURI())); assertThat(locations.get(1).getStorageType(), is(Storage
hadoop git commit: HDFS-8807. dfs.datanode.data.dir does not handle spaces between storageType and URI correctly. Contributed by Anu Engineer
Repository: hadoop Updated Branches: refs/heads/branch-2 3a2f5d632 -> 79e55fb47 HDFS-8807. dfs.datanode.data.dir does not handle spaces between storageType and URI correctly. Contributed by Anu Engineer Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79e55fb4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79e55fb4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79e55fb4 Branch: refs/heads/branch-2 Commit: 79e55fb47f16c9ed3dfb0ff2b103ed920b75cf62 Parents: 3a2f5d6 Author: Tsz-Wo Nicholas Sze Authored: Tue Nov 24 16:01:55 2015 -0800 Committer: Tsz-Wo Nicholas Sze Committed: Tue Nov 24 16:05:21 2015 -0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/server/datanode/StorageLocation.java | 2 +- .../hdfs/server/datanode/TestDataDirs.java | 29 ++-- 3 files changed, 25 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e55fb4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 097c8e8..0732893 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -814,6 +814,9 @@ Release 2.8.0 - UNRELEASED HDFS-9314. Improve BlockPlacementPolicyDefault's picking of excess replicas. (Xiao Chen via mingma) +HDFS-8807. dfs.datanode.data.dir does not handle spaces between +storageType and URI correctly. (Anu Engineer via szetszwo) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e55fb4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java index 5c8dd85..4cae381 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/StorageLocation.java @@ -87,7 +87,7 @@ public class StorageLocation { if (matcher.matches()) { String classString = matcher.group(1); - location = matcher.group(2); + location = matcher.group(2).trim(); if (!classString.isEmpty()) { storageType = StorageType.valueOf(StringUtils.toUpperCase(classString)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/79e55fb4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java index 396945e..d41c13e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataDirs.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode.DataNodeDiskChecker; public class TestDataDirs { - @Test (timeout = 3) + @Test(timeout = 3) public void testDataDirParsing() throws Throwable { Configuration conf = new Configuration(); List locations; @@ -46,12 +46,16 @@ public class TestDataDirs { File dir3 = new File("/dir3"); File dir4 = new File("/dir4"); +File dir5 = new File("/dir5"); +File dir6 = new File("/dir6"); // Verify that a valid string is correctly parsed, and that storage -// type is not case-sensitive -String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3,[ram_disk]/dir4"; +// type is not case-sensitive and we are able to handle white-space between +// storage type and URI. +String locations1 = "[disk]/dir0,[DISK]/dir1,[sSd]/dir2,[disK]/dir3," + +"[ram_disk]/dir4,[disk]/dir5, [disk] /dir6, [disk] "; conf.set(DFS_DATANODE_DATA_DIR_KEY, locations1); locations = DataNode.getStorageLocations(conf); -assertThat(locations.size(), is(5)); +assertThat(locations.size(), is(8)); assertThat(locations.get(0).getStorageType(), is(StorageType.DISK)); assertThat(locations.get(0).getUri(), is(dir0.toURI())); assertThat(locations.get(1).getStorageType(), is(Sto
hadoop git commit: YARN-4384. updateNodeResource CLI should not accept negative values for resource. (Junping Du via wangda)
Repository: hadoop Updated Branches: refs/heads/trunk 78ec38b2e -> 23c625ec5 YARN-4384. updateNodeResource CLI should not accept negative values for resource. (Junping Du via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23c625ec Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23c625ec Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23c625ec Branch: refs/heads/trunk Commit: 23c625ec571b01c0a2811728608890da38f86890 Parents: 78ec38b Author: Wangda Tan Authored: Tue Nov 24 16:35:56 2015 -0800 Committer: Wangda Tan Committed: Tue Nov 24 16:35:56 2015 -0800 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/yarn/client/cli/RMAdminCLI.java | 11 +++ .../apache/hadoop/yarn/client/cli/TestRMAdminCLI.java | 14 ++ 3 files changed, 28 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c625ec/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 155d5c9..e036335 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1069,6 +1069,9 @@ Release 2.8.0 - UNRELEASED YARN-4387. Fix typo in FairScheduler log message. (Xin Wang via ozawa) +YARN-4384. updateNodeResource CLI should not accept negative values for resource. +(Junping Du via wangda) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c625ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 21ba7a8..a5e53e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -418,11 +418,17 @@ public class RMAdminCLI extends HAAdmin { private int updateNodeResource(String nodeIdStr, int memSize, int cores, int overCommitTimeout) throws IOException, YarnException { +// check resource value first +if (invalidResourceValue(memSize, cores)) { + throw new IllegalArgumentException("Invalid resource value: " + "(" + + memSize + "," + cores + ") for updateNodeResource."); +} // Refresh the nodes ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); UpdateNodeResourceRequest request = recordFactory.newRecordInstance(UpdateNodeResourceRequest.class); NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr); + Resource resource = Resources.createResource(memSize, cores); Map resourceMap = new HashMap(); @@ -433,6 +439,11 @@ public class RMAdminCLI extends HAAdmin { return 0; } + // complain negative value for cpu or memory. + private boolean invalidResourceValue(int memValue, int coreValue) { +return (memValue < 0) || (coreValue < 0); + } + private int getGroups(String[] usernames) throws IOException { // Get groups users belongs to ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/23c625ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index 085cf02..f01441d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -219,6 +219,20 @@ public class TestRMAdminCLI { } @Test(timeout=500) + public void testUpdateNodeResourceWithInvalidValue() throws Exception { +String nodeIdStr = "0.0.0.0:0"; +int memSize = -2048; +int cores = 2; +String[] args = { "-updateNodeResource", nodeIdStr, +Integer.toString(memSize), Integer.toString(cores) }; +// execution of command line is expected to be failed +assertEquals(-1, rmAdmi
hadoop git commit: YARN-4384. updateNodeResource CLI should not accept negative values for resource. (Junping Du via wangda)
Repository: hadoop Updated Branches: refs/heads/branch-2 79e55fb47 -> 7b9f7eaf5 YARN-4384. updateNodeResource CLI should not accept negative values for resource. (Junping Du via wangda) (cherry picked from commit 23c625ec571b01c0a2811728608890da38f86890) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b9f7eaf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b9f7eaf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b9f7eaf Branch: refs/heads/branch-2 Commit: 7b9f7eaf5c2be6589f3ed1fbf13a64b2b61f2a1b Parents: 79e55fb Author: Wangda Tan Authored: Tue Nov 24 16:35:56 2015 -0800 Committer: Wangda Tan Committed: Tue Nov 24 16:36:30 2015 -0800 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/yarn/client/cli/RMAdminCLI.java | 11 +++ .../apache/hadoop/yarn/client/cli/TestRMAdminCLI.java | 14 ++ 3 files changed, 28 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b9f7eaf/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fe1d3f0..feef3a3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1017,6 +1017,9 @@ Release 2.8.0 - UNRELEASED YARN-4387. Fix typo in FairScheduler log message. (Xin Wang via ozawa) +YARN-4384. updateNodeResource CLI should not accept negative values for resource. +(Junping Du via wangda) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b9f7eaf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 21ba7a8..a5e53e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -418,11 +418,17 @@ public class RMAdminCLI extends HAAdmin { private int updateNodeResource(String nodeIdStr, int memSize, int cores, int overCommitTimeout) throws IOException, YarnException { +// check resource value first +if (invalidResourceValue(memSize, cores)) { + throw new IllegalArgumentException("Invalid resource value: " + "(" + + memSize + "," + cores + ") for updateNodeResource."); +} // Refresh the nodes ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); UpdateNodeResourceRequest request = recordFactory.newRecordInstance(UpdateNodeResourceRequest.class); NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr); + Resource resource = Resources.createResource(memSize, cores); Map resourceMap = new HashMap(); @@ -433,6 +439,11 @@ public class RMAdminCLI extends HAAdmin { return 0; } + // complain negative value for cpu or memory. + private boolean invalidResourceValue(int memValue, int coreValue) { +return (memValue < 0) || (coreValue < 0); + } + private int getGroups(String[] usernames) throws IOException { // Get groups users belongs to ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b9f7eaf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index 085cf02..f01441d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -219,6 +219,20 @@ public class TestRMAdminCLI { } @Test(timeout=500) + public void testUpdateNodeResourceWithInvalidValue() throws Exception { +String nodeIdStr = "0.0.0.0:0"; +int memSize = -2048; +int cores = 2; +String[] args = { "-updateNodeResource", nodeIdStr, +Integer.toString(memSize), Integer.toString(cores) }; +// exe
hadoop git commit: HADOOP-12415. Add io.netty dependency to hadoop-nfs and to hadoop-hdfs, needed to build Bigtop successfuly, see BIGTOP-2049
Repository: hadoop Updated Branches: refs/heads/trunk 23c625ec5 -> b4c6b511e HADOOP-12415. Add io.netty dependency to hadoop-nfs and to hadoop-hdfs, needed to build Bigtop successfuly, see BIGTOP-2049 Signed-off-by: Konstantin Boudnik Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4c6b511 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4c6b511 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4c6b511 Branch: refs/heads/trunk Commit: b4c6b511e705230dfe4580288addacb81f5c5c5f Parents: 23c625e Author: Tom Zeng Authored: Mon Sep 21 09:32:25 2015 -0700 Committer: Konstantin Boudnik Committed: Tue Nov 24 18:36:34 2015 -0800 -- hadoop-common-project/hadoop-nfs/pom.xml | 5 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 + 2 files changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c6b511/hadoop-common-project/hadoop-nfs/pom.xml -- diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml index e8156d9..16e53f3 100644 --- a/hadoop-common-project/hadoop-nfs/pom.xml +++ b/hadoop-common-project/hadoop-nfs/pom.xml @@ -82,6 +82,11 @@ runtime + io.netty + netty + compile + + com.google.guava guava http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4c6b511/hadoop-hdfs-project/hadoop-hdfs/pom.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index c8a0dc6..cffc11b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -176,6 +176,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> io.netty + netty + compile + + + io.netty netty-all compile
hadoop git commit: HADOOP-12415. Add io.netty dependency to hadoop-nfs and to hadoop-hdfs, needed to build Bigtop successfuly, see BIGTOP-2049
Repository: hadoop Updated Branches: refs/heads/branch-2 7b9f7eaf5 -> d6f7d2292 HADOOP-12415. Add io.netty dependency to hadoop-nfs and to hadoop-hdfs, needed to build Bigtop successfuly, see BIGTOP-2049 Signed-off-by: Konstantin Boudnik (cherry picked from commit b4c6b511e705230dfe4580288addacb81f5c5c5f) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6f7d229 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6f7d229 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6f7d229 Branch: refs/heads/branch-2 Commit: d6f7d22929d556783fd682227d86d5bf9aba7275 Parents: 7b9f7ea Author: Tom Zeng Authored: Mon Sep 21 09:32:25 2015 -0700 Committer: Konstantin Boudnik Committed: Tue Nov 24 18:39:01 2015 -0800 -- hadoop-common-project/hadoop-nfs/pom.xml | 5 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 5 + 2 files changed, 10 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6f7d229/hadoop-common-project/hadoop-nfs/pom.xml -- diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml index f101669..932c23e 100644 --- a/hadoop-common-project/hadoop-nfs/pom.xml +++ b/hadoop-common-project/hadoop-nfs/pom.xml @@ -82,6 +82,11 @@ runtime + io.netty + netty + compile + + com.google.guava guava http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6f7d229/hadoop-hdfs-project/hadoop-hdfs/pom.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 0d1856b..b4c3c7e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -175,6 +175,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> io.netty + netty + compile + + + io.netty netty-all compile
hadoop git commit: MAPREDUCE-6553. Replace '\u2b05' with '<-' in rendering job configuration. Contributed by Gabor Liptak.
Repository: hadoop Updated Branches: refs/heads/trunk b4c6b511e -> 5ec44fc35 MAPREDUCE-6553. Replace '\u2b05' with '<-' in rendering job configuration. Contributed by Gabor Liptak. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ec44fc3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ec44fc3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ec44fc3 Branch: refs/heads/trunk Commit: 5ec44fc35c129d266edaebbc6fa17754a89b2ad6 Parents: b4c6b51 Author: Akira Ajisaka Authored: Wed Nov 25 16:30:23 2015 +0900 Committer: Akira Ajisaka Committed: Wed Nov 25 16:30:23 2015 +0900 -- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec44fc3/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ef4bb18..170ae7b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -642,6 +642,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6533. testDetermineCacheVisibilities of TestClientDistributedCacheManager is broken (Chang Li via jlowe) + MAPREDUCE-6553. Replace '\u2b05' with '<-' in rendering job configuration. + (Gabor Liptak via aajisaka) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ec44fc3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java index fd2edb4..4cb79bf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java @@ -90,8 +90,7 @@ public class ConfBlock extends HtmlBlock { boolean first = true; for(int i = (sources.length - 2); i >= 0; i--) { if(!first) { -// \u2B05 is an arrow <-- -buffer.append(" \u2B05 "); +buffer.append(" <- "); } first = false; buffer.append(sources[i]);
hadoop git commit: MAPREDUCE-6553. Replace '\u2b05' with '<-' in rendering job configuration. Contributed by Gabor Liptak.
Repository: hadoop Updated Branches: refs/heads/branch-2 d6f7d2292 -> c6cf7910a MAPREDUCE-6553. Replace '\u2b05' with '<-' in rendering job configuration. Contributed by Gabor Liptak. (cherry picked from commit 5ec44fc35c129d266edaebbc6fa17754a89b2ad6) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6cf7910 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6cf7910 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6cf7910 Branch: refs/heads/branch-2 Commit: c6cf7910a461618433a0c23e2a7aa06d53b2a4c1 Parents: d6f7d22 Author: Akira Ajisaka Authored: Wed Nov 25 16:30:23 2015 +0900 Committer: Akira Ajisaka Committed: Wed Nov 25 16:33:18 2015 +0900 -- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6cf7910/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 324f3a4..42139ff 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -352,6 +352,9 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6540. TestMRTimelineEventHandling fails (sjlee) +MAPREDUCE-6553. Replace '\u2b05' with '<-' in rendering job configuration. +(Gabor Liptak via aajisaka) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6cf7910/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java index fd2edb4..4cb79bf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java @@ -90,8 +90,7 @@ public class ConfBlock extends HtmlBlock { boolean first = true; for(int i = (sources.length - 2); i >= 0; i--) { if(!first) { -// \u2B05 is an arrow <-- -buffer.append(" \u2B05 "); +buffer.append(" <- "); } first = false; buffer.append(sources[i]);
hadoop git commit: Fix indents in the 2.8.0 section of MapReduce CHANGES.txt.
Repository: hadoop Updated Branches: refs/heads/trunk 5ec44fc35 -> 177975e96 Fix indents in the 2.8.0 section of MapReduce CHANGES.txt. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/177975e9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/177975e9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/177975e9 Branch: refs/heads/trunk Commit: 177975e962284a2bbd954ebb32bf8f6da5fc975e Parents: 5ec44fc Author: Akira Ajisaka Authored: Wed Nov 25 16:39:49 2015 +0900 Committer: Akira Ajisaka Committed: Wed Nov 25 16:42:22 2015 +0900 -- hadoop-mapreduce-project/CHANGES.txt | 52 +++ 1 file changed, 26 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/177975e9/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 170ae7b..1631668 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -307,17 +307,17 @@ Release 2.8.0 - UNRELEASED NEW FEATURES - MAPREDUCE-6284. Add Task Attempt State API to MapReduce Application - Master REST API. (Ryu Kobayashi via ozawa) +MAPREDUCE-6284. Add Task Attempt State API to MapReduce Application +Master REST API. (Ryu Kobayashi via ozawa) - MAPREDUCE-6364. Add a "Kill" link to Task Attempts page. (Ryu Kobayashi - via ozawa) +MAPREDUCE-6364. Add a "Kill" link to Task Attempts page. (Ryu Kobayashi +via ozawa) - MAPREDUCE-6304. Specifying node labels when submitting MR jobs. - (Naganarasimha G R via wangda) +MAPREDUCE-6304. Specifying node labels when submitting MR jobs. +(Naganarasimha G R via wangda) - MAPREDUCE-6415. Create a tool to combine aggregated logs into HAR files. - (Robert Kanter via kasha) +MAPREDUCE-6415. Create a tool to combine aggregated logs into HAR files. +(Robert Kanter via kasha) IMPROVEMENTS @@ -617,33 +617,33 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6484. Yarn Client uses local address instead of RM address as token renewer in a secure cluster when RM HA is enabled. (Zhihai Xu) - MAPREDUCE-6480. archive-logs tool may miss applications (rkanter) +MAPREDUCE-6480. archive-logs tool may miss applications (rkanter) - MAPREDUCE-6494. Permission issue when running archive-logs tool as - different users (rkanter) +MAPREDUCE-6494. Permission issue when running archive-logs tool as +different users (rkanter) - MAPREDUCE-6485. Create a new task attempt with failed map task priority - if in-progress attempts are unassigned. (Xianyin Xin via rohithsharmaks) +MAPREDUCE-6485. Create a new task attempt with failed map task priority +if in-progress attempts are unassigned. (Xianyin Xin via rohithsharmaks) - MAPREDUCE-6503. archive-logs tool should use HADOOP_PREFIX instead - of HADOOP_HOME (rkanter) +MAPREDUCE-6503. archive-logs tool should use HADOOP_PREFIX instead +of HADOOP_HOME (rkanter) - MAPREDUCE-6302. Preempt reducers after a configurable timeout irrespective - of headroom. (kasha) +MAPREDUCE-6302. Preempt reducers after a configurable timeout irrespective +of headroom. (kasha) - MAPREDUCE-6495. Docs for archive-logs tool (rkanter) +MAPREDUCE-6495. Docs for archive-logs tool (rkanter) - MAPREDUCE-6508. TestNetworkedJob fails consistently due to delegation - token changes on RM. (Akira AJISAKA via junping_du) +MAPREDUCE-6508. TestNetworkedJob fails consistently due to delegation +token changes on RM. (Akira AJISAKA via junping_du) - MAPREDUCE-6515. Update Application priority in AM side from AM-RM heartbeat - (Sunil G via jlowe) +MAPREDUCE-6515. Update Application priority in AM side from AM-RM heartbeat +(Sunil G via jlowe) - MAPREDUCE-6533. testDetermineCacheVisibilities of - TestClientDistributedCacheManager is broken (Chang Li via jlowe) +MAPREDUCE-6533. testDetermineCacheVisibilities of +TestClientDistributedCacheManager is broken (Chang Li via jlowe) - MAPREDUCE-6553. Replace '\u2b05' with '<-' in rendering job configuration. - (Gabor Liptak via aajisaka) +MAPREDUCE-6553. Replace '\u2b05' with '<-' in rendering job configuration. +(Gabor Liptak via aajisaka) Release 2.7.3 - UNRELEASED
hadoop git commit: Fix indents in the 2.8.0 section of MapReduce CHANGES.txt.
Repository: hadoop Updated Branches: refs/heads/branch-2 c6cf7910a -> 5794dc83b Fix indents in the 2.8.0 section of MapReduce CHANGES.txt. (cherry picked from commit 177975e962284a2bbd954ebb32bf8f6da5fc975e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5794dc83 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5794dc83 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5794dc83 Branch: refs/heads/branch-2 Commit: 5794dc83b05fee6fab222c983a1c440eddf923df Parents: c6cf791 Author: Akira Ajisaka Authored: Wed Nov 25 16:39:49 2015 +0900 Committer: Akira Ajisaka Committed: Wed Nov 25 16:44:25 2015 +0900 -- hadoop-mapreduce-project/CHANGES.txt | 48 +++ 1 file changed, 24 insertions(+), 24 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5794dc83/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 42139ff..8046e88 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -6,17 +6,17 @@ Release 2.8.0 - UNRELEASED NEW FEATURES - MAPREDUCE-6284. Add Task Attempt State API to MapReduce Application - Master REST API. (Ryu Kobayashi via ozawa) +MAPREDUCE-6284. Add Task Attempt State API to MapReduce Application +Master REST API. (Ryu Kobayashi via ozawa) - MAPREDUCE-6364. Add a "Kill" link to Task Attempts page. (Ryu Kobayashi - via ozawa) +MAPREDUCE-6364. Add a "Kill" link to Task Attempts page. (Ryu Kobayashi +via ozawa) - MAPREDUCE-6304. Specifying node labels when submitting MR jobs. - (Naganarasimha G R via wangda) +MAPREDUCE-6304. Specifying node labels when submitting MR jobs. +(Naganarasimha G R via wangda) - MAPREDUCE-6415. Create a tool to combine aggregated logs into HAR files. - (Robert Kanter via kasha) +MAPREDUCE-6415. Create a tool to combine aggregated logs into HAR files. +(Robert Kanter via kasha) IMPROVEMENTS @@ -325,30 +325,30 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6484. Yarn Client uses local address instead of RM address as token renewer in a secure cluster when RM HA is enabled. (Zhihai Xu) - MAPREDUCE-6480. archive-logs tool may miss applications (rkanter) +MAPREDUCE-6480. archive-logs tool may miss applications (rkanter) - MAPREDUCE-6494. Permission issue when running archive-logs tool as - different users (rkanter) +MAPREDUCE-6494. Permission issue when running archive-logs tool as +different users (rkanter) - MAPREDUCE-6485. Create a new task attempt with failed map task priority - if in-progress attempts are unassigned. (Xianyin Xin via rohithsharmaks) +MAPREDUCE-6485. Create a new task attempt with failed map task priority +if in-progress attempts are unassigned. (Xianyin Xin via rohithsharmaks) - MAPREDUCE-6503. archive-logs tool should use HADOOP_PREFIX instead - of HADOOP_HOME (rkanter) +MAPREDUCE-6503. archive-logs tool should use HADOOP_PREFIX instead +of HADOOP_HOME (rkanter) - MAPREDUCE-6302. Preempt reducers after a configurable timeout irrespective - of headroom. (kasha) +MAPREDUCE-6302. Preempt reducers after a configurable timeout irrespective +of headroom. (kasha) - MAPREDUCE-6495. Docs for archive-logs tool (rkanter) +MAPREDUCE-6495. Docs for archive-logs tool (rkanter) - MAPREDUCE-6508. TestNetworkedJob fails consistently due to delegation - token changes on RM. (Akira AJISAKA via junping_du) +MAPREDUCE-6508. TestNetworkedJob fails consistently due to delegation +token changes on RM. (Akira AJISAKA via junping_du) - MAPREDUCE-6515. Update Application priority in AM side from AM-RM heartbeat - (Sunil G via jlowe) +MAPREDUCE-6515. Update Application priority in AM side from AM-RM heartbeat +(Sunil G via jlowe) - MAPREDUCE-6533. testDetermineCacheVisibilities of - TestClientDistributedCacheManager is broken (Chang Li via jlowe) +MAPREDUCE-6533. testDetermineCacheVisibilities of +TestClientDistributedCacheManager is broken (Chang Li via jlowe) MAPREDUCE-6540. TestMRTimelineEventHandling fails (sjlee)