hadoop git commit: YARN-8827. Plumb aggregated application resource utilization from the NM to RM. (asuresh)
Repository: hadoop Updated Branches: refs/heads/YARN-1011 bb5991423 -> 608f00998 YARN-8827. Plumb aggregated application resource utilization from the NM to RM. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/608f0099 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/608f0099 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/608f0099 Branch: refs/heads/YARN-1011 Commit: 608f009980b779857b3660a3bd6c70ee44738b8f Parents: bb59914 Author: Arun Suresh Authored: Tue Oct 9 21:09:50 2018 -0700 Committer: Arun Suresh Committed: Tue Oct 9 21:09:50 2018 -0700 -- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 + .../yarn/sls/scheduler/RMNodeWrapper.java | 6 + .../yarn/api/records/ResourceUtilization.java | 24 ++ .../yarn/server/api/records/NodeStatus.java | 19 + .../api/records/impl/pb/NodeStatusPBImpl.java | 59 +++ .../main/proto/yarn_server_common_protos.proto | 6 + .../nodemanager/NodeStatusUpdaterImpl.java | 9 + .../monitor/ContainersMonitor.java | 33 ++ .../monitor/ContainersMonitorImpl.java | 53 ++- .../server/resourcemanager/ResourceManager.java | 8 + .../ResourceUtilizationAggregator.java | 178 + .../server/resourcemanager/rmnode/RMNode.java | 7 + .../resourcemanager/rmnode/RMNodeImpl.java | 26 +- .../rmnode/RMNodeStatusEvent.java | 7 +- .../yarn/server/resourcemanager/MockNM.java | 26 +- .../yarn/server/resourcemanager/MockNodes.java | 16 + .../TestResourceUtilizationAggregator.java | 357 +++ 17 files changed, 820 insertions(+), 20 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/608f0099/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 350f4a3..716a1d8 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -199,6 +199,12 @@ public class NodeInfo { } @Override +public Map +getAggregatedAppUtilizations() { + return null; +} + +@Override public ResourceUtilization getNodeUtilization() { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/608f0099/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index bb6fb9d..0e2a84e 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -225,6 +225,12 @@ public class RMNodeWrapper implements RMNode { } @Override + public Map + getAggregatedAppUtilizations() { +return node.getAggregatedAppUtilizations(); + } + + @Override public Resource getPhysicalResource() { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/608f0099/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java index 2ae4872..c340093 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java @@ -53,6 +53,16 @@ public abstract class ResourceUtilization implements } /** + * Helper function to return a zero-ed out Utilization. + * @return New Resource Utilization. + */ + @Public + @Unstable + public static ResourceUtilization newZero() { +return newInstance(0, 0, 0.0f); + } + + /** * Get used virtual memory. * * @return virtual memory in MB @@ -157,6 +167,20 @@ public abstract class ResourceUtilization implements } /** + *
hadoop git commit: YARN-8808. Use aggregate container utilization instead of node utilization to determine resources available for oversubscription. (Haibo Chen via asuresh)
Repository: hadoop Updated Branches: refs/heads/YARN-1011 8d217ee3c -> efd852449 YARN-8808. Use aggregate container utilization instead of node utilization to determine resources available for oversubscription. (Haibo Chen via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efd85244 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efd85244 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efd85244 Branch: refs/heads/YARN-1011 Commit: efd852449f8db899553fa351ece06cbb53823fc0 Parents: 8d217ee Author: Arun Suresh Authored: Wed Oct 3 11:22:16 2018 -0700 Committer: Arun Suresh Committed: Wed Oct 3 11:22:16 2018 -0700 -- .../scheduler/SchedulerNode.java| 10 - .../yarn/server/resourcemanager/MockNodes.java | 8 +--- .../scheduler/fair/TestFairScheduler.java | 42 ++-- 3 files changed, 32 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 9e38d52..8d47f34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -731,8 +731,16 @@ public abstract class SchedulerNode { return Resources.none(); } +ResourceUtilization aggregateContainersUtilization = +getAggregatedContainersUtilization(); +if (aggregateContainersUtilization == null) { + // be conservative if the aggregate container utilization is unknown + return Resources.none(); + +} + ResourceUtilization projectedNodeUtilization = ResourceUtilization. -newInstance(getNodeUtilization()); +newInstance(aggregateContainersUtilization); // account for resources allocated in this heartbeat projectedNodeUtilization.addTo( (int) (resourceAllocatedPendingLaunch.getMemorySize()), 0, http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index bff6125..d841ff0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -330,17 +330,13 @@ public class MockNodes { return this.physicalResource; } -public void updateResourceUtilization(ResourceUtilization utilization) { - this.nodeUtilization = utilization; -} - -public void updateContainersAndNodeUtilization( +public void updateContainersInfoAndUtilization( UpdatedContainerInfo updatedContainerInfo, ResourceUtilization resourceUtilization) { if (updatedContainerInfo != null) { containerUpdates = Collections.singletonList(updatedContainerInfo); } - this.nodeUtilization = resourceUtilization; + this.containersUtilization = resourceUtilization; } }; http://git-wip-us.apache.org/repos/asf/hadoop/blob/efd85244/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/
hadoop git commit: YARN-7974. Allow updating application tracking url after registration. (Jonathan Hung via asuresh)
Repository: hadoop Updated Branches: refs/heads/branch-2 c617dba49 -> 4d69741a6 YARN-7974. Allow updating application tracking url after registration. (Jonathan Hung via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d69741a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d69741a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d69741a Branch: refs/heads/branch-2 Commit: 4d69741a61b8d29774a1e5f5075406808fc0594a Parents: c617dba Author: Arun Suresh Authored: Wed Sep 26 00:08:10 2018 -0700 Committer: Arun Suresh Committed: Wed Sep 26 00:08:10 2018 -0700 -- .../api/protocolrecords/AllocateRequest.java| 47 +++- .../src/main/proto/yarn_service_protos.proto| 1 + .../hadoop/yarn/client/api/AMRMClient.java | 11 +++ .../yarn/client/api/async/AMRMClientAsync.java | 11 +++ .../api/async/impl/AMRMClientAsyncImpl.java | 5 ++ .../yarn/client/api/impl/AMRMClientImpl.java| 11 +++ .../yarn/client/api/impl/TestAMRMClient.java| 77 .../impl/pb/AllocateRequestPBImpl.java | 27 ++- .../resourcemanager/DefaultAMSProcessor.java| 2 +- .../rmapp/attempt/RMAppAttemptImpl.java | 22 ++ .../event/RMAppAttemptStatusupdateEvent.java| 11 +++ .../TestApplicationMasterService.java | 34 + .../server/resourcemanager/TestRMRestart.java | 45 13 files changed, 300 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index da68d8c..8f6a204 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -100,7 +100,21 @@ public abstract class AllocateRequest { allocateRequest.setIncreaseRequests(increaseRequests); return allocateRequest; } - + + @Public + @Unstable + public static AllocateRequest newInstance(int responseID, float appProgress, + List resourceAsk, + List containersToBeReleased, + ResourceBlacklistRequest resourceBlacklistRequest, + String trackingUrl) { +return AllocateRequest.newBuilder().responseId(responseID) +.progress(appProgress).askList(resourceAsk) +.releaseList(containersToBeReleased) +.resourceBlacklistRequest(resourceBlacklistRequest) +.trackingUrl(trackingUrl).build(); + } + @Public @Unstable public static AllocateRequest newInstance(int responseID, float appProgress, @@ -256,6 +270,22 @@ public abstract class AllocateRequest { public abstract void setUpdateRequests( List updateRequests); + /** + * Get the tracking url update for this heartbeat. + * @return tracking url to update this application with + */ + @Public + @Unstable + public abstract String getTrackingUrl(); + + /** + * Set the new tracking url for this application. + * @param trackingUrl the new tracking url + */ + @Public + @Unstable + public abstract void setTrackingUrl(String trackingUrl); + @Public @Unstable public static AllocateRequestBuilder newBuilder() { @@ -358,6 +388,19 @@ public abstract class AllocateRequest { } /** + * Set the trackingUrl of the request. + * @see AllocateRequest#setTrackingUrl(String) + * @param trackingUrl new tracking url + * @return {@link AllocateRequestBuilder} + */ +@Public +@Unstable +public AllocateRequestBuilder trackingUrl(String trackingUrl) { + allocateRequest.setTrackingUrl(trackingUrl); + return this; +} + +/** * Return generated {@link AllocateRequest} object. * @return {@link AllocateRequest} */ @@ -367,4 +410,4 @@ public abstract class AllocateRequest { return allocateRequest; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d69741a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/pr
hadoop git commit: YARN-8809. Refactor AbstractYarnScheduler and CapacityScheduler OPPORTUNISTIC container completion codepaths. (Haibo Chen via asuresh)
Repository: hadoop Updated Branches: refs/heads/YARN-1011 36ec27ed3 -> 4d858dd92 YARN-8809. Refactor AbstractYarnScheduler and CapacityScheduler OPPORTUNISTIC container completion codepaths. (Haibo Chen via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d858dd9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d858dd9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d858dd9 Branch: refs/heads/YARN-1011 Commit: 4d858dd92d0cf388330c41e984fde5fd0c5aa95a Parents: 36ec27e Author: Arun Suresh Authored: Fri Sep 21 12:02:00 2018 -0700 Committer: Arun Suresh Committed: Fri Sep 21 12:02:00 2018 -0700 -- .../scheduler/AbstractYarnScheduler.java| 32 .../scheduler/capacity/CapacityScheduler.java | 25 ++- .../scheduler/fair/FairScheduler.java | 11 --- .../scheduler/fifo/FifoScheduler.java | 2 +- .../scheduler/fair/TestFairScheduler.java | 25 +++ 5 files changed, 70 insertions(+), 25 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d858dd9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index a05ee26..d382263 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -677,25 +677,12 @@ public abstract class AbstractYarnScheduler } if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) { - completedContainerInternal(rmContainer, containerStatus, event); + completeGuaranteedContainerInternal(rmContainer, containerStatus, event); completeOustandingUpdatesWhichAreReserved( rmContainer, containerStatus, event); } else { - ContainerId containerId = rmContainer.getContainerId(); - // Inform the container - rmContainer.handle( - new RMContainerFinishedEvent(containerId, containerStatus, event)); - SchedulerApplicationAttempt schedulerAttempt = - getCurrentAttemptForContainer(containerId); - if (schedulerAttempt != null) { -schedulerAttempt.removeRMContainer(containerId); - } - if (LOG.isDebugEnabled()) { -LOG.debug("Completed container: " + rmContainer.getContainerId() + -" in state: " + rmContainer.getState() + " event:" + event); - } - getSchedulerNode(rmContainer.getNodeId()).releaseContainer( - rmContainer.getContainerId(), false); + completeOpportunisticContainerInternal(rmContainer, containerStatus, + event); } // If the container is getting killed in ACQUIRED state, the requester (AM @@ -705,6 +692,12 @@ public abstract class AbstractYarnScheduler recoverResourceRequestForContainer(rmContainer); } + protected void completeOpportunisticContainerInternal( + RMContainer rmContainer, ContainerStatus containerStatus, + RMContainerEventType event) { +completeGuaranteedContainerInternal(rmContainer, containerStatus, event); + } + // Optimization: // Check if there are in-flight container updates and complete the // associated temp containers. These are removed when the app completes, @@ -722,7 +715,7 @@ public abstract class AbstractYarnScheduler .getReservedSchedulerKey().getContainerToUpdate(); if (containerToUpdate != null && containerToUpdate.equals(containerStatus.getContainerId())) { - completedContainerInternal(resContainer, + completeGuaranteedContainerInternal(resContainer, ContainerStatus.newInstance(resContainer.getContainerId(), containerStatus.getState(), containerStatus .getDiagnostics(), @@ -732,8 +725,9 @@ public abstract class AbstractYarnScheduler } } - // clean up a completed container - protected abstract void completedContainerInternal(RMContainer rmContainer, + // clean up a co
hadoop git commit: MAPREDUCE-7101. Add config parameter to allow JHS to alway scan user dir irrespective of modTime. (Thomas Marquardt via asuresh)
Repository: hadoop Updated Branches: refs/heads/branch-2 b3f4c2766 -> 358fe4c61 MAPREDUCE-7101. Add config parameter to allow JHS to alway scan user dir irrespective of modTime. (Thomas Marquardt via asuresh) (cherry picked from commit 5670e89b2ec69ab71e32dcd5acbd3a57ca6abea5) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/358fe4c6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/358fe4c6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/358fe4c6 Branch: refs/heads/branch-2 Commit: 358fe4c61a563ec70dbf96164abd28f28677ef9a Parents: b3f4c27 Author: Arun Suresh Authored: Tue Jun 12 15:36:52 2018 -0700 Committer: Arun Suresh Committed: Tue Jun 12 16:25:01 2018 -0700 -- .../hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java | 9 +++-- .../src/main/resources/mapred-default.xml | 9 + .../apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java | 8 +++- 3 files changed, 23 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/358fe4c6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index 644727f..9c3e668 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -61,8 +61,13 @@ public class JHAdminConfig { MR_HISTORY_PREFIX + "cleaner.interval-ms"; public static final long DEFAULT_MR_HISTORY_CLEANER_INTERVAL_MS = 1 * 24 * 60 * 60 * 1000l; //1 day - - + + /** Always scan user dir, irrespective of dir modification time.*/ + public static final String MR_HISTORY_ALWAYS_SCAN_USER_DIR = + MR_HISTORY_PREFIX + "always-scan-user-dir"; + public static final boolean DEFAULT_MR_HISTORY_ALWAYS_SCAN_USER_DIR = + false; + /** The number of threads to handle client API requests.*/ public static final String MR_HISTORY_CLIENT_THREAD_COUNT = MR_HISTORY_PREFIX + "client.thread-count"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/358fe4c6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index a33960c..8b133ed 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1719,6 +1719,15 @@ + mapreduce.jobhistory.always-scan-user-dir + false + Some Cloud FileSystems do not currently update the + modification time of directories. To support these filesystems, this + configuration value should be set to 'true'. + + + + mapreduce.jobhistory.done-dir ${yarn.app.mapreduce.am.staging-dir}/history/done http://git-wip-us.apache.org/repos/asf/hadoop/blob/358fe4c6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index 6fd1c0e..9ba00ad 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -322,7 +322,13 @@ public class HistoryFileManager extends AbstractServi
hadoop git commit: MAPREDUCE-7101. Add config parameter to allow JHS to alway scan user dir irrespective of modTime. (Thomas Marquardt via asuresh)
Repository: hadoop Updated Branches: refs/heads/trunk aeaf9fec6 -> 5670e89b2 MAPREDUCE-7101. Add config parameter to allow JHS to alway scan user dir irrespective of modTime. (Thomas Marquardt via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5670e89b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5670e89b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5670e89b Branch: refs/heads/trunk Commit: 5670e89b2ec69ab71e32dcd5acbd3a57ca6abea5 Parents: aeaf9fe Author: Arun Suresh Authored: Tue Jun 12 15:36:52 2018 -0700 Committer: Arun Suresh Committed: Tue Jun 12 15:36:52 2018 -0700 -- .../hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java | 9 +++-- .../src/main/resources/mapred-default.xml | 9 + .../apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java | 8 +++- 3 files changed, 23 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5670e89b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java index 1cadf84..9e964e1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java @@ -61,8 +61,13 @@ public class JHAdminConfig { MR_HISTORY_PREFIX + "cleaner.interval-ms"; public static final long DEFAULT_MR_HISTORY_CLEANER_INTERVAL_MS = 1 * 24 * 60 * 60 * 1000l; //1 day - - + + /** Always scan user dir, irrespective of dir modification time.*/ + public static final String MR_HISTORY_ALWAYS_SCAN_USER_DIR = + MR_HISTORY_PREFIX + "always-scan-user-dir"; + public static final boolean DEFAULT_MR_HISTORY_ALWAYS_SCAN_USER_DIR = + false; + /** The number of threads to handle client API requests.*/ public static final String MR_HISTORY_CLIENT_THREAD_COUNT = MR_HISTORY_PREFIX + "client.thread-count"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/5670e89b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index dcb312c..9f33d65 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -1775,6 +1775,15 @@ + mapreduce.jobhistory.always-scan-user-dir + false + Some Cloud FileSystems do not currently update the + modification time of directories. To support these filesystems, this + configuration value should be set to 'true'. + + + + mapreduce.jobhistory.done-dir ${yarn.app.mapreduce.am.staging-dir}/history/done http://git-wip-us.apache.org/repos/asf/hadoop/blob/5670e89b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index a07ca26..7fe99a2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -324,7 +324,13 @@ public class HistoryFileManager extends AbstractService { // so we need to have additional check. // Note: modTi
hadoop git commit: YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. (Botong Huang via asuresh)
Repository: hadoop Updated Branches: refs/heads/branch-2 734cbdd6b -> 113e2d680 YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. (Botong Huang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/113e2d68 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/113e2d68 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/113e2d68 Branch: refs/heads/branch-2 Commit: 113e2d6801e06f90dcd2c1110bcebab9be06a60b Parents: 734cbdd Author: Arun Suresh <asur...@apache.org> Authored: Mon May 21 11:26:32 2018 -0700 Committer: Arun Suresh <asur...@apache.org> Committed: Mon May 21 11:26:32 2018 -0700 -- .../hadoop/yarn/client/AMRMClientUtils.java | 190 +++ .../hadoop/yarn/server/AMRMClientRelayer.java | 336 +++ .../failover/FederationProxyProviderUtil.java | 3 +- .../apache/hadoop/yarn/server/package-info.java | 18 + .../server/scheduler/ResourceRequestSet.java| 206 .../server/scheduler/ResourceRequestSetKey.java | 133 .../server/scheduler/SchedulerRequestKey.java | 4 +- .../yarn/server/uam/UnmanagedAMPoolManager.java | 2 +- .../server/uam/UnmanagedApplicationManager.java | 2 +- .../yarn/server/utils/AMRMClientUtils.java | 191 --- .../yarn/server/MockResourceManagerFacade.java | 2 +- .../yarn/server/TestAMRMClientRelayer.java | 275 +++ .../amrmproxy/FederationInterceptor.java| 2 +- .../ApplicationMasterService.java | 2 +- .../TestApplicationMasterLauncher.java | 2 +- 15 files changed, 1166 insertions(+), 202 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/113e2d68/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java new file mode 100644 index 000..7f624859 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AMRMClientUtils.java @@ -0,0 +1,190 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.security.SaslRpcServer; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; +import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; +import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; +import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for
[1/2] hadoop git commit: YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. (Botong Huang via asuresh)
Repository: hadoop Updated Branches: refs/heads/trunk f749517cc -> 3159bffce http://git-wip-us.apache.org/repos/asf/hadoop/blob/3159bffc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index e518b90..38181e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.client.AMRMClientUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException; @@ -78,7 +79,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.server.utils.AMRMClientUtils; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.log4j.Level; import org.apache.log4j.LogManager; - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[2/2] hadoop git commit: YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. (Botong Huang via asuresh)
YARN-7900. [AMRMProxy] AMRMClientRelayer for stateful FederationInterceptor. (Botong Huang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3159bffc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3159bffc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3159bffc Branch: refs/heads/trunk Commit: 3159bffce23abf35754da2d7d51de7d8c2631ae3 Parents: f749517 Author: Arun Suresh <asur...@apache.org> Authored: Thu May 17 20:00:52 2018 -0700 Committer: Arun Suresh <asur...@apache.org> Committed: Thu May 17 20:00:52 2018 -0700 -- .../yarn/client/api/impl/AMRMClientImpl.java| 151 ++-- .../hadoop/yarn/client/AMRMClientUtils.java | 262 + .../hadoop/yarn/server/AMRMClientRelayer.java | 364 +++ .../failover/FederationProxyProviderUtil.java | 2 +- .../apache/hadoop/yarn/server/package-info.java | 18 + .../server/scheduler/ResourceRequestSet.java| 206 +++ .../server/scheduler/ResourceRequestSetKey.java | 133 +++ .../server/scheduler/SchedulerRequestKey.java | 4 +- .../yarn/server/uam/UnmanagedAMPoolManager.java | 2 +- .../server/uam/UnmanagedApplicationManager.java | 2 +- .../yarn/server/utils/AMRMClientUtils.java | 191 -- .../yarn/server/MockResourceManagerFacade.java | 2 +- .../yarn/server/TestAMRMClientRelayer.java | 275 ++ .../amrmproxy/FederationInterceptor.java| 2 +- .../ApplicationMasterService.java | 2 +- .../TestApplicationMasterLauncher.java | 2 +- 16 files changed, 1299 insertions(+), 319 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3159bffc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java index ef849b2..36c3cf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java @@ -31,11 +31,9 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; import java.util.Set; import java.util.TreeSet; import java.util.AbstractMap.SimpleEntry; -import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -68,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.api.records.UpdatedContainer; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.client.AMRMClientUtils; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; @@ -113,13 +112,9 @@ public class AMRMClientImpl extends AMRMClient { protected final Set blacklistRemovals = new HashSet(); private Map<Set, PlacementConstraint> placementConstraints = new HashMap<>(); - private Queue<Collection> batchedSchedulingRequests = - new LinkedList<>(); - private Map<Set, List> outstandingSchedRequests = - new ConcurrentHashMap<>(); protected Map<String, Resource> resourceProfilesMap; - + static class ResourceRequestInfo { ResourceRequest remoteRequest; LinkedHashSet containerRequests; @@ -168,6 +163,10 @@ public class AMRMClientImpl extends AMRMClient { SimpleEntry<Container, UpdateContainerRequest>> pendingChange = new HashMap<>(); + private List schedulingRequests = new ArrayList<>(); + private Map<Set, List> outstandingSchedRequests = + new HashMap<>(); + public AMRMClientImpl() { super(AMRMClientImpl.class.getName()); } @@ -252,18 +251,18 @@ public class AMRMClientImpl extends AMRMClient { this.resourceProfilesMap = response.getResourceProfiles(); List prevContainers = response.getContainersFromPreviousAttempts(); - removeFromOutstandingSchedulingRequests(prevContainers); - recreateSchedulingRequestBatch(); + AMRMClientUtils.removeFromOutstandingSchedulingRequests
hadoop git commit: HADOOP-15446. WASB: PageBlobInputStream.skip breaks HBASE replication. Contributed by Thomas Marquardt
Repository: hadoop Updated Branches: refs/heads/branch-2.9 019ba5998 -> 1f32345cd HADOOP-15446. WASB: PageBlobInputStream.skip breaks HBASE replication. Contributed by Thomas Marquardt (cherry picked from commit 5b11b9fd413470e134ecdc7c50468f8c7b39fa50) (cherry picked from commit 67468651b19ae9de04d20decb79c23e2541df584) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f32345c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f32345c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f32345c Branch: refs/heads/branch-2.9 Commit: 1f32345cd0d59a14a264933509d143aecb82db5e Parents: 019ba59 Author: Steve LoughranAuthored: Mon May 7 11:54:08 2018 +0100 Committer: Arun Suresh Committed: Thu May 10 10:46:14 2018 -0700 -- .../hadoop/fs/azure/PageBlobInputStream.java| 123 +++-- .../fs/azure/ITestPageBlobInputStream.java | 527 +++ 2 files changed, 605 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f32345c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java -- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java index 097201b..40bf6f4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java @@ -25,12 +25,14 @@ import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.toShort; import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.withMD5Checking; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper; import com.microsoft.azure.storage.OperationContext; @@ -58,7 +60,9 @@ final class PageBlobInputStream extends InputStream { // The buffer holding the current data we last read from the server. private byte[] currentBuffer; // The current byte offset we're at in the buffer. - private int currentOffsetInBuffer; + private int currentBufferOffset; + // The current buffer length + private int currentBufferLength; // Maximum number of pages to get per any one request. private static final int MAX_PAGES_PER_DOWNLOAD = 4 * 1024 * 1024 / PAGE_SIZE; @@ -174,7 +178,7 @@ final class PageBlobInputStream extends InputStream { private boolean dataAvailableInBuffer() { return currentBuffer != null -&& currentOffsetInBuffer < currentBuffer.length; +&& currentBufferOffset < currentBufferLength; } /** @@ -194,6 +198,8 @@ final class PageBlobInputStream extends InputStream { return true; } currentBuffer = null; +currentBufferOffset = 0; +currentBufferLength = 0; if (numberOfPagesRemaining == 0) { // No more data to read. return false; @@ -209,43 +215,48 @@ final class PageBlobInputStream extends InputStream { ByteArrayOutputStream baos = new ByteArrayOutputStream(bufferSize); blob.downloadRange(currentOffsetInBlob, bufferSize, baos, withMD5Checking(), opContext); - currentBuffer = baos.toByteArray(); + validateDataIntegrity(baos.toByteArray()); } catch (StorageException e) { throw new IOException(e); } numberOfPagesRemaining -= pagesToRead; currentOffsetInBlob += bufferSize; -currentOffsetInBuffer = PAGE_HEADER_SIZE; - -// Since we just downloaded a new buffer, validate its consistency. -validateCurrentBufferConsistency(); return true; } - private void validateCurrentBufferConsistency() + private void validateDataIntegrity(byte[] buffer) throws IOException { -if (currentBuffer.length % PAGE_SIZE != 0) { + +if (buffer.length % PAGE_SIZE != 0) { throw new AssertionError("Unexpected buffer size: " - + currentBuffer.length); + + buffer.length); } -int numberOfPages = currentBuffer.length / PAGE_SIZE; + +int bufferLength = 0; +int numberOfPages = buffer.length / PAGE_SIZE; +long totalPagesAfterCurrent = numberOfPagesRemaining; + for (int page = 0; page < numberOfPages; page++) { - short currentPageSize = getPageSize(blob, currentBuffer, - page * PAGE_SIZE); - // Calculate the number of pages that exist after this one
hadoop git commit: HADOOP-15446. WASB: PageBlobInputStream.skip breaks HBASE replication. Contributed by Thomas Marquardt
Repository: hadoop Updated Branches: refs/heads/branch-2 ed592a210 -> 67468651b HADOOP-15446. WASB: PageBlobInputStream.skip breaks HBASE replication. Contributed by Thomas Marquardt (cherry picked from commit 5b11b9fd413470e134ecdc7c50468f8c7b39fa50) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67468651 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67468651 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67468651 Branch: refs/heads/branch-2 Commit: 67468651b19ae9de04d20decb79c23e2541df584 Parents: ed592a2 Author: Steve LoughranAuthored: Mon May 7 11:54:08 2018 +0100 Committer: Arun Suresh Committed: Thu May 10 10:33:03 2018 -0700 -- .../hadoop/fs/azure/PageBlobInputStream.java| 123 +++-- .../fs/azure/ITestPageBlobInputStream.java | 527 +++ 2 files changed, 605 insertions(+), 45 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/67468651/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java -- diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java index aaac490..40bf6f4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobInputStream.java @@ -25,12 +25,14 @@ import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.toShort; import static org.apache.hadoop.fs.azure.PageBlobFormatHelpers.withMD5Checking; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper; import com.microsoft.azure.storage.OperationContext; @@ -58,7 +60,9 @@ final class PageBlobInputStream extends InputStream { // The buffer holding the current data we last read from the server. private byte[] currentBuffer; // The current byte offset we're at in the buffer. - private int currentOffsetInBuffer; + private int currentBufferOffset; + // The current buffer length + private int currentBufferLength; // Maximum number of pages to get per any one request. private static final int MAX_PAGES_PER_DOWNLOAD = 4 * 1024 * 1024 / PAGE_SIZE; @@ -174,7 +178,7 @@ final class PageBlobInputStream extends InputStream { private boolean dataAvailableInBuffer() { return currentBuffer != null -&& currentOffsetInBuffer < currentBuffer.length; +&& currentBufferOffset < currentBufferLength; } /** @@ -194,6 +198,8 @@ final class PageBlobInputStream extends InputStream { return true; } currentBuffer = null; +currentBufferOffset = 0; +currentBufferLength = 0; if (numberOfPagesRemaining == 0) { // No more data to read. return false; @@ -209,43 +215,48 @@ final class PageBlobInputStream extends InputStream { ByteArrayOutputStream baos = new ByteArrayOutputStream(bufferSize); blob.downloadRange(currentOffsetInBlob, bufferSize, baos, withMD5Checking(), opContext); - currentBuffer = baos.toByteArray(); + validateDataIntegrity(baos.toByteArray()); } catch (StorageException e) { throw new IOException(e); } numberOfPagesRemaining -= pagesToRead; currentOffsetInBlob += bufferSize; -currentOffsetInBuffer = PAGE_HEADER_SIZE; - -// Since we just downloaded a new buffer, validate its consistency. -validateCurrentBufferConsistency(); return true; } - private void validateCurrentBufferConsistency() + private void validateDataIntegrity(byte[] buffer) throws IOException { -if (currentBuffer.length % PAGE_SIZE != 0) { + +if (buffer.length % PAGE_SIZE != 0) { throw new AssertionError("Unexpected buffer size: " - + currentBuffer.length); + + buffer.length); } -int numberOfPages = currentBuffer.length / PAGE_SIZE; + +int bufferLength = 0; +int numberOfPages = buffer.length / PAGE_SIZE; +long totalPagesAfterCurrent = numberOfPagesRemaining; + for (int page = 0; page < numberOfPages; page++) { - short currentPageSize = getPageSize(blob, currentBuffer, - page * PAGE_SIZE); - // Calculate the number of pages that exist after this one - // in the blob. - long totalPagesAfterCurrent = -
hadoop git commit: YARN-7972. Support inter-app placement constraints for allocation tags by application ID. (Weiwei Yang via asuresh)
Repository: hadoop Updated Branches: refs/heads/trunk 2e1e049bd -> 1054b48c2 YARN-7972. Support inter-app placement constraints for allocation tags by application ID. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1054b48c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1054b48c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1054b48c Branch: refs/heads/trunk Commit: 1054b48c27f3158110bd0512afecded36eecb8ad Parents: 2e1e049 Author: Arun Suresh <asur...@apache.org> Authored: Mon Mar 5 11:24:17 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Mon Mar 5 11:24:17 2018 -0800 -- .../api/records/AllocationTagNamespace.java | 336 +++ .../api/records/AllocationTagNamespaceType.java | 74 .../hadoop/yarn/api/records/AllocationTags.java | 50 +++ .../hadoop/yarn/api/records/Evaluable.java | 38 +++ .../yarn/api/records/TargetApplications.java| 53 +++ .../yarn/api/resource/PlacementConstraints.java | 26 +- .../InvalidAllocationTagException.java | 34 ++ .../constraint/AllocationTagsManager.java | 9 + .../InvalidAllocationTagsQueryException.java| 4 + .../constraint/PlacementConstraintsUtil.java| 71 +++- .../SingleConstraintAppPlacementAllocator.java | 31 +- .../constraint/TestAllocationTagsNamespace.java | 147 .../TestPlacementConstraintsUtil.java | 151 + 13 files changed, 1001 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1054b48c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java new file mode 100644 index 000..25f8761 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AllocationTagNamespace.java @@ -0,0 +1,336 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.yarn.exceptions.InvalidAllocationTagException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.SELF; +import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.NOT_SELF; +import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.APP_LABEL; +import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.APP_ID; +import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.ALL; +import static org.apache.hadoop.yarn.api.records.AllocationTagNamespaceType.fromString; + +/** + * Class to describe the namespace of an allocation tag. + * Each namespace can be evaluated against a set of applications. + * After evaluation, the namespace should have an implicit set of + * applications which defines its scope. + */ +public abstract class AllocationTagNamespace implements +Evaluable { + + public final static String NAMESPACE_DELIMITER = "/"; + + private AllocationTagNamespaceType nsType; + // Namespace scope value will be delay binding by eval method. + private Set nsScope; + + public AllocationTagNamespace(AllocationTagNamespaceType + allocationTagNamespaceType) { +this.nsType = allocationTagNamespaceType; + } + + protected void setScopeIfNotNull(Set appIds) { +if (appIds != null) { + this.nsScope = appIds; +} + } + + /** + * Get the type of the namespace. + * @return namespace type
hadoop git commit: YARN-7918. Fix TestAMRMClientPlacementConstraints. (Gergely Novák via asuresh)
Repository: hadoop Updated Branches: refs/heads/branch-3.1 f878ddb06 -> 950566230 YARN-7918. Fix TestAMRMClientPlacementConstraints. (Gergely Novák via asuresh) (cherry picked from commit a1e56a62863d8d494af309ec5f476c4b7e4d5ef9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95056623 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95056623 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95056623 Branch: refs/heads/branch-3.1 Commit: 950566230c5fb2a7e5c5b1eb5328ca155e6d0f45 Parents: f878ddb Author: Arun Suresh <asur...@apache.org> Authored: Sat Feb 17 03:24:55 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Sat Feb 17 03:26:13 2018 -0800 -- .../constraint/processor/PlacementConstraintProcessor.java| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/95056623/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java index f089a19..cf944a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor; +import com.google.common.collect.Lists; import org.apache.hadoop.yarn.ams.ApplicationMasterServiceContext; import org.apache.hadoop.yarn.ams.ApplicationMasterServiceProcessor; import org.apache.hadoop.yarn.ams.ApplicationMasterServiceUtils; @@ -329,7 +330,7 @@ public class PlacementConstraintProcessor extends AbstractPlacementProcessor { if (!isAdded) { BatchedRequests br = new BatchedRequests(iteratorType, schedulerResponse.getApplicationId(), - Collections.singleton(schedulerResponse.getSchedulingRequest()), + Lists.newArrayList(schedulerResponse.getSchedulingRequest()), placementAttempt + 1); reqsToRetry.add(br); br.addToBlacklist( - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7918. Fix TestAMRMClientPlacementConstraints. (Gergely Novák via asuresh)
Repository: hadoop Updated Branches: refs/heads/trunk 31db977cb -> a1e56a628 YARN-7918. Fix TestAMRMClientPlacementConstraints. (Gergely Novák via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1e56a62 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1e56a62 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1e56a62 Branch: refs/heads/trunk Commit: a1e56a62863d8d494af309ec5f476c4b7e4d5ef9 Parents: 31db977 Author: Arun Suresh <asur...@apache.org> Authored: Sat Feb 17 03:24:55 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Sat Feb 17 03:24:55 2018 -0800 -- .../constraint/processor/PlacementConstraintProcessor.java| 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1e56a62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java index f089a19..cf944a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementConstraintProcessor.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor; +import com.google.common.collect.Lists; import org.apache.hadoop.yarn.ams.ApplicationMasterServiceContext; import org.apache.hadoop.yarn.ams.ApplicationMasterServiceProcessor; import org.apache.hadoop.yarn.ams.ApplicationMasterServiceUtils; @@ -329,7 +330,7 @@ public class PlacementConstraintProcessor extends AbstractPlacementProcessor { if (!isAdded) { BatchedRequests br = new BatchedRequests(iteratorType, schedulerResponse.getApplicationId(), - Collections.singleton(schedulerResponse.getSchedulingRequest()), + Lists.newArrayList(schedulerResponse.getSchedulingRequest()), placementAttempt + 1); reqsToRetry.add(br); br.addToBlacklist( - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7839. Modify PlacementAlgorithm to Check node capacity before placing request on node. (Panagiotis Garefalakis via asuresh)
Repository: hadoop Updated Branches: refs/heads/trunk 460d77bd6 -> 6e5ba9366 YARN-7839. Modify PlacementAlgorithm to Check node capacity before placing request on node. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e5ba936 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e5ba936 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e5ba936 Branch: refs/heads/trunk Commit: 6e5ba9366fc05719906ff2789b1a0fd26001182b Parents: 460d77b Author: Arun Suresh <asur...@apache.org> Authored: Fri Feb 2 10:28:22 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Fri Feb 2 10:28:22 2018 -0800 -- .../scheduler/capacity/CapacityScheduler.java | 4 - .../algorithm/DefaultPlacementAlgorithm.java| 61 ++ .../api/ConstraintPlacementAlgorithmOutput.java | 5 +- .../SchedulingRequestWithPlacementAttempt.java | 52 .../constraint/processor/BatchedRequests.java | 2 +- .../processor/PlacementDispatcher.java | 12 +-- .../processor/PlacementProcessor.java | 28 +-- .../constraint/TestPlacementProcessor.java | 87 +++- 8 files changed, 215 insertions(+), 36 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e5ba936/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index cb01351..d3aa5cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2609,10 +2609,6 @@ public class CapacityScheduler extends " but only 1 will be attempted !!"); } if (!appAttempt.isStopped()) { -Resource resource = -schedulingRequest.getResourceSizing().getResources(); -schedulingRequest.getResourceSizing().setResources( -getNormalizedResource(resource)); ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> resourceCommitRequest = createResourceCommitRequest( appAttempt, schedulingRequest, schedulerNode); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e5ba936/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java index 4e6473f..710e6c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java @@ -18,10 +18,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm; import java.util.ArrayList; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceSizing; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.server.reso
[hadoop] Git Push Summary
Repository: hadoop Updated Branches: refs/heads/YARN-7812 [deleted] e6d2d26a1 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
hadoop git commit: YARN-7822. Constraint satisfaction checker support for composite OR and AND constraints. (Weiwei Yang via asuresh)
Repository: hadoop Updated Branches: refs/heads/trunk 8d1e2c640 -> d48134478 YARN-7822. Constraint satisfaction checker support for composite OR and AND constraints. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4813447 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4813447 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4813447 Branch: refs/heads/trunk Commit: d4813447831770446399f2d6501860141551ff33 Parents: 8d1e2c6 Author: Arun Suresh <asur...@apache.org> Authored: Tue Jan 30 10:15:33 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:51:08 2018 -0800 -- .../TestPlacementConstraintTransformations.java | 2 +- .../constraint/PlacementConstraintsUtil.java| 53 +++- .../TestPlacementConstraintsUtil.java | 278 --- .../constraint/TestPlacementProcessor.java | 159 ++- 4 files changed, 444 insertions(+), 48 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4813447/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java index 62da092..aa92d7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java @@ -156,7 +156,7 @@ public class TestPlacementConstraintTransformations { SingleConstraintTransformer singleTransformer = new SingleConstraintTransformer(specConstraint); PlacementConstraint simConstraint = singleTransformer.transform(); -Assert.assertTrue(constraintExpr instanceof Or); +Assert.assertTrue(simConstraint.getConstraintExpr() instanceof Or); Or simOrExpr = (Or) specConstraint.getConstraintExpr(); for (AbstractConstraint child : simOrExpr.getChildren()) { Assert.assertTrue(child instanceof SingleConstraint); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4813447/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index 199dd62..6396e57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; @@ -149,6 +151,48 @@ public final class PlacementConstraintsUtil { return true; } + /** + * Returns true if all child constraints are satisfied. + * @param appId application id + * @param constraint Or constraint + * @param node node + * @param atm allocation tags manager + * @return true if all child constraints are satisfied, false otherwise + * @throws InvalidAllocationTagsQueryException + */ + private
[09/32] hadoop git commit: YARN-7522. Introduce AllocationTagsManager to associate allocation tags to nodes. (Wangda Tan via asuresh)
YARN-7522. Introduce AllocationTagsManager to associate allocation tags to nodes. (Wangda Tan via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/801c0988 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/801c0988 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/801c0988 Branch: refs/heads/trunk Commit: 801c0988b5ad1eff1e896a2635c2937721c96b04 Parents: 69de9a1 Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 8 00:24:00 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../resourcemanager/RMActiveServiceContext.java | 15 + .../yarn/server/resourcemanager/RMContext.java | 5 + .../server/resourcemanager/RMContextImpl.java | 12 + .../server/resourcemanager/ResourceManager.java | 9 + .../constraint/AllocationTagsManager.java | 431 +++ .../constraint/AllocationTagsNamespaces.java| 31 ++ .../InvalidAllocationTagsQueryException.java| 35 ++ .../rmcontainer/RMContainer.java| 8 + .../rmcontainer/RMContainerImpl.java| 21 + .../constraint/TestAllocationTagsManager.java | 328 ++ .../rmcontainer/TestRMContainerImpl.java| 124 ++ .../scheduler/capacity/TestUtils.java | 9 + .../scheduler/fifo/TestFifoScheduler.java | 5 + 13 files changed, 1033 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/801c0988/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 9dc5945..6ee3a4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; @@ -107,6 +108,7 @@ public class RMActiveServiceContext { private RMAppLifetimeMonitor rmAppLifetimeMonitor; private QueueLimitCalculator queueLimitCalculator; + private AllocationTagsManager allocationTagsManager; public RMActiveServiceContext() { queuePlacementManager = new PlacementManager(); @@ -398,6 +400,19 @@ public class RMActiveServiceContext { @Private @Unstable + public AllocationTagsManager getAllocationTagsManager() { +return allocationTagsManager; + } + + @Private + @Unstable + public void setAllocationTagsManager( + AllocationTagsManager allocationTagsManager) { +this.allocationTagsManager = allocationTagsManager; + } + + @Private + @Unstable public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() { return rmDelegatedNodeLabelsUpdater; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/801c0988/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index ec94030..62899d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext
[30/32] hadoop git commit: YARN-7779. Display allocation tags in RM web UI and expose same through REST API. Contributed by Weiwei Yang.
YARN-7779. Display allocation tags in RM web UI and expose same through REST API. Contributed by Weiwei Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b81cb05 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b81cb05 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b81cb05 Branch: refs/heads/trunk Commit: 9b81cb0537e5b731581e6a375bf0a59abf61c359 Parents: adbe87a Author: Sunil GAuthored: Tue Jan 23 17:09:58 2018 +0530 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 ++ .../yarn/sls/scheduler/RMNodeWrapper.java | 6 ++ .../server/resourcemanager/rmnode/RMNode.java | 7 ++ .../resourcemanager/rmnode/RMNodeImpl.java | 6 ++ .../constraint/AllocationTagsManager.java | 11 +++ .../resourcemanager/webapp/NodesPage.java | 3 + .../webapp/dao/AllocationTagInfo.java | 56 ++ .../webapp/dao/AllocationTagsInfo.java | 59 +++ .../resourcemanager/webapp/dao/NodeInfo.java| 15 .../yarn/server/resourcemanager/MockNodes.java | 6 ++ .../resourcemanager/webapp/TestNodesPage.java | 4 +- .../webapp/TestRMWebServicesNodes.java | 77 +++- 12 files changed, 253 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 1016ce1..0c99139 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.sls.nodemanager; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -213,6 +214,11 @@ public class NodeInfo { } @Override +public Map getAllocationTagsWithCount() { + return null; +} + +@Override public Resource getPhysicalResource() { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index fdad826..92f9b0f 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; @Private @@ -203,6 +204,11 @@ public class RMNodeWrapper implements RMNode { } @Override + public Map getAllocationTagsWithCount() { +return node.getAllocationTagsWithCount(); + } + + @Override public Resource getPhysicalResource() { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index a5615ef..872f2a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmnode; import java.util.List; +import java.util.Map; import java.util.Set;
[23/32] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index 73b4f9e..24c5a5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; import java.util.Iterator; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -30,9 +32,12 @@ import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE_PARTITION; + /** * This class contains various static methods used by the Placement Algorithms * to simplify constrained placement. @@ -41,16 +46,20 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algori @Public @Unstable public final class PlacementConstraintsUtil { + private static final Log LOG = + LogFactory.getLog(PlacementConstraintsUtil.class); // Suppresses default constructor, ensuring non-instantiability. private PlacementConstraintsUtil() { } /** - * Returns true if **single** application constraint with associated + * Returns true if **single** placement constraint with associated * allocationTags and scope is satisfied by a specific scheduler Node. * - * @param appId the application id + * @param targetApplicationId the application id, which could be override by + * target application id specified inside allocation + * tags. * @param sc the placement constraint * @param te the target expression * @param node the scheduler node @@ -59,32 +68,123 @@ public final class PlacementConstraintsUtil { * @throws InvalidAllocationTagsQueryException */ private static boolean canSatisfySingleConstraintExpression( - ApplicationId appId, SingleConstraint sc, TargetExpression te, - SchedulerNode node, AllocationTagsManager tm) + ApplicationId targetApplicationId, SingleConstraint sc, + TargetExpression te, SchedulerNode node, AllocationTagsManager tm) throws InvalidAllocationTagsQueryException { long minScopeCardinality = 0; long maxScopeCardinality = 0; + +// Optimizations to only check cardinality if necessary. +int desiredMinCardinality = sc.getMinCardinality(); +int desiredMaxCardinality = sc.getMaxCardinality(); +boolean checkMinCardinality = desiredMinCardinality > 0; +boolean checkMaxCardinality = desiredMaxCardinality < Integer.MAX_VALUE; + if (sc.getScope().equals(PlacementConstraints.NODE)) { - minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId, - te.getTargetValues(), Long::max); - maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId, - te.getTargetValues(), Long::min); + if (checkMinCardinality) { +minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), +targetApplicationId, te.getTargetValues(), Long::max); + } + if (checkMaxCardinality) { +maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), +targetApplicationId, te.getTargetValues(),
[28/32] hadoop git commit: YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh)
YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5c1fc88 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5c1fc88 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5c1fc88 Branch: refs/heads/trunk Commit: a5c1fc881e21ebf43da7ead5f3852808fce25492 Parents: 4eda58c Author: Arun Suresh <asur...@apache.org> Authored: Fri Jan 12 14:37:06 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/api/records/Container.java | 15 + .../src/main/proto/yarn_protos.proto| 1 + .../api/records/impl/pb/ContainerPBImpl.java| 31 + .../yarn/security/ContainerTokenIdentifier.java | 69 +++- .../src/main/proto/yarn_security_token.proto| 1 + .../api/protocolrecords/NMContainerStatus.java | 14 .../impl/pb/NMContainerStatusPBImpl.java| 33 ++ .../yarn_server_common_service_protos.proto | 1 + .../containermanager/ContainerManagerImpl.java | 3 +- .../container/ContainerImpl.java| 19 +++--- .../rmcontainer/RMContainerImpl.java| 10 ++- .../scheduler/SchedulerApplicationAttempt.java | 3 +- .../security/RMContainerTokenSecretManager.java | 21 ++ .../capacity/TestContainerAllocation.java | 5 +- 14 files changed, 194 insertions(+), 32 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 4fdc803..b9ca3f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.Collections; +import java.util.Set; + /** * {@code Container} represents an allocated resource in the cluster. * @@ -256,4 +259,16 @@ public abstract class Container implements Comparable { public void setVersion(int version) { throw new UnsupportedOperationException(); } + + @Private + @Unstable + public Set getAllocationTags() { +return Collections.EMPTY_SET; + } + + @Private + @Unstable + public void setAllocationTags(Set allocationTags) { + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5cb1177..25c8569 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -152,6 +152,7 @@ message ContainerProto { optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED]; optional int64 allocation_request_id = 8 [default = -1]; optional int32 version = 9 [default = 0]; + repeated string allocation_tags = 10; } message ContainerReportProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index be84938..47be2f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -36,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.PriorityPr
[18/32] hadoop git commit: YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh)
YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4eda58c1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4eda58c1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4eda58c1 Branch: refs/heads/trunk Commit: 4eda58c13641c14c4b248843a2589781cbcd343f Parents: bdba01f Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 10 09:04:30 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/capacity/CapacityScheduler.java | 23 1 file changed, 23 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eda58c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index e682d0f..d2713c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -124,6 +124,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; @@ -2574,6 +2576,27 @@ public class CapacityScheduler extends ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> resourceCommitRequest = createResourceCommitRequest( appAttempt, schedulingRequest, schedulerNode); + +// Validate placement constraint is satisfied before +// committing the request. +try { + if (!PlacementConstraintsUtil.canSatisfyConstraints( + appAttempt.getApplicationId(), + schedulingRequest.getAllocationTags(), + schedulerNode, + rmContext.getPlacementConstraintManager(), + rmContext.getAllocationTagsManager())) { +LOG.debug("Failed to allocate container for application " ++ appAttempt.getApplicationId() + " on node " ++ schedulerNode.getNodeName() ++ " because this allocation violates the" ++ " placement constraint."); +return false; + } +} catch (InvalidAllocationTagsQueryException e) { + LOG.warn("Unable to allocate container", e); + return false; +} return tryCommit(getClusterResource(), resourceCommitRequest, false); } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[21/32] hadoop git commit: YARN-7745. Allow DistributedShell to take a placement specification for containers it wants to launch. (Arun Suresh via wangda)
YARN-7745. Allow DistributedShell to take a placement specification for containers it wants to launch. (Arun Suresh via wangda) Change-Id: Ided146d662e944a8a4692e5d6885f23fd9bbcad5 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e60f5129 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e60f5129 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e60f5129 Branch: refs/heads/trunk Commit: e60f51299dba360d13aa39f9ab714fdfc666b532 Parents: 38af237 Author: Wangda TanAuthored: Thu Jan 18 14:22:45 2018 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../distributedshell/ApplicationMaster.java | 124 +++-- .../applications/distributedshell/Client.java | 14 ++ .../distributedshell/PlacementSpec.java | 137 +++ 3 files changed, 263 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e60f5129/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 270ef1b..9ba2138 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -42,6 +42,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -87,8 +88,11 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ProfileCapability; +import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.ResourceSizing; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.UpdatedContainer; import org.apache.hadoop.yarn.api.records.ExecutionType; @@ -99,6 +103,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.TimelineV2Client; @@ -274,6 +279,10 @@ public class ApplicationMaster { @VisibleForTesting protected AtomicInteger numRequestedContainers = new AtomicInteger(); + protected AtomicInteger numIgnore = new AtomicInteger(); + + protected AtomicInteger totalRetries = new AtomicInteger(10); + // Shell command to be executed private String shellCommand = ""; // Args to be passed to the shell command @@ -289,6 +298,9 @@ public class ApplicationMaster { // File length needed for local resource private long shellScriptPathLen = 0; + // Placement Specifications + private Map placementSpecs = null; + // Container retry options private ContainerRetryPolicy containerRetryPolicy = ContainerRetryPolicy.NEVER_RETRY; @@ -334,6 +346,7 @@ public class ApplicationMaster { private final String windows_command = "cmd /c"; private int yarnShellIdCounter = 1; + private final AtomicLong allocIdCounter = new AtomicLong(1); @VisibleForTesting protected final Set launchedContainers = @@ -457,6 +470,7 @@ public class ApplicationMaster { "If container could retry, it specifies max
[17/32] hadoop git commit: YARN-7682. Expose canSatisfyConstraints utility function to validate a placement against a constraint. (Panagiotis Garefalakis via asuresh)
YARN-7682. Expose canSatisfyConstraints utility function to validate a placement against a constraint. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdba01f7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdba01f7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdba01f7 Branch: refs/heads/trunk Commit: bdba01f73b58d2228e808c6f61377f101b6bac1c Parents: a52d11f Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 3 08:00:50 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../constraint/PlacementConstraintsUtil.java| 132 + .../algorithm/DefaultPlacementAlgorithm.java| 55 +--- .../TestPlacementConstraintsUtil.java | 287 +++ .../constraint/TestPlacementProcessor.java | 204 +++-- 4 files changed, 601 insertions(+), 77 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdba01f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java new file mode 100644 index 000..956a3c9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; + +import java.util.Iterator; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer; +import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm; + +/** + * This class contains various static methods used by the Placement Algorithms + * to simplify constrained placement. + * (see also {@link DefaultPlacementAlgorithm}). + */ +@Public +@Unstable +public final class PlacementConstraintsUtil { + + // Suppresses default constructor, ensuring non-instantiability. + private PlacementConstraintsUtil() { + } + + /** + * Returns true if **single** application constraint with associated + * allocationTags and scope is satisfied by a specific scheduler Node. + * + * @param appId the application id + * @param sc the placement constraint + * @param te the target expression + * @param node the scheduler node + * @param tm the allocation tags store + * @return true if single application constraint is satisfied by node + * @throws InvalidAllocationTagsQueryException + */ + private static boolean canS
[19/32] hadoop git commit: YARN-7709. Remove SELF from TargetExpression type. (Konstantinos Karanasos via asuresh)
YARN-7709. Remove SELF from TargetExpression type. (Konstantinos Karanasos via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8779a357 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8779a357 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8779a357 Branch: refs/heads/trunk Commit: 8779a35742085fadddccc21342b55d4f17fae5c2 Parents: 29d9e4d Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 18 04:29:57 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../yarn/api/resource/PlacementConstraint.java | 32 ++ .../yarn/api/resource/PlacementConstraints.java | 35 +--- .../api/resource/TestPlacementConstraints.java | 3 +- .../PlacementConstraintTransformations.java | 19 +++ .../TestPlacementConstraintTransformations.java | 35 +--- .../constraint/PlacementConstraintsUtil.java| 10 -- 6 files changed, 64 insertions(+), 70 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8779a357/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java index b6e851a..4d998ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -242,7 +242,7 @@ public class PlacementConstraint { * Enum specifying the type of the target expression. */ public enum TargetType { - NODE_ATTRIBUTE, ALLOCATION_TAG, SELF + NODE_ATTRIBUTE, ALLOCATION_TAG } private TargetType targetType; @@ -418,23 +418,25 @@ public class PlacementConstraint { } /** - * Class that represents a cardinality constraint. Such a constraint the - * number of allocations within a given scope to some minimum and maximum - * values. + * Class that represents a cardinality constraint. Such a constraint allows + * the number of allocations with a specific set of tags and within a given + * scope to be between some minimum and maximum values. * * It is a specialized version of the {@link SingleConstraint}, where the - * target is self (i.e., the allocation to which the constraint is attached). + * target is a set of allocation tags. */ public static class CardinalityConstraint extends AbstractConstraint { private String scope; private int minCardinality; private int maxCardinality; +private Set allocationTags; public CardinalityConstraint(String scope, int minCardinality, -int maxCardinality) { +int maxCardinality, Set allocationTags) { this.scope = scope; this.minCardinality = minCardinality; this.maxCardinality = maxCardinality; + this.allocationTags = allocationTags; } /** @@ -464,11 +466,21 @@ public class PlacementConstraint { return maxCardinality; } +/** + * Get the allocation tags of the constraint. + * + * @return the allocation tags of the constraint + */ +public Set getAllocationTags() { + return allocationTags; +} + @Override public T accept(Visitor visitor) { return visitor.visit(this); } + @Override public boolean equals(Object o) { if (this == o) { @@ -486,7 +498,11 @@ public class PlacementConstraint { if (maxCardinality != that.maxCardinality) { return false; } - return scope != null ? scope.equals(that.scope) : that.scope == null; + if (scope != null ? !scope.equals(that.scope) : that.scope != null) { +return false; + } + return allocationTags != null ? allocationTags.equals(that.allocationTags) + : that.allocationTags == null; } @Override @@ -494,6 +510,8 @@ public class PlacementConstraint { int result = scope != null ? scope.hashCode() : 0; result = 31 * result + minCardinality; result = 31 * result + maxCardinality; + result = 31 * result + + (allocationTags != null ? allocationTags.hashCode() : 0); return result; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8779a357/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementCons
[16/32] hadoop git commit: YARN-7613. Implement Basic algorithm for constraint based placement. (Panagiotis Garefalakis via asuresh)
YARN-7613. Implement Basic algorithm for constraint based placement. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a52d11fb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a52d11fb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a52d11fb Branch: refs/heads/trunk Commit: a52d11fb8c103f14e42692600a058ba3b56e2ecf Parents: f9af15d Author: Arun Suresh <asur...@apache.org> Authored: Wed Dec 27 22:59:22 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 4 + .../src/main/resources/yarn-default.xml | 8 +- .../rmcontainer/RMContainerImpl.java| 10 +- .../constraint/AllocationTagsManager.java | 121 ++--- .../algorithm/DefaultPlacementAlgorithm.java| 172 +++ .../iterators/PopularTagsIterator.java | 71 .../algorithm/iterators/SerialIterator.java | 53 ++ .../algorithm/iterators/package-info.java | 29 .../constraint/algorithm/package-info.java | 29 .../constraint/processor/BatchedRequests.java | 45 - .../processor/PlacementProcessor.java | 32 ++-- .../processor/SamplePlacementAlgorithm.java | 144 .../constraint/TestAllocationTagsManager.java | 156 - .../TestBatchedRequestsIterators.java | 82 + .../constraint/TestPlacementProcessor.java | 4 +- 15 files changed, 721 insertions(+), 239 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8fb3c2e..367b1ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -536,6 +536,10 @@ public class YarnConfiguration extends Configuration { public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS = RM_PREFIX + "placement-constraints.algorithm.class"; + /** Used for BasicPlacementAlgorithm - default SERIAL. **/ + public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR = + RM_PREFIX + "placement-constraints.algorithm.iterator"; + public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED = RM_PREFIX + "placement-constraints.enabled"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 6d52ace..509a040 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -145,7 +145,13 @@ Constraint Placement Algorithm to be used. yarn.resourcemanager.placement-constraints.algorithm.class - org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.SamplePlacementAlgorithm + org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm + + + +Placement Algorithm Requests Iterator to be used. +yarn.resourcemanager.placement-constraints.algorithm.iterator +SERIAL http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index c873509..2c4ef7b 100644 --- a/hadoop-yarn-project/
[15/32] hadoop git commit: YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh)
YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9af15d6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9af15d6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9af15d6 Branch: refs/heads/trunk Commit: f9af15d659f59fd0cf564fe1ecc8e06c6429ba68 Parents: 1efb2b6 Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 22 15:51:20 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 26 ++ .../src/main/resources/yarn-default.xml | 30 ++ .../ApplicationMasterService.java | 15 + .../rmcontainer/RMContainerImpl.java| 7 +- .../scheduler/capacity/CapacityScheduler.java | 2 + .../constraint/processor/BatchedRequests.java | 105 + .../processor/NodeCandidateSelector.java| 38 ++ .../processor/PlacementDispatcher.java | 145 +++ .../processor/PlacementProcessor.java | 343 .../processor/SamplePlacementAlgorithm.java | 144 +++ .../constraint/processor/package-info.java | 29 ++ .../yarn/server/resourcemanager/MockAM.java | 26 ++ .../yarn/server/resourcemanager/MockRM.java | 14 + .../constraint/TestPlacementProcessor.java | 394 +++ 14 files changed, 1316 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9af15d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index bbbfc52..8fb3c2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -531,6 +531,32 @@ public class YarnConfiguration extends Configuration { /** The class to use as the resource scheduler.*/ public static final String RM_SCHEDULER = RM_PREFIX + "scheduler.class"; + + /** Placement Algorithm. */ + public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS = + RM_PREFIX + "placement-constraints.algorithm.class"; + + public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED = + RM_PREFIX + "placement-constraints.enabled"; + + public static final boolean DEFAULT_RM_PLACEMENT_CONSTRAINTS_ENABLED = true; + + public static final String RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS = + RM_PREFIX + "placement-constraints.retry-attempts"; + + public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS = 3; + + public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE = + RM_PREFIX + "placement-constraints.algorithm.pool-size"; + + public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE = + 1; + + public static final String RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE = + RM_PREFIX + "placement-constraints.scheduler.pool-size"; + + public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE = + 1; public static final String DEFAULT_RM_SCHEDULER = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9af15d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0bb4fca..6d52ace 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -131,6 +131,36 @@ +Enable Constraint Placement. +yarn.resourcemanager.placement-constraints.enabled +false + + + +Number of times to retry placing of rejected SchedulingRequests +yarn.resourcemanager.placement-constraints.retry-attempts +3 + + + +Constraint Placement Algorithm to be used. +yarn.resourcemanager.placement-constraints.algorithm.class + org.apache.hadoop.yarn.server.resourcemanager
[20/32] hadoop git commit: YARN-7807. Assume intra-app anti-affinity as default for scheduling request inside AppPlacementAllocator. (Wangda Tan via asuresh)
YARN-7807. Assume intra-app anti-affinity as default for scheduling request inside AppPlacementAllocator. (Wangda Tan via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/644afe5f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/644afe5f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/644afe5f Branch: refs/heads/trunk Commit: 644afe5fd800ac4f2b873a99f9b3868c3a8c5c40 Parents: a4c539f Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 24 12:55:01 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../placement/SingleConstraintAppPlacementAllocator.java| 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/644afe5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java index 9e7d71c..b02cb00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java @@ -334,14 +334,15 @@ public class SingleConstraintAppPlacementAllocator targetAllocationTags = new HashSet<>( targetExpression.getTargetValues()); -if (targetExpression.getTargetKey() == null || !targetExpression +if (targetExpression.getTargetKey() != null && !targetExpression .getTargetKey().equals(APPLICATION_LABEL_INTRA_APPLICATION)) { throwExceptionWithMetaInfo( "As of now, the only accepted target key for targetKey of " + "allocation_tag target expression is: [" + APPLICATION_LABEL_INTRA_APPLICATION + "]. Please make changes to placement constraints " - + "accordingly."); + + "accordingly. If this is null, it will be set to " + + APPLICATION_LABEL_INTRA_APPLICATION + " by default."); } } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[32/32] hadoop git commit: Merge branch 'YARN-6592' into trunk
Merge branch 'YARN-6592' into trunk Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d1e2c64 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d1e2c64 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d1e2c64 Branch: refs/heads/trunk Commit: 8d1e2c6409a44f4515a1549ae82c7e2597e96467 Parents: 7288b08 8df7666 Author: Arun SureshAuthored: Wed Jan 31 01:42:42 2018 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:42:42 2018 -0800 -- .../v2/app/rm/TestRMContainerAllocator.java | 15 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 + .../yarn/sls/scheduler/RMNodeWrapper.java | 6 + .../sls/scheduler/SLSCapacityScheduler.java | 15 +- .../yarn/sls/scheduler/SLSFairScheduler.java| 12 +- .../dev-support/findbugs-exclude.xml| 8 + .../yarn/ams/ApplicationMasterServiceUtils.java | 16 + .../api/protocolrecords/AllocateRequest.java| 42 ++ .../api/protocolrecords/AllocateResponse.java | 23 + .../RegisterApplicationMasterRequest.java | 42 +- .../hadoop/yarn/api/records/Container.java | 15 + .../api/records/RejectedSchedulingRequest.java | 70 ++ .../yarn/api/records/RejectionReason.java | 44 ++ .../hadoop/yarn/api/records/ResourceSizing.java | 91 +++ .../yarn/api/records/SchedulingRequest.java | 206 + .../yarn/api/resource/PlacementConstraint.java | 744 +++ .../yarn/api/resource/PlacementConstraints.java | 320 .../hadoop/yarn/api/resource/package-info.java | 23 + .../hadoop/yarn/conf/YarnConfiguration.java | 31 + ...SchedulerInvalidResoureRequestException.java | 47 ++ .../src/main/proto/yarn_protos.proto| 86 +++ .../src/main/proto/yarn_service_protos.proto| 3 + .../api/resource/TestPlacementConstraints.java | 107 +++ .../distributedshell/ApplicationMaster.java | 124 +++- .../applications/distributedshell/Client.java | 14 + .../distributedshell/PlacementSpec.java | 137 .../hadoop/yarn/client/api/AMRMClient.java | 38 +- .../yarn/client/api/async/AMRMClientAsync.java | 48 ++ .../api/async/impl/AMRMClientAsyncImpl.java | 49 +- .../yarn/client/api/impl/AMRMClientImpl.java| 142 +++- .../client/api/impl/BaseAMRMClientTest.java | 212 ++ .../yarn/client/api/impl/TestAMRMClient.java| 156 +--- .../api/impl/TestAMRMClientOnRMRestart.java | 9 +- .../TestAMRMClientPlacementConstraints.java | 204 + .../PlacementConstraintFromProtoConverter.java | 116 +++ .../pb/PlacementConstraintToProtoConverter.java | 174 + .../apache/hadoop/yarn/api/pb/package-info.java | 23 + .../impl/pb/AllocateRequestPBImpl.java | 84 +++ .../impl/pb/AllocateResponsePBImpl.java | 85 +++ .../RegisterApplicationMasterRequestPBImpl.java | 106 ++- .../api/records/impl/pb/ContainerPBImpl.java| 31 + .../yarn/api/records/impl/pb/ProtoUtils.java| 43 ++ .../pb/RejectedSchedulingRequestPBImpl.java | 148 .../records/impl/pb/ResourceSizingPBImpl.java | 128 .../impl/pb/SchedulingRequestPBImpl.java| 296 .../PlacementConstraintTransformations.java | 200 + .../hadoop/yarn/api/resource/package-info.java | 23 + .../yarn/security/ContainerTokenIdentifier.java | 69 +- .../src/main/proto/yarn_security_token.proto| 1 + .../src/main/resources/yarn-default.xml | 36 + .../hadoop/yarn/api/BasePBImplRecordsTest.java | 11 + .../hadoop/yarn/api/TestPBImplRecords.java | 21 + .../TestPlacementConstraintPBConversion.java| 195 + .../TestPlacementConstraintTransformations.java | 166 + .../api/protocolrecords/NMContainerStatus.java | 14 + .../impl/pb/NMContainerStatusPBImpl.java| 33 + .../server/scheduler/SchedulerRequestKey.java | 11 + .../yarn_server_common_service_protos.proto | 1 + .../containermanager/ContainerManagerImpl.java | 3 +- .../container/ContainerImpl.java| 19 +- .../ApplicationMasterService.java | 15 + .../resourcemanager/DefaultAMSProcessor.java| 13 +- .../resourcemanager/RMActiveServiceContext.java | 30 + .../yarn/server/resourcemanager/RMContext.java | 11 + .../server/resourcemanager/RMContextImpl.java | 25 + .../server/resourcemanager/ResourceManager.java | 22 + .../rmapp/attempt/RMAppAttemptImpl.java | 5 +- .../rmcontainer/RMContainer.java| 8 + .../rmcontainer/RMContainerImpl.java| 37 +- .../server/resourcemanager/rmnode/RMNode.java | 7 + .../resourcemanager/rmnode/RMNodeImpl.java | 6 + .../scheduler/AbstractYarnScheduler.java| 21 +- .../scheduler/AppSchedulingInfo.java| 214 -- .../ApplicationPlacementAllocatorFactory.java | 68 ++
[22/32] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index a3b88c0..01d5e6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -170,7 +170,7 @@ public class TestCapacitySchedulerAutoQueueCreation 1 * GB, 1, true, priority, recordFactory); cs.allocate(appAttemptId, Collections.singletonList(r1), - Collections.emptyList(), Collections.singletonList(host), + null, Collections.emptyList(), Collections.singletonList(host), null, NULL_UPDATE_REQUESTS); //And this will result in container assignment for app1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java new file mode 100644 index 000..b6ac4b6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.ResourceSizing; +import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Test; + +import java.util.Arrays; + +public class
[10/32] hadoop git commit: YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh)
YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88d8d3f4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88d8d3f4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88d8d3f4 Branch: refs/heads/trunk Commit: 88d8d3f40b2923fab23a933bce1cd2e9c320ae84 Parents: 801c098 Author: Arun Suresh <asur...@apache.org> Authored: Tue Dec 19 08:59:23 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/AbstractYarnScheduler.java| 18 + .../scheduler/ResourceScheduler.java| 13 .../scheduler/capacity/CapacityScheduler.java | 78 ++-- .../common/ResourceAllocationCommitter.java | 12 ++- .../scheduler/common/fica/FiCaSchedulerApp.java | 30 +--- .../TestCapacitySchedulerAsyncScheduling.java | 10 +-- 6 files changed, 138 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/88d8d3f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 4b76327..213d784 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -295,6 +296,10 @@ public abstract class AbstractYarnScheduler return nodeTracker.getNodes(nodeFilter); } + public List getNodes(final NodeFilter filter) { +return nodeTracker.getNodes(filter); + } + public boolean shouldContainersBeAutoUpdated() { return this.autoUpdateContainers; } @@ -1443,4 +1448,17 @@ public abstract class AbstractYarnScheduler throw new IOException(e); } } + + /** + * Default implementation. Always returns false. + * @param appAttempt ApplicationAttempt. + * @param schedulingRequest SchedulingRequest. + * @param schedulerNode SchedulerNode. + * @return Success or not. + */ + @Override + public boolean attemptAllocationOnNode(SchedulerApplicationAttempt appAttempt, + SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) { +return false; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/88d8d3f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java index d96d625..5a56ac7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.re
[13/32] hadoop git commit: YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37f1a7b6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37f1a7b6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37f1a7b6 Branch: refs/heads/trunk Commit: 37f1a7b64fcc93191367330cd59d4d71d7b29ac7 Parents: 06eb63e Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 22 07:24:37 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../server/resourcemanager/ResourceManager.java | 2 +- .../constraint/AllocationTagsManager.java | 282 ++- .../rmcontainer/TestRMContainerImpl.java| 2 +- .../constraint/TestAllocationTagsManager.java | 269 -- 4 files changed, 392 insertions(+), 163 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37f1a7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index a1d3dfc..1d838f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -496,7 +496,7 @@ public class ResourceManager extends CompositeService implements Recoverable { } protected AllocationTagsManager createAllocationTagsManager() { -return new AllocationTagsManager(); +return new AllocationTagsManager(this.rmContext); } protected DelegationTokenRenewer createDelegationTokenRenewer() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/37f1a7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java index c278606..7b0b959 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.log4j.Logger; import java.util.HashMap; @@ -38,9 +39,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongBinaryOperator; /** - * Support storing maps between container-tags/applications and - * nodes. This will be required by affinity/anti-affinity implementation and - * cardinality. + * In-memory mapping between applications/container-tags and nodes/racks. + * Required by constrained affinity/anti-affinity and cardinality placement. */ @InterfaceAudience.Private @InterfaceStability.Unstable @@ -51,48 +51,54 @@ public class AllocationTagsManager { private ReentrantReadWriteLock.ReadLock readLock; private ReentrantReadWriteLock.WriteLock writeLock; + private final RMContext rmContext; - // Application's tags to node - private Map<ApplicationId, NodeToCountedTags> perAppMappings = + // Application's tags to Node + private Map<ApplicationId, NodeToCountedTags> perAppNodeMappings = + new HashMap<>(); + // Application's tags to Rac
[25/32] hadoop git commit: YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh)
YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28fe7f33 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28fe7f33 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28fe7f33 Branch: refs/heads/trunk Commit: 28fe7f331837b36e78fa34ed990993677dddeaee Parents: e60f512 Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 18 11:01:36 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/SchedulerNode.java| 16 +++- .../scheduler/capacity/CapacityScheduler.java | 4 + .../constraint/PlacementConstraintsUtil.java| 5 +- .../constraint/algorithm/CircularIterator.java | 86 .../algorithm/DefaultPlacementAlgorithm.java| 50 ++-- .../constraint/processor/BatchedRequests.java | 8 ++ .../SingleConstraintAppPlacementAllocator.java | 2 +- .../yarn/server/resourcemanager/MockAM.java | 4 +- .../constraint/TestPlacementProcessor.java | 24 +++--- .../algorithm/TestCircularIterator.java | 84 +++ ...stSingleConstraintAppPlacementAllocator.java | 28 +++ 11 files changed, 271 insertions(+), 40 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28fe7f33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 89f748d..96a8e34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -469,6 +469,20 @@ public abstract class SchedulerNode { this.lastHeartbeatMonotonicTime = Time.monotonicNow(); } + @Override + public boolean equals(Object o) { +if (this == o) return true; +if (!(o instanceof SchedulerNode)) return false; + +SchedulerNode that = (SchedulerNode) o; + +return getNodeID().equals(that.getNodeID()); + } + + @Override + public int hashCode() { +return getNodeID().hashCode(); + } private static class ContainerInfo { private final RMContainer container; @@ -479,4 +493,4 @@ public abstract class SchedulerNode { this.launchedOnNode = launchedOnNode; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/28fe7f33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index c713036..429f9f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2610,6 +2610,10 @@ public class CapacityScheduler extends " but only 1 will be attempted !!"); } if (!appAttempt.isStopped()) { +Resource resource = +schedulingRequest.getResourceSizing().getResources(); +schedulingRequest.getResourceSizing().setResources( +getNormalizedResource(resource)); ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> resourceCommitRequest = createResourceCommitRequest( appAttempt, schedulingRequest, schedulerNode); http://git-wip-us.apache.org/repos/asf/hadoop/blob/
[04/32] hadoop git commit: YARN-6619. AMRMClient Changes to use the PlacementConstraint and SchcedulingRequest objects. (Arun Suresh via wangda)
YARN-6619. AMRMClient Changes to use the PlacementConstraint and SchcedulingRequest objects. (Arun Suresh via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29d9e4d5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29d9e4d5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29d9e4d5 Branch: refs/heads/trunk Commit: 29d9e4d5814900d5c59d77fe05d32186d4ad9385 Parents: a5c1fc8 Author: Wangda TanAuthored: Wed Jan 17 11:36:26 2018 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/client/api/AMRMClient.java | 38 +++- .../yarn/client/api/async/AMRMClientAsync.java | 48 + .../api/async/impl/AMRMClientAsyncImpl.java | 49 - .../yarn/client/api/impl/AMRMClientImpl.java| 142 - .../client/api/impl/BaseAMRMClientTest.java | 212 +++ .../yarn/client/api/impl/TestAMRMClient.java| 156 +- .../TestAMRMClientPlacementConstraints.java | 204 ++ .../rmcontainer/RMContainerImpl.java| 3 + .../scheduler/AbstractYarnScheduler.java| 1 + .../scheduler/SchedulerApplicationAttempt.java | 1 + .../constraint/PlacementConstraintsUtil.java| 4 +- 11 files changed, 700 insertions(+), 158 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d9e4d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java index d3d1974..914a146 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.client.api; import java.io.IOException; import java.util.Collection; +import java.util.Map; +import java.util.Set; import java.util.function.Supplier; import java.util.List; @@ -39,7 +41,9 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ProfileCapability; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.resource.Resources; @@ -554,6 +558,18 @@ public abstract class AMRMClient extends } /** + * Add a Collection of SchedulingRequests. The AMRMClient will ensure that + * all requests in the same batch are sent in the same allocate call. + * @param schedulingRequests Collection of Scheduling Requests. + */ + @Public + @InterfaceStability.Unstable + public void addSchedulingRequests( + Collection schedulingRequests) { + + } + + /** * Register the application master. This must be called before any * other interaction * @param appHostName Name of the host on which master is running @@ -568,7 +584,27 @@ public abstract class AMRMClient extends int appHostPort, String appTrackingUrl) throws YarnException, IOException; - + + /** + * Register the application master. This must be called before any + * other interaction + * @param appHostName Name of the host on which master is running + * @param appHostPort Port master is listening on + * @param appTrackingUrl URL at which the master info can be seen + * @param placementConstraints Placement Constraints mappings. + * @return RegisterApplicationMasterResponse + * @throws YarnException + * @throws IOException + */ + @Public + @InterfaceStability.Unstable + public RegisterApplicationMasterResponse registerApplicationMaster( + String appHostName, int appHostPort, String appTrackingUrl, + Map placementConstraints) + throws YarnException, IOException { +throw new YarnException("Not supported"); + } + /** * Request additional containers and receive new container allocations. * Requests made via addContainerRequest are sent to the
[12/32] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)
YARN-7669. API and interface modifications for placement constraint processor. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06eb63e6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06eb63e6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06eb63e6 Branch: refs/heads/trunk Commit: 06eb63e64b05e2e8bb8a76c15360ab0495f11317 Parents: 88d8d3f Author: Arun Suresh <asur...@apache.org> Authored: Tue Dec 19 22:47:46 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../yarn/ams/ApplicationMasterServiceUtils.java | 16 + .../api/protocolrecords/AllocateResponse.java | 23 + .../api/records/RejectedSchedulingRequest.java | 70 +++ .../yarn/api/records/RejectionReason.java | 44 ++ .../src/main/proto/yarn_protos.proto| 10 + .../src/main/proto/yarn_service_protos.proto| 1 + .../impl/pb/AllocateResponsePBImpl.java | 85 .../yarn/api/records/impl/pb/ProtoUtils.java| 16 + .../pb/RejectedSchedulingRequestPBImpl.java | 148 +++ .../records/impl/pb/ResourceSizingPBImpl.java | 8 + .../impl/pb/SchedulingRequestPBImpl.java| 11 + .../hadoop/yarn/api/TestPBImplRecords.java | 2 + .../resourcemanager/RMActiveServiceContext.java | 2 +- .../yarn/server/resourcemanager/RMContext.java | 2 +- .../server/resourcemanager/RMContextImpl.java | 2 +- .../server/resourcemanager/ResourceManager.java | 2 +- .../constraint/AllocationTagsManager.java | 431 --- .../constraint/AllocationTagsNamespaces.java| 31 -- .../InvalidAllocationTagsQueryException.java| 35 -- .../constraint/AllocationTagsManager.java | 431 +++ .../constraint/AllocationTagsNamespaces.java| 31 ++ .../InvalidAllocationTagsQueryException.java| 35 ++ .../api/ConstraintPlacementAlgorithm.java | 43 ++ .../api/ConstraintPlacementAlgorithmInput.java | 32 ++ .../api/ConstraintPlacementAlgorithmOutput.java | 58 +++ ...traintPlacementAlgorithmOutputCollector.java | 32 ++ .../constraint/api/PlacedSchedulingRequest.java | 79 .../constraint/api/SchedulingResponse.java | 70 +++ .../scheduler/constraint/api/package-info.java | 28 ++ .../constraint/TestAllocationTagsManager.java | 328 -- .../rmcontainer/TestRMContainerImpl.java| 2 +- .../scheduler/capacity/TestUtils.java | 2 +- .../constraint/TestAllocationTagsManager.java | 328 ++ .../scheduler/fifo/TestFifoScheduler.java | 2 +- 34 files changed, 1608 insertions(+), 832 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java index 476da8b..8bdfaf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.ams; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerUpdateType; +import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdatedContainer; @@ -86,4 +87,19 @@ public final class ApplicationMasterServiceUtils { } allocateResponse.setAllocatedContainers(allocatedContainers); } + + /** + * Add rejected Scheduling Requests to {@link AllocateResponse}. + * @param allocateResponse Allocate Response. + * @param rejectedRequests Rejected SchedulingRequests. + */ + public static void addToRejectedSchedulingRequests( + AllocateResponse allocateResponse, + List rejectedRequests) { +if (allocateResponse.getRejectedSchedulingRequests() != null +&& !allocateResponse.getRejectedSchedulingRequests().isEmpty()) { + rejectedRequests.addAll(allocateResponse.getRejectedSchedulingRequests()); +} +allocateResponse.setRejectedSchedulingRequests(rejectedRequests); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hado
[14/32] hadoop git commit: YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh)
YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1efb2b6f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1efb2b6f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1efb2b6f Branch: refs/heads/trunk Commit: 1efb2b6f250022f41fe5911c1bb3028ec15c5447 Parents: 37f1a7b Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 22 13:26:30 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../resourcemanager/RMActiveServiceContext.java | 15 + .../yarn/server/resourcemanager/RMContext.java | 6 + .../server/resourcemanager/RMContextImpl.java | 13 + .../server/resourcemanager/ResourceManager.java | 13 + .../MemoryPlacementConstraintManager.java | 282 +++ .../constraint/PlacementConstraintManager.java | 151 ++ .../PlacementConstraintManagerService.java | 93 ++ .../scheduler/constraint/package-info.java | 29 ++ .../TestPlacementConstraintManagerService.java | 182 9 files changed, 784 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1efb2b6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 4d0c230..06a1d00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; @@ -109,6 +110,7 @@ public class RMActiveServiceContext { private RMAppLifetimeMonitor rmAppLifetimeMonitor; private QueueLimitCalculator queueLimitCalculator; private AllocationTagsManager allocationTagsManager; + private PlacementConstraintManager placementConstraintManager; public RMActiveServiceContext() { queuePlacementManager = new PlacementManager(); @@ -413,6 +415,19 @@ public class RMActiveServiceContext { @Private @Unstable + public PlacementConstraintManager getPlacementConstraintManager() { +return placementConstraintManager; + } + + @Private + @Unstable + public void setPlacementConstraintManager( + PlacementConstraintManager placementConstraintManager) { +this.placementConstraintManager = placementConstraintManager; + } + + @Private + @Unstable public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() { return rmDelegatedNodeLabelsUpdater; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1efb2b6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 00da108..eb91a31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-s
[24/32] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)
YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38af2379 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38af2379 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38af2379 Branch: refs/heads/trunk Commit: 38af23796971193fa529c3d08ffde8fcd6e607b6 Parents: 8779a35 Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 18 14:10:30 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../v2/app/rm/TestRMContainerAllocator.java | 15 +- .../sls/scheduler/SLSCapacityScheduler.java | 15 +- .../yarn/sls/scheduler/SLSFairScheduler.java| 12 +- .../dev-support/findbugs-exclude.xml| 8 + .../yarn/api/resource/PlacementConstraints.java | 43 +- .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- ...SchedulerInvalidResoureRequestException.java | 47 ++ .../api/impl/TestAMRMClientOnRMRestart.java | 9 +- .../impl/pb/AllocateRequestPBImpl.java | 1 + .../server/scheduler/SchedulerRequestKey.java | 11 + .../resourcemanager/DefaultAMSProcessor.java| 13 +- .../rmapp/attempt/RMAppAttemptImpl.java | 5 +- .../scheduler/AbstractYarnScheduler.java| 3 +- .../scheduler/AppSchedulingInfo.java| 205 +-- .../ApplicationPlacementAllocatorFactory.java | 68 +++ .../scheduler/ApplicationPlacementFactory.java | 63 --- .../scheduler/ContainerUpdateContext.java | 4 +- .../scheduler/SchedulerApplicationAttempt.java | 20 +- .../scheduler/YarnScheduler.java| 15 +- .../scheduler/capacity/CapacityScheduler.java | 54 +- .../CapacitySchedulerConfiguration.java | 5 + .../allocator/RegularContainerAllocator.java| 3 +- .../scheduler/common/ContainerRequest.java | 12 + .../scheduler/common/PendingAsk.java| 6 + .../scheduler/common/fica/FiCaSchedulerApp.java | 6 + .../constraint/AllocationTagsManager.java | 71 +-- .../constraint/AllocationTagsNamespaces.java| 31 -- .../constraint/PlacementConstraintsUtil.java| 165 -- .../algorithm/DefaultPlacementAlgorithm.java| 2 +- .../processor/PlacementProcessor.java | 8 +- .../scheduler/fair/FairScheduler.java | 12 +- .../scheduler/fifo/FifoScheduler.java | 7 +- .../placement/AppPlacementAllocator.java| 66 ++- .../LocalityAppPlacementAllocator.java | 35 +- .../SingleConstraintAppPlacementAllocator.java | 531 +++ .../server/resourcemanager/Application.java | 9 +- .../yarn/server/resourcemanager/MockAM.java | 51 ++ .../attempt/TestRMAppAttemptTransitions.java| 10 +- .../rmcontainer/TestRMContainerImpl.java| 6 +- .../scheduler/TestAppSchedulingInfo.java| 4 +- .../capacity/CapacitySchedulerTestBase.java | 79 +++ .../capacity/TestCapacityScheduler.java | 90 +--- .../TestCapacitySchedulerAsyncScheduling.java | 2 +- .../TestCapacitySchedulerAutoQueueCreation.java | 2 +- ...apacitySchedulerSchedulingRequestUpdate.java | 260 + .../capacity/TestIncreaseAllocationExpirer.java | 2 +- ...estSchedulingRequestContainerAllocation.java | 277 ++ ...hedulingRequestContainerAllocationAsync.java | 139 + .../scheduler/capacity/TestUtils.java | 2 + .../constraint/TestAllocationTagsManager.java | 30 +- .../TestPlacementConstraintsUtil.java | 36 +- .../scheduler/fair/FairSchedulerTestBase.java | 6 +- .../fair/TestContinuousScheduling.java | 10 +- .../scheduler/fair/TestFairScheduler.java | 30 +- .../scheduler/fifo/TestFifoScheduler.java | 28 +- ...stSingleConstraintAppPlacementAllocator.java | 403 ++ 56 files changed, 2557 insertions(+), 492 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index 85e4181..7875917 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hado
[29/32] hadoop git commit: YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)
YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8c5f5b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8c5f5b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8c5f5b2 Branch: refs/heads/trunk Commit: f8c5f5b23732a1e35f012c1a6850bed09c8a5180 Parents: c23980c Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 25 19:09:21 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/AppSchedulingInfo.java| 10 - .../scheduler/common/fica/FiCaSchedulerApp.java | 7 +++ .../constraint/TestPlacementProcessor.java | 45 3 files changed, 60 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c5f5b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 0389895..1efdd8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -694,6 +694,12 @@ public class AppSchedulingInfo { metrics.runAppAttempt(applicationId, user); } +updateMetrics(applicationId, type, node, containerAllocated, user, queue); + } + + public static void updateMetrics(ApplicationId applicationId, NodeType type, + SchedulerNode node, Container containerAllocated, String user, + Queue queue) { if (LOG.isDebugEnabled()) { LOG.debug("allocate: applicationId=" + applicationId + " container=" + containerAllocated.getId() + " host=" + containerAllocated @@ -702,10 +708,10 @@ public class AppSchedulingInfo { + type); } if(node != null) { - metrics.allocateResources(node.getPartition(), user, 1, + queue.getMetrics().allocateResources(node.getPartition(), user, 1, containerAllocated.getResource(), true); } -metrics.incrNodeTypeAggregations(user, type); +queue.getMetrics().incrNodeTypeAggregations(user, type); } // Get AppPlacementAllocator by specified schedulerKey http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c5f5b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 7eb1e31..f3da0a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRese import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; @@ -548,6 +549,1
[02/32] hadoop git commit: YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/add993e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/add993e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/add993e2 Branch: refs/heads/trunk Commit: add993e26a3c96f77dfd42086f186a139966019e Parents: f8c5f5b Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 25 23:01:43 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../rmcontainer/RMContainerImpl.java| 8 +++ .../rmcontainer/TestRMContainerImpl.java| 25 ++-- 2 files changed, 26 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/add993e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index a504221..541621b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -40,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; @@ -533,7 +531,7 @@ public class RMContainerImpl implements RMContainer { RMContainerEvent event) { NMContainerStatus report = ((RMContainerRecoverEvent) event).getContainerReport(); - // Set the allocation tags from the + // Set the allocation tags from the NMContainerStatus container.setAllocationTags(report.getAllocationTags()); // Notify AllocationTagsManager container.rmContext.getAllocationTagsManager().addContainer( @@ -689,7 +687,7 @@ public class RMContainerImpl implements RMContainer { // Something wrong happened, kill the container LOG.warn("Something wrong happened, container size reported by NM" + " is not expected, ContainerID=" + container.getContainerId() -+ " rm-size-resource:" + rmContainerResource + " nm-size-reosurce:" ++ " rm-size-resource:" + rmContainerResource + " nm-size-resource:" + nmContainerResource); container.eventHandler.handle(new RMNodeCleanContainerEvent( container.nodeId, container.getContainerId())); @@ -702,7 +700,7 @@ public class RMContainerImpl implements RMContainer { @Override public void transition(RMContainerImpl container, RMContainerEvent event) { - // Notify placementManager + // Notify AllocationTagsManager container.rmContext.getAllocationTagsManager().removeContainer( container.getNodeId(), container.getContainerId(), container.getAllocationTags()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/add993e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-p
[27/32] hadoop git commit: YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos)
YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adbe87ab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adbe87ab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adbe87ab Branch: refs/heads/trunk Commit: adbe87abf8b2814e0e2988d09ef8a8569190c80e Parents: 8bf7c44 Author: Konstantinos KaranasosAuthored: Mon Jan 22 23:51:02 2018 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../constraint/AllocationTagsManager.java | 110 +++- .../algorithm/DefaultPlacementAlgorithm.java| 8 +- .../algorithm/LocalAllocationTagsManager.java | 167 +++ .../constraint/TestAllocationTagsManager.java | 82 - .../TestLocalAllocationTagsManager.java | 139 +++ 5 files changed, 336 insertions(+), 170 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/adbe87ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java index 962e548..7ad5e8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java @@ -24,17 +24,14 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; -import org.apache.hadoop.yarn.api.resource.PlacementConstraints; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.log4j.Logger; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -61,9 +58,6 @@ public class AllocationTagsManager { // Application's tags to Rack private Map perAppRackMappings = new HashMap<>(); - // Application's Temporary containers mapping - private Map >> - appTempMappings = new HashMap<>(); // Global tags to node mapping (used to fast return aggregated tags // cardinality across apps) @@ -76,7 +70,7 @@ public class AllocationTagsManager { * Currently used both for NodeId to Tag, Count and Rack to Tag, Count */ @VisibleForTesting - static class TypeToCountedTags { + public static class TypeToCountedTags { // Map > private Map > typeToTagsWithCount = new HashMap<>(); @@ -214,7 +208,7 @@ public class AllocationTagsManager { } @VisibleForTesting - Map getPerAppNodeMappings() { + public Map getPerAppNodeMappings() { return perAppNodeMappings; } @@ -233,12 +227,6 @@ public class AllocationTagsManager { return globalRackMapping; } - @VisibleForTesting - public Map > getAppTempMappings( - ApplicationId applicationId) { -return appTempMappings.get(applicationId); - } - public AllocationTagsManager(RMContext context) { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); @@ -246,39 +234,6 @@ public class AllocationTagsManager { rmContext = context; } - // - - /** - * Method adds a temporary fake-container tag to Node mapping. - * Used by the constrained placement algorithm to keep track of containers - * that are currently placed on nodes but are not yet
[26/32] hadoop git commit: YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh)
YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf7c444 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf7c444 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf7c444 Branch: refs/heads/trunk Commit: 8bf7c444368f48f63f8011cf155f551c6b51ee21 Parents: 28fe7f3 Author: Arun Suresh <asur...@apache.org> Authored: Sun Jan 21 19:11:17 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/capacity/CapacityScheduler.java | 4 +- .../constraint/PlacementConstraintsUtil.java| 98 +++- .../algorithm/DefaultPlacementAlgorithm.java| 4 +- .../SingleConstraintAppPlacementAllocator.java | 10 +- .../TestPlacementConstraintsUtil.java | 94 --- 5 files changed, 123 insertions(+), 87 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf7c444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 429f9f3..a096e2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2621,9 +2621,9 @@ public class CapacityScheduler extends // Validate placement constraint is satisfied before // committing the request. try { - if (!PlacementConstraintsUtil.canSatisfySingleConstraint( + if (!PlacementConstraintsUtil.canSatisfyConstraints( appAttempt.getApplicationId(), - schedulingRequest.getAllocationTags(), schedulerNode, + schedulingRequest, schedulerNode, rmContext.getPlacementConstraintManager(), rmContext.getAllocationTagsManager())) { LOG.debug("Failed to allocate container for application " http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf7c444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index ff5cb67..c07c16f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; @@ -54,7 +55,7 @@ public final class PlacementConstraintsUtil { } /** - * Returns true if **single** placement constraint with associated + * Returns true if single placement constraint with associated * allocationTags and scope is satisfied by a s
[31/32] hadoop git commit: YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)
YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c23980c4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c23980c4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c23980c4 Branch: refs/heads/trunk Commit: c23980c4f2cf4c751a99fd310e60149cb32ea7c7 Parents: 644afe5 Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 24 14:18:32 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/api/protocolrecords/AllocateRequest.java | 4 ++-- .../apache/hadoop/yarn/api/records/SchedulingRequest.java| 8 .../apache/hadoop/yarn/api/resource/PlacementConstraint.java | 3 +++ .../hadoop/yarn/api/resource/PlacementConstraints.java | 3 --- .../yarn/api/records/impl/pb/ResourceSizingPBImpl.java | 3 +++ .../yarn/api/records/impl/pb/SchedulingRequestPBImpl.java| 3 +++ .../resourcemanager/scheduler/AbstractYarnScheduler.java | 1 - .../server/resourcemanager/scheduler/AppSchedulingInfo.java | 3 +-- .../yarn/server/resourcemanager/scheduler/SchedulerNode.java | 8 ++-- .../scheduler/capacity/CapacityScheduler.java| 3 +-- .../scheduler/constraint/AllocationTagsManager.java | 8 +--- .../scheduler/constraint/PlacementConstraintsUtil.java | 4 ++-- .../scheduler/placement/AppPlacementAllocator.java | 4 ++-- .../placement/SingleConstraintAppPlacementAllocator.java | 1 - .../scheduler/capacity/TestCapacityScheduler.java| 1 - .../TestCapacitySchedulerSchedulingRequestUpdate.java| 4 +++- .../capacity/TestSchedulingRequestContainerAllocation.java | 8 .../TestSchedulingRequestContainerAllocationAsync.java | 1 - 18 files changed, 35 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c23980c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index d8d2347..876957e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -229,8 +229,8 @@ public abstract class AllocateRequest { /** * Set the list of Scheduling requests to inform the * ResourceManager about the application's resource requirements - * (potentially including allocation tags & placement constraints). - * @param schedulingRequests list of SchedulingRequest to update + * (potentially including allocation tags and placement constraints). + * @param schedulingRequests list of {@link SchedulingRequest} to update * the ResourceManager about the application's resource * requirements. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/c23980c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java index e32dd24..4bb2b84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java @@ -77,7 +77,7 @@ public abstract class SchedulingRequest { /** * Set the allocationRequestId of the request. - * + * * @see SchedulingRequest#setAllocationRequestId(long) * @param allocationRequestId allocationRequestId of the * request @@ -107,7 +107,7 @@ public abstract class SchedulingRequest { /** * Set the executionType of the request. - * + * * @see SchedulingRequest#setExecutionType(ExecutionTypeRequest) * @param executionType executionType of the request * @return {@link SchedulingRequest.SchedulingRequestBuilder} @@ -119,7 +119,7 @@ public abstract class SchedulingRequest { schedulingRequest.
[07/32] hadoop git commit: YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos)
YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db928556 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db928556 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db928556 Branch: refs/heads/trunk Commit: db928556c81e5950b3fe374fa5b99ab26791ef3a Parents: b57e8bc Author: Konstantinos KaranasosAuthored: Mon Nov 13 15:25:24 2017 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../RegisterApplicationMasterRequest.java | 42 - .../yarn/api/resource/PlacementConstraint.java | 156 +++ .../src/main/proto/yarn_protos.proto| 6 + .../src/main/proto/yarn_service_protos.proto| 1 + .../RegisterApplicationMasterRequestPBImpl.java | 106 - .../hadoop/yarn/api/BasePBImplRecordsTest.java | 11 ++ 6 files changed, 313 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db928556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java index 395e190..f2d537a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java @@ -18,11 +18,16 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.util.Records; - /** * The request sent by the {@code ApplicationMaster} to {@code ResourceManager} * on registration. @@ -132,4 +137,39 @@ public abstract class RegisterApplicationMasterRequest { @Public @Stable public abstract void setTrackingUrl(String trackingUrl); + + /** + * Return all Placement Constraints specified at the Application level. The + * mapping is from a set of allocation tags to a + * PlacementConstraint associated with the tags, i.e., each + * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those + * tags will be placed taking into account the corresponding constraint. + * + * @return A map of Placement Constraints. + */ + @Public + @Unstable + public Map getPlacementConstraints() { +return new HashMap<>(); + } + + /** + * Set Placement Constraints applicable to the + * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s + * of this application. + * The mapping is from a set of allocation tags to a + * PlacementConstraint associated with the tags. + * For example: + * Map + * hb_regionserver - node_anti_affinity, + * hb_regionserver, hb_master - rack_affinity, + * ... + * + * @param placementConstraints Placement Constraint Mapping. + */ + @Public + @Unstable + public void setPlacementConstraints( + Map placementConstraints) { + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db928556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java index f0e3982..b6e851a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -54,6 +54,26 @@ public class PlacementConstraint { return constraintExpr; } + @Override +
[06/32] hadoop git commit: YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos via wangda)
YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57e8bc3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57e8bc3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57e8bc3 Branch: refs/heads/trunk Commit: b57e8bc3002a95d2f2f328554d792151cdc1120d Parents: 33a796d Author: Wangda TanAuthored: Mon Oct 30 16:54:02 2017 -0700 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/api/records/ResourceSizing.java | 64 + .../yarn/api/records/SchedulingRequest.java | 205 ++ .../src/main/proto/yarn_protos.proto| 14 + .../records/impl/pb/ResourceSizingPBImpl.java | 117 .../impl/pb/SchedulingRequestPBImpl.java| 266 +++ 5 files changed, 666 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57e8bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java new file mode 100644 index 000..d82be11 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@code ResourceSizing} contains information for the size of a + * {@link SchedulingRequest}, such as the number of requested allocations and + * the resources for each allocation. + */ +@Public +@Unstable +public abstract class ResourceSizing { + + @Public + @Unstable + public static ResourceSizing newInstance(Resource resources) { +return ResourceSizing.newInstance(1, resources); + } + + @Public + @Unstable + public static ResourceSizing newInstance(int numAllocations, Resource resources) { +ResourceSizing resourceSizing = Records.newRecord(ResourceSizing.class); +resourceSizing.setNumAllocations(numAllocations); +resourceSizing.setResources(resources); +return resourceSizing; + } + + @Public + @Unstable + public abstract int getNumAllocations(); + + @Public + @Unstable + public abstract void setNumAllocations(int numAllocations); + + @Public + @Unstable + public abstract Resource getResources(); + + @Public + @Unstable + public abstract void setResources(Resource resources); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57e8bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java new file mode 100644 index 000..47a0697 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in
[05/32] hadoop git commit: YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos via wangda)
YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos via wangda) Change-Id: Id00edb7185fdf01cce6e40f920cac3585f8cbe9c Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33a796d9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33a796d9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33a796d9 Branch: refs/heads/trunk Commit: 33a796d9b778bf7350e87a4e36ca30c925cf7036 Parents: 1453a04 Author: Wangda TanAuthored: Thu Aug 3 14:03:55 2017 -0700 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../yarn/api/resource/PlacementConstraint.java | 567 +++ .../yarn/api/resource/PlacementConstraints.java | 286 ++ .../hadoop/yarn/api/resource/package-info.java | 23 + .../src/main/proto/yarn_protos.proto| 55 ++ .../api/resource/TestPlacementConstraints.java | 106 .../PlacementConstraintFromProtoConverter.java | 116 .../pb/PlacementConstraintToProtoConverter.java | 174 ++ .../apache/hadoop/yarn/api/pb/package-info.java | 23 + .../yarn/api/records/impl/pb/ProtoUtils.java| 27 + .../PlacementConstraintTransformations.java | 209 +++ .../hadoop/yarn/api/resource/package-info.java | 23 + .../TestPlacementConstraintPBConversion.java| 195 +++ .../TestPlacementConstraintTransformations.java | 183 ++ 13 files changed, 1987 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/33a796d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java new file mode 100644 index 000..f0e3982 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -0,0 +1,567 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * {@code PlacementConstraint} represents a placement constraint for a resource + * allocation. + */ +@Public +@Unstable +public class PlacementConstraint { + + /** + * The constraint expression tree. + */ + private AbstractConstraint constraintExpr; + + public PlacementConstraint(AbstractConstraint constraintExpr) { +this.constraintExpr = constraintExpr; + } + + /** + * Get the constraint expression of the placement constraint. + * + * @return the constraint expression + */ + public AbstractConstraint getConstraintExpr() { +return constraintExpr; + } + + /** + * Interface used to enable the elements of the constraint tree to be visited. + */ + @Private + public interface Visitable { +/** + * Visitor pattern. + * + * @param visitor visitor to be used + * @param defines the type that the visitor will use and the return type + * of the accept. + * @return the result of visiting a given object. + */ + T accept(Visitor visitor); + + } + + /** + * Visitor API for a constraint tree. + * + * @param determines the return type of the visit methods. + */ + @Private + public interface Visitor { +T visit(SingleConstraint constraint); + +T visit(TargetExpression target); + +T visit(TargetConstraint constraint); + +T visit(CardinalityConstraint constraint); + +T visit(And constraint); +
[03/32] hadoop git commit: YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh)
YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4c539fc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4c539fc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4c539fc Branch: refs/heads/trunk Commit: a4c539fcdba817e313b2375abf2c4c9a1d13a4fd Parents: 9b81cb0 Author: Arun Suresh <asur...@apache.org> Authored: Tue Jan 23 08:15:58 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../algorithm/DefaultPlacementAlgorithm.java| 119 +-- .../constraint/TestPlacementProcessor.java | 49 2 files changed, 155 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4c539fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java index 9887749..4e6473f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.List; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ResourceSizing; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; @@ -69,13 +70,9 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { public boolean attemptPlacementOnNode(ApplicationId appId, SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) throws InvalidAllocationTagsQueryException { -int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations(); -if (numAllocs > 0) { - if (PlacementConstraintsUtil.canSatisfyConstraints(appId, - schedulingRequest, schedulerNode, - constraintManager, tagsManager)) { -return true; - } +if (PlacementConstraintsUtil.canSatisfyConstraints(appId, +schedulingRequest, schedulerNode, constraintManager, tagsManager)) { + return true; } return false; } @@ -93,6 +90,9 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { int rePlacementCount = RE_ATTEMPT_COUNT; while (rePlacementCount > 0) { doPlacement(requests, resp, allNodes, rejectedRequests); + // Double check if placement constraints are really satisfied + validatePlacement(requests.getApplicationId(), resp, + rejectedRequests); if (rejectedRequests.size() == 0 || rePlacementCount == 1) { break; } @@ -122,9 +122,14 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { break; } SchedulingRequest schedulingRequest = requestIterator.next(); + PlacedSchedulingRequest placedReq = + new PlacedSchedulingRequest(schedulingRequest); + placedReq.setPlacementAttempt(requests.getPlacementAttempt()); + resp.getPlacedRequests().add(placedReq); CircularIterator nodeIter = new CircularIterator(lastSatisfiedNode, nIter, allNodes); - int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations(); + int numAllocs = + schedulingRequest.getResourceSizing().getNumAllocations(); while (nodeIter.hasNext() && numAllocs > 0) { SchedulerNode node = nodeIter.next(); try { @@ -135,11 +140,7 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { requests.getApplicationId(), schedulingRequest, node)) { schedulingRequest.getResourceSizing() .
[08/32] hadoop git commit: YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis Garefalakis via asuresh)
YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69de9a1b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69de9a1b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69de9a1b Branch: refs/heads/trunk Commit: 69de9a1ba9a587c7e03ae7c7aeae93e04c36d665 Parents: db92855 Author: Arun Suresh <asur...@apache.org> Authored: Fri Nov 17 10:42:43 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../api/protocolrecords/AllocateRequest.java| 42 ++ .../hadoop/yarn/api/records/ResourceSizing.java | 27 +++ .../yarn/api/records/SchedulingRequest.java | 1 + .../src/main/proto/yarn_service_protos.proto| 1 + .../impl/pb/AllocateRequestPBImpl.java | 83 .../records/impl/pb/ResourceSizingPBImpl.java | 2 +- .../impl/pb/SchedulingRequestPBImpl.java| 16 .../hadoop/yarn/api/TestPBImplRecords.java | 19 + 8 files changed, 190 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/69de9a1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index ae0891e..d8d2347 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.Collections; import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -28,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.util.Records; @@ -212,6 +214,32 @@ public abstract class AllocateRequest { public abstract void setUpdateRequests( List updateRequests); + /** + * Get the list of Scheduling requests being sent by the + * ApplicationMaster. + * @return list of {@link SchedulingRequest} being sent by the + * ApplicationMaster. + */ + @Public + @Unstable + public List getSchedulingRequests() { +return Collections.EMPTY_LIST; + } + + /** + * Set the list of Scheduling requests to inform the + * ResourceManager about the application's resource requirements + * (potentially including allocation tags & placement constraints). + * @param schedulingRequests list of SchedulingRequest to update + * the ResourceManager about the application's resource + * requirements. + */ + @Public + @Unstable + public void setSchedulingRequests( + List schedulingRequests) { + } + @Public @Unstable public static AllocateRequestBuilder newBuilder() { @@ -314,6 +342,20 @@ public abstract class AllocateRequest { } /** + * Set the schedulingRequests of the request. + * @see AllocateRequest#setSchedulingRequests(List) + * @param schedulingRequests SchedulingRequest of the request + * @return {@link AllocateRequestBuilder} + */ +@Public +@Unstable +public AllocateRequestBuilder schedulingRequests( +List schedulingRequests) { + allocateRequest.setSchedulingRequests(schedulingRequests); + return this; +} + +/** * Return generated {@link AllocateRequest} object. * @return {@link AllocateRequest} */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/69de9a1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java index d82be11..8cdc63f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-y
[11/32] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java new file mode 100644 index 000..9571f0e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; + +import java.util.ArrayList; +import java.util.List; + +/** + * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm + * is free to produce multiple of output objects at the end of each run and it + * must use the provided ConstraintPlacementAlgorithmOutputCollector to + * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer + * which is provided a collector to collect output. + */ +public class ConstraintPlacementAlgorithmOutput { + + private final ApplicationId applicationId; + + public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) { +this.applicationId = applicationId; + } + + private final List placedRequests = + new ArrayList<>(); + + private final List rejectedRequests = + new ArrayList<>(); + + public List getPlacedRequests() { +return placedRequests; + } + + public List getRejectedRequests() { +return rejectedRequests; + } + + public ApplicationId getApplicationId() { +return applicationId; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java new file mode 100644 index 000..131fd42 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing
[01/32] hadoop git commit: YARN-7780. Documentation for Placement Constraints. (Konstantinos Karanasos via asuresh)
Repository: hadoop Updated Branches: refs/heads/trunk 7288b0833 -> 8d1e2c640 YARN-7780. Documentation for Placement Constraints. (Konstantinos Karanasos via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8df7666f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8df7666f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8df7666f Branch: refs/heads/trunk Commit: 8df7666fe19f124e80bcc63c496607e085fcf804 Parents: add993e Author: Arun Suresh <asur...@apache.org> Authored: Tue Jan 30 07:38:27 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../yarn/api/resource/PlacementConstraints.java | 17 ++- .../hadoop/yarn/conf/YarnConfiguration.java | 11 +- .../site/markdown/PlacementConstraints.md.vm| 149 +++ 3 files changed, 164 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df7666f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java index 70a8080..c1549c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java @@ -96,8 +96,9 @@ public final class PlacementConstraints { * Creates a constraint that restricts the number of allocations within a * given scope (e.g., node or rack). * - * For example, {@code cardinality(NODE, 3, 10)}, restricts the number of - * allocations per node to be no less than 3 and no more than 10. + * For example, {@code cardinality(NODE, 3, 10, "zk")} is satisfied on nodes + * where there are no less than 3 allocations with tag "zk" and no more than + * 10. * * @param scope the scope of the constraint * @param minCardinality determines the minimum number of allocations within @@ -132,7 +133,7 @@ public final class PlacementConstraints { /** * Similar to {@link #cardinality(String, int, int, String...)}, but - * determines only the maximum cardinality (the minimum can be as low as 0). + * determines only the maximum cardinality (the minimum cardinality is 0). * * @param scope the scope of the constraint * @param maxCardinality determines the maximum number of allocations within @@ -150,7 +151,7 @@ public final class PlacementConstraints { * * Consider a set of nodes N that belongs to the scope specified in the * constraint. If the target expressions are satisfied at least minCardinality - * times and at most max-cardinality times in the node set N, then the + * times and at most maxCardinality times in the node set N, then the * constraint is satisfied. * * For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))}, @@ -197,7 +198,7 @@ public final class PlacementConstraints { /** * Constructs a target expression on a node partition. It is satisfied if - * the specified node partition has one of the specified nodePartitions + * the specified node partition has one of the specified nodePartitions. * * @param nodePartitions the set of values that the attribute should take * values from @@ -211,7 +212,7 @@ public final class PlacementConstraints { /** * Constructs a target expression on an allocation tag. It is satisfied if - * the there are allocations with one of the given tags. + * there are allocations with one of the given tags. * * @param allocationTags the set of tags that the attribute should take * values from @@ -224,8 +225,8 @@ public final class PlacementConstraints { /** * Constructs a target expression on an allocation tag. It is satisfied if - * the there are allocations with one of the given tags. Comparing to - * {@link PlacementTargets#allocationTag(String...)}, this only check tags + * there are allocations with one of the given tags. Comparing to + * {@link PlacementTargets#allocationTag(String...)}, this only checks tags * within the application. * * @param allocationTags the set of tags that the attribute should take http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df7666f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-
[37/37] hadoop git commit: YARN-7861. [UI2] Logs page shows duplicated containers with ATS. (Sunil G via wangda)
YARN-7861. [UI2] Logs page shows duplicated containers with ATS. (Sunil G via wangda) Change-Id: Id07d1ff8d519e0d9291ab037236641ceb901582c Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1453a04e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1453a04e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1453a04e Branch: refs/heads/YARN-6592 Commit: 1453a04e92ce88b65995248c5d6a2bc934cbe65f Parents: 5ca4bf2 Author: Wangda TanAuthored: Wed Jan 31 21:44:42 2018 +0800 Committer: Wangda Tan Committed: Wed Jan 31 21:44:42 2018 +0800 -- .../src/main/webapp/app/controllers/yarn-app/logs.js | 13 +++-- .../src/main/webapp/app/routes/yarn-app/logs.js| 1 + 2 files changed, 12 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1453a04e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js index 90ecc12..3b75f76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/logs.js @@ -47,10 +47,14 @@ export default Ember.Controller.extend({ this.fetchContainersForAttemptId(attemptId) .then(hash => { let containers = null; +let containerIdArr = []; if ( hash.rmContainers.get("length") > 0 && hash.rmContainers.get("content") ) { + hash.rmContainers.get("content").forEach(function(o) { +containerIdArr.push(o.id); + }.bind(this)); containers = (containers || []).concat( hash.rmContainers.get("content") ); @@ -59,9 +63,14 @@ export default Ember.Controller.extend({ hash.tsContainers.get("length") > 0 && hash.tsContainers.get("content") ) { + let tscontainer = []; + hash.tsContainers.get("content").forEach(function(o) { +if(!containerIdArr.contains(o.id)) { + tscontainer.push(o); +} + }.bind(this)); containers = (containers || []).concat( -hash.tsContainers.get("content") - ); +tscontainer); } this.set("attemptContainerList", containers); this.initializeSelect(".js-fetch-logs-containers"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/1453a04e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js index d92fc3c..8837503 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-app/logs.js @@ -36,6 +36,7 @@ export default AbstractRoute.extend(AppAttemptMixin, { activate() { const controller = this.controllerFor("yarn-app.logs"); +controller.resetAfterRefresh(); controller.initializeSelect(); }, - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[19/37] hadoop git commit: YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37f1a7b6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37f1a7b6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37f1a7b6 Branch: refs/heads/YARN-6592 Commit: 37f1a7b64fcc93191367330cd59d4d71d7b29ac7 Parents: 06eb63e Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 22 07:24:37 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../server/resourcemanager/ResourceManager.java | 2 +- .../constraint/AllocationTagsManager.java | 282 ++- .../rmcontainer/TestRMContainerImpl.java| 2 +- .../constraint/TestAllocationTagsManager.java | 269 -- 4 files changed, 392 insertions(+), 163 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37f1a7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index a1d3dfc..1d838f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -496,7 +496,7 @@ public class ResourceManager extends CompositeService implements Recoverable { } protected AllocationTagsManager createAllocationTagsManager() { -return new AllocationTagsManager(); +return new AllocationTagsManager(this.rmContext); } protected DelegationTokenRenewer createDelegationTokenRenewer() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/37f1a7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java index c278606..7b0b959 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.log4j.Logger; import java.util.HashMap; @@ -38,9 +39,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongBinaryOperator; /** - * Support storing maps between container-tags/applications and - * nodes. This will be required by affinity/anti-affinity implementation and - * cardinality. + * In-memory mapping between applications/container-tags and nodes/racks. + * Required by constrained affinity/anti-affinity and cardinality placement. */ @InterfaceAudience.Private @InterfaceStability.Unstable @@ -51,48 +51,54 @@ public class AllocationTagsManager { private ReentrantReadWriteLock.ReadLock readLock; private ReentrantReadWriteLock.WriteLock writeLock; + private final RMContext rmContext; - // Application's tags to node - private Map<ApplicationId, NodeToCountedTags> perAppMappings = + // Application's tags to Node + private Map<ApplicationId, NodeToCountedTags> perAppNodeMappings = + new HashMap<>(); + // Application's tags to Rac
[34/37] hadoop git commit: YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh)
YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf7c444 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf7c444 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf7c444 Branch: refs/heads/YARN-6592 Commit: 8bf7c444368f48f63f8011cf155f551c6b51ee21 Parents: 28fe7f3 Author: Arun Suresh <asur...@apache.org> Authored: Sun Jan 21 19:11:17 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/capacity/CapacityScheduler.java | 4 +- .../constraint/PlacementConstraintsUtil.java| 98 +++- .../algorithm/DefaultPlacementAlgorithm.java| 4 +- .../SingleConstraintAppPlacementAllocator.java | 10 +- .../TestPlacementConstraintsUtil.java | 94 --- 5 files changed, 123 insertions(+), 87 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf7c444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 429f9f3..a096e2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2621,9 +2621,9 @@ public class CapacityScheduler extends // Validate placement constraint is satisfied before // committing the request. try { - if (!PlacementConstraintsUtil.canSatisfySingleConstraint( + if (!PlacementConstraintsUtil.canSatisfyConstraints( appAttempt.getApplicationId(), - schedulingRequest.getAllocationTags(), schedulerNode, + schedulingRequest, schedulerNode, rmContext.getPlacementConstraintManager(), rmContext.getAllocationTagsManager())) { LOG.debug("Failed to allocate container for application " http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf7c444/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index ff5cb67..c07c16f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; @@ -54,7 +55,7 @@ public final class PlacementConstraintsUtil { } /** - * Returns true if **single** placement constraint with associated + * Returns true if single placement constraint with associated * allocationTags and scope is satisfied by a s
[17/37] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java new file mode 100644 index 000..9571f0e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; + +import java.util.ArrayList; +import java.util.List; + +/** + * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm + * is free to produce multiple of output objects at the end of each run and it + * must use the provided ConstraintPlacementAlgorithmOutputCollector to + * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer + * which is provided a collector to collect output. + */ +public class ConstraintPlacementAlgorithmOutput { + + private final ApplicationId applicationId; + + public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) { +this.applicationId = applicationId; + } + + private final List placedRequests = + new ArrayList<>(); + + private final List rejectedRequests = + new ArrayList<>(); + + public List getPlacedRequests() { +return placedRequests; + } + + public List getRejectedRequests() { +return rejectedRequests; + } + + public ApplicationId getApplicationId() { +return applicationId; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java new file mode 100644 index 000..131fd42 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing
[33/37] hadoop git commit: YARN-7780. Documentation for Placement Constraints. (Konstantinos Karanasos via asuresh)
YARN-7780. Documentation for Placement Constraints. (Konstantinos Karanasos via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8df7666f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8df7666f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8df7666f Branch: refs/heads/YARN-6592 Commit: 8df7666fe19f124e80bcc63c496607e085fcf804 Parents: add993e Author: Arun Suresh <asur...@apache.org> Authored: Tue Jan 30 07:38:27 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../yarn/api/resource/PlacementConstraints.java | 17 ++- .../hadoop/yarn/conf/YarnConfiguration.java | 11 +- .../site/markdown/PlacementConstraints.md.vm| 149 +++ 3 files changed, 164 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df7666f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java index 70a8080..c1549c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java @@ -96,8 +96,9 @@ public final class PlacementConstraints { * Creates a constraint that restricts the number of allocations within a * given scope (e.g., node or rack). * - * For example, {@code cardinality(NODE, 3, 10)}, restricts the number of - * allocations per node to be no less than 3 and no more than 10. + * For example, {@code cardinality(NODE, 3, 10, "zk")} is satisfied on nodes + * where there are no less than 3 allocations with tag "zk" and no more than + * 10. * * @param scope the scope of the constraint * @param minCardinality determines the minimum number of allocations within @@ -132,7 +133,7 @@ public final class PlacementConstraints { /** * Similar to {@link #cardinality(String, int, int, String...)}, but - * determines only the maximum cardinality (the minimum can be as low as 0). + * determines only the maximum cardinality (the minimum cardinality is 0). * * @param scope the scope of the constraint * @param maxCardinality determines the maximum number of allocations within @@ -150,7 +151,7 @@ public final class PlacementConstraints { * * Consider a set of nodes N that belongs to the scope specified in the * constraint. If the target expressions are satisfied at least minCardinality - * times and at most max-cardinality times in the node set N, then the + * times and at most maxCardinality times in the node set N, then the * constraint is satisfied. * * For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))}, @@ -197,7 +198,7 @@ public final class PlacementConstraints { /** * Constructs a target expression on a node partition. It is satisfied if - * the specified node partition has one of the specified nodePartitions + * the specified node partition has one of the specified nodePartitions. * * @param nodePartitions the set of values that the attribute should take * values from @@ -211,7 +212,7 @@ public final class PlacementConstraints { /** * Constructs a target expression on an allocation tag. It is satisfied if - * the there are allocations with one of the given tags. + * there are allocations with one of the given tags. * * @param allocationTags the set of tags that the attribute should take * values from @@ -224,8 +225,8 @@ public final class PlacementConstraints { /** * Constructs a target expression on an allocation tag. It is satisfied if - * the there are allocations with one of the given tags. Comparing to - * {@link PlacementTargets#allocationTag(String...)}, this only check tags + * there are allocations with one of the given tags. Comparing to + * {@link PlacementTargets#allocationTag(String...)}, this only checks tags * within the application. * * @param allocationTags the set of tags that the attribute should take http://git-wip-us.apache.org/repos/asf/hadoop/blob/8df7666f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apac
[36/37] hadoop git commit: YARN-7828. Clicking on yarn service should take to component tab. (Sunil G via wangda)
YARN-7828. Clicking on yarn service should take to component tab. (Sunil G via wangda) Change-Id: Ib52d6727c863bed65de30b02bf60080255af11b8 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ca4bf22 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ca4bf22 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ca4bf22 Branch: refs/heads/YARN-6592 Commit: 5ca4bf22dd45d9ac1328c74204413d90725d1405 Parents: 64344c3 Author: Wangda TanAuthored: Wed Jan 31 21:43:08 2018 +0800 Committer: Wangda Tan Committed: Wed Jan 31 21:43:08 2018 +0800 -- .../main/webapp/app/controllers/app-table-columns.js| 2 +- .../src/main/webapp/app/controllers/yarn-app-attempt.js | 2 +- .../src/main/webapp/app/controllers/yarn-app.js | 2 +- .../webapp/app/controllers/yarn-component-instance.js | 4 ++-- .../webapp/app/controllers/yarn-component-instances.js | 2 +- .../src/main/webapp/app/templates/yarn-app.hbs | 12 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca4bf22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js index fb002f9..33e3bb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/app-table-columns.js @@ -122,7 +122,7 @@ export default Ember.Controller.extend({ getCellContent: function(row) { return { displayText: row.get('appName'), - href: `#/yarn-app/${row.id}/attempts?service=${row.get('appName')}` + href: `#/yarn-app/${row.id}/components?service=${row.get('appName')}` }; } }, { http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca4bf22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js index 504b66e..4c8b8a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app-attempt.js @@ -50,7 +50,7 @@ export default Ember.Controller.extend({ routeName: 'yarn-services' }, { text: `${serviceName} [${appId}]`, -href: `#/yarn-app/${appId}/info?service=${serviceName}` +href: `#/yarn-app/${appId}/components?service=${serviceName}` }, { text: "Attempts", href: `#/yarn-app/${appId}/attempts?service=${serviceName}` http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca4bf22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js index b84f328..b2b99b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app.js @@ -102,7 +102,7 @@ export default Ember.Controller.extend({ routeName: 'yarn-services' }, { text: `${serviceName} [${appId}]`, -href: `#/yarn-app/${appId}/info?service=${serviceName}` +href: `#/yarn-app/${appId}/components?service=${serviceName}` }); } else { breadcrumbs.push({ http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca4bf22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance.js index 4b8dbf4..358c6b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instance.js +++
[26/37] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index 73b4f9e..24c5a5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; import java.util.Iterator; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -30,9 +32,12 @@ import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm; +import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE_PARTITION; + /** * This class contains various static methods used by the Placement Algorithms * to simplify constrained placement. @@ -41,16 +46,20 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algori @Public @Unstable public final class PlacementConstraintsUtil { + private static final Log LOG = + LogFactory.getLog(PlacementConstraintsUtil.class); // Suppresses default constructor, ensuring non-instantiability. private PlacementConstraintsUtil() { } /** - * Returns true if **single** application constraint with associated + * Returns true if **single** placement constraint with associated * allocationTags and scope is satisfied by a specific scheduler Node. * - * @param appId the application id + * @param targetApplicationId the application id, which could be override by + * target application id specified inside allocation + * tags. * @param sc the placement constraint * @param te the target expression * @param node the scheduler node @@ -59,32 +68,123 @@ public final class PlacementConstraintsUtil { * @throws InvalidAllocationTagsQueryException */ private static boolean canSatisfySingleConstraintExpression( - ApplicationId appId, SingleConstraint sc, TargetExpression te, - SchedulerNode node, AllocationTagsManager tm) + ApplicationId targetApplicationId, SingleConstraint sc, + TargetExpression te, SchedulerNode node, AllocationTagsManager tm) throws InvalidAllocationTagsQueryException { long minScopeCardinality = 0; long maxScopeCardinality = 0; + +// Optimizations to only check cardinality if necessary. +int desiredMinCardinality = sc.getMinCardinality(); +int desiredMaxCardinality = sc.getMaxCardinality(); +boolean checkMinCardinality = desiredMinCardinality > 0; +boolean checkMaxCardinality = desiredMaxCardinality < Integer.MAX_VALUE; + if (sc.getScope().equals(PlacementConstraints.NODE)) { - minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId, - te.getTargetValues(), Long::max); - maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId, - te.getTargetValues(), Long::min); + if (checkMinCardinality) { +minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), +targetApplicationId, te.getTargetValues(), Long::max); + } + if (checkMaxCardinality) { +maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), +targetApplicationId, te.getTargetValues(),
[02/37] hadoop git commit: HDFS-12528. Add an option to not disable short-circuit reads on failures. Contributed by Xiao Chen.
HDFS-12528. Add an option to not disable short-circuit reads on failures. Contributed by Xiao Chen. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e7331ca Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e7331ca Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e7331ca Branch: refs/heads/YARN-6592 Commit: 2e7331ca264dd366b975f3c8e610cf84eb8cc155 Parents: f9dd5b6 Author: Weiwei YangAuthored: Wed Jan 31 07:25:03 2018 +0800 Committer: Weiwei Yang Committed: Wed Jan 31 07:25:03 2018 +0800 -- .../hdfs/client/HdfsClientConfigKeys.java | 3 + .../hdfs/client/impl/BlockReaderFactory.java| 16 ++- .../hadoop/hdfs/client/impl/DfsClientConf.java | 19 +++- .../hdfs/shortcircuit/DomainSocketFactory.java | 14 ++- .../hdfs/shortcircuit/ShortCircuitCache.java| 16 ++- .../src/main/resources/hdfs-default.xml | 11 ++ .../client/impl/TestBlockReaderFactory.java | 111 ++- 7 files changed, 175 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e7331ca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java index d6c47b5..38c2435 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java @@ -127,6 +127,9 @@ public interface HdfsClientConfigKeys { boolean DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT = false; String DFS_DOMAIN_SOCKET_PATH_KEY = "dfs.domain.socket.path"; String DFS_DOMAIN_SOCKET_PATH_DEFAULT = ""; + String DFS_DOMAIN_SOCKET_DISABLE_INTERVAL_SECOND_KEY = + "dfs.domain.socket.disable.interval.seconds"; + long DFS_DOMAIN_SOCKET_DISABLE_INTERVAL_SECOND_DEFAULT = 600; String DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS = "dfs.short.circuit.shared.memory.watcher.interrupt.check.ms"; int DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT = http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e7331ca/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java index 6f3fc61..60dde82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hdfs.client.impl; +import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DOMAIN_SOCKET_DISABLE_INTERVAL_SECOND_KEY; import static org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitFdResponse.USE_RECEIPT_VERIFICATION; import java.io.BufferedOutputStream; @@ -644,10 +645,17 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator { LOG.debug("{}:{}", this, msg); return new ShortCircuitReplicaInfo(new InvalidToken(msg)); default: - LOG.warn(this + ": unknown response code " + resp.getStatus() + - " while attempting to set up short-circuit access. " + - resp.getMessage() + ". Disabling short-circuit read for DataNode " - + datanode + " temporarily."); + final long expiration = + clientContext.getDomainSocketFactory().getPathExpireSeconds(); + String disableMsg = "disabled temporarily for " + expiration + " seconds"; + if (expiration == 0) { +disableMsg = "not disabled"; + } + LOG.warn("{}: unknown response code {} while attempting to set up " + + "short-circuit access. {}. Short-circuit read for " + + "DataNode {} is {} based on {}.", + this, resp.getStatus(), resp.getMessage(), datanode, + disableMsg, DFS_DOMAIN_SOCKET_DISABLE_INTERVAL_SECOND_KEY); clientContext.getDomainSocketFactory() .disableShortCircuitForPath(pathInfo.getPath()); return null;
[21/37] hadoop git commit: YARN-7682. Expose canSatisfyConstraints utility function to validate a placement against a constraint. (Panagiotis Garefalakis via asuresh)
YARN-7682. Expose canSatisfyConstraints utility function to validate a placement against a constraint. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdba01f7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdba01f7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdba01f7 Branch: refs/heads/YARN-6592 Commit: bdba01f73b58d2228e808c6f61377f101b6bac1c Parents: a52d11f Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 3 08:00:50 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../constraint/PlacementConstraintsUtil.java| 132 + .../algorithm/DefaultPlacementAlgorithm.java| 55 +--- .../TestPlacementConstraintsUtil.java | 287 +++ .../constraint/TestPlacementProcessor.java | 204 +++-- 4 files changed, 601 insertions(+), 77 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdba01f7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java new file mode 100644 index 000..956a3c9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint; + +import java.util.Iterator; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer; +import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm; + +/** + * This class contains various static methods used by the Placement Algorithms + * to simplify constrained placement. + * (see also {@link DefaultPlacementAlgorithm}). + */ +@Public +@Unstable +public final class PlacementConstraintsUtil { + + // Suppresses default constructor, ensuring non-instantiability. + private PlacementConstraintsUtil() { + } + + /** + * Returns true if **single** application constraint with associated + * allocationTags and scope is satisfied by a specific scheduler Node. + * + * @param appId the application id + * @param sc the placement constraint + * @param te the target expression + * @param node the scheduler node + * @param tm the allocation tags store + * @return true if single application constraint is satisfied by node + * @throws InvalidAllocationTagsQueryException + */ + private static boolean canS
[22/37] hadoop git commit: YARN-7613. Implement Basic algorithm for constraint based placement. (Panagiotis Garefalakis via asuresh)
YARN-7613. Implement Basic algorithm for constraint based placement. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a52d11fb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a52d11fb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a52d11fb Branch: refs/heads/YARN-6592 Commit: a52d11fb8c103f14e42692600a058ba3b56e2ecf Parents: f9af15d Author: Arun Suresh <asur...@apache.org> Authored: Wed Dec 27 22:59:22 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 4 + .../src/main/resources/yarn-default.xml | 8 +- .../rmcontainer/RMContainerImpl.java| 10 +- .../constraint/AllocationTagsManager.java | 121 ++--- .../algorithm/DefaultPlacementAlgorithm.java| 172 +++ .../iterators/PopularTagsIterator.java | 71 .../algorithm/iterators/SerialIterator.java | 53 ++ .../algorithm/iterators/package-info.java | 29 .../constraint/algorithm/package-info.java | 29 .../constraint/processor/BatchedRequests.java | 45 - .../processor/PlacementProcessor.java | 32 ++-- .../processor/SamplePlacementAlgorithm.java | 144 .../constraint/TestAllocationTagsManager.java | 156 - .../TestBatchedRequestsIterators.java | 82 + .../constraint/TestPlacementProcessor.java | 4 +- 15 files changed, 721 insertions(+), 239 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8fb3c2e..367b1ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -536,6 +536,10 @@ public class YarnConfiguration extends Configuration { public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS = RM_PREFIX + "placement-constraints.algorithm.class"; + /** Used for BasicPlacementAlgorithm - default SERIAL. **/ + public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR = + RM_PREFIX + "placement-constraints.algorithm.iterator"; + public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED = RM_PREFIX + "placement-constraints.enabled"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 6d52ace..509a040 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -145,7 +145,13 @@ Constraint Placement Algorithm to be used. yarn.resourcemanager.placement-constraints.algorithm.class - org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.SamplePlacementAlgorithm + org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm + + + +Placement Algorithm Requests Iterator to be used. +yarn.resourcemanager.placement-constraints.algorithm.iterator +SERIAL http://git-wip-us.apache.org/repos/asf/hadoop/blob/a52d11fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index c873509..2c4ef7b 100644 --- a/hadoop-yarn-pro
[04/37] hadoop git commit: YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)
YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8c5f5b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8c5f5b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8c5f5b2 Branch: refs/heads/YARN-6592 Commit: f8c5f5b23732a1e35f012c1a6850bed09c8a5180 Parents: c23980c Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 25 19:09:21 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/AppSchedulingInfo.java| 10 - .../scheduler/common/fica/FiCaSchedulerApp.java | 7 +++ .../constraint/TestPlacementProcessor.java | 45 3 files changed, 60 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c5f5b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 0389895..1efdd8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -694,6 +694,12 @@ public class AppSchedulingInfo { metrics.runAppAttempt(applicationId, user); } +updateMetrics(applicationId, type, node, containerAllocated, user, queue); + } + + public static void updateMetrics(ApplicationId applicationId, NodeType type, + SchedulerNode node, Container containerAllocated, String user, + Queue queue) { if (LOG.isDebugEnabled()) { LOG.debug("allocate: applicationId=" + applicationId + " container=" + containerAllocated.getId() + " host=" + containerAllocated @@ -702,10 +708,10 @@ public class AppSchedulingInfo { + type); } if(node != null) { - metrics.allocateResources(node.getPartition(), user, 1, + queue.getMetrics().allocateResources(node.getPartition(), user, 1, containerAllocated.getResource(), true); } -metrics.incrNodeTypeAggregations(user, type); +queue.getMetrics().incrNodeTypeAggregations(user, type); } // Get AppPlacementAllocator by specified schedulerKey http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8c5f5b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 7eb1e31..f3da0a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRese import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; @@ -548,6 +549,1
[10/37] hadoop git commit: YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh)
YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4c539fc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4c539fc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4c539fc Branch: refs/heads/YARN-6592 Commit: a4c539fcdba817e313b2375abf2c4c9a1d13a4fd Parents: 9b81cb0 Author: Arun Suresh <asur...@apache.org> Authored: Tue Jan 23 08:15:58 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../algorithm/DefaultPlacementAlgorithm.java| 119 +-- .../constraint/TestPlacementProcessor.java | 49 2 files changed, 155 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4c539fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java index 9887749..4e6473f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.List; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ResourceSizing; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; @@ -69,13 +70,9 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { public boolean attemptPlacementOnNode(ApplicationId appId, SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) throws InvalidAllocationTagsQueryException { -int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations(); -if (numAllocs > 0) { - if (PlacementConstraintsUtil.canSatisfyConstraints(appId, - schedulingRequest, schedulerNode, - constraintManager, tagsManager)) { -return true; - } +if (PlacementConstraintsUtil.canSatisfyConstraints(appId, +schedulingRequest, schedulerNode, constraintManager, tagsManager)) { + return true; } return false; } @@ -93,6 +90,9 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { int rePlacementCount = RE_ATTEMPT_COUNT; while (rePlacementCount > 0) { doPlacement(requests, resp, allNodes, rejectedRequests); + // Double check if placement constraints are really satisfied + validatePlacement(requests.getApplicationId(), resp, + rejectedRequests); if (rejectedRequests.size() == 0 || rePlacementCount == 1) { break; } @@ -122,9 +122,14 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { break; } SchedulingRequest schedulingRequest = requestIterator.next(); + PlacedSchedulingRequest placedReq = + new PlacedSchedulingRequest(schedulingRequest); + placedReq.setPlacementAttempt(requests.getPlacementAttempt()); + resp.getPlacedRequests().add(placedReq); CircularIterator nodeIter = new CircularIterator(lastSatisfiedNode, nIter, allNodes); - int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations(); + int numAllocs = + schedulingRequest.getResourceSizing().getNumAllocations(); while (nodeIter.hasNext() && numAllocs > 0) { SchedulerNode node = nodeIter.next(); try { @@ -135,11 +140,7 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { requests.getApplicationId(), schedulingRequest, node)) { schedulingRequest.getResourceSizing() .
[06/37] hadoop git commit: YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh)
YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5c1fc88 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5c1fc88 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5c1fc88 Branch: refs/heads/YARN-6592 Commit: a5c1fc881e21ebf43da7ead5f3852808fce25492 Parents: 4eda58c Author: Arun Suresh <asur...@apache.org> Authored: Fri Jan 12 14:37:06 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/api/records/Container.java | 15 + .../src/main/proto/yarn_protos.proto| 1 + .../api/records/impl/pb/ContainerPBImpl.java| 31 + .../yarn/security/ContainerTokenIdentifier.java | 69 +++- .../src/main/proto/yarn_security_token.proto| 1 + .../api/protocolrecords/NMContainerStatus.java | 14 .../impl/pb/NMContainerStatusPBImpl.java| 33 ++ .../yarn_server_common_service_protos.proto | 1 + .../containermanager/ContainerManagerImpl.java | 3 +- .../container/ContainerImpl.java| 19 +++--- .../rmcontainer/RMContainerImpl.java| 10 ++- .../scheduler/SchedulerApplicationAttempt.java | 3 +- .../security/RMContainerTokenSecretManager.java | 21 ++ .../capacity/TestContainerAllocation.java | 5 +- 14 files changed, 194 insertions(+), 32 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 4fdc803..b9ca3f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.Collections; +import java.util.Set; + /** * {@code Container} represents an allocated resource in the cluster. * @@ -256,4 +259,16 @@ public abstract class Container implements Comparable { public void setVersion(int version) { throw new UnsupportedOperationException(); } + + @Private + @Unstable + public Set getAllocationTags() { +return Collections.EMPTY_SET; + } + + @Private + @Unstable + public void setAllocationTags(Set allocationTags) { + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5cb1177..25c8569 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -152,6 +152,7 @@ message ContainerProto { optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED]; optional int64 allocation_request_id = 8 [default = -1]; optional int32 version = 9 [default = 0]; + repeated string allocation_tags = 10; } message ContainerReportProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5c1fc88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index be84938..47be2f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -36,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.PriorityPr
[13/37] hadoop git commit: YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos via wangda)
YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57e8bc3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57e8bc3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57e8bc3 Branch: refs/heads/YARN-6592 Commit: b57e8bc3002a95d2f2f328554d792151cdc1120d Parents: 33a796d Author: Wangda TanAuthored: Mon Oct 30 16:54:02 2017 -0700 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/api/records/ResourceSizing.java | 64 + .../yarn/api/records/SchedulingRequest.java | 205 ++ .../src/main/proto/yarn_protos.proto| 14 + .../records/impl/pb/ResourceSizingPBImpl.java | 117 .../impl/pb/SchedulingRequestPBImpl.java| 266 +++ 5 files changed, 666 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57e8bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java new file mode 100644 index 000..d82be11 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@code ResourceSizing} contains information for the size of a + * {@link SchedulingRequest}, such as the number of requested allocations and + * the resources for each allocation. + */ +@Public +@Unstable +public abstract class ResourceSizing { + + @Public + @Unstable + public static ResourceSizing newInstance(Resource resources) { +return ResourceSizing.newInstance(1, resources); + } + + @Public + @Unstable + public static ResourceSizing newInstance(int numAllocations, Resource resources) { +ResourceSizing resourceSizing = Records.newRecord(ResourceSizing.class); +resourceSizing.setNumAllocations(numAllocations); +resourceSizing.setResources(resources); +return resourceSizing; + } + + @Public + @Unstable + public abstract int getNumAllocations(); + + @Public + @Unstable + public abstract void setNumAllocations(int numAllocations); + + @Public + @Unstable + public abstract Resource getResources(); + + @Public + @Unstable + public abstract void setResources(Resource resources); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57e8bc3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java new file mode 100644 index 000..47a0697 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java @@ -0,0 +1,205 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in
[03/37] hadoop git commit: HDFS-13083. RBF: Fix doc error setting up client. Contributed by tartarus.
HDFS-13083. RBF: Fix doc error setting up client. Contributed by tartarus. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5206b2c7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5206b2c7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5206b2c7 Branch: refs/heads/YARN-6592 Commit: 5206b2c7ca479dd53f614d75bab594043a9866e1 Parents: 2e7331c Author: Inigo GoiriAuthored: Tue Jan 30 18:27:18 2018 -0800 Committer: Inigo Goiri Committed: Tue Jan 30 18:27:18 2018 -0800 -- .../hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5206b2c7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md index 6b21123..b5fbcc4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSRouterFederation.md @@ -236,7 +236,7 @@ For example, a cluster with 4 namespaces **ns0, ns1, ns2, ns3**, can add a new o ns0,ns1,ns2,ns3,ns-fed -dfs.namenodes.ns-fed +dfs.ha.namenodes.ns-fed r1,r2 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[14/37] hadoop git commit: YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis Garefalakis via asuresh)
YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69de9a1b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69de9a1b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69de9a1b Branch: refs/heads/YARN-6592 Commit: 69de9a1ba9a587c7e03ae7c7aeae93e04c36d665 Parents: db92855 Author: Arun Suresh <asur...@apache.org> Authored: Fri Nov 17 10:42:43 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../api/protocolrecords/AllocateRequest.java| 42 ++ .../hadoop/yarn/api/records/ResourceSizing.java | 27 +++ .../yarn/api/records/SchedulingRequest.java | 1 + .../src/main/proto/yarn_service_protos.proto| 1 + .../impl/pb/AllocateRequestPBImpl.java | 83 .../records/impl/pb/ResourceSizingPBImpl.java | 2 +- .../impl/pb/SchedulingRequestPBImpl.java| 16 .../hadoop/yarn/api/TestPBImplRecords.java | 19 + 8 files changed, 190 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/69de9a1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index ae0891e..d8d2347 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.Collections; import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -28,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.util.Records; @@ -212,6 +214,32 @@ public abstract class AllocateRequest { public abstract void setUpdateRequests( List updateRequests); + /** + * Get the list of Scheduling requests being sent by the + * ApplicationMaster. + * @return list of {@link SchedulingRequest} being sent by the + * ApplicationMaster. + */ + @Public + @Unstable + public List getSchedulingRequests() { +return Collections.EMPTY_LIST; + } + + /** + * Set the list of Scheduling requests to inform the + * ResourceManager about the application's resource requirements + * (potentially including allocation tags & placement constraints). + * @param schedulingRequests list of SchedulingRequest to update + * the ResourceManager about the application's resource + * requirements. + */ + @Public + @Unstable + public void setSchedulingRequests( + List schedulingRequests) { + } + @Public @Unstable public static AllocateRequestBuilder newBuilder() { @@ -314,6 +342,20 @@ public abstract class AllocateRequest { } /** + * Set the schedulingRequests of the request. + * @see AllocateRequest#setSchedulingRequests(List) + * @param schedulingRequests SchedulingRequest of the request + * @return {@link AllocateRequestBuilder} + */ +@Public +@Unstable +public AllocateRequestBuilder schedulingRequests( +List schedulingRequests) { + allocateRequest.setSchedulingRequests(schedulingRequests); + return this; +} + +/** * Return generated {@link AllocateRequest} object. * @return {@link AllocateRequest} */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/69de9a1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java index d82be11..8cdc63f 100644 --- a/hadoop-yarn-project/hadoop-yarn/had
[11/37] hadoop git commit: YARN-7709. Remove SELF from TargetExpression type. (Konstantinos Karanasos via asuresh)
YARN-7709. Remove SELF from TargetExpression type. (Konstantinos Karanasos via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8779a357 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8779a357 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8779a357 Branch: refs/heads/YARN-6592 Commit: 8779a35742085fadddccc21342b55d4f17fae5c2 Parents: 29d9e4d Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 18 04:29:57 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../yarn/api/resource/PlacementConstraint.java | 32 ++ .../yarn/api/resource/PlacementConstraints.java | 35 +--- .../api/resource/TestPlacementConstraints.java | 3 +- .../PlacementConstraintTransformations.java | 19 +++ .../TestPlacementConstraintTransformations.java | 35 +--- .../constraint/PlacementConstraintsUtil.java| 10 -- 6 files changed, 64 insertions(+), 70 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8779a357/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java index b6e851a..4d998ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -242,7 +242,7 @@ public class PlacementConstraint { * Enum specifying the type of the target expression. */ public enum TargetType { - NODE_ATTRIBUTE, ALLOCATION_TAG, SELF + NODE_ATTRIBUTE, ALLOCATION_TAG } private TargetType targetType; @@ -418,23 +418,25 @@ public class PlacementConstraint { } /** - * Class that represents a cardinality constraint. Such a constraint the - * number of allocations within a given scope to some minimum and maximum - * values. + * Class that represents a cardinality constraint. Such a constraint allows + * the number of allocations with a specific set of tags and within a given + * scope to be between some minimum and maximum values. * * It is a specialized version of the {@link SingleConstraint}, where the - * target is self (i.e., the allocation to which the constraint is attached). + * target is a set of allocation tags. */ public static class CardinalityConstraint extends AbstractConstraint { private String scope; private int minCardinality; private int maxCardinality; +private Set allocationTags; public CardinalityConstraint(String scope, int minCardinality, -int maxCardinality) { +int maxCardinality, Set allocationTags) { this.scope = scope; this.minCardinality = minCardinality; this.maxCardinality = maxCardinality; + this.allocationTags = allocationTags; } /** @@ -464,11 +466,21 @@ public class PlacementConstraint { return maxCardinality; } +/** + * Get the allocation tags of the constraint. + * + * @return the allocation tags of the constraint + */ +public Set getAllocationTags() { + return allocationTags; +} + @Override public T accept(Visitor visitor) { return visitor.visit(this); } + @Override public boolean equals(Object o) { if (this == o) { @@ -486,7 +498,11 @@ public class PlacementConstraint { if (maxCardinality != that.maxCardinality) { return false; } - return scope != null ? scope.equals(that.scope) : that.scope == null; + if (scope != null ? !scope.equals(that.scope) : that.scope != null) { +return false; + } + return allocationTags != null ? allocationTags.equals(that.allocationTags) + : that.allocationTags == null; } @Override @@ -494,6 +510,8 @@ public class PlacementConstraint { int result = scope != null ? scope.hashCode() : 0; result = 31 * result + minCardinality; result = 31 * result + maxCardinality; + result = 31 * result + + (allocationTags != null ? allocationTags.hashCode() : 0); return result; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8779a357/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementCons
[20/37] hadoop git commit: YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh)
YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1efb2b6f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1efb2b6f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1efb2b6f Branch: refs/heads/YARN-6592 Commit: 1efb2b6f250022f41fe5911c1bb3028ec15c5447 Parents: 37f1a7b Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 22 13:26:30 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../resourcemanager/RMActiveServiceContext.java | 15 + .../yarn/server/resourcemanager/RMContext.java | 6 + .../server/resourcemanager/RMContextImpl.java | 13 + .../server/resourcemanager/ResourceManager.java | 13 + .../MemoryPlacementConstraintManager.java | 282 +++ .../constraint/PlacementConstraintManager.java | 151 ++ .../PlacementConstraintManagerService.java | 93 ++ .../scheduler/constraint/package-info.java | 29 ++ .../TestPlacementConstraintManagerService.java | 182 9 files changed, 784 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1efb2b6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 4d0c230..06a1d00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; @@ -109,6 +110,7 @@ public class RMActiveServiceContext { private RMAppLifetimeMonitor rmAppLifetimeMonitor; private QueueLimitCalculator queueLimitCalculator; private AllocationTagsManager allocationTagsManager; + private PlacementConstraintManager placementConstraintManager; public RMActiveServiceContext() { queuePlacementManager = new PlacementManager(); @@ -413,6 +415,19 @@ public class RMActiveServiceContext { @Private @Unstable + public PlacementConstraintManager getPlacementConstraintManager() { +return placementConstraintManager; + } + + @Private + @Unstable + public void setPlacementConstraintManager( + PlacementConstraintManager placementConstraintManager) { +this.placementConstraintManager = placementConstraintManager; + } + + @Private + @Unstable public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() { return rmDelegatedNodeLabelsUpdater; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/1efb2b6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 00da108..eb91a31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop
[24/37] hadoop git commit: YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)
YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c23980c4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c23980c4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c23980c4 Branch: refs/heads/YARN-6592 Commit: c23980c4f2cf4c751a99fd310e60149cb32ea7c7 Parents: 644afe5 Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 24 14:18:32 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/api/protocolrecords/AllocateRequest.java | 4 ++-- .../apache/hadoop/yarn/api/records/SchedulingRequest.java| 8 .../apache/hadoop/yarn/api/resource/PlacementConstraint.java | 3 +++ .../hadoop/yarn/api/resource/PlacementConstraints.java | 3 --- .../yarn/api/records/impl/pb/ResourceSizingPBImpl.java | 3 +++ .../yarn/api/records/impl/pb/SchedulingRequestPBImpl.java| 3 +++ .../resourcemanager/scheduler/AbstractYarnScheduler.java | 1 - .../server/resourcemanager/scheduler/AppSchedulingInfo.java | 3 +-- .../yarn/server/resourcemanager/scheduler/SchedulerNode.java | 8 ++-- .../scheduler/capacity/CapacityScheduler.java| 3 +-- .../scheduler/constraint/AllocationTagsManager.java | 8 +--- .../scheduler/constraint/PlacementConstraintsUtil.java | 4 ++-- .../scheduler/placement/AppPlacementAllocator.java | 4 ++-- .../placement/SingleConstraintAppPlacementAllocator.java | 1 - .../scheduler/capacity/TestCapacityScheduler.java| 1 - .../TestCapacitySchedulerSchedulingRequestUpdate.java| 4 +++- .../capacity/TestSchedulingRequestContainerAllocation.java | 8 .../TestSchedulingRequestContainerAllocationAsync.java | 1 - 18 files changed, 35 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c23980c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index d8d2347..876957e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -229,8 +229,8 @@ public abstract class AllocateRequest { /** * Set the list of Scheduling requests to inform the * ResourceManager about the application's resource requirements - * (potentially including allocation tags & placement constraints). - * @param schedulingRequests list of SchedulingRequest to update + * (potentially including allocation tags and placement constraints). + * @param schedulingRequests list of {@link SchedulingRequest} to update * the ResourceManager about the application's resource * requirements. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/c23980c4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java index e32dd24..4bb2b84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java @@ -77,7 +77,7 @@ public abstract class SchedulingRequest { /** * Set the allocationRequestId of the request. - * + * * @see SchedulingRequest#setAllocationRequestId(long) * @param allocationRequestId allocationRequestId of the * request @@ -107,7 +107,7 @@ public abstract class SchedulingRequest { /** * Set the executionType of the request. - * + * * @see SchedulingRequest#setExecutionType(ExecutionTypeRequest) * @param executionType executionType of the request * @return {@link SchedulingRequest.SchedulingRequestBuilder} @@ -119,7 +119,7 @@ public abstract class SchedulingRequest { schedulingRequest.
[27/37] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)
YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38af2379 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38af2379 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38af2379 Branch: refs/heads/YARN-6592 Commit: 38af23796971193fa529c3d08ffde8fcd6e607b6 Parents: 8779a35 Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 18 14:10:30 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../v2/app/rm/TestRMContainerAllocator.java | 15 +- .../sls/scheduler/SLSCapacityScheduler.java | 15 +- .../yarn/sls/scheduler/SLSFairScheduler.java| 12 +- .../dev-support/findbugs-exclude.xml| 8 + .../yarn/api/resource/PlacementConstraints.java | 43 +- .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- ...SchedulerInvalidResoureRequestException.java | 47 ++ .../api/impl/TestAMRMClientOnRMRestart.java | 9 +- .../impl/pb/AllocateRequestPBImpl.java | 1 + .../server/scheduler/SchedulerRequestKey.java | 11 + .../resourcemanager/DefaultAMSProcessor.java| 13 +- .../rmapp/attempt/RMAppAttemptImpl.java | 5 +- .../scheduler/AbstractYarnScheduler.java| 3 +- .../scheduler/AppSchedulingInfo.java| 205 +-- .../ApplicationPlacementAllocatorFactory.java | 68 +++ .../scheduler/ApplicationPlacementFactory.java | 63 --- .../scheduler/ContainerUpdateContext.java | 4 +- .../scheduler/SchedulerApplicationAttempt.java | 20 +- .../scheduler/YarnScheduler.java| 15 +- .../scheduler/capacity/CapacityScheduler.java | 54 +- .../CapacitySchedulerConfiguration.java | 5 + .../allocator/RegularContainerAllocator.java| 3 +- .../scheduler/common/ContainerRequest.java | 12 + .../scheduler/common/PendingAsk.java| 6 + .../scheduler/common/fica/FiCaSchedulerApp.java | 6 + .../constraint/AllocationTagsManager.java | 71 +-- .../constraint/AllocationTagsNamespaces.java| 31 -- .../constraint/PlacementConstraintsUtil.java| 165 -- .../algorithm/DefaultPlacementAlgorithm.java| 2 +- .../processor/PlacementProcessor.java | 8 +- .../scheduler/fair/FairScheduler.java | 12 +- .../scheduler/fifo/FifoScheduler.java | 7 +- .../placement/AppPlacementAllocator.java| 66 ++- .../LocalityAppPlacementAllocator.java | 35 +- .../SingleConstraintAppPlacementAllocator.java | 531 +++ .../server/resourcemanager/Application.java | 9 +- .../yarn/server/resourcemanager/MockAM.java | 51 ++ .../attempt/TestRMAppAttemptTransitions.java| 10 +- .../rmcontainer/TestRMContainerImpl.java| 6 +- .../scheduler/TestAppSchedulingInfo.java| 4 +- .../capacity/CapacitySchedulerTestBase.java | 79 +++ .../capacity/TestCapacityScheduler.java | 90 +--- .../TestCapacitySchedulerAsyncScheduling.java | 2 +- .../TestCapacitySchedulerAutoQueueCreation.java | 2 +- ...apacitySchedulerSchedulingRequestUpdate.java | 260 + .../capacity/TestIncreaseAllocationExpirer.java | 2 +- ...estSchedulingRequestContainerAllocation.java | 277 ++ ...hedulingRequestContainerAllocationAsync.java | 139 + .../scheduler/capacity/TestUtils.java | 2 + .../constraint/TestAllocationTagsManager.java | 30 +- .../TestPlacementConstraintsUtil.java | 36 +- .../scheduler/fair/FairSchedulerTestBase.java | 6 +- .../fair/TestContinuousScheduling.java | 10 +- .../scheduler/fair/TestFairScheduler.java | 30 +- .../scheduler/fifo/TestFifoScheduler.java | 28 +- ...stSingleConstraintAppPlacementAllocator.java | 403 ++ 56 files changed, 2557 insertions(+), 492 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java index 85e4181..7875917 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-pr
[25/37] hadoop git commit: YARN-6599. Support anti-affinity constraint via AppPlacementAllocator. (Wangda Tan via asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java index a3b88c0..01d5e6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java @@ -170,7 +170,7 @@ public class TestCapacitySchedulerAutoQueueCreation 1 * GB, 1, true, priority, recordFactory); cs.allocate(appAttemptId, Collections.singletonList(r1), - Collections.emptyList(), Collections.singletonList(host), + null, Collections.emptyList(), Collections.singletonList(host), null, NULL_UPDATE_REQUESTS); //And this will result in container assignment for app1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/38af2379/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java new file mode 100644 index 000..b6ac4b6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSchedulingRequestUpdate.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.ResourceSizing; +import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockAM; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Test; + +import java.util.Arrays; + +public class
[15/37] hadoop git commit: YARN-7522. Introduce AllocationTagsManager to associate allocation tags to nodes. (Wangda Tan via asuresh)
YARN-7522. Introduce AllocationTagsManager to associate allocation tags to nodes. (Wangda Tan via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/801c0988 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/801c0988 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/801c0988 Branch: refs/heads/YARN-6592 Commit: 801c0988b5ad1eff1e896a2635c2937721c96b04 Parents: 69de9a1 Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 8 00:24:00 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../resourcemanager/RMActiveServiceContext.java | 15 + .../yarn/server/resourcemanager/RMContext.java | 5 + .../server/resourcemanager/RMContextImpl.java | 12 + .../server/resourcemanager/ResourceManager.java | 9 + .../constraint/AllocationTagsManager.java | 431 +++ .../constraint/AllocationTagsNamespaces.java| 31 ++ .../InvalidAllocationTagsQueryException.java| 35 ++ .../rmcontainer/RMContainer.java| 8 + .../rmcontainer/RMContainerImpl.java| 21 + .../constraint/TestAllocationTagsManager.java | 328 ++ .../rmcontainer/TestRMContainerImpl.java| 124 ++ .../scheduler/capacity/TestUtils.java | 9 + .../scheduler/fifo/TestFifoScheduler.java | 5 + 13 files changed, 1033 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/801c0988/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 9dc5945..6ee3a4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager; +import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager; import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; @@ -107,6 +108,7 @@ public class RMActiveServiceContext { private RMAppLifetimeMonitor rmAppLifetimeMonitor; private QueueLimitCalculator queueLimitCalculator; + private AllocationTagsManager allocationTagsManager; public RMActiveServiceContext() { queuePlacementManager = new PlacementManager(); @@ -398,6 +400,19 @@ public class RMActiveServiceContext { @Private @Unstable + public AllocationTagsManager getAllocationTagsManager() { +return allocationTagsManager; + } + + @Private + @Unstable + public void setAllocationTagsManager( + AllocationTagsManager allocationTagsManager) { +this.allocationTagsManager = allocationTagsManager; + } + + @Private + @Unstable public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() { return rmDelegatedNodeLabelsUpdater; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/801c0988/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index ec94030..62899d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext
[23/37] hadoop git commit: YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/add993e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/add993e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/add993e2 Branch: refs/heads/YARN-6592 Commit: add993e26a3c96f77dfd42086f186a139966019e Parents: f8c5f5b Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 25 23:01:43 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../rmcontainer/RMContainerImpl.java| 8 +++ .../rmcontainer/TestRMContainerImpl.java| 25 ++-- 2 files changed, 26 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/add993e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index a504221..541621b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -40,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; @@ -533,7 +531,7 @@ public class RMContainerImpl implements RMContainer { RMContainerEvent event) { NMContainerStatus report = ((RMContainerRecoverEvent) event).getContainerReport(); - // Set the allocation tags from the + // Set the allocation tags from the NMContainerStatus container.setAllocationTags(report.getAllocationTags()); // Notify AllocationTagsManager container.rmContext.getAllocationTagsManager().addContainer( @@ -689,7 +687,7 @@ public class RMContainerImpl implements RMContainer { // Something wrong happened, kill the container LOG.warn("Something wrong happened, container size reported by NM" + " is not expected, ContainerID=" + container.getContainerId() -+ " rm-size-resource:" + rmContainerResource + " nm-size-reosurce:" ++ " rm-size-resource:" + rmContainerResource + " nm-size-resource:" + nmContainerResource); container.eventHandler.handle(new RMNodeCleanContainerEvent( container.nodeId, container.getContainerId())); @@ -702,7 +700,7 @@ public class RMContainerImpl implements RMContainer { @Override public void transition(RMContainerImpl container, RMContainerEvent event) { - // Notify placementManager + // Notify AllocationTagsManager container.rmContext.getAllocationTagsManager().removeContainer( container.getNodeId(), container.getContainerId(), container.getAllocationTags()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/add993e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
[31/37] hadoop git commit: YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh)
YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88d8d3f4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88d8d3f4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88d8d3f4 Branch: refs/heads/YARN-6592 Commit: 88d8d3f40b2923fab23a933bce1cd2e9c320ae84 Parents: 801c098 Author: Arun Suresh <asur...@apache.org> Authored: Tue Dec 19 08:59:23 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/AbstractYarnScheduler.java| 18 + .../scheduler/ResourceScheduler.java| 13 .../scheduler/capacity/CapacityScheduler.java | 78 ++-- .../common/ResourceAllocationCommitter.java | 12 ++- .../scheduler/common/fica/FiCaSchedulerApp.java | 30 +--- .../TestCapacitySchedulerAsyncScheduling.java | 10 +-- 6 files changed, 138 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/88d8d3f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 4b76327..213d784 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -295,6 +296,10 @@ public abstract class AbstractYarnScheduler return nodeTracker.getNodes(nodeFilter); } + public List getNodes(final NodeFilter filter) { +return nodeTracker.getNodes(filter); + } + public boolean shouldContainersBeAutoUpdated() { return this.autoUpdateContainers; } @@ -1443,4 +1448,17 @@ public abstract class AbstractYarnScheduler throw new IOException(e); } } + + /** + * Default implementation. Always returns false. + * @param appAttempt ApplicationAttempt. + * @param schedulingRequest SchedulingRequest. + * @param schedulerNode SchedulerNode. + * @return Success or not. + */ + @Override + public boolean attemptAllocationOnNode(SchedulerApplicationAttempt appAttempt, + SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) { +return false; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/88d8d3f4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java index d96d625..5a56ac7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configurat
[01/37] hadoop git commit: YARN-7811. Fixed a bug in user defined docker network settings. (Contributed by Billie Rinaldi) [Forced Update!]
Repository: hadoop Updated Branches: refs/heads/YARN-6592 6ae4cc995 -> 8df7666fe (forced update) YARN-7811. Fixed a bug in user defined docker network settings. (Contributed by Billie Rinaldi) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9dd5b61 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9dd5b61 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9dd5b61 Branch: refs/heads/YARN-6592 Commit: f9dd5b61f4ed0288cc01cb1a676df8c9cd69cdd9 Parents: 901d15a Author: Eric YangAuthored: Tue Jan 30 12:42:38 2018 -0500 Committer: Eric Yang Committed: Tue Jan 30 12:42:38 2018 -0500 -- .../containerlaunch/AbstractLauncher.java | 22 ++-- .../service/provider/docker/DockerKeys.java | 7 --- .../provider/docker/DockerProviderService.java | 2 +- 3 files changed, 12 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9dd5b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java index e1e88cd..f497985 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java @@ -39,8 +39,6 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK; - /** * Launcher of applications: base class */ @@ -60,9 +58,9 @@ public class AbstractLauncher { private final Map serviceData = new HashMap<>(); protected boolean yarnDockerMode = false; protected String dockerImage; - protected String dockerNetwork = DEFAULT_DOCKER_NETWORK; + protected String dockerNetwork; protected String dockerHostname; - protected String runPrivilegedContainer; + protected boolean runPrivilegedContainer = false; private ServiceContext context; public AbstractLauncher(ServiceContext context) { @@ -145,10 +143,16 @@ public class AbstractLauncher { Map env = containerLaunchContext.getEnvironment(); env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker"); env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage); - env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", dockerNetwork); + if (ServiceUtils.isSet(dockerNetwork)) { +env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", +dockerNetwork); + } env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME", dockerHostname); - env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer); + if (runPrivilegedContainer) { +env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", +"true"); + } StringBuilder sb = new StringBuilder(); for (Entry mount : mountPaths.entrySet()) { if (sb.length() > 0) { @@ -238,11 +242,7 @@ public class AbstractLauncher { } public void setRunPrivilegedContainer(boolean runPrivilegedContainer) { -if (runPrivilegedContainer) { - this.runPrivilegedContainer = Boolean.toString(true); -} else { - this.runPrivilegedContainer = Boolean.toString(false); -} +this.runPrivilegedContainer = runPrivilegedContainer; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9dd5b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerKeys.java
[29/37] hadoop git commit: YARN-7745. Allow DistributedShell to take a placement specification for containers it wants to launch. (Arun Suresh via wangda)
YARN-7745. Allow DistributedShell to take a placement specification for containers it wants to launch. (Arun Suresh via wangda) Change-Id: Ided146d662e944a8a4692e5d6885f23fd9bbcad5 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e60f5129 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e60f5129 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e60f5129 Branch: refs/heads/YARN-6592 Commit: e60f51299dba360d13aa39f9ab714fdfc666b532 Parents: 38af237 Author: Wangda TanAuthored: Thu Jan 18 14:22:45 2018 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../distributedshell/ApplicationMaster.java | 124 +++-- .../applications/distributedshell/Client.java | 14 ++ .../distributedshell/PlacementSpec.java | 137 +++ 3 files changed, 263 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e60f5129/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 270ef1b..9ba2138 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -42,6 +42,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; @@ -87,8 +88,11 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ProfileCapability; +import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.ResourceSizing; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.UpdatedContainer; import org.apache.hadoop.yarn.api.records.ExecutionType; @@ -99,6 +103,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.client.api.TimelineV2Client; @@ -274,6 +279,10 @@ public class ApplicationMaster { @VisibleForTesting protected AtomicInteger numRequestedContainers = new AtomicInteger(); + protected AtomicInteger numIgnore = new AtomicInteger(); + + protected AtomicInteger totalRetries = new AtomicInteger(10); + // Shell command to be executed private String shellCommand = ""; // Args to be passed to the shell command @@ -289,6 +298,9 @@ public class ApplicationMaster { // File length needed for local resource private long shellScriptPathLen = 0; + // Placement Specifications + private Map placementSpecs = null; + // Container retry options private ContainerRetryPolicy containerRetryPolicy = ContainerRetryPolicy.NEVER_RETRY; @@ -334,6 +346,7 @@ public class ApplicationMaster { private final String windows_command = "cmd /c"; private int yarnShellIdCounter = 1; + private final AtomicLong allocIdCounter = new AtomicLong(1); @VisibleForTesting protected final Set launchedContainers = @@ -457,6 +470,7 @@ public class ApplicationMaster { "If container could retry, it specifies max
[08/37] hadoop git commit: YARN-6619. AMRMClient Changes to use the PlacementConstraint and SchcedulingRequest objects. (Arun Suresh via wangda)
YARN-6619. AMRMClient Changes to use the PlacementConstraint and SchcedulingRequest objects. (Arun Suresh via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29d9e4d5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29d9e4d5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29d9e4d5 Branch: refs/heads/YARN-6592 Commit: 29d9e4d5814900d5c59d77fe05d32186d4ad9385 Parents: a5c1fc8 Author: Wangda TanAuthored: Wed Jan 17 11:36:26 2018 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/client/api/AMRMClient.java | 38 +++- .../yarn/client/api/async/AMRMClientAsync.java | 48 + .../api/async/impl/AMRMClientAsyncImpl.java | 49 - .../yarn/client/api/impl/AMRMClientImpl.java| 142 - .../client/api/impl/BaseAMRMClientTest.java | 212 +++ .../yarn/client/api/impl/TestAMRMClient.java| 156 +- .../TestAMRMClientPlacementConstraints.java | 204 ++ .../rmcontainer/RMContainerImpl.java| 3 + .../scheduler/AbstractYarnScheduler.java| 1 + .../scheduler/SchedulerApplicationAttempt.java | 1 + .../constraint/PlacementConstraintsUtil.java| 4 +- 11 files changed, 700 insertions(+), 158 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/29d9e4d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java index d3d1974..914a146 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.client.api; import java.io.IOException; import java.util.Collection; +import java.util.Map; +import java.util.Set; import java.util.function.Supplier; import java.util.List; @@ -39,7 +41,9 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.ProfileCapability; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.resource.Resources; @@ -554,6 +558,18 @@ public abstract class AMRMClient extends } /** + * Add a Collection of SchedulingRequests. The AMRMClient will ensure that + * all requests in the same batch are sent in the same allocate call. + * @param schedulingRequests Collection of Scheduling Requests. + */ + @Public + @InterfaceStability.Unstable + public void addSchedulingRequests( + Collection schedulingRequests) { + + } + + /** * Register the application master. This must be called before any * other interaction * @param appHostName Name of the host on which master is running @@ -568,7 +584,27 @@ public abstract class AMRMClient extends int appHostPort, String appTrackingUrl) throws YarnException, IOException; - + + /** + * Register the application master. This must be called before any + * other interaction + * @param appHostName Name of the host on which master is running + * @param appHostPort Port master is listening on + * @param appTrackingUrl URL at which the master info can be seen + * @param placementConstraints Placement Constraints mappings. + * @return RegisterApplicationMasterResponse + * @throws YarnException + * @throws IOException + */ + @Public + @InterfaceStability.Unstable + public RegisterApplicationMasterResponse registerApplicationMaster( + String appHostName, int appHostPort, String appTrackingUrl, + Map placementConstraints) + throws YarnException, IOException { +throw new YarnException("Not supported"); + } + /** * Request additional containers and receive new container allocations. * Requests made via addContainerRequest are sent to the
[32/37] hadoop git commit: YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh)
YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9af15d6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9af15d6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9af15d6 Branch: refs/heads/YARN-6592 Commit: f9af15d659f59fd0cf564fe1ecc8e06c6429ba68 Parents: 1efb2b6 Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 22 15:51:20 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/conf/YarnConfiguration.java | 26 ++ .../src/main/resources/yarn-default.xml | 30 ++ .../ApplicationMasterService.java | 15 + .../rmcontainer/RMContainerImpl.java| 7 +- .../scheduler/capacity/CapacityScheduler.java | 2 + .../constraint/processor/BatchedRequests.java | 105 + .../processor/NodeCandidateSelector.java| 38 ++ .../processor/PlacementDispatcher.java | 145 +++ .../processor/PlacementProcessor.java | 343 .../processor/SamplePlacementAlgorithm.java | 144 +++ .../constraint/processor/package-info.java | 29 ++ .../yarn/server/resourcemanager/MockAM.java | 26 ++ .../yarn/server/resourcemanager/MockRM.java | 14 + .../constraint/TestPlacementProcessor.java | 394 +++ 14 files changed, 1316 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9af15d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index bbbfc52..8fb3c2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -531,6 +531,32 @@ public class YarnConfiguration extends Configuration { /** The class to use as the resource scheduler.*/ public static final String RM_SCHEDULER = RM_PREFIX + "scheduler.class"; + + /** Placement Algorithm. */ + public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS = + RM_PREFIX + "placement-constraints.algorithm.class"; + + public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED = + RM_PREFIX + "placement-constraints.enabled"; + + public static final boolean DEFAULT_RM_PLACEMENT_CONSTRAINTS_ENABLED = true; + + public static final String RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS = + RM_PREFIX + "placement-constraints.retry-attempts"; + + public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS = 3; + + public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE = + RM_PREFIX + "placement-constraints.algorithm.pool-size"; + + public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE = + 1; + + public static final String RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE = + RM_PREFIX + "placement-constraints.scheduler.pool-size"; + + public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE = + 1; public static final String DEFAULT_RM_SCHEDULER = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9af15d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0bb4fca..6d52ace 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -131,6 +131,36 @@ +Enable Constraint Placement. +yarn.resourcemanager.placement-constraints.enabled +false + + + +Number of times to retry placing of rejected SchedulingRequests +yarn.resourcemanager.placement-constraints.retry-attempts +3 + + + +Constraint Placement Algorithm to be used. +yarn.resourcemanager.placement-constraints.algorithm.class + org.apache.hadoop.y
[16/37] hadoop git commit: YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos)
YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adbe87ab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adbe87ab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adbe87ab Branch: refs/heads/YARN-6592 Commit: adbe87abf8b2814e0e2988d09ef8a8569190c80e Parents: 8bf7c44 Author: Konstantinos KaranasosAuthored: Mon Jan 22 23:51:02 2018 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../constraint/AllocationTagsManager.java | 110 +++- .../algorithm/DefaultPlacementAlgorithm.java| 8 +- .../algorithm/LocalAllocationTagsManager.java | 167 +++ .../constraint/TestAllocationTagsManager.java | 82 - .../TestLocalAllocationTagsManager.java | 139 +++ 5 files changed, 336 insertions(+), 170 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/adbe87ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java index 962e548..7ad5e8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java @@ -24,17 +24,14 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; -import org.apache.hadoop.yarn.api.resource.PlacementConstraints; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.log4j.Logger; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -61,9 +58,6 @@ public class AllocationTagsManager { // Application's tags to Rack private Map perAppRackMappings = new HashMap<>(); - // Application's Temporary containers mapping - private Map >> - appTempMappings = new HashMap<>(); // Global tags to node mapping (used to fast return aggregated tags // cardinality across apps) @@ -76,7 +70,7 @@ public class AllocationTagsManager { * Currently used both for NodeId to Tag, Count and Rack to Tag, Count */ @VisibleForTesting - static class TypeToCountedTags { + public static class TypeToCountedTags { // Map > private Map > typeToTagsWithCount = new HashMap<>(); @@ -214,7 +208,7 @@ public class AllocationTagsManager { } @VisibleForTesting - Map getPerAppNodeMappings() { + public Map getPerAppNodeMappings() { return perAppNodeMappings; } @@ -233,12 +227,6 @@ public class AllocationTagsManager { return globalRackMapping; } - @VisibleForTesting - public Map > getAppTempMappings( - ApplicationId applicationId) { -return appTempMappings.get(applicationId); - } - public AllocationTagsManager(RMContext context) { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); @@ -246,39 +234,6 @@ public class AllocationTagsManager { rmContext = context; } - // - - /** - * Method adds a temporary fake-container tag to Node mapping. - * Used by the constrained placement algorithm to keep track of containers - * that are currently placed on nodes but are not yet
[28/37] hadoop git commit: YARN-7807. Assume intra-app anti-affinity as default for scheduling request inside AppPlacementAllocator. (Wangda Tan via asuresh)
YARN-7807. Assume intra-app anti-affinity as default for scheduling request inside AppPlacementAllocator. (Wangda Tan via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/644afe5f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/644afe5f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/644afe5f Branch: refs/heads/YARN-6592 Commit: 644afe5fd800ac4f2b873a99f9b3868c3a8c5c40 Parents: a4c539f Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 24 12:55:01 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../placement/SingleConstraintAppPlacementAllocator.java| 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/644afe5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java index 9e7d71c..b02cb00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java @@ -334,14 +334,15 @@ public class SingleConstraintAppPlacementAllocator targetAllocationTags = new HashSet<>( targetExpression.getTargetValues()); -if (targetExpression.getTargetKey() == null || !targetExpression +if (targetExpression.getTargetKey() != null && !targetExpression .getTargetKey().equals(APPLICATION_LABEL_INTRA_APPLICATION)) { throwExceptionWithMetaInfo( "As of now, the only accepted target key for targetKey of " + "allocation_tag target expression is: [" + APPLICATION_LABEL_INTRA_APPLICATION + "]. Please make changes to placement constraints " - + "accordingly."); + + "accordingly. If this is null, it will be set to " + + APPLICATION_LABEL_INTRA_APPLICATION + " by default."); } } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[07/37] hadoop git commit: YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh)
YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4eda58c1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4eda58c1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4eda58c1 Branch: refs/heads/YARN-6592 Commit: 4eda58c13641c14c4b248843a2589781cbcd343f Parents: bdba01f Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 10 09:04:30 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/capacity/CapacityScheduler.java | 23 1 file changed, 23 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eda58c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index e682d0f..d2713c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -124,6 +124,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; @@ -2574,6 +2576,27 @@ public class CapacityScheduler extends ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> resourceCommitRequest = createResourceCommitRequest( appAttempt, schedulingRequest, schedulerNode); + +// Validate placement constraint is satisfied before +// committing the request. +try { + if (!PlacementConstraintsUtil.canSatisfyConstraints( + appAttempt.getApplicationId(), + schedulingRequest.getAllocationTags(), + schedulerNode, + rmContext.getPlacementConstraintManager(), + rmContext.getAllocationTagsManager())) { +LOG.debug("Failed to allocate container for application " ++ appAttempt.getApplicationId() + " on node " ++ schedulerNode.getNodeName() ++ " because this allocation violates the" ++ " placement constraint."); +return false; + } +} catch (InvalidAllocationTagsQueryException e) { + LOG.warn("Unable to allocate container", e); + return false; +} return tryCommit(getClusterResource(), resourceCommitRequest, false); } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[12/37] hadoop git commit: YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos)
YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db928556 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db928556 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db928556 Branch: refs/heads/YARN-6592 Commit: db928556c81e5950b3fe374fa5b99ab26791ef3a Parents: b57e8bc Author: Konstantinos KaranasosAuthored: Mon Nov 13 15:25:24 2017 -0800 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../RegisterApplicationMasterRequest.java | 42 - .../yarn/api/resource/PlacementConstraint.java | 156 +++ .../src/main/proto/yarn_protos.proto| 6 + .../src/main/proto/yarn_service_protos.proto| 1 + .../RegisterApplicationMasterRequestPBImpl.java | 106 - .../hadoop/yarn/api/BasePBImplRecordsTest.java | 11 ++ 6 files changed, 313 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db928556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java index 395e190..f2d537a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java @@ -18,11 +18,16 @@ package org.apache.hadoop.yarn.api.protocolrecords; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.util.Records; - /** * The request sent by the {@code ApplicationMaster} to {@code ResourceManager} * on registration. @@ -132,4 +137,39 @@ public abstract class RegisterApplicationMasterRequest { @Public @Stable public abstract void setTrackingUrl(String trackingUrl); + + /** + * Return all Placement Constraints specified at the Application level. The + * mapping is from a set of allocation tags to a + * PlacementConstraint associated with the tags, i.e., each + * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those + * tags will be placed taking into account the corresponding constraint. + * + * @return A map of Placement Constraints. + */ + @Public + @Unstable + public Map getPlacementConstraints() { +return new HashMap<>(); + } + + /** + * Set Placement Constraints applicable to the + * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s + * of this application. + * The mapping is from a set of allocation tags to a + * PlacementConstraint associated with the tags. + * For example: + * Map + * hb_regionserver - node_anti_affinity, + * hb_regionserver, hb_master - rack_affinity, + * ... + * + * @param placementConstraints Placement Constraint Mapping. + */ + @Public + @Unstable + public void setPlacementConstraints( + Map placementConstraints) { + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/db928556/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java index f0e3982..b6e851a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -54,6 +54,26 @@ public class PlacementConstraint { return constraintExpr; } + @Override +
[30/37] hadoop git commit: YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh)
YARN-7774. Miscellaneous fixes to the PlacementProcessor. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28fe7f33 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28fe7f33 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28fe7f33 Branch: refs/heads/YARN-6592 Commit: 28fe7f331837b36e78fa34ed990993677dddeaee Parents: e60f512 Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 18 11:01:36 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../scheduler/SchedulerNode.java| 16 +++- .../scheduler/capacity/CapacityScheduler.java | 4 + .../constraint/PlacementConstraintsUtil.java| 5 +- .../constraint/algorithm/CircularIterator.java | 86 .../algorithm/DefaultPlacementAlgorithm.java| 50 ++-- .../constraint/processor/BatchedRequests.java | 8 ++ .../SingleConstraintAppPlacementAllocator.java | 2 +- .../yarn/server/resourcemanager/MockAM.java | 4 +- .../constraint/TestPlacementProcessor.java | 24 +++--- .../algorithm/TestCircularIterator.java | 84 +++ ...stSingleConstraintAppPlacementAllocator.java | 28 +++ 11 files changed, 271 insertions(+), 40 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28fe7f33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java index 89f748d..96a8e34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java @@ -469,6 +469,20 @@ public abstract class SchedulerNode { this.lastHeartbeatMonotonicTime = Time.monotonicNow(); } + @Override + public boolean equals(Object o) { +if (this == o) return true; +if (!(o instanceof SchedulerNode)) return false; + +SchedulerNode that = (SchedulerNode) o; + +return getNodeID().equals(that.getNodeID()); + } + + @Override + public int hashCode() { +return getNodeID().hashCode(); + } private static class ContainerInfo { private final RMContainer container; @@ -479,4 +493,4 @@ public abstract class SchedulerNode { this.launchedOnNode = launchedOnNode; } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/28fe7f33/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index c713036..429f9f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2610,6 +2610,10 @@ public class CapacityScheduler extends " but only 1 will be attempted !!"); } if (!appAttempt.isStopped()) { +Resource resource = +schedulingRequest.getResourceSizing().getResources(); +schedulingRequest.getResourceSizing().setResources( +getNormalizedResource(resource)); ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> resourceCommitRequest = createResourceCommitRequest( appAttempt, schedulingRequest, schedulerNode); http://git-wip-us.apache.org/repos/asf/hado
[05/37] hadoop git commit: YARN-7779. Display allocation tags in RM web UI and expose same through REST API. Contributed by Weiwei Yang.
YARN-7779. Display allocation tags in RM web UI and expose same through REST API. Contributed by Weiwei Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b81cb05 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b81cb05 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b81cb05 Branch: refs/heads/YARN-6592 Commit: 9b81cb0537e5b731581e6a375bf0a59abf61c359 Parents: adbe87a Author: Sunil GAuthored: Tue Jan 23 17:09:58 2018 +0530 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 6 ++ .../yarn/sls/scheduler/RMNodeWrapper.java | 6 ++ .../server/resourcemanager/rmnode/RMNode.java | 7 ++ .../resourcemanager/rmnode/RMNodeImpl.java | 6 ++ .../constraint/AllocationTagsManager.java | 11 +++ .../resourcemanager/webapp/NodesPage.java | 3 + .../webapp/dao/AllocationTagInfo.java | 56 ++ .../webapp/dao/AllocationTagsInfo.java | 59 +++ .../resourcemanager/webapp/dao/NodeInfo.java| 15 .../yarn/server/resourcemanager/MockNodes.java | 6 ++ .../resourcemanager/webapp/TestNodesPage.java | 4 +- .../webapp/TestRMWebServicesNodes.java | 77 +++- 12 files changed, 253 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 1016ce1..0c99139 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.sls.nodemanager; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -213,6 +214,11 @@ public class NodeInfo { } @Override +public Map getAllocationTagsWithCount() { + return null; +} + +@Override public Resource getPhysicalResource() { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java -- diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index fdad826..92f9b0f 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; @Private @@ -203,6 +204,11 @@ public class RMNodeWrapper implements RMNode { } @Override + public Map getAllocationTagsWithCount() { +return node.getAllocationTagsWithCount(); + } + + @Override public Resource getPhysicalResource() { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b81cb05/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index a5615ef..872f2a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmnode; import java.util.List; +import java.util.Map; import
[35/37] hadoop git commit: YARN-7802. [UI2] Application regex search did not work properly with app name. Contributed by Sreenath Somarajapuram.
YARN-7802. [UI2] Application regex search did not work properly with app name. Contributed by Sreenath Somarajapuram. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64344c34 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64344c34 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64344c34 Branch: refs/heads/YARN-6592 Commit: 64344c345d69bcdcf72a08cec326e6d1a5c25fab Parents: 5206b2c Author: Sunil GAuthored: Wed Jan 31 17:40:36 2018 +0530 Committer: Sunil G Committed: Wed Jan 31 17:40:36 2018 +0530 -- .../src/main/webapp/app/components/timeline-view.js| 5 + .../src/main/webapp/app/controllers/yarn-app/components.js | 5 + .../src/main/webapp/app/controllers/yarn-apps/apps.js | 1 + .../webapp/app/controllers/yarn-component-instances/info.js| 5 + .../src/main/webapp/app/controllers/yarn-flowrun/info.js | 5 + .../src/main/webapp/app/controllers/yarn-flowrun/metrics.js| 5 + .../src/main/webapp/app/controllers/yarn-nodes/table.js| 4 +++- .../src/main/webapp/app/controllers/yarn-queue/apps.js | 1 + .../src/main/webapp/app/controllers/yarn-services.js | 1 + .../src/main/webapp/app/controllers/yarn-tools/yarn-conf.js| 4 .../src/main/webapp/app/templates/components/timeline-view.hbs | 2 +- .../src/main/webapp/app/templates/yarn-app/components.hbs | 2 +- .../webapp/app/templates/yarn-component-instances/info.hbs | 2 +- .../src/main/webapp/app/templates/yarn-flowrun/info.hbs| 2 +- .../src/main/webapp/app/templates/yarn-flowrun/metrics.hbs | 6 +++--- .../src/main/webapp/app/templates/yarn-tools/yarn-conf.hbs | 6 +++--- .../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json| 6 +++--- .../hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock | 6 +++--- 18 files changed, 51 insertions(+), 17 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/64344c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js index e23d991..65a8cb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/timeline-view.js @@ -19,8 +19,13 @@ import Ember from 'ember'; import Converter from 'yarn-ui/utils/converter'; import ColumnDef from 'em-table/utils/column-definition'; +import TableDefinition from 'em-table/utils/table-definition'; export default Ember.Component.extend({ + tableDefinition: TableDefinition.create({ +searchType: 'manual', + }), + canvas: { svg: undefined, h: 0, http://git-wip-us.apache.org/repos/asf/hadoop/blob/64344c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js index 812f1e6..5981eb5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js @@ -18,11 +18,16 @@ import Ember from 'ember'; import ColumnDef from 'em-table/utils/column-definition'; +import TableDefinition from 'em-table/utils/table-definition'; export default Ember.Controller.extend({ queryParams: ["service"], service: undefined, + tableDefinition: TableDefinition.create({ +searchType: 'manual', + }), + tableColumns: Ember.computed('model.appId', 'model.serviceName', function() { var cols = []; var service = this.get('model.serviceName'); http://git-wip-us.apache.org/repos/asf/hadoop/blob/64344c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js index bfd6299..55d5a88 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-apps/apps.js +++
[18/37] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)
YARN-7669. API and interface modifications for placement constraint processor. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06eb63e6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06eb63e6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06eb63e6 Branch: refs/heads/YARN-6592 Commit: 06eb63e64b05e2e8bb8a76c15360ab0495f11317 Parents: 88d8d3f Author: Arun Suresh <asur...@apache.org> Authored: Tue Dec 19 22:47:46 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../yarn/ams/ApplicationMasterServiceUtils.java | 16 + .../api/protocolrecords/AllocateResponse.java | 23 + .../api/records/RejectedSchedulingRequest.java | 70 +++ .../yarn/api/records/RejectionReason.java | 44 ++ .../src/main/proto/yarn_protos.proto| 10 + .../src/main/proto/yarn_service_protos.proto| 1 + .../impl/pb/AllocateResponsePBImpl.java | 85 .../yarn/api/records/impl/pb/ProtoUtils.java| 16 + .../pb/RejectedSchedulingRequestPBImpl.java | 148 +++ .../records/impl/pb/ResourceSizingPBImpl.java | 8 + .../impl/pb/SchedulingRequestPBImpl.java| 11 + .../hadoop/yarn/api/TestPBImplRecords.java | 2 + .../resourcemanager/RMActiveServiceContext.java | 2 +- .../yarn/server/resourcemanager/RMContext.java | 2 +- .../server/resourcemanager/RMContextImpl.java | 2 +- .../server/resourcemanager/ResourceManager.java | 2 +- .../constraint/AllocationTagsManager.java | 431 --- .../constraint/AllocationTagsNamespaces.java| 31 -- .../InvalidAllocationTagsQueryException.java| 35 -- .../constraint/AllocationTagsManager.java | 431 +++ .../constraint/AllocationTagsNamespaces.java| 31 ++ .../InvalidAllocationTagsQueryException.java| 35 ++ .../api/ConstraintPlacementAlgorithm.java | 43 ++ .../api/ConstraintPlacementAlgorithmInput.java | 32 ++ .../api/ConstraintPlacementAlgorithmOutput.java | 58 +++ ...traintPlacementAlgorithmOutputCollector.java | 32 ++ .../constraint/api/PlacedSchedulingRequest.java | 79 .../constraint/api/SchedulingResponse.java | 70 +++ .../scheduler/constraint/api/package-info.java | 28 ++ .../constraint/TestAllocationTagsManager.java | 328 -- .../rmcontainer/TestRMContainerImpl.java| 2 +- .../scheduler/capacity/TestUtils.java | 2 +- .../constraint/TestAllocationTagsManager.java | 328 ++ .../scheduler/fifo/TestFifoScheduler.java | 2 +- 34 files changed, 1608 insertions(+), 832 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java index 476da8b..8bdfaf3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.ams; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerUpdateType; +import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdatedContainer; @@ -86,4 +87,19 @@ public final class ApplicationMasterServiceUtils { } allocateResponse.setAllocatedContainers(allocatedContainers); } + + /** + * Add rejected Scheduling Requests to {@link AllocateResponse}. + * @param allocateResponse Allocate Response. + * @param rejectedRequests Rejected SchedulingRequests. + */ + public static void addToRejectedSchedulingRequests( + AllocateResponse allocateResponse, + List rejectedRequests) { +if (allocateResponse.getRejectedSchedulingRequests() != null +&& !allocateResponse.getRejectedSchedulingRequests().isEmpty()) { + rejectedRequests.addAll(allocateResponse.getRejectedSchedulingRequests()); +} +allocateResponse.setRejectedSchedulingRequests(rejectedRequests); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/06eb63e6/hadoop-yarn-project/hadoop-yarn
[09/37] hadoop git commit: YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos via wangda)
YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos via wangda) Change-Id: Id00edb7185fdf01cce6e40f920cac3585f8cbe9c Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33a796d9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33a796d9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33a796d9 Branch: refs/heads/YARN-6592 Commit: 33a796d9b778bf7350e87a4e36ca30c925cf7036 Parents: 1453a04 Author: Wangda TanAuthored: Thu Aug 3 14:03:55 2017 -0700 Committer: Arun Suresh Committed: Wed Jan 31 01:30:17 2018 -0800 -- .../yarn/api/resource/PlacementConstraint.java | 567 +++ .../yarn/api/resource/PlacementConstraints.java | 286 ++ .../hadoop/yarn/api/resource/package-info.java | 23 + .../src/main/proto/yarn_protos.proto| 55 ++ .../api/resource/TestPlacementConstraints.java | 106 .../PlacementConstraintFromProtoConverter.java | 116 .../pb/PlacementConstraintToProtoConverter.java | 174 ++ .../apache/hadoop/yarn/api/pb/package-info.java | 23 + .../yarn/api/records/impl/pb/ProtoUtils.java| 27 + .../PlacementConstraintTransformations.java | 209 +++ .../hadoop/yarn/api/resource/package-info.java | 23 + .../TestPlacementConstraintPBConversion.java| 195 +++ .../TestPlacementConstraintTransformations.java | 183 ++ 13 files changed, 1987 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/33a796d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java new file mode 100644 index 000..f0e3982 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java @@ -0,0 +1,567 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.resource; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * {@code PlacementConstraint} represents a placement constraint for a resource + * allocation. + */ +@Public +@Unstable +public class PlacementConstraint { + + /** + * The constraint expression tree. + */ + private AbstractConstraint constraintExpr; + + public PlacementConstraint(AbstractConstraint constraintExpr) { +this.constraintExpr = constraintExpr; + } + + /** + * Get the constraint expression of the placement constraint. + * + * @return the constraint expression + */ + public AbstractConstraint getConstraintExpr() { +return constraintExpr; + } + + /** + * Interface used to enable the elements of the constraint tree to be visited. + */ + @Private + public interface Visitable { +/** + * Visitor pattern. + * + * @param visitor visitor to be used + * @param defines the type that the visitor will use and the return type + * of the accept. + * @return the result of visiting a given object. + */ + T accept(Visitor visitor); + + } + + /** + * Visitor API for a constraint tree. + * + * @param determines the return type of the visit methods. + */ + @Private + public interface Visitor { +T visit(SingleConstraint constraint); + +T visit(TargetExpression target); + +T visit(TargetConstraint constraint); + +T visit(CardinalityConstraint constraint); + +T visit(And
hadoop git commit: YARN-7822. Constraint satisfaction checker support for composite OR and AND constraints. (Weiwei Yang via asuresh)
Repository: hadoop Updated Branches: refs/heads/YARN-7812 [created] e6d2d26a1 YARN-7822. Constraint satisfaction checker support for composite OR and AND constraints. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6d2d26a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6d2d26a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6d2d26a Branch: refs/heads/YARN-7812 Commit: e6d2d26a13337adc995efa7bcb77181871930796 Parents: 6ae4cc9 Author: Arun Suresh <asur...@apache.org> Authored: Tue Jan 30 10:15:33 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 10:15:33 2018 -0800 -- .../TestPlacementConstraintTransformations.java | 2 +- .../constraint/PlacementConstraintsUtil.java| 53 +++- .../TestPlacementConstraintsUtil.java | 278 --- .../constraint/TestPlacementProcessor.java | 159 ++- 4 files changed, 444 insertions(+), 48 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6d2d26a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java index 62da092..aa92d7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java @@ -156,7 +156,7 @@ public class TestPlacementConstraintTransformations { SingleConstraintTransformer singleTransformer = new SingleConstraintTransformer(specConstraint); PlacementConstraint simConstraint = singleTransformer.transform(); -Assert.assertTrue(constraintExpr instanceof Or); +Assert.assertTrue(simConstraint.getConstraintExpr() instanceof Or); Or simOrExpr = (Or) specConstraint.getConstraintExpr(); for (AbstractConstraint child : simOrExpr.getChildren()) { Assert.assertTrue(child instanceof SingleConstraint); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6d2d26a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index 199dd62..6396e57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And; +import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType; @@ -149,6 +151,48 @@ public final class PlacementConstraintsUtil { return true; } + /** + * Returns true if all child constraints are satisfied. + * @param appId application id + * @param constraint Or constraint + * @param node node + * @param atm allocation tags manager + * @return true if all child constraints are satisfied, false otherwise + * @throws InvalidAllocationTagsQueryException + */ + private st
[40/50] [abbrv] hadoop git commit: YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh)
YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30b8d4fc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30b8d4fc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30b8d4fc Branch: refs/heads/YARN-6592 Commit: 30b8d4fc57bf9d25ef9093bb6c7ef5b2868c3602 Parents: 5df2556 Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 10 09:04:30 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:53:34 2018 -0800 -- .../scheduler/capacity/CapacityScheduler.java | 23 1 file changed, 23 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/30b8d4fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index e682d0f..d2713c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -124,6 +124,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent; @@ -2574,6 +2576,27 @@ public class CapacityScheduler extends ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> resourceCommitRequest = createResourceCommitRequest( appAttempt, schedulingRequest, schedulerNode); + +// Validate placement constraint is satisfied before +// committing the request. +try { + if (!PlacementConstraintsUtil.canSatisfyConstraints( + appAttempt.getApplicationId(), + schedulingRequest.getAllocationTags(), + schedulerNode, + rmContext.getPlacementConstraintManager(), + rmContext.getAllocationTagsManager())) { +LOG.debug("Failed to allocate container for application " ++ appAttempt.getApplicationId() + " on node " ++ schedulerNode.getNodeName() ++ " because this allocation violates the" ++ " placement constraint."); +return false; + } +} catch (InvalidAllocationTagsQueryException e) { + LOG.warn("Unable to allocate container", e); + return false; +} return tryCommit(getClusterResource(), resourceCommitRequest, false); } } - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh)
YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f9139cc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f9139cc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f9139cc Branch: refs/heads/YARN-6592 Commit: 9f9139ccad3de707d6c5a1bc4664794468dcb1dd Parents: 2615da8 Author: Arun Suresh <asur...@apache.org> Authored: Tue Dec 19 08:59:23 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:53:34 2018 -0800 -- .../scheduler/AbstractYarnScheduler.java| 18 + .../scheduler/ResourceScheduler.java| 13 .../scheduler/capacity/CapacityScheduler.java | 78 ++-- .../common/ResourceAllocationCommitter.java | 12 ++- .../scheduler/common/fica/FiCaSchedulerApp.java | 30 +--- .../TestCapacitySchedulerAsyncScheduling.java | 10 +-- 6 files changed, 138 insertions(+), 23 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f9139cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 4b76327..213d784 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.records.UpdateContainerError; import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -295,6 +296,10 @@ public abstract class AbstractYarnScheduler return nodeTracker.getNodes(nodeFilter); } + public List getNodes(final NodeFilter filter) { +return nodeTracker.getNodes(filter); + } + public boolean shouldContainersBeAutoUpdated() { return this.autoUpdateContainers; } @@ -1443,4 +1448,17 @@ public abstract class AbstractYarnScheduler throw new IOException(e); } } + + /** + * Default implementation. Always returns false. + * @param appAttempt ApplicationAttempt. + * @param schedulingRequest SchedulingRequest. + * @param schedulerNode SchedulerNode. + * @return Success or not. + */ + @Override + public boolean attemptAllocationOnNode(SchedulerApplicationAttempt appAttempt, + SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) { +return false; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f9139cc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java index d96d625..5a56ac7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Evolving; import org.apache.hadoop.conf.Configurat
[45/50] [abbrv] hadoop git commit: YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
YARN-6597. Add RMContainer recovery test to verify tag population in the AllocationTagsManager. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06d22eb2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06d22eb2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06d22eb2 Branch: refs/heads/YARN-6592 Commit: 06d22eb2784d4e2386b50926286409adb8b47973 Parents: 3663239 Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 25 23:01:43 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:54:37 2018 -0800 -- .../rmcontainer/RMContainerImpl.java| 8 +++ .../rmcontainer/TestRMContainerImpl.java| 25 ++-- 2 files changed, 26 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/06d22eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java index a504221..541621b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer; import java.util.Collections; import java.util.EnumSet; -import java.util.List; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; @@ -40,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; @@ -533,7 +531,7 @@ public class RMContainerImpl implements RMContainer { RMContainerEvent event) { NMContainerStatus report = ((RMContainerRecoverEvent) event).getContainerReport(); - // Set the allocation tags from the + // Set the allocation tags from the NMContainerStatus container.setAllocationTags(report.getAllocationTags()); // Notify AllocationTagsManager container.rmContext.getAllocationTagsManager().addContainer( @@ -689,7 +687,7 @@ public class RMContainerImpl implements RMContainer { // Something wrong happened, kill the container LOG.warn("Something wrong happened, container size reported by NM" + " is not expected, ContainerID=" + container.getContainerId() -+ " rm-size-resource:" + rmContainerResource + " nm-size-reosurce:" ++ " rm-size-resource:" + rmContainerResource + " nm-size-resource:" + nmContainerResource); container.eventHandler.handle(new RMNodeCleanContainerEvent( container.nodeId, container.getContainerId())); @@ -702,7 +700,7 @@ public class RMContainerImpl implements RMContainer { @Override public void transition(RMContainerImpl container, RMContainerEvent event) { - // Notify placementManager + // Notify AllocationTagsManager container.rmContext.getAllocationTagsManager().removeContainer( container.getNodeId(), container.getContainerId(), container.getAllocationTags()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/06d22eb2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
[46/50] [abbrv] hadoop git commit: YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos)
YARN-7788. Factor out management of temp tags from AllocationTagsManager. (Arun Suresh via kkaranasos) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fca7371f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fca7371f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fca7371f Branch: refs/heads/YARN-6592 Commit: fca7371f48cbd1faad6d06287b89a2526a0ff9d3 Parents: 3803dea Author: Konstantinos KaranasosAuthored: Mon Jan 22 23:51:02 2018 -0800 Committer: Arun Suresh Committed: Tue Jan 30 07:54:37 2018 -0800 -- .../constraint/AllocationTagsManager.java | 110 +++- .../algorithm/DefaultPlacementAlgorithm.java| 8 +- .../algorithm/LocalAllocationTagsManager.java | 167 +++ .../constraint/TestAllocationTagsManager.java | 82 - .../TestLocalAllocationTagsManager.java | 139 +++ 5 files changed, 336 insertions(+), 170 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fca7371f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java index 962e548..7ad5e8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java @@ -24,17 +24,14 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; -import org.apache.hadoop.yarn.api.resource.PlacementConstraints; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.log4j.Logger; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -61,9 +58,6 @@ public class AllocationTagsManager { // Application's tags to Rack private Map perAppRackMappings = new HashMap<>(); - // Application's Temporary containers mapping - private Map >> - appTempMappings = new HashMap<>(); // Global tags to node mapping (used to fast return aggregated tags // cardinality across apps) @@ -76,7 +70,7 @@ public class AllocationTagsManager { * Currently used both for NodeId to Tag, Count and Rack to Tag, Count */ @VisibleForTesting - static class TypeToCountedTags { + public static class TypeToCountedTags { // Map > private Map > typeToTagsWithCount = new HashMap<>(); @@ -214,7 +208,7 @@ public class AllocationTagsManager { } @VisibleForTesting - Map getPerAppNodeMappings() { + public Map getPerAppNodeMappings() { return perAppNodeMappings; } @@ -233,12 +227,6 @@ public class AllocationTagsManager { return globalRackMapping; } - @VisibleForTesting - public Map > getAppTempMappings( - ApplicationId applicationId) { -return appTempMappings.get(applicationId); - } - public AllocationTagsManager(RMContext context) { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); @@ -246,39 +234,6 @@ public class AllocationTagsManager { rmContext = context; } - // - - /** - * Method adds a temporary fake-container tag to Node mapping. - * Used by the constrained placement algorithm to keep track of containers - * that are currently placed on nodes but are not yet
[34/50] [abbrv] hadoop git commit: YARN-7669. API and interface modifications for placement constraint processor. (asuresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c83ef6f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java new file mode 100644 index 000..9571f0e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; + +import java.util.ArrayList; +import java.util.List; + +/** + * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm + * is free to produce multiple of output objects at the end of each run and it + * must use the provided ConstraintPlacementAlgorithmOutputCollector to + * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer + * which is provided a collector to collect output. + */ +public class ConstraintPlacementAlgorithmOutput { + + private final ApplicationId applicationId; + + public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) { +this.applicationId = applicationId; + } + + private final List placedRequests = + new ArrayList<>(); + + private final List rejectedRequests = + new ArrayList<>(); + + public List getPlacedRequests() { +return placedRequests; + } + + public List getRejectedRequests() { +return rejectedRequests; + } + + public ApplicationId getApplicationId() { +return applicationId; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/c83ef6f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java new file mode 100644 index 000..131fd42 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing
[47/50] [abbrv] hadoop git commit: YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh)
YARN-7783. Add validation step to ensure constraints are not violated due to order in which a request is processed. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d04ec492 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d04ec492 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d04ec492 Branch: refs/heads/YARN-6592 Commit: d04ec492f2d9e8a7d6ad1490ff6e13cf07d8fd7c Parents: 416f2aa Author: Arun Suresh <asur...@apache.org> Authored: Tue Jan 23 08:15:58 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:54:37 2018 -0800 -- .../algorithm/DefaultPlacementAlgorithm.java| 119 +-- .../constraint/TestPlacementProcessor.java | 49 2 files changed, 155 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d04ec492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java index 9887749..4e6473f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.List; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ResourceSizing; import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler; @@ -69,13 +70,9 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { public boolean attemptPlacementOnNode(ApplicationId appId, SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) throws InvalidAllocationTagsQueryException { -int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations(); -if (numAllocs > 0) { - if (PlacementConstraintsUtil.canSatisfyConstraints(appId, - schedulingRequest, schedulerNode, - constraintManager, tagsManager)) { -return true; - } +if (PlacementConstraintsUtil.canSatisfyConstraints(appId, +schedulingRequest, schedulerNode, constraintManager, tagsManager)) { + return true; } return false; } @@ -93,6 +90,9 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { int rePlacementCount = RE_ATTEMPT_COUNT; while (rePlacementCount > 0) { doPlacement(requests, resp, allNodes, rejectedRequests); + // Double check if placement constraints are really satisfied + validatePlacement(requests.getApplicationId(), resp, + rejectedRequests); if (rejectedRequests.size() == 0 || rePlacementCount == 1) { break; } @@ -122,9 +122,14 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { break; } SchedulingRequest schedulingRequest = requestIterator.next(); + PlacedSchedulingRequest placedReq = + new PlacedSchedulingRequest(schedulingRequest); + placedReq.setPlacementAttempt(requests.getPlacementAttempt()); + resp.getPlacedRequests().add(placedReq); CircularIterator nodeIter = new CircularIterator(lastSatisfiedNode, nIter, allNodes); - int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations(); + int numAllocs = + schedulingRequest.getResourceSizing().getNumAllocations(); while (nodeIter.hasNext() && numAllocs > 0) { SchedulerNode node = nodeIter.next(); try { @@ -135,11 +140,7 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm { requests.getApplicationId(), schedulingRequest, node)) { schedulingRequest.getResourceSizing() .
[36/50] [abbrv] hadoop git commit: YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh)
YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80af031d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80af031d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80af031d Branch: refs/heads/YARN-6592 Commit: 80af031dccf5b264949d5a996d40ed993f0e21c3 Parents: e724972 Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 22 13:26:30 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:53:34 2018 -0800 -- .../resourcemanager/RMActiveServiceContext.java | 15 + .../yarn/server/resourcemanager/RMContext.java | 6 + .../server/resourcemanager/RMContextImpl.java | 13 + .../server/resourcemanager/ResourceManager.java | 13 + .../MemoryPlacementConstraintManager.java | 282 +++ .../constraint/PlacementConstraintManager.java | 151 ++ .../PlacementConstraintManagerService.java | 93 ++ .../scheduler/constraint/package-info.java | 29 ++ .../TestPlacementConstraintManagerService.java | 182 9 files changed, 784 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/80af031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java index 4d0c230..06a1d00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator; import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; @@ -109,6 +110,7 @@ public class RMActiveServiceContext { private RMAppLifetimeMonitor rmAppLifetimeMonitor; private QueueLimitCalculator queueLimitCalculator; private AllocationTagsManager allocationTagsManager; + private PlacementConstraintManager placementConstraintManager; public RMActiveServiceContext() { queuePlacementManager = new PlacementManager(); @@ -413,6 +415,19 @@ public class RMActiveServiceContext { @Private @Unstable + public PlacementConstraintManager getPlacementConstraintManager() { +return placementConstraintManager; + } + + @Private + @Unstable + public void setPlacementConstraintManager( + PlacementConstraintManager placementConstraintManager) { +this.placementConstraintManager = placementConstraintManager; + } + + @Private + @Unstable public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() { return rmDelegatedNodeLabelsUpdater; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/80af031d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 00da108..eb91a31 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop
[48/50] [abbrv] hadoop git commit: YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh)
YARN-7795. Fix jenkins issues of YARN-6592 branch. (Sunil G via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c6644fe Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c6644fe Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c6644fe Branch: refs/heads/YARN-6592 Commit: 7c6644fedf8243e907d26e2d3d2194ab13fbf213 Parents: c7cee3e Author: Arun Suresh <asur...@apache.org> Authored: Wed Jan 24 14:18:32 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:54:37 2018 -0800 -- .../hadoop/yarn/api/protocolrecords/AllocateRequest.java | 4 ++-- .../apache/hadoop/yarn/api/records/SchedulingRequest.java| 8 .../apache/hadoop/yarn/api/resource/PlacementConstraint.java | 3 +++ .../hadoop/yarn/api/resource/PlacementConstraints.java | 3 --- .../yarn/api/records/impl/pb/ResourceSizingPBImpl.java | 3 +++ .../yarn/api/records/impl/pb/SchedulingRequestPBImpl.java| 3 +++ .../resourcemanager/scheduler/AbstractYarnScheduler.java | 1 - .../server/resourcemanager/scheduler/AppSchedulingInfo.java | 3 +-- .../yarn/server/resourcemanager/scheduler/SchedulerNode.java | 8 ++-- .../scheduler/capacity/CapacityScheduler.java| 3 +-- .../scheduler/constraint/AllocationTagsManager.java | 8 +--- .../scheduler/constraint/PlacementConstraintsUtil.java | 4 ++-- .../scheduler/placement/AppPlacementAllocator.java | 4 ++-- .../placement/SingleConstraintAppPlacementAllocator.java | 1 - .../scheduler/capacity/TestCapacityScheduler.java| 1 - .../TestCapacitySchedulerSchedulingRequestUpdate.java| 4 +++- .../capacity/TestSchedulingRequestContainerAllocation.java | 8 .../TestSchedulingRequestContainerAllocationAsync.java | 1 - 18 files changed, 35 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c6644fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java index d8d2347..876957e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java @@ -229,8 +229,8 @@ public abstract class AllocateRequest { /** * Set the list of Scheduling requests to inform the * ResourceManager about the application's resource requirements - * (potentially including allocation tags & placement constraints). - * @param schedulingRequests list of SchedulingRequest to update + * (potentially including allocation tags and placement constraints). + * @param schedulingRequests list of {@link SchedulingRequest} to update * the ResourceManager about the application's resource * requirements. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c6644fe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java index e32dd24..4bb2b84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java @@ -77,7 +77,7 @@ public abstract class SchedulingRequest { /** * Set the allocationRequestId of the request. - * + * * @see SchedulingRequest#setAllocationRequestId(long) * @param allocationRequestId allocationRequestId of the * request @@ -107,7 +107,7 @@ public abstract class SchedulingRequest { /** * Set the executionType of the request. - * + * * @see SchedulingRequest#setExecutionType(ExecutionTypeRequest) * @param executionType executionType of the request * @return {@link SchedulingRequest.SchedulingRequestBuilder} @@ -119,7 +119,7 @@ public abstract class SchedulingRequest { schedulingRequest.
[42/50] [abbrv] hadoop git commit: YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh)
YARN-7784. Fix Cluster metrics when placement processor is enabled. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3663239d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3663239d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3663239d Branch: refs/heads/YARN-6592 Commit: 3663239d8726a435bd4dda71c0553e7965b5d628 Parents: 7c6644f Author: Arun Suresh <asur...@apache.org> Authored: Thu Jan 25 19:09:21 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:54:37 2018 -0800 -- .../scheduler/AppSchedulingInfo.java| 10 - .../scheduler/common/fica/FiCaSchedulerApp.java | 7 +++ .../constraint/TestPlacementProcessor.java | 45 3 files changed, 60 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3663239d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 0389895..1efdd8b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -694,6 +694,12 @@ public class AppSchedulingInfo { metrics.runAppAttempt(applicationId, user); } +updateMetrics(applicationId, type, node, containerAllocated, user, queue); + } + + public static void updateMetrics(ApplicationId applicationId, NodeType type, + SchedulerNode node, Container containerAllocated, String user, + Queue queue) { if (LOG.isDebugEnabled()) { LOG.debug("allocate: applicationId=" + applicationId + " container=" + containerAllocated.getId() + " host=" + containerAllocated @@ -702,10 +708,10 @@ public class AppSchedulingInfo { + type); } if(node != null) { - metrics.allocateResources(node.getPartition(), user, 1, + queue.getMetrics().allocateResources(node.getPartition(), user, 1, containerAllocated.getResource(), true); } -metrics.incrNodeTypeAggregations(user, type); +queue.getMetrics().incrNodeTypeAggregations(user, type); } // Get AppPlacementAllocator by specified schedulerKey http://git-wip-us.apache.org/repos/asf/hadoop/blob/3663239d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 7eb1e31..f3da0a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerRese import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; @@ -548,6 +549,1
[32/50] [abbrv] hadoop git commit: YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e724972b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e724972b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e724972b Branch: refs/heads/YARN-6592 Commit: e724972bf756bafaf395d05748d940d8d68b09bd Parents: c83ef6f Author: Arun Suresh <asur...@apache.org> Authored: Fri Dec 22 07:24:37 2017 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:53:34 2018 -0800 -- .../server/resourcemanager/ResourceManager.java | 2 +- .../constraint/AllocationTagsManager.java | 282 ++- .../rmcontainer/TestRMContainerImpl.java| 2 +- .../constraint/TestAllocationTagsManager.java | 269 -- 4 files changed, 392 insertions(+), 163 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e724972b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index a1d3dfc..1d838f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -496,7 +496,7 @@ public class ResourceManager extends CompositeService implements Recoverable { } protected AllocationTagsManager createAllocationTagsManager() { -return new AllocationTagsManager(); +return new AllocationTagsManager(this.rmContext); } protected DelegationTokenRenewer createDelegationTokenRenewer() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e724972b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java index c278606..7b0b959 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.SchedulingRequest; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.log4j.Logger; import java.util.HashMap; @@ -38,9 +39,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongBinaryOperator; /** - * Support storing maps between container-tags/applications and - * nodes. This will be required by affinity/anti-affinity implementation and - * cardinality. + * In-memory mapping between applications/container-tags and nodes/racks. + * Required by constrained affinity/anti-affinity and cardinality placement. */ @InterfaceAudience.Private @InterfaceStability.Unstable @@ -51,48 +51,54 @@ public class AllocationTagsManager { private ReentrantReadWriteLock.ReadLock readLock; private ReentrantReadWriteLock.WriteLock writeLock; + private final RMContext rmContext; - // Application's tags to node - private Map<ApplicationId, NodeToCountedTags> perAppMappings = + // Application's tags to Node + private Map<ApplicationId, NodeToCountedTags> perAppNodeMappings = + new HashMap<>(); + // Application's tags to Rac
[21/50] [abbrv] hadoop git commit: YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh)
YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/126eb8d7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/126eb8d7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/126eb8d7 Branch: refs/heads/YARN-6592 Commit: 126eb8d7abfb5b97a50fa9a1be3d0e630bc6259e Parents: 30b8d4f Author: Arun Suresh <asur...@apache.org> Authored: Fri Jan 12 14:37:06 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:53:34 2018 -0800 -- .../hadoop/yarn/api/records/Container.java | 15 + .../src/main/proto/yarn_protos.proto| 1 + .../api/records/impl/pb/ContainerPBImpl.java| 31 + .../yarn/security/ContainerTokenIdentifier.java | 69 +++- .../src/main/proto/yarn_security_token.proto| 1 + .../api/protocolrecords/NMContainerStatus.java | 14 .../impl/pb/NMContainerStatusPBImpl.java| 33 ++ .../yarn_server_common_service_protos.proto | 1 + .../containermanager/ContainerManagerImpl.java | 3 +- .../container/ContainerImpl.java| 19 +++--- .../rmcontainer/RMContainerImpl.java| 10 ++- .../scheduler/SchedulerApplicationAttempt.java | 3 +- .../security/RMContainerTokenSecretManager.java | 21 ++ .../capacity/TestContainerAllocation.java | 5 +- 14 files changed, 194 insertions(+), 32 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/126eb8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 4fdc803..b9ca3f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.util.Records; +import java.util.Collections; +import java.util.Set; + /** * {@code Container} represents an allocated resource in the cluster. * @@ -256,4 +259,16 @@ public abstract class Container implements Comparable { public void setVersion(int version) { throw new UnsupportedOperationException(); } + + @Private + @Unstable + public Set getAllocationTags() { +return Collections.EMPTY_SET; + } + + @Private + @Unstable + public void setAllocationTags(Set allocationTags) { + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/126eb8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 5cb1177..25c8569 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -152,6 +152,7 @@ message ContainerProto { optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED]; optional int64 allocation_request_id = 8 [default = -1]; optional int32 version = 9 [default = 0]; + repeated string allocation_tags = 10; } message ContainerReportProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/126eb8d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index be84938..47be2f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -36,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.PriorityPr
[05/50] [abbrv] hadoop git commit: HDFS-13054. Handling PathIsNotEmptyDirectoryException in DFSClient delete call. Contributed by Nanda kumar.
HDFS-13054. Handling PathIsNotEmptyDirectoryException in DFSClient delete call. Contributed by Nanda kumar. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e990904d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e990904d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e990904d Branch: refs/heads/YARN-6592 Commit: e990904dd568a1d8f98efb55c1dd2d598ae4752b Parents: a37e7f0 Author: Arpit AgarwalAuthored: Fri Jan 26 11:42:27 2018 -0800 Committer: Arpit Agarwal Committed: Fri Jan 26 13:09:13 2018 -0800 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +++- .../hadoop/hdfs/protocol/ClientProtocol.java | 3 +++ .../hadoop/hdfs/TestDistributedFileSystem.java | 17 + 3 files changed, 23 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index f0769c1..92bb99e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -82,6 +82,7 @@ import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.fs.QuotaUsage; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.StorageType; @@ -1620,7 +1621,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, FileNotFoundException.class, SafeModeException.class, UnresolvedPathException.class, - SnapshotAccessControlException.class); + SnapshotAccessControlException.class, + PathIsNotEmptyDirectoryException.class); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index fbef037..0d77037 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.crypto.CryptoProtocolVersion; import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; import org.apache.hadoop.hdfs.AddBlockFlag; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; @@ -625,6 +626,8 @@ public interface ClientProtocol { * @throws org.apache.hadoop.fs.UnresolvedLinkException If src * contains a symlink * @throws SnapshotAccessControlException if path is in RO snapshot + * @throws PathIsNotEmptyDirectoryException if path is a non-empty directory + * and recursive is set to false * @throws IOException If an I/O error occurred */ @AtMostOnce http://git-wip-us.apache.org/repos/asf/hadoop/blob/e990904d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 823c747..072ee9f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -67,6 +67,7 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; +import
[50/50] [abbrv] hadoop git commit: YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh)
YARN-7763. Allow Constraints specified in the SchedulingRequest to override application level constraints. (Weiwei Yang via asuresh) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3803dea4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3803dea4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3803dea4 Branch: refs/heads/YARN-6592 Commit: 3803dea47f3f498f5b4165bef677fed12e66a36e Parents: 7445139 Author: Arun Suresh <asur...@apache.org> Authored: Sun Jan 21 19:11:17 2018 -0800 Committer: Arun Suresh <asur...@apache.org> Committed: Tue Jan 30 07:54:37 2018 -0800 -- .../scheduler/capacity/CapacityScheduler.java | 4 +- .../constraint/PlacementConstraintsUtil.java| 98 +++- .../algorithm/DefaultPlacementAlgorithm.java| 4 +- .../SingleConstraintAppPlacementAllocator.java | 10 +- .../TestPlacementConstraintsUtil.java | 94 --- 5 files changed, 123 insertions(+), 87 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3803dea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 429f9f3..a096e2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2621,9 +2621,9 @@ public class CapacityScheduler extends // Validate placement constraint is satisfied before // committing the request. try { - if (!PlacementConstraintsUtil.canSatisfySingleConstraint( + if (!PlacementConstraintsUtil.canSatisfyConstraints( appAttempt.getApplicationId(), - schedulingRequest.getAllocationTags(), schedulerNode, + schedulingRequest, schedulerNode, rmContext.getPlacementConstraintManager(), rmContext.getAllocationTagsManager())) { LOG.debug("Failed to allocate container for application " http://git-wip-us.apache.org/repos/asf/hadoop/blob/3803dea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java index ff5cb67..c07c16f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.SchedulingRequest; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint; @@ -54,7 +55,7 @@ public final class PlacementConstraintsUtil { } /** - * Returns true if **single** placement constraint with associated + * Returns true if single placement constraint with associated * allocationTags and scope is satisfied by a s