[hadoop] branch branch-2.10 updated: YARN-1115: Provide optional means for a scheduler to check real user ACLs. Contributed by Eric Payne (epayne)
This is an automated email from the ASF dual-hosted git repository.
ahussein pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new de120b1 YARN-1115: Provide optional means for a scheduler to check
real user ACLs. Contributed by Eric Payne (epayne)
de120b1 is described below
commit de120b16ad8710176b97907677f0f4b8eda133d4
Author: Ahmed Hussein
AuthorDate: Fri Oct 22 17:02:38 2021 +
YARN-1115: Provide optional means for a scheduler to check real user ACLs.
Contributed by Eric Payne (epayne)
---
.../server/resourcemanager/ClientRMService.java| 6 +-
.../yarn/server/resourcemanager/RMAppManager.java | 33 -
.../resourcemanager/recovery/RMStateStore.java | 5 +-
.../recovery/records/ApplicationStateData.java | 45 ++-
.../impl/pb/ApplicationStateDataPBImpl.java| 17 ++-
.../yarn/server/resourcemanager/rmapp/RMApp.java | 2 +
.../server/resourcemanager/rmapp/RMAppImpl.java| 23 +++-
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../yarn/server/resourcemanager/RMHATestBase.java | 4 +-
.../applicationsmanager/MockAsm.java | 4 +
.../server/resourcemanager/rmapp/MockRMApp.java| 5 +
.../scheduler/capacity/TestLeafQueue.java | 147 +
.../src/site/markdown/CapacityScheduler.md | 4 +-
14 files changed, 283 insertions(+), 19 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index da77b30..3cd9154 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -572,10 +572,12 @@ public class ClientRMService extends AbstractService
implements
// checked here, those that are dependent on RM configuration are validated
// in RMAppManager.
+UserGroupInformation userUgi = null;
String user = null;
try {
// Safety
- user = UserGroupInformation.getCurrentUser().getShortUserName();
+ userUgi = UserGroupInformation.getCurrentUser();
+ user = userUgi.getShortUserName();
} catch (IOException ie) {
LOG.warn("Unable to get the current user.", ie);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
@@ -665,7 +667,7 @@ public class ClientRMService extends AbstractService
implements
try {
// call RMAppManager to submit application directly
rmAppManager.submitApplication(submissionContext,
- System.currentTimeMillis(), user);
+ System.currentTimeMillis(), userUgi);
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 6746c7c..ca5a270 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -322,16 +322,26 @@ public class RMAppManager implements
EventHandler,
}
}
- @SuppressWarnings("unchecked")
+ @VisibleForTesting
+ @Deprecated
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
String user) throws YarnException {
+submitApplication(submissionContext, submitTime,
+UserGroupInformation.createRemoteUser(user));
+ }
+
+ @VisibleForTesting
+ @SuppressWarnings("unchecked")
+ protected void submitApplication(
+ ApplicationSubmissionContext submissionContext, long submitTime,
+ UserGroupInformation userUgi) throws YarnException {
ApplicationId applicationId = submissionContext.getApplicationId();
// Passing start time as -1. It will be eventually set in RMAppImpl
// constructor.
RMAppImpl application = createAndPopulateNewRMApp(
-submissionContext, submitTime, user, false,
[hadoop] branch branch-3.2 updated: YARN-1115: Provide optional means for a scheduler to check real user ACLs. Contributed by Eric Payne (epayne)
This is an automated email from the ASF dual-hosted git repository.
ahussein pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new c36d8a1 YARN-1115: Provide optional means for a scheduler to check
real user ACLs. Contributed by Eric Payne (epayne)
c36d8a1 is described below
commit c36d8a1395b5de1a75b7c689fea432f3691497c4
Author: Ahmed Hussein
AuthorDate: Fri Oct 22 15:54:13 2021 +
YARN-1115: Provide optional means for a scheduler to check real user ACLs.
Contributed by Eric Payne (epayne)
---
.../server/resourcemanager/ClientRMService.java| 6 +-
.../yarn/server/resourcemanager/RMAppManager.java | 32 -
.../resourcemanager/recovery/RMStateStore.java | 5 +-
.../recovery/records/ApplicationStateData.java | 45 ++-
.../impl/pb/ApplicationStateDataPBImpl.java| 17 ++-
.../yarn/server/resourcemanager/rmapp/RMApp.java | 2 +
.../server/resourcemanager/rmapp/RMAppImpl.java| 25 +++-
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../server/resourcemanager/AppManagerTestBase.java | 3 +-
.../yarn/server/resourcemanager/RMHATestBase.java | 4 +-
.../applicationsmanager/MockAsm.java | 4 +
.../server/resourcemanager/rmapp/MockRMApp.java| 5 +
.../scheduler/capacity/TestLeafQueue.java | 146 +
.../src/site/markdown/CapacityScheduler.md | 4 +-
15 files changed, 285 insertions(+), 20 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 87e70b3..eedf274 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -597,10 +597,12 @@ public class ClientRMService extends AbstractService
implements
// checked here, those that are dependent on RM configuration are validated
// in RMAppManager.
+UserGroupInformation userUgi = null;
String user = null;
try {
// Safety
- user = UserGroupInformation.getCurrentUser().getShortUserName();
+ userUgi = UserGroupInformation.getCurrentUser();
+ user = userUgi.getShortUserName();
} catch (IOException ie) {
LOG.warn("Unable to get the current user.", ie);
RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
@@ -691,7 +693,7 @@ public class ClientRMService extends AbstractService
implements
try {
// call RMAppManager to submit application directly
rmAppManager.submitApplication(submissionContext,
- System.currentTimeMillis(), user);
+ System.currentTimeMillis(), userUgi);
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 458ea66..bd84222 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -348,16 +348,26 @@ public class RMAppManager implements
EventHandler,
}
}
- @SuppressWarnings("unchecked")
+ @VisibleForTesting
+ @Deprecated
protected void submitApplication(
ApplicationSubmissionContext submissionContext, long submitTime,
String user) throws YarnException {
+submitApplication(submissionContext, submitTime,
+UserGroupInformation.createRemoteUser(user));
+ }
+
+ @VisibleForTesting
+ @SuppressWarnings("unchecked")
+ protected void submitApplication(
+ ApplicationSubmissionContext submissionContext, long submitTime,
+ UserGroupInformation userUgi) throws YarnException {
ApplicationId applicationId = submissionContext.getApplicationId();
// Passing start time as -1. It will be eventually set in RMAppImpl
// constructor.
RMAppImpl application =
[hadoop] branch branch-3.3 updated: HDFS-16282. Remove duplicate generic usage information to hdfs debug command (#3576). Contributed by daimin.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 847cff7 HDFS-16282. Remove duplicate generic usage information to
hdfs debug command (#3576). Contributed by daimin.
847cff7 is described below
commit 847cff7d91c98e9e20f88b6c0a75e2e077391d05
Author: daimin
AuthorDate: Fri Oct 22 23:39:37 2021 +0800
HDFS-16282. Remove duplicate generic usage information to hdfs debug
command (#3576). Contributed by daimin.
Signed-off-by: Ayush Saxena
---
.../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index f5967e1..efe355d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -459,9 +459,9 @@ public class DebugAdmin extends Configured implements Tool {
if (!command.name.equals("help")) {
System.out.println(command.usageText);
}
- System.out.println();
- ToolRunner.printGenericCommandUsage(System.out);
}
+System.out.println();
+ToolRunner.printGenericCommandUsage(System.out);
}
public static void main(String[] argsArray) throws Exception {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 02/02: HDFS-16282. Remove duplicate generic usage information to hdfs debug command (#3576). Contributed by daimin.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 6e574493508d69cefcfb1b05b78e73d342adb1b2
Author: daimin
AuthorDate: Fri Oct 22 23:39:37 2021 +0800
HDFS-16282. Remove duplicate generic usage information to hdfs debug
command (#3576). Contributed by daimin.
Signed-off-by: Ayush Saxena
---
.../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index f5967e1..efe355d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -459,9 +459,9 @@ public class DebugAdmin extends Configured implements Tool {
if (!command.name.equals("help")) {
System.out.println(command.usageText);
}
- System.out.println();
- ToolRunner.printGenericCommandUsage(System.out);
}
+System.out.println();
+ToolRunner.printGenericCommandUsage(System.out);
}
public static void main(String[] argsArray) throws Exception {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated (7b913bc -> 6e57449)
This is an automated email from the ASF dual-hosted git repository. ayushsaxena pushed a change to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git. from 7b913bc Remove duplicate generic usage information to hdfs debug command (#3576). Contributed by daimin. new f67e6c3 Revert "Remove duplicate generic usage information to hdfs debug command (#3576). Contributed by daimin." new 6e57449 HDFS-16282. Remove duplicate generic usage information to hdfs debug command (#3576). Contributed by daimin. The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] 01/02: Revert "Remove duplicate generic usage information to hdfs debug command (#3576). Contributed by daimin."
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit f67e6c315b1c6173c7c3b6ea72586987a60872df
Author: Ayush Saxena
AuthorDate: Fri Oct 22 21:10:38 2021 +0530
Revert "Remove duplicate generic usage information to hdfs debug command
(#3576). Contributed by daimin."
Misses Jira ID in the commit Message: HDFS-16282
This reverts commit 7b913bc6bc70cf71cd9fb16bf71f239e594d3f71.
---
.../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index efe355d..f5967e1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -459,9 +459,9 @@ public class DebugAdmin extends Configured implements Tool {
if (!command.name.equals("help")) {
System.out.println(command.usageText);
}
+ System.out.println();
+ ToolRunner.printGenericCommandUsage(System.out);
}
-System.out.println();
-ToolRunner.printGenericCommandUsage(System.out);
}
public static void main(String[] argsArray) throws Exception {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: Remove duplicate generic usage information to hdfs debug command (#3576). Contributed by daimin.
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7b913bc Remove duplicate generic usage information to hdfs debug
command (#3576). Contributed by daimin.
7b913bc is described below
commit 7b913bc6bc70cf71cd9fb16bf71f239e594d3f71
Author: daimin
AuthorDate: Fri Oct 22 23:39:37 2021 +0800
Remove duplicate generic usage information to hdfs debug command (#3576).
Contributed by daimin.
Signed-off-by: Ayush Saxena
---
.../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java| 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
index f5967e1..efe355d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java
@@ -459,9 +459,9 @@ public class DebugAdmin extends Configured implements Tool {
if (!command.name.equals("help")) {
System.out.println(command.usageText);
}
- System.out.println();
- ToolRunner.printGenericCommandUsage(System.out);
}
+System.out.println();
+ToolRunner.printGenericCommandUsage(System.out);
}
public static void main(String[] argsArray) throws Exception {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: YARN-10930. Introduce universal capacity resource vector. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 32ecaed YARN-10930. Introduce universal capacity resource vector.
Contributed by Andras Gyori
32ecaed is described below
commit 32ecaed9c3c06a48ef01d0437e62e8faccd3e9f3
Author: 9uapaw
AuthorDate: Fri Oct 22 17:32:33 2021 +0200
YARN-10930. Introduce universal capacity resource vector. Contributed by
Andras Gyori
---
.../scheduler/capacity/AbstractCSQueue.java| 12 +-
.../scheduler/capacity/CSQueue.java| 8 +
.../capacity/CapacitySchedulerConfiguration.java | 19 +-
.../scheduler/capacity/QueueCapacityVector.java| 258 +
.../scheduler/capacity/ResourceVector.java | 129 +++
.../capacity/conf/QueueCapacityConfigParser.java | 215 +
.../capacity/TestQueueCapacityVector.java | 111 +
.../scheduler/capacity/TestResourceVector.java | 118 ++
.../conf/TestQueueCapacityConfigParser.java| 241 +++
9 files changed, 1108 insertions(+), 3 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 2d9bf85..e3feb51 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -131,6 +131,8 @@ public abstract class AbstractCSQueue implements CSQueue {
protected CapacityConfigType capacityConfigType =
CapacityConfigType.NONE;
+ protected Map configuredCapacityVectors;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
protected CapacitySchedulerContext csContext;
@@ -374,6 +376,8 @@ public abstract class AbstractCSQueue implements CSQueue {
this.reservationsContinueLooking =
configuration.getReservationContinueLook();
+ this.configuredCapacityVectors = csContext.getConfiguration()
+ .parseConfiguredResourceVector(queuePath, configuredNodeLabels);
// Update metrics
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
@@ -688,6 +692,12 @@ public abstract class AbstractCSQueue implements CSQueue {
minimumAllocation);
}
+ @Override
+ public QueueCapacityVector getConfiguredCapacityVector(
+ String label) {
+return configuredCapacityVectors.get(label);
+ }
+
private void initializeQueueState(CapacitySchedulerConfiguration
configuration) {
QueueState previousState = getState();
QueueState configuredState = configuration
@@ -978,7 +988,7 @@ public abstract class AbstractCSQueue implements CSQueue {
"Default lifetime " + defaultAppLifetime
+ " can't exceed maximum lifetime " + myMaxAppLifetime);
}
-
+
if (defaultAppLifetime <= 0) {
defaultAppLifetime = myMaxAppLifetime;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 03a5afb..2acc1d4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -420,6 +420,14 @@ public interface CSQueue extends SchedulerQueue {
Resource getEffectiveCapacity(String label);
/**
+ * Get configured capacity resource vector parsed from the capacity config
+ * of the queue.
+ * @param label node label (partition)
+ * @return capacity resource vector
+ */
+ QueueCapacityVector getConfiguredCapacityVector(String label);
+
+ /**
* Get effective capacity of queue. If min/max resource is configured,
* preference will be given to absolute configuration over normal capacity.
* Also round down the result to normalizeDown.
diff --git
[hadoop] branch trunk updated: YARN-10948. Rename SchedulerQueue#activeQueue to activateQueue. Contributed by Adam Antal
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 23772d9 YARN-10948. Rename SchedulerQueue#activeQueue to
activateQueue. Contributed by Adam Antal
23772d9 is described below
commit 23772d946bdf26a72fb611a9719afe743cdebd52
Author: Adam Antal
AuthorDate: Fri Oct 22 16:33:03 2021 +0200
YARN-10948. Rename SchedulerQueue#activeQueue to activateQueue. Contributed
by Adam Antal
---
.../yarn/server/resourcemanager/scheduler/QueueStateManager.java | 2 +-
.../hadoop/yarn/server/resourcemanager/scheduler/SchedulerQueue.java | 4 ++--
.../server/resourcemanager/scheduler/capacity/AbstractCSQueue.java| 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
index 0127e1e..ab0f445 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueStateManager.java
@@ -77,7 +77,7 @@ public class QueueStateManager
extends Queue {
void stopQueue();
/**
- * Active the queue.
+ * Activate the queue.
* @throws YarnException if the queue can not be activated.
*/
- void activeQueue() throws YarnException;
+ void activateQueue() throws YarnException;
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index ca861fb..2d9bf85 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -733,7 +733,7 @@ public abstract class AbstractCSQueue implements CSQueue {
} else {
if (configuredState == QueueState.RUNNING) {
try {
-activeQueue();
+activateQueue();
} catch (YarnException ex) {
throw new IllegalArgumentException(ex.getMessage());
}
@@ -1322,7 +1322,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public void activeQueue() throws YarnException {
+ public void activateQueue() throws YarnException {
this.writeLock.lock();
try {
if (getState() == QueueState.RUNNING) {
-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch branch-3.3 updated: HADOOP-17770 WASB : Support disabling buffered reads in positional reads (#3233)
This is an automated email from the ASF dual-hosted git repository.
surendralilhore pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 913d06a HADOOP-17770 WASB : Support disabling buffered reads in
positional reads (#3233)
913d06a is described below
commit 913d06ad4d43f4943e89f834658275bf60309284
Author: Anoop Sam John
AuthorDate: Fri Oct 22 11:45:42 2021 +0530
HADOOP-17770 WASB : Support disabling buffered reads in positional reads
(#3233)
---
.../hadoop-azure/dev-support/findbugs-exclude.xml | 13 +
.../fs/azure/AzureNativeFileSystemStore.java | 31 +--
.../hadoop/fs/azure/BlockBlobInputStream.java | 41 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 64 +-
.../hadoop/fs/azure/NativeFileSystemStore.java | 4 ++
.../hadoop-azure/src/site/markdown/index.md| 11
.../hadoop/fs/azure/ITestBlockBlobInputStream.java | 56 +++
7 files changed, 212 insertions(+), 8 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml
b/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml
index b750b8b..fa6085f 100644
--- a/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml
+++ b/hadoop-tools/hadoop-azure/dev-support/findbugs-exclude.xml
@@ -83,4 +83,17 @@
+
+
+
+
+
+
diff --git
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index c613468..368283a 100644
---
a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++
b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -40,6 +40,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
@@ -236,6 +237,16 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
public static final String KEY_ENABLE_FLAT_LISTING =
"fs.azure.flatlist.enable";
/**
+ * Optional config to enable a lock free pread which will bypass buffer in
+ * BlockBlobInputStream.
+ * This is not a config which can be set at cluster level. It can be used as
+ * an option on FutureDataInputStreamBuilder.
+ * @see FileSystem#openFile(org.apache.hadoop.fs.Path)
+ */
+ public static final String FS_AZURE_BLOCK_BLOB_BUFFERED_PREAD_DISABLE =
+ "fs.azure.block.blob.buffered.pread.disable";
+
+ /**
* The set of directories where we should apply atomic folder rename
* synchronized with createNonRecursive.
*/
@@ -1577,8 +1588,8 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
* Opens a new input stream for the given blob (page or block blob)
* to read its data.
*/
- private InputStream openInputStream(CloudBlobWrapper blob)
- throws StorageException, IOException {
+ private InputStream openInputStream(CloudBlobWrapper blob,
+ Optional options) throws StorageException, IOException {
if (blob instanceof CloudBlockBlobWrapper) {
LOG.debug("Using stream seek algorithm {}", inputStreamVersion);
switch(inputStreamVersion) {
@@ -1586,9 +1597,13 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
return blob.openInputStream(getDownloadOptions(),
getInstrumentedContext(isConcurrentOOBAppendAllowed()));
case 2:
+boolean bufferedPreadDisabled = options.map(c -> c
+.getBoolean(FS_AZURE_BLOCK_BLOB_BUFFERED_PREAD_DISABLE, false))
+.orElse(false);
return new BlockBlobInputStream((CloudBlockBlobWrapper) blob,
getDownloadOptions(),
-getInstrumentedContext(isConcurrentOOBAppendAllowed()));
+getInstrumentedContext(isConcurrentOOBAppendAllowed()),
+bufferedPreadDisabled);
default:
throw new IOException("Unknown seek algorithm: " + inputStreamVersion);
}
@@ -2262,6 +2277,12 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
@Override
public InputStream retrieve(String key, long startByteOffset)
throws AzureException, IOException {
+return retrieve(key, startByteOffset, Optional.empty());
+ }
+
+ @Override
+ public InputStream retrieve(String key, long startByteOffset,
+ Optional options) throws AzureException, IOException {
try {
// Check if a session exists, if not create a session with the
// Azure storage server.
@@ -2273,7 +2294,7 @@ public class AzureNativeFileSystemStore implements
NativeFileSystemStore {
}
