[hadoop] branch trunk updated: YARN-9460. QueueACLsManager and ReservationsACLManager should not use instanceof checks. Contributed by Bilwa S T.
This is an automated email from the ASF dual-hosted git repository. surendralilhore pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b2facc8 YARN-9460. QueueACLsManager and ReservationsACLManager should not use instanceof checks. Contributed by Bilwa S T. b2facc8 is described below commit b2facc84a1b48b9dcbe0816e120778d2100b320e Author: Surendra Singh Lilhore AuthorDate: Sat Jun 20 19:55:23 2020 +0530 YARN-9460. QueueACLsManager and ReservationsACLManager should not use instanceof checks. Contributed by Bilwa S T. --- .../server/resourcemanager/ResourceManager.java| 2 +- .../reservation/AbstractReservationSystem.java | 10 +- .../security/CapacityQueueACLsManager.java | 111 .../security/CapacityReservationsACLsManager.java | 46 .../security/FairQueueACLsManager.java | 72 + .../security/FairReservationsACLsManager.java | 42 .../security/GenericQueueACLsManager.java | 55 ++ .../resourcemanager/security/QueueACLsManager.java | 116 + .../security/ReservationsACLsManager.java | 44 ++-- .../resourcemanager/security/package-info.java | 28 + .../server/resourcemanager/TestClientRMTokens.java | 5 +- 11 files changed, 402 insertions(+), 129 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 48cbd8f..836a5ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -438,7 +438,7 @@ public class ResourceManager extends CompositeService protected QueueACLsManager createQueueACLsManager(ResourceScheduler scheduler, Configuration conf) { -return new QueueACLsManager(scheduler, conf); +return QueueACLsManager.getQueueACLsManager(scheduler, conf); } @VisibleForTesting diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java index 5b8772c..d9e4be9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java @@ -50,6 +50,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.CapacityReservationsACLsManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.FairReservationsACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.security.ReservationsACLsManager; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.UTCClock; @@ -173,7 +175,13 @@ public abstract class AbstractReservationSystem extends AbstractService YarnConfiguration.DEFAULT_YARN_RESERVATION_ACL_ENABLE) && conf.getBoolean(YarnConfiguration.YARN_ACL_ENABLE, YarnConfiguration.DEFAULT_YARN_ACL_ENABLE)) { - reservationsACLsManager = new ReservationsACLsManager(scheduler, conf); + if (scheduler instanceof CapacityScheduler) { +reservationsACLsManager = new CapacityReservationsACLsManager(scheduler, +conf); + } else if (scheduler instanceof FairScheduler) { +reservationsACLsManager = new FairReservationsACLsManager(scheduler, +conf); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/CapacityQueueACLsManager.java b/hadoop-yarn-project/hadoop-yarn/
[hadoop] branch branch-3.3 updated: MAPREDUCE-7281. Fix NoClassDefFoundError on 'mapred minicluster'. (#2077)
This is an automated email from the ASF dual-hosted git repository. iwasakims pushed a commit to branch branch-3.3 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.3 by this push: new 56d72ad MAPREDUCE-7281. Fix NoClassDefFoundError on 'mapred minicluster'. (#2077) 56d72ad is described below commit 56d72adbdd2a1c8647ebd974a2ecb2904f92799c Author: Masatake Iwasaki AuthorDate: Sat Jun 20 07:37:55 2020 +0900 MAPREDUCE-7281. Fix NoClassDefFoundError on 'mapred minicluster'. (#2077) (cherry picked from commit 8fd0fdf8890b4c0cf3ea977be8fae8fa17e6599b) --- hadoop-mapreduce-project/bin/mapred | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 9773ec8..3e52556 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -105,6 +105,8 @@ function mapredcmd_case minicluster) hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}/timelineservice"'/*' hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}/test"'/*' + junitjar=$(echo "${HADOOP_TOOLS_LIB_JARS_DIR}"/junit-[0-9]*.jar) + hadoop_add_classpath "${junitjar}" HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.MiniHadoopClusterManager ;; *) - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
[hadoop] branch trunk updated: HDFS-15418. ViewFileSystemOverloadScheme should represent mount links as non symlinks. Contributed by Uma Maheswara Rao G.
This is an automated email from the ASF dual-hosted git repository. umamahesh pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new b27810a HDFS-15418. ViewFileSystemOverloadScheme should represent mount links as non symlinks. Contributed by Uma Maheswara Rao G. b27810a is described below commit b27810aa6015253866ccc0ccc7247ad7024c0730 Author: Uma Maheswara Rao G AuthorDate: Sat Jun 20 00:32:02 2020 -0700 HDFS-15418. ViewFileSystemOverloadScheme should represent mount links as non symlinks. Contributed by Uma Maheswara Rao G. --- .../org/apache/hadoop/fs/viewfs/Constants.java | 8 ++ .../apache/hadoop/fs/viewfs/ViewFileSystem.java| 71 +++ .../fs/viewfs/ViewFileSystemOverloadScheme.java| 20 +++- .../java/org/apache/hadoop/fs/viewfs/ViewFs.java | 80 - .../viewfs/TestViewFsOverloadSchemeListStatus.java | 132 + .../hadoop/fs/viewfs/TestViewfsFileStatus.java | 4 +- .../src/site/markdown/ViewFsOverloadScheme.md | 42 +++ ...SystemOverloadSchemeHdfsFileSystemContract.java | 5 + ...ViewFileSystemOverloadSchemeWithHdfsScheme.java | 9 ++ 9 files changed, 295 insertions(+), 76 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java index 0a5d4b4..f454f63 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java @@ -90,4 +90,12 @@ public interface Constants { String CONFIG_VIEWFS_ENABLE_INNER_CACHE = "fs.viewfs.enable.inner.cache"; boolean CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT = true; + + /** + * Enable ViewFileSystem to show mountlinks as symlinks. + */ + String CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS = + "fs.viewfs.mount.links.as.symlinks"; + + boolean CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT = true; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 895edc0..1ee06e0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -20,6 +20,8 @@ package org.apache.hadoop.fs.viewfs; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE; import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS; +import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINKS_DEFAULT; import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555; import java.io.FileNotFoundException; @@ -527,10 +529,18 @@ public class ViewFileSystem extends FileSystem { * the target path FileStatus object. The target path will be available via * getSymlink on that children's FileStatus object. Since it represents as * symlink, isDirectory on that children's FileStatus will return false. + * This behavior can be changed by setting an advanced configuration + * fs.viewfs.mount.links.as.symlinks to false. In this case, mount points will + * be represented as non-symlinks and all the file/directory attributes like + * permissions, isDirectory etc will be assigned from it's resolved target + * directory/file. * * If you want to get the FileStatus of target path for that children, you may * want to use GetFileStatus API with that children's symlink path. Please see * {@link ViewFileSystem#getFileStatus(Path f)} + * + * Note: In ViewFileSystem, by default the mount links are represented as + * symlinks. */ @Override public FileStatus[] listStatus(final Path f) throws AccessControlException, @@ -1114,6 +1124,7 @@ public class ViewFileSystem extends FileSystem { final long creationTime; // of the the mount table final UserGroupInformation ugi; // the user/group of user who created mtable final URI myUri; +private final boolean showMountLinksAsSymlinks; public InternalDirOfViewFs(final InodeTree.INodeDir dir, final long cTime, final UserGroupInformation ugi, URI uri, @@ -1127,6 +1138,9 @@ public class ViewFileSystem extends FileSystem { theInternalDir = dir; creationTime = cTime; this.ugi = ugi; + showMountLinksAsSymlinks = config + .getBoolean(CONFIG_VIEWFS_MOUNT_LINKS_AS_SYMLINK