Repository: incubator-slider Updated Branches: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it 81a85495d -> 621a981cb (forced update)
SLIDER-162: home dir created by hdfs if user lacks permissions (and hdfs can be impersonated) Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e3c5d9d8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e3c5d9d8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e3c5d9d8 Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it Commit: e3c5d9d8e7198e3a30027a412b9348926900c14a Parents: 9619c33 Author: Steve Loughran <ste...@apache.org> Authored: Mon Jun 30 16:14:15 2014 +0100 Committer: Steve Loughran <ste...@apache.org> Committed: Mon Jun 30 16:14:15 2014 +0100 ---------------------------------------------------------------------- .../funtest/framework/AgentUploads.groovy | 13 ++--- .../funtest/framework/FileUploader.groovy | 51 +++++++++++++++++--- .../lifecycle/AgentCommandTestBase.groovy | 10 +++- 3 files changed, 57 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c5d9d8/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy index 3570183..2cec5c2 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy @@ -27,11 +27,9 @@ import org.apache.hadoop.fs.FileSystem as HadoopFS @Slf4j class AgentUploads implements FuntestProperties { final Configuration conf - private final FileUploader uploader - private final HadoopFS clusterFS - private final Path homeDir - - + public final FileUploader uploader + public final HadoopFS clusterFS + public final Path homeDir AgentUploads(Configuration conf) { this.conf = conf @@ -52,6 +50,8 @@ class AgentUploads implements FuntestProperties { homeDir, AGENT_TAR_FILENAME) + //create the home dir or fail + uploader.mkHomeDir() // Upload the agent tarball uploader.copyIfOutOfDate(localAgentTar, agentTarballPath, force) @@ -59,10 +59,7 @@ class AgentUploads implements FuntestProperties { // Upload the agent.ini def agentIniPath = new Path(homeDir, AGENT_INI) uploader.copyIfOutOfDate(localAgentIni, agentIniPath, force) - return [agentTarballPath, agentIniPath] - - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c5d9d8/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy index 4f61730..2dc85be 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy @@ -20,12 +20,14 @@ package org.apache.slider.funtest.framework import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.FileSystem as HadoopFS import org.apache.hadoop.fs.FileUtil import org.apache.hadoop.fs.Path import org.apache.hadoop.fs.permission.FsPermission import org.apache.hadoop.security.AccessControlException import org.apache.hadoop.security.UserGroupInformation -import org.apache.hadoop.fs.FileSystem as HadoopFS + +@SuppressWarnings("GroovyOctalInteger") @Slf4j class FileUploader { final Configuration conf @@ -60,18 +62,20 @@ class FileUploader { } if (toCopy) { log.info("Copying $src to $destPath") + def dir = destPath.getParent() try { fs.delete(destPath, true) - fs.mkdirs(destPath.getParent(), FsPermission.dirDefault) + fs.mkdirs(dir, FsPermission.dirDefault) return FileUtil.copy(src, fs, destPath, false, conf) } catch (AccessControlException ace) { - log.error("No write access to test user home directory. " + + log.error("No write access to destination directory $dir" + "Ensure home directory exists and has correct permissions." + ace, ace) throw ace } } else { - log.debug("Skipping copy as the destination $destPath considered up to date") + log.debug( + "Skipping copy as the destination $destPath considered up to date") return false; } } @@ -79,7 +83,7 @@ class FileUploader { public HadoopFS getFileSystem(Path dest) { getFileSystem(user, dest) } - + public HadoopFS getFileSystem() { getFileSystem(user, HadoopFS.getDefaultUri(conf)) } @@ -88,13 +92,46 @@ class FileUploader { public def getFileSystem( UserGroupInformation user, final Path path) { return getFileSystem(user, path.toUri()) - + } + public def getFileSystem( UserGroupInformation user, final URI uri) { - + SudoClosure.sudo(user) { HadoopFS.get(uri, conf); } } + + public def getFileSystemAsUserHdfs() { + def hdfs = UserGroupInformation.createRemoteUser("hdfs") + getFileSystem(hdfs, HadoopFS.getDefaultUri(conf)) + } + + /** + * Create the home dir. If it can't be created as the user, + * try to become the user 'hdfs' and try there, setting the + * user and group after. + * @return the home dir + */ + public def mkHomeDir() { + def fs = fileSystem + def home = fs.homeDirectory + if (!fs.exists(home)) { + try { + fs.mkdirs(home) + } catch (AccessControlException ace) { + log.info("Failed to mkdir $home as $user -impersonating 'hdfs") + if (UserGroupInformation.securityEnabled) { + // in a secure cluster, we cannot impersonate HDFS, so rethrow + throw ace; + } + //now create as hdfs + def FsAsUserHDFS = fileSystemAsUserHdfs + FsAsUserHDFS.mkdirs(home, new FsPermission((short) 00755)) + FsAsUserHDFS.setOwner(home, user.userName, user.primaryGroupName) + } + } + return home + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e3c5d9d8/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy index c3d6bd6..33c0b81 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy @@ -80,6 +80,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { @Before public void setupApplicationPackage() { + AgentUploads agentUploads = new AgentUploads(SLIDER_CONFIG) + agentUploads.uploader.mkHomeDir() + appPkgPath = new Path(clusterFS.homeDirectory, "cmd_log_app_pkg.zip") if (!clusterFS.exists(appPkgPath)) { clusterFS.delete(appPkgPath, false) @@ -87,14 +90,17 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { def pkgPath = folder.newFolder("testpkg") File zipFileName = new File(pkgPath, "cmd_log_app_pkg.zip").canonicalFile - assume(new File(APP_PKG_DIR).exists(), "App pkg dir not found at $APP_PKG_DIR") + + def localZipDirectory = new File(APP_PKG_DIR) + assume(localZipDirectory.exists(), "App pkg dir not found at $APP_PKG_DIR") zipDir(zipFileName.canonicalPath, APP_PKG_DIR) // Verify and upload the app pkg assume(zipFileName.exists(), "App pkg not found at $zipFileName") Path localAppPkg = new Path(zipFileName.toURI()); - clusterFS.copyFromLocalFile(false, true, localAppPkg, appPkgPath) + agentUploads.uploader.copyIfOutOfDate(zipFileName, appPkgPath, false) + } public static void logShell(SliderShell shell) {