Repository: oozie Updated Branches: refs/heads/master 2dd10ff0f -> 3c03d03fe
OOZIE-2829 Improve sharelib upload to accept multiple source folders (kmarton via gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3c03d03f Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3c03d03f Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3c03d03f Branch: refs/heads/master Commit: 3c03d03fe606659fd80c430b13f413fbdfa21faa Parents: 2dd10ff Author: Gezapeti Cseh <gezap...@apache.org> Authored: Mon Jul 30 14:42:51 2018 +0200 Committer: Gezapeti Cseh <gezap...@apache.org> Committed: Mon Jul 30 14:42:51 2018 +0200 ---------------------------------------------------------------------- docs/src/site/twiki/AG_Install.twiki | 8 +- release-log.txt | 1 + .../apache/oozie/tools/OozieSharelibCLI.java | 113 +++++++-- .../tools/IntegrationTestOozieSharelibCLI.java | 199 ++++++++++++++++ .../tools/OozieSharelibFileOperations.java | 7 +- .../tools/TestConcurrentCopyFromLocal.java | 4 +- .../oozie/tools/TestCopyTaskCallable.java | 3 +- .../oozie/tools/TestOozieSharelibCLI.java | 237 +++++++------------ .../TestOozieSharelibCLIExtraArgsParser.java | 66 ++++++ 9 files changed, 463 insertions(+), 175 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/docs/src/site/twiki/AG_Install.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki index 46363a3..b8031c8 100644 --- a/docs/src/site/twiki/AG_Install.twiki +++ b/docs/src/site/twiki/AG_Install.twiki @@ -47,7 +47,7 @@ The =oozie-setup.sh= script options are: <verbatim> Usage : oozie-setup.sh <Command and OPTIONS> - sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] [-concurrency CONCURRENCY] + sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] [-extralib EXTRA_SHARED_LIBRARY] [-concurrency CONCURRENCY] (create sharelib for oozie, FS_URI is the fs.default.name for hdfs uri; SHARED_LIBRARY, path to the @@ -55,6 +55,12 @@ Usage : oozie-setup.sh <Command and OPTIONS> or an expanded version of it. If omitted, the Oozie sharelib tarball from the Oozie installation directory will be used. + EXTRA_SHARED_LIBRARY represents extra sharelib resources. + This option requires a pair of sharelibname + and comma-separated list of pathnames in the following format: + sharelib-name=path-name-1,path-name-2 + In case of more than one sharelib, this option can be specified + multiple times. CONCURRENCY is a number of threads to be used for copy operations. By default 1 thread will be used) http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 733ea3d..9096017 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.1.0 release (trunk - unreleased) +OOZIE-2829 Improve sharelib upload to accept multiple source folders (kmarton via gezapeti) OOZIE-3309 Runtime error during /v2/sla filtering for bundle (asalamon74 via andras.piros) OOZIE-3306 Make it possible to override maven dependency plugin version number (asalamon74 via andras.piros) OOZIE-3208 "It should never happen" error messages should be more specific to root cause (kmarton via andras.piros) http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java index 75e932c..312770c 100644 --- a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java +++ b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java @@ -26,7 +26,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; @@ -37,10 +39,12 @@ import java.util.concurrent.Future; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -64,10 +68,29 @@ public class OozieSharelibCLI { public static final String CREATE_CMD = "create"; public static final String UPGRADE_CMD = "upgrade"; public static final String LIB_OPT = "locallib"; + public static final String EXTRALIBS = "extralib"; public static final String FS_OPT = "fs"; public static final String CONCURRENCY_OPT = "concurrency"; public static final String OOZIE_HOME = "oozie.home.dir"; public static final String SHARE_LIB_PREFIX = "lib_"; + public static final String NEW_LINE = System.lineSeparator(); + public static final String EXTRALIBS_USAGE = "Extra sharelib resources. " + + "This option requires a pair of sharelibname and coma-separated list of pathnames" + + " in the following format:" + NEW_LINE + + "\"sharelib_name=pathname[,pathname...]\"" + NEW_LINE + + "Caveats:" + NEW_LINE + + "* Each pathname is either a directory or a regular file (compressed files are not extracted prior to " + + "the upload operation)." + NEW_LINE + + "* Sharelibname shall be specified only once." + NEW_LINE + NEW_LINE + + "* Do not upload multiple conflicting library versions for an extra sharelib directory as it may " + + "cause runtime issues." + NEW_LINE + + "This option can be present multiple times, in case of more than one sharelib" + NEW_LINE + + "Example command:" + NEW_LINE + NEW_LINE + + "$ oozie-setup.sh sharelib create -fs hdfs://localhost:9000 -locallib oozie-sharelib.tar.gz " + + "-extralib share2=dir2,file2 -extralib share3=file3"; + public static final String EXTRALIBS_PATH_SEPARATOR = ","; + public static final String EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR = "="; + private boolean used; public static void main(String[] args) throws Exception{ @@ -86,9 +109,12 @@ public class OozieSharelibCLI { options.addOption(sharelib); options.addOption(uri); options.addOption(concurrency); + Option addLibsOption = new Option(EXTRALIBS, true, EXTRALIBS_USAGE); + options.addOption(addLibsOption); return options; } + @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "False positive") public synchronized int run(String[] args) throws Exception{ if (used) { throw new IllegalStateException("CLI instance already used"); @@ -141,6 +167,12 @@ public class OozieSharelibCLI { srcFile = files.iterator().next(); } + Map<String, String> extraLibs = new HashMap<>(); + if (command.getCommandLine().hasOption(EXTRALIBS)) { + String[] param = command.getCommandLine().getOptionValues(EXTRALIBS); + extraLibs = getExtraLibs(param); + } + File temp = File.createTempFile("oozie", ".dir"); temp.delete(); temp.mkdir(); @@ -183,21 +215,9 @@ public class OozieSharelibCLI { System.out.println("the destination path for sharelib is: " + dstPath); - if (!srcFile.exists()){ - throw new IOException(srcPath + " cannot be found"); - } - - if (threadPoolSize > 1) { - long fsLimitsMinBlockSize = fs.getConf() - .getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT); - long bytesPerChecksum = fs.getConf() - .getLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT); - new ConcurrentCopyFromLocal(threadPoolSize, fsLimitsMinBlockSize, bytesPerChecksum) - .concurrentCopyFromLocal(fs, srcFile, dstPath); - - } else { - fs.copyFromLocalFile(false, srcPath, dstPath); - } + checkIfSourceFilesExist(srcFile); + copyToSharelib(threadPoolSize, srcFile, srcPath, dstPath, fs); + copyExtraLibs(threadPoolSize, extraLibs, dstPath, fs); services.destroy(); FileUtils.deleteDirectory(temp); @@ -220,6 +240,69 @@ public class OozieSharelibCLI { } } + @VisibleForTesting + static Map<String,String> getExtraLibs(String[] param) { + Map<String, String> extraLibs = new HashMap<>(); + + for (String lib : param) { + String[] addLibParts = lib.split(EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR); + if (addLibParts.length != 2) { + printExtraSharelibUsage(); + throw new IllegalArgumentException(String + .format("Argument of extralibs '%s' is in a wrong format. Exiting.", param)); + } + String sharelibName = addLibParts[0]; + String sharelibPaths = addLibParts[1]; + if (extraLibs.containsKey(sharelibName)) { + printExtraSharelibUsage(); + throw new IllegalArgumentException(String + .format("Extra sharelib, '%s', has been specified multiple times. " + "Exiting.", param)); + } + extraLibs.put(sharelibName, sharelibPaths); + } + return extraLibs; + } + + private static void printExtraSharelibUsage() { + System.err.println(EXTRALIBS_USAGE); + } + + + @VisibleForTesting + @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "FilenameUtils is used to filter user input. JDK8+ is used.") + void copyExtraLibs(int threadPoolSize, Map<String, String> extraLibs, Path dstPath, FileSystem fs) throws IOException { + for (Map.Entry<String, String> sharelib : extraLibs.entrySet()) { + Path libDestPath = new Path(dstPath.toString() + Path.SEPARATOR + sharelib.getKey()); + for (String libPath : sharelib.getValue().split(EXTRALIBS_PATH_SEPARATOR)) { + File srcFile = new File(FilenameUtils.getFullPath(libPath) + FilenameUtils.getName(libPath)); + Path srcPath = new Path(FilenameUtils.getFullPath(libPath) + FilenameUtils.getName(libPath)); + checkIfSourceFilesExist(srcFile); + copyToSharelib(threadPoolSize, srcFile, srcPath, libDestPath, fs); + } + } + } + + @VisibleForTesting + protected void copyToSharelib(int threadPoolSize, File srcFile, Path srcPath, Path dstPath, FileSystem fs) throws IOException { + if (threadPoolSize > 1) { + long fsLimitsMinBlockSize = fs.getConf() + .getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT); + long bytesPerChecksum = fs.getConf() + .getLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT); + new ConcurrentCopyFromLocal(threadPoolSize, fsLimitsMinBlockSize, bytesPerChecksum) + .concurrentCopyFromLocal(fs, srcFile, dstPath); + + } else { + fs.copyFromLocalFile(false, srcPath, dstPath); + } + } + + @VisibleForTesting + protected void checkIfSourceFilesExist(File srcFile) throws IOException { + if (!srcFile.exists()){ + throw new IOException(srcFile + " cannot be found"); + } + } private static void logError(String errorMessage, Throwable ex) { http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java b/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java new file mode 100644 index 0000000..12cb665 --- /dev/null +++ b/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.tools; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.action.hadoop.security.LauncherSecurityManager; +import org.apache.oozie.service.ConfigurationService; +import org.apache.oozie.service.HadoopAccessorService; +import org.apache.oozie.service.ServiceException; +import org.apache.oozie.service.Services; +import org.apache.oozie.service.ShareLibService; +import org.apache.oozie.service.WorkflowAppService; +import org.apache.oozie.test.XTestCase; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.net.URI; +import java.util.List; + +import static org.apache.oozie.tools.OozieSharelibCLI.EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR; + + +public class IntegrationTestOozieSharelibCLI extends XTestCase { + + private final TemporaryFolder tmpFolder = new TemporaryFolder(); + private File libDirectory; + private Services services = null; + private Path dstPath = null; + private FileSystem fs; + private LauncherSecurityManager launcherSecurityManager; + public final String outPath = "outFolder"; + + @Override + protected void setUp() throws Exception { + launcherSecurityManager = new LauncherSecurityManager(); + launcherSecurityManager.enable(); + tmpFolder.create(); + libDirectory = tmpFolder.newFolder("lib"); + super.setUp(false); + } + + @Override + protected void tearDown() throws Exception { + launcherSecurityManager.disable(); + if (services != null) { + services.destroy(); + } + super.tearDown(); + } + + + /** + * test copy libraries + */ + public void testOozieSharelibCLICreate() throws Exception { + + final int fileNr = 2; + List<File> sharelibFiles = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, fileNr); + + String[] argsCreate = { "create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath() }; + assertEquals("Exit code mismatch", 0, execOozieSharelibCLICommands(argsCreate)); + + ShareLibService sharelibService = getServices().get(ShareLibService.class); + Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(), + ShareLibService.SHARE_LIB_PREFIX); + + checkCopiedSharelibFiles(sharelibFiles, latestLibPath); + } + + /** + * test parallel copy libraries + */ + public void testOozieSharelibCLICreateConcurrent() throws Exception { + + final int testFiles = 7; + final int concurrency = 5; + + List<File> sharelibFiles = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles); + String[] argsCreate = {"create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath(), + "-concurrency", String.valueOf(concurrency)}; + assertEquals("Exit code mismatch",0, execOozieSharelibCLICommands(argsCreate)); + + getTargetFileSysyem(); + ShareLibService sharelibService = getServices().get(ShareLibService.class); + Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(), + ShareLibService.SHARE_LIB_PREFIX); + + checkCopiedSharelibFiles(sharelibFiles, latestLibPath); + } + + public void testOozieSharelibCreateExtraLibs () throws Exception { + File extraLibBirectory1 = tmpFolder.newFolder("extralib1"); + File extraLibBirectory2 = tmpFolder.newFolder("extralib2"); + final int sharelibFileNr = 3; + final int extraSharelib1FileNr = 4; + final int extraSharelib2FileNr = 4; + List<File> shareLibFiles = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, sharelibFileNr); + List<File> extraShareLibFiles1 = OozieSharelibFileOperations + .generateAndWriteFiles(extraLibBirectory1, extraSharelib1FileNr); + List<File> extraShareLibFiles2 = OozieSharelibFileOperations + .generateAndWriteFiles(extraLibBirectory2, extraSharelib2FileNr); + + String extraLib1 = extraLibBirectory1.getName() + EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR + + extraLibBirectory1.getAbsolutePath(); + String extraLib2 = extraLibBirectory2.getName() + EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR + + extraLibBirectory2.getAbsolutePath(); + String[] argsCreate = { "create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath(), + "-" + OozieSharelibCLI.EXTRALIBS, extraLib1, "-" + OozieSharelibCLI.EXTRALIBS, extraLib2}; + + assertEquals("Exit code mismatch",0, execOozieSharelibCLICommands(argsCreate)); + + ShareLibService sharelibService = getServices().get(ShareLibService.class); + Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(), ShareLibService.SHARE_LIB_PREFIX); + Path extraSharelibPath1 = new Path(latestLibPath + Path.SEPARATOR + extraLibBirectory1.getName()); + Path extraSharelibPath2 = new Path(latestLibPath + Path.SEPARATOR + extraLibBirectory2.getName()); + + checkCopiedSharelibFiles(shareLibFiles, latestLibPath); + checkCopiedSharelibFiles(extraShareLibFiles1, extraSharelibPath1); + checkCopiedSharelibFiles(extraShareLibFiles2, extraSharelibPath2); + } + + private void checkCopiedSharelibFiles(List<File> fileList, Path libPath) throws Exception { + FileSystem fileSystem = getTargetFileSysyem(); + for (File f: fileList) { + try (InputStream originalFileStream = new FileInputStream(f); + InputStream copiedFileStream = fileSystem.open(new Path(libPath, f.getName()))) { + assertTrue("The content of the files must be equal", IOUtils.contentEquals(originalFileStream, copiedFileStream)); + } + } + } + + private FileSystem getTargetFileSysyem() throws Exception { + if (fs == null) { + HadoopAccessorService has = getServices().get(HadoopAccessorService.class); + URI uri = new Path(outPath).toUri(); + Configuration fsConf = has.createConfiguration(uri.getAuthority()); + fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf); + } + return fs; + + } + + private Services getServices() throws ServiceException { + if (services == null) { + services = new Services(); + services.get(ConfigurationService.class).getConf() + .set(Services.CONF_SERVICE_CLASSES,"org.apache.oozie.service.LiteWorkflowAppService," + + "org.apache.oozie.service.SchedulerService," + + "org.apache.oozie.service.HadoopAccessorService," + + "org.apache.oozie.service.ShareLibService"); + services.init(); + } + return services; + } + + private Path getDistPath() throws Exception { + if (dstPath == null) { + WorkflowAppService lwas = getServices().get(WorkflowAppService.class); + dstPath = lwas.getSystemLibPath(); + } + return dstPath; + } + + private int execOozieSharelibCLICommands(String[] args) throws Exception { + try { + OozieSharelibCLI.main(args); + } catch (SecurityException ex) { + if (launcherSecurityManager.getExitInvoked()) { + System.out.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")"); + System.err.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")"); + return launcherSecurityManager.getExitCode(); + } else { + throw ex; + } + } + return 1; + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java b/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java index d344300..ca31e96 100644 --- a/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java +++ b/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.io.Writer; +import java.util.ArrayList; import java.util.List; public final class OozieSharelibFileOperations { @@ -36,15 +37,17 @@ public final class OozieSharelibFileOperations { /** * generate a number of files equals with fileNr, and save the fileList parameter * @param fileNr number of files to be generated - * @param fileList a list of the generated files + * @return a list of the generated files * @throws Exception */ - public static void generateAndWriteFiles(File libDirectory, int fileNr, List<File> fileList) throws IOException { + public static List<File> generateAndWriteFiles(File libDirectory, int fileNr) throws IOException { + List<File> fileList = new ArrayList<>(); for (int i=0; i<fileNr; i++) { String fileName = generateFileName(i); String fileContent = generateFileContent(i); fileList.add(writeFile(libDirectory, fileName, fileContent)); } + return fileList; } /** http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java b/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java index d77eba6..51cf93d 100644 --- a/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java +++ b/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java @@ -100,9 +100,7 @@ public class TestConcurrentCopyFromLocal extends XTestCase { private void performAndCheckConcurrentCopy(final int testFiles, final int threadPoolSize, final long fsLimitsMinBlockSize, final long bytesPerChecksum) throws Exception { - List<File> fileList = new ArrayList<>(); - - OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles, fileList); + List<File> fileList = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles); File srcFile = new File(libDirectory.getParentFile().getAbsolutePath()); OozieSharelibCLI.ConcurrentCopyFromLocal concurrentCopy = new OozieSharelibCLI .ConcurrentCopyFromLocal(threadPoolSize, fsLimitsMinBlockSize, bytesPerChecksum); http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java b/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java index bce0433..02f937a 100644 --- a/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java +++ b/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java @@ -110,8 +110,7 @@ public class TestCopyTaskCallable extends XTestCase { private void performAndCheckCallCopyTask(final long blockSize, final int poolSize, final int testFiles) throws Exception { Set<OozieSharelibCLI.CopyTaskConfiguration> failedCopyTasks = new ConcurrentHashSet<>(); - List<File> fileList = new ArrayList<>(); - OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles, fileList); + List<File> fileList = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles); File srcFile = new File(libDirectory.getParentFile().getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java index 5929e5c..18ab8f2 100644 --- a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java +++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java @@ -19,206 +19,139 @@ package org.apache.oozie.tools; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.oozie.action.hadoop.security.LauncherSecurityManager; -import org.apache.oozie.service.ConfigurationService; -import org.apache.oozie.service.HadoopAccessorService; -import org.apache.oozie.service.ServiceException; -import org.apache.oozie.service.Services; -import org.apache.oozie.service.ShareLibService; -import org.apache.oozie.service.WorkflowAppService; -import org.apache.oozie.test.XTestCase; -import org.junit.rules.TemporaryFolder; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.InputStream; +import java.io.IOException; import java.io.PrintStream; -import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.apache.oozie.tools.OozieSharelibCLI.getExtraLibs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyMap; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** * Test OozieSharelibCLI */ -public class TestOozieSharelibCLI extends XTestCase { - public final String outPath = "outFolder"; - private final TemporaryFolder tmpFolder = new TemporaryFolder(); - private File libDirectory; - private Services services = null; - private Path dstPath = null; - private FileSystem fs; +public class TestOozieSharelibCLI { + + private final static String TEST_SHAERELIBNAME1 = "sharelibName"; + private final static String TEST_SHAERELIBNAME2 = "sharelibName2"; + private final static String TEST_EXTRALIBS_PATHS1 = "/path/to/source/,/path/to/some/file"; + private final static String TEST_EXTRALIBS_PATHS2 = "hdfs://my/jar.jar#myjar.jar"; + private LauncherSecurityManager launcherSecurityManager; - @Override - protected void setUp() throws Exception { + + @Before + public void setUp() { launcherSecurityManager = new LauncherSecurityManager(); launcherSecurityManager.enable(); - tmpFolder.create(); - libDirectory = tmpFolder.newFolder("lib"); - super.setUp(false); - } - @Override - protected void tearDown() throws Exception { + @After + public void tearDown() { launcherSecurityManager.disable(); - if (services != null) { - services.destroy(); - } - super.tearDown(); } - /** - * Test help command - */ - public void testHelp() throws Exception { + @Test + public void testHelpCommand() throws Exception { ByteArrayOutputStream data = new ByteArrayOutputStream(); PrintStream oldPrintStream = System.out; System.setOut(new PrintStream(data)); try { - String[] argsHelp = { "help" }; - assertEquals(0, execOozieSharelibCLICommands(argsHelp)); + String[] argsHelp = {"help"}; + assertEquals("Exit code mismatch",0, execOozieSharelibCLICommands(argsHelp)); String helpMessage = data.toString(); - assertTrue(helpMessage.contains( + assertTrue("Missing create <OPTIONS> description from help message", helpMessage.contains( "oozie-setup.sh create <OPTIONS> : create a new timestamped version of oozie sharelib")); - assertTrue(helpMessage.contains( - "oozie-setup.sh upgrade <OPTIONS> : [deprecated][use command \"create\" " - + "to create new version] upgrade oozie sharelib ")); - assertTrue(helpMessage.contains(" oozie-setup.sh help")); - assertTrue(helpMessage.contains( - "-concurrency <arg> Number of threads to be used for copy operations.")); - } - finally { + assertTrue("Missing upgrade <OPTIONS> description from help message", + helpMessage.contains("oozie-setup.sh upgrade <OPTIONS> : [deprecated][use command \"create\" " + + "to create new version] upgrade oozie sharelib ")); + assertTrue("Help message mismatch", helpMessage.contains(" oozie-setup.sh help")); + assertTrue("Missing -concurrency from help message", + helpMessage.contains("-concurrency <arg> Number of threads to be used for copy operations.")); + } finally { System.setOut(oldPrintStream); } - - } - - /** - * test copy libraries - */ - public void testOozieSharelibCLICreate() throws Exception { - - OozieSharelibFileOperations.writeFile(libDirectory, "file1", "test File"); - OozieSharelibFileOperations.writeFile(libDirectory, "file2", "test File2"); - - String[] argsCreate = { "create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath() }; - assertEquals(0, execOozieSharelibCLICommands(argsCreate)); - - FileSystem fs = getTargetFileSysyem(); - ShareLibService sharelibService = getServices().get(ShareLibService.class); - - // test files in new folder - assertEquals(9, fs.getFileStatus(new Path(sharelibService.getLatestLibPath(getDistPath(), - ShareLibService.SHARE_LIB_PREFIX), "file1")).getLen()); - assertEquals(10, fs.getFileStatus(new Path(sharelibService.getLatestLibPath(getDistPath(), - ShareLibService.SHARE_LIB_PREFIX), "file2")).getLen()); - - } - - /** - * test parallel copy libraries - */ - public void testOozieSharelibCLICreateConcurrent() throws Exception { - - final int testFiles = 7; - final int concurrency = 5; - - for (int i = 0; i < testFiles; i++) { - OozieSharelibFileOperations.writeFile(libDirectory, OozieSharelibFileOperations.generateFileName(i), - OozieSharelibFileOperations.generateFileContent(i)); - } - - String[] argsCreate = {"create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath(), - "-concurrency", String.valueOf(concurrency)}; - assertEquals(0, execOozieSharelibCLICommands(argsCreate)); - - getTargetFileSysyem(); - ShareLibService sharelibService = getServices().get(ShareLibService.class); - Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(), - ShareLibService.SHARE_LIB_PREFIX); - - // test files in new folder - - for (int i = 0; i < testFiles; i++) { - String fileName = OozieSharelibFileOperations.generateFileName(i); - String expectedFileContent = OozieSharelibFileOperations.generateFileContent(i); - InputStream in = null; - try { - in = getTargetFileSysyem().open(new Path(latestLibPath, fileName)); - String actualFileContent = IOUtils.toString(in); - assertEquals(fileName, expectedFileContent, actualFileContent); - } finally { - IOUtils.closeQuietly(in); - } - } - } - /** - * test fake command - */ + @Test public void testFakeCommand() throws Exception { ByteArrayOutputStream data = new ByteArrayOutputStream(); PrintStream oldPrintStream = System.err; System.setErr(new PrintStream(data)); try { - String[] argsFake = { "fakeCommand" }; - assertEquals(1, execOozieSharelibCLICommands(argsFake)); - assertTrue(data.toString().contains("Invalid sub-command: invalid sub-command [fakeCommand]")); - assertTrue(data.toString().contains("use 'help [sub-command]' for help details")); - } - finally { + String[] argsFake = {"fakeCommand"}; + assertEquals("Exit code mismatch", 1, execOozieSharelibCLICommands(argsFake)); + assertTrue("Error message missing", + data.toString().contains("Invalid sub-command: invalid sub-command [fakeCommand]")); + assertTrue("Help message missing", data.toString().contains("use 'help [sub-command]' for help details")); + } finally { System.setErr(oldPrintStream); } - - } - - private FileSystem getTargetFileSysyem() throws Exception { - if (fs == null) { - HadoopAccessorService has = getServices().get(HadoopAccessorService.class); - URI uri = new Path(outPath).toUri(); - Configuration fsConf = has.createConfiguration(uri.getAuthority()); - fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf); - } - return fs; - - } - - private Services getServices() throws ServiceException { - if (services == null) { - services = new Services(); - services.get(ConfigurationService.class).getConf() - .set(Services.CONF_SERVICE_CLASSES,"org.apache.oozie.service.LiteWorkflowAppService," - + "org.apache.oozie.service.SchedulerService," - + "org.apache.oozie.service.HadoopAccessorService," - + "org.apache.oozie.service.ShareLibService"); - services.init(); - } - return services; } - private Path getDistPath() throws Exception { - if (dstPath == null) { - WorkflowAppService lwas = getServices().get(WorkflowAppService.class); - dstPath = lwas.getSystemLibPath(); - } - return dstPath; + @Test + public void testCopyingExtraSharelibs() throws IOException { + OozieSharelibCLI oozieSharelibCLI = spy(new OozieSharelibCLI() { + @Override + protected void checkIfSourceFilesExist(File srcFile) { } + + @Override + protected void copyToSharelib(int threadPoolSize, File srcFile, Path srcPath, Path dstPath, FileSystem fs) { + String dstStr = dstPath.toString(); + String actualSharelibName = dstStr.substring(dstStr.lastIndexOf(Path.SEPARATOR) + 1); + Assert.assertTrue("Expected sharelib name missing", + Arrays.asList(TEST_SHAERELIBNAME1, TEST_SHAERELIBNAME2).contains(actualSharelibName)); + + List<File> expectedTestFiles = new ArrayList<>(); + for (String s : Arrays.asList(TEST_EXTRALIBS_PATHS1.split(","))) { + expectedTestFiles.add(new File(s)); + } + for (String s : Arrays.asList(TEST_EXTRALIBS_PATHS2.split(","))) { + expectedTestFiles.add(new File(s)); + } + Assert.assertTrue("Expected file missing", expectedTestFiles.contains(srcFile)); + } + }); + + final FileSystem fs = mock(FileSystem.class); + final Path dst = new Path("/share/lib"); + String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + TEST_EXTRALIBS_PATHS1; + String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + TEST_EXTRALIBS_PATHS2; + Map<String, String> addLibs = getExtraLibs(new String[] {extraLibOption1, extraLibOption2}); + oozieSharelibCLI.copyExtraLibs(1, addLibs, dst, fs); + verify(oozieSharelibCLI, times(1)) + .copyExtraLibs(anyInt(), anyMap(), any(Path.class), any(FileSystem.class)); } private int execOozieSharelibCLICommands(String[] args) throws Exception { try { OozieSharelibCLI.main(args); - } - catch (SecurityException ex) { + } catch (SecurityException ex) { if (launcherSecurityManager.getExitInvoked()) { System.out.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")"); System.err.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")"); return launcherSecurityManager.getExitCode(); - } - else { + } else { throw ex; } } http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java ---------------------------------------------------------------------- diff --git a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java new file mode 100644 index 0000000..f18db4d --- /dev/null +++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.tools; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +import static org.apache.oozie.tools.OozieSharelibCLI.getExtraLibs; + +public class TestOozieSharelibCLIExtraArgsParser { + + private final static String TEST_SHAERELIBNAME1 = "sharelibName"; + private final static String TEST_SHAERELIBNAME2 = "sharelibName2"; + private final static String TEST_EXTRALIBS_PATHS1 = "/path/to/source/,/path/to/some/file"; + private final static String TEST_EXTRALIBS_PATHS2 = "hdfs://my/jar.jar#myjar.jar,path/to/source/"; + + @Test + public void testParsingExtraSharelibs() { + String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + TEST_EXTRALIBS_PATHS1; + String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + TEST_EXTRALIBS_PATHS2; + Map<String, String> addLibs = getExtraLibs(new String[] {extraLibOption1, extraLibOption2}); + + Assert.assertEquals("Extra libs count mismatch", 2, addLibs.size()); + Assert.assertEquals("Missing extra lib", TEST_EXTRALIBS_PATHS1, addLibs.get(TEST_SHAERELIBNAME1)); + Assert.assertEquals("Missing extra lib", TEST_EXTRALIBS_PATHS2, addLibs.get(TEST_SHAERELIBNAME2)); + } + + @Test (expected = IllegalArgumentException.class) + public void testParsingExtraSharelibsMissingValue() { + String extraLibOption1 = TEST_SHAERELIBNAME1 + "="; + String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + TEST_EXTRALIBS_PATHS2; + getExtraLibs(new String[] {extraLibOption1, extraLibOption2}); + } + + @Test (expected = IllegalArgumentException.class) + public void testParsingExtraSharelibsMissingKey() { + String extraLibOption1 = TEST_EXTRALIBS_PATHS1; + String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + TEST_EXTRALIBS_PATHS2; + getExtraLibs(new String[] {extraLibOption1, extraLibOption2}); + } + + @Test(expected = IllegalArgumentException.class) + public void testParsingSharelibNamePresentMultipleTimes() { + String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + TEST_EXTRALIBS_PATHS1; + String extraLibOption2 = TEST_SHAERELIBNAME1 + "=" + TEST_EXTRALIBS_PATHS2; + getExtraLibs(new String[] {extraLibOption1, extraLibOption2}); + } +} \ No newline at end of file
