This is an automated email from the ASF dual-hosted git repository. asalamon74 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push: new 0eb006f OOZIE-3574 JavaAction create incorrect fileSystem instance in addActionLibs method (zuston via asalamon74) 0eb006f is described below commit 0eb006f655e6fe92b471dcc76cd85f8329f7c8f2 Author: Andras Salamon <asalamo...@apache.org> AuthorDate: Fri Jan 31 13:33:42 2020 +0100 OOZIE-3574 JavaAction create incorrect fileSystem instance in addActionLibs method (zuston via asalamon74) --- .../oozie/action/hadoop/JavaActionExecutor.java | 4 +- .../action/hadoop/ActionExecutorTestCase.java | 65 +++++++++++++--------- .../hadoop/TestJavaActionExecutorLibAddition.java | 53 +++++++++++++----- release-log.txt | 1 + 4 files changed, 81 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java index 0940703..f4f1902 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java @@ -718,7 +718,7 @@ public class JavaActionExecutor extends ActionExecutor { if (files == null) return; for (FileStatus file : files) { if (!shareLibExcluder.shouldExclude(file.getPath().toUri())) { - addToCache(conf, appPath, file.getPath().toUri().getPath(), false); + addToCache(conf, appPath, file.getPath().toString(), false); } } } @@ -734,7 +734,7 @@ public class JavaActionExecutor extends ActionExecutor { Path actionLibsPath = new Path(actionLibsStr); String user = conf.get("user.name"); FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, - appPath.toUri(), conf); + actionLibsPath.toUri(), conf); if (fs.exists(actionLibsPath)) { addFilesToCacheIfNotExcluded(appPath, conf, fs.listStatus(actionLibsPath)); } diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java index 6f118bd..33c5382 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/ActionExecutorTestCase.java @@ -18,6 +18,31 @@ package org.apache.oozie.action.hadoop; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.Reader; +import java.io.StringReader; +import java.io.Writer; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -42,8 +67,8 @@ import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.LiteWorkflowStoreService; import org.apache.oozie.service.Services; -import org.apache.oozie.service.UUIDService; import org.apache.oozie.service.ShareLibService; +import org.apache.oozie.service.UUIDService; import org.apache.oozie.service.WorkflowAppService; import org.apache.oozie.service.WorkflowStoreService; import org.apache.oozie.test.XHCatTestCase; @@ -58,32 +83,6 @@ import org.apache.oozie.workflow.lite.EndNodeDef; import org.apache.oozie.workflow.lite.LiteWorkflowApp; import org.apache.oozie.workflow.lite.StartNodeDef; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.io.Reader; -import java.io.StringReader; -import java.io.Writer; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Collection; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.FileInputStream; - -import java.text.SimpleDateFormat; - public abstract class ActionExecutorTestCase extends XHCatTestCase { protected static final int JOB_TIMEOUT = 100_000; @@ -517,12 +516,24 @@ public abstract class ActionExecutorTestCase extends XHCatTestCase { } } + void createFilesWithFs2(Collection<Path> paths) throws Exception{ + for(Path p : paths){ + getFileSystem2().create(p); + } + } + void makeDirs(Path... dirs) throws Exception{ for(Path p : dirs){ getFileSystem().mkdirs(p); } } + void makeDirWithFs2(Path... dirs) throws IOException { + for(Path p : dirs){ + getFileSystem2().mkdirs(p); + } + } + private void assertContainsJarsOrNot(boolean contains, String cacheFilesStr, Collection<Path> jars) { for (Path jar : jars) { assertEquals("Unexpected distributed cache file content", contains, cacheFilesStr.contains(jar.toString())); diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java index 00e7777..ed2e20c 100644 --- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java +++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutorLibAddition.java @@ -18,6 +18,18 @@ package org.apache.oozie.action.hadoop; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.Path; @@ -33,19 +45,6 @@ import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XmlUtils; import org.jdom.Element; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - - import static org.apache.oozie.action.hadoop.JavaActionExecutor.ACTION_SHARELIB_FOR; import static org.apache.oozie.action.hadoop.JavaActionExecutor.SHARELIB_EXCLUDE_SUFFIX; @@ -98,6 +97,12 @@ public class TestJavaActionExecutorLibAddition extends ActionExecutorTestCase { }; @Override + protected void beforeSetUp() throws Exception { + super.beforeSetUp(); + setSystemProperty("oozie.test.hadoop.minicluster2", "true"); + } + + @Override protected void setSystemProps() throws Exception { super.setSystemProps(); setHadoopSystemProps(); @@ -303,6 +308,12 @@ public class TestJavaActionExecutorLibAddition extends ActionExecutorTestCase { return actionLibPath; } + private Path getActionLibPathWithFs2() throws Exception { + Path actionLibPath = new Path(getFs2TestCaseDir(), "actionlibs"); + makeDirWithFs2(actionLibPath); + return actionLibPath; + } + private List<Path> createTestActionLibPaths(Path... paths) throws Exception{ final Path actionLibPath = new Path(getFsTestCaseDir(), "actionlibs"); makeDirs(actionLibPath); @@ -310,6 +321,11 @@ public class TestJavaActionExecutorLibAddition extends ActionExecutorTestCase { return Arrays.asList(paths); } + private List<Path> createTestActionLibPathsWithFs2(Path... paths) throws Exception{ + createFilesWithFs2(Arrays.asList(paths)); + return Arrays.asList(paths); + } + public void testAddingActionLibDir() throws Exception{ makeDirs(getActionLibPath()); List<Path> expectedJars = createTestActionLibPaths( @@ -321,6 +337,17 @@ public class TestJavaActionExecutorLibAddition extends ActionExecutorTestCase { assertContainsJars(getDistributedCacheFilesStr(jobConf), expectedJars); } + public void testAddingActionLibDirWhenActionJarsWithFs2() throws Exception { + Path baseActionLibPath = getActionLibPathWithFs2(); + List<Path> expectedJars = createTestActionLibPathsWithFs2( + new Path(baseActionLibPath, "jar1.jar"), + new Path(baseActionLibPath, "jar2.jar")); + + Context context = createContextUsingSharelib(baseActionLibPath); + Configuration jobConf = createActionExecutorAndSetLibFilesArchives(context); + assertContainsJars(getDistributedCacheFilesStr(jobConf), expectedJars); + } + public void testAddingActionLibFile() throws Exception{ List<Path> expectedJars = createTestActionLibPaths(new Path(getFsTestCaseDir(), "jar3.jar")); diff --git a/release-log.txt b/release-log.txt index 21f70f7..31eeda3 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.3.0 release (trunk - unreleased) +OOZIE-3574 JavaAction create incorrect fileSystem instance in addActionLibs method (zuston via asalamon74) OOZIE-3569 SSH Action should add checking success file (zuston via asalamon74) OOZIE-3305 Prometheus /metrics http endpoint for monitoring integration (qsbao via gezapeti) OOZIE-3575 Add credential support for cloud file systems (matijhs via gezapeti)