[2/2] oozie git commit: OOZIE-2601 Ability to use local paths for the sharelib
OOZIE-2601 Ability to use local paths for the sharelib
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/25a8b99d
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/25a8b99d
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/25a8b99d
Branch: refs/heads/master
Commit: 25a8b99d5197c4e18acf0fd332c4396450d3d551
Parents: 18694b6
Author: Attila Sasvari
Authored: Thu Jul 20 13:20:47 2017 +0200
Committer: Attila Sasvari
Committed: Thu Jul 20 17:18:20 2017 +0200
--
.../oozie/action/hadoop/JavaActionExecutor.java | 109 ++-
.../apache/oozie/service/ShareLibService.java | 75 +-
.../org/apache/oozie/util/ClasspathUtils.java | 27 +
.../java/org/apache/oozie/util/FSUtils.java | 14 +
core/src/main/resources/oozie-default.xml | 3 +-
.../service/TestShareLibMappingFileInput.java | 88 ++
.../oozie/service/TestShareLibService.java | 873 ++-
docs/src/site/twiki/AG_Install.twiki| 30 +-
release-log.txt | 1 +
9 files changed, 724 insertions(+), 496 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
--
diff --git
a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 89bac95..23e1f69 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -18,44 +18,24 @@
package org.apache.oozie.action.hadoop;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.StringReader;
-import java.net.ConnectException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.commons.io.IOUtils;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.Closeables;
import com.google.common.primitives.Ints;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -95,6 +75,7 @@ import org.apache.oozie.service.WorkflowAppService;
import org.apache.oozie.util.ClasspathUtils;
import org.apache.oozie.util.ELEvaluationException;
import org.apache.oozie.util.ELEvaluator;
+import org.apache.oozie.util.FSUtils;
import org.apache.oozie.util.JobUtils;
import org.apache.oozie.util.LogUtils;
import org.apache.oozie.util.PropertiesUtils;
@@ -105,8 +86,28 @@ import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
-import com.google.common.collect.ImmutableList;
-import com.google.common.io.Closeables;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.ConnectException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.Set;
public class JavaActionExecutor extends ActionExecutor {
@@ -463,8 +464,11 @@ public class JavaActionExecutor extends ActionExecutor {
[1/2] oozie git commit: OOZIE-2601 Ability to use local paths for the sharelib
Repository: oozie
Updated Branches:
refs/heads/master 18694b616 -> 25a8b99d5
http://git-wip-us.apache.org/repos/asf/oozie/blob/25a8b99d/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
--
diff --git
a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
index 390545d..a4f3919 100644
--- a/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestShareLibService.java
@@ -26,14 +26,18 @@ import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
+import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
@@ -54,23 +58,28 @@ import org.junit.Test;
import com.google.common.collect.Lists;
public class TestShareLibService extends XFsTestCase {
+private static final String HDFS_SCHEME_PREFIX = "hdfs";
+private static final String TEST_HDFS_HOME = "/user/test/";
+private static final String TEST_MAPPING_FILENAME = "config.properties";
+private static final String TEST_HDFS_MAPPING_FILE_PATH = TEST_HDFS_HOME +
TEST_MAPPING_FILENAME;
+private static final String SHARELIB_PATH = "shareLibPath/";
Services services;
private static String testCaseDirPath;
-String shareLibPath = "shareLibPath";
SimpleDateFormat dt = new SimpleDateFormat("MMddHHmmss");
-final String sharelibPath = "sharelib";
-final String metaFile = "/user/test/config.properties";
@Override
protected void setUp() throws Exception {
super.setUp();
testCaseDirPath = getTestCaseDir();
+services = new Services();
+setSystemProps();
}
@Override
protected void tearDown() throws Exception {
super.tearDown();
+services.destroy();
}
private void setSystemProps() throws IOException {
@@ -78,22 +87,21 @@ public class TestShareLibService extends XFsTestCase {
IOUtils.createJar(new File(getTestCaseDir()), MyPig.class.getName() +
".jar", MyPig.class);
IOUtils.createJar(new File(getTestCaseDir()), TestHive.class.getName()
+ ".jar", TestHive.class);
-Configuration conf =
services.get(ConfigurationService.class).getConf();
+Configuration conf = getOozieConfig();
conf.set(WorkflowAppService.SYSTEM_LIB_PATH, getFsTestCaseDir() +
"/share/lib");
conf.set(Services.CONF_SERVICE_CLASSES,
conf.get(Services.CONF_SERVICE_CLASSES) + ","
+ DummyShareLibService.class.getName());
conf.setStrings(ActionService.CONF_ACTION_EXECUTOR_CLASSES,
DummyPigActionExecutor.class.getName(),
DummyHiveActionExecutor.class.getName());
-
}
public static class DummyShareLibService extends ShareLibService {
@Override
public String findContainingJar(Class clazz) {
if (JavaActionExecutor.getCommonLauncherClasses().contains(clazz))
{
-return testCaseDirPath + "/" + MyOozie.class.getName() +
".jar";
+return testCaseDirPath + Path.SEPARATOR +
MyOozie.class.getName() + ".jar";
}
-return testCaseDirPath + "/" + clazz.getName() + ".jar";
+return testCaseDirPath + Path.SEPARATOR + clazz.getName() + ".jar";
}
}
@@ -128,9 +136,7 @@ public class TestShareLibService extends XFsTestCase {
@Test
public void testfailFast() throws Exception {
-services = new Services();
-setSystemProps();
-Configuration conf =
services.get(ConfigurationService.class).getConf();
+Configuration conf = getOozieConfig();
conf.set(ShareLibService.FAIL_FAST_ON_STARTUP, "true");
// Set dummyfile as metafile which doesn't exist.
conf.set(ShareLibService.SHARELIB_MAPPING_FILE, String.valueOf(new
Date().getTime()));
@@ -141,99 +147,68 @@ public class TestShareLibService extends XFsTestCase {
catch (Throwable e) {
assertTrue(e.getMessage().contains("E0104: Could not fully
initialize service"));
}
-finally {
-services.destroy();
-}
}
@Test
public void testCreateLauncherLibPath() throws Exception {
-services = new Services();
-setSystemProps();
-Configuration conf =
services.get(ConfigurationService.class).getConf();
-conf.set(S
