This is an automated email from the ASF dual-hosted git repository. asalamon74 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push: new 6bdca1d OOZIE-3179 Adding a configurable config-default.xml location to a workflow (jphelps via asalamon74) 6bdca1d is described below commit 6bdca1d1c043d6ad688f53602667ca5201af5a28 Author: Andras Salamon <asalamo...@apache.org> AuthorDate: Fri Sep 20 13:57:41 2019 +0200 OOZIE-3179 Adding a configurable config-default.xml location to a workflow (jphelps via asalamon74) --- .../java/org/apache/oozie/client/OozieClient.java | 2 + .../apache/oozie/command/wf/SubmitXCommand.java | 54 ++++++++--- .../oozie/command/wf/TestSubmitXCommand.java | 107 +++++++++++++++++++++ release-log.txt | 1 + 4 files changed, 149 insertions(+), 15 deletions(-) diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index a0f586d..1c6966f 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -174,6 +174,8 @@ public class OozieClient { public static final String LIBPATH = "oozie.libpath"; + public static final String CONFIG_PATH = "oozie.default.configuration.path"; + public static final String USE_SYSTEM_LIBPATH = "oozie.use.system.libpath"; public static final String OOZIE_SUSPEND_ON_NODES = "oozie.suspend.on.nodes"; diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java index 1352db9..c63a24e 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitXCommand.java @@ -135,30 +135,54 @@ public class SubmitXCommand extends WorkflowXCommand<String> { Configuration fsConf = has.createConfiguration(uri.getAuthority()); FileSystem fs = has.createFileSystem(user, uri, fsConf); - Path configDefault = null; - Configuration defaultConf = null; + + + // Generate a list of all default configuration files to be processed + // A default-conf.xml in the workflow directory should have higher precedence than service defaults + // Adding oozie.default.configuration.path files first will allow default-conf.xml to override duplicates + ArrayList<Path> configDefault = new ArrayList<>(); + Configuration defaultConf = new XConfiguration(); + + String[] defaultConfFiles = conf.getStrings(OozieClient.CONFIG_PATH); + if (defaultConfFiles != null && defaultConfFiles.length > 0) { + for ( String defaultConfFile : defaultConfFiles ) { + if (defaultConfFile.trim().length() > 0) { + configDefault.add(new Path(defaultConfFile.trim())); + } + + } + } + // app path could be a directory Path path = new Path(uri.getPath()); if (!fs.isFile(path)) { - configDefault = new Path(path, CONFIG_DEFAULT); + configDefault.add(new Path(path, CONFIG_DEFAULT)); } else { - configDefault = new Path(path.getParent(), CONFIG_DEFAULT); + configDefault.add(new Path(path.getParent(), CONFIG_DEFAULT)); } - if (fs.exists(configDefault)) { - try { - defaultConf = new XConfiguration(fs.open(configDefault)); - PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_USER_PROPERTIES); - PropertiesUtils.checkDefaultDisallowedProperties(defaultConf); - XConfiguration.injectDefaults(defaultConf, conf); + // + for (Path configDefaultFile: configDefault) { + + LOG.debug("Loading Configuration file {0}", configDefaultFile.getName()); + Configuration defaultConfigs = null; + if (fs.exists(configDefaultFile)) { + try { + defaultConfigs = new XConfiguration(fs.open(configDefaultFile)); + PropertiesUtils.checkDisallowedProperties(defaultConfigs, DISALLOWED_USER_PROPERTIES); + PropertiesUtils.checkDefaultDisallowedProperties(defaultConfigs); + XConfiguration.injectDefaults(defaultConfigs, conf); + + } + catch (IOException ex) { + throw new IOException("Failed Loading default configuration file: " + + configDefaultFile.getName() + ex.getMessage(), ex); + } } - catch (IOException ex) { - throw new IOException("default configuration file, " + ex.getMessage(), ex); + if (defaultConfigs != null) { + XConfiguration.copy(resolveDefaultConfVariables(defaultConfigs),defaultConf); } } - if (defaultConf != null) { - defaultConf = resolveDefaultConfVariables(defaultConf); - } WorkflowApp app = wps.parseDef(conf, defaultConf); XConfiguration protoActionConf = wps.createProtoActionConf(conf, true); diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java index 2cd263f..c27ce23 100644 --- a/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/wf/TestSubmitXCommand.java @@ -421,6 +421,113 @@ public class TestSubmitXCommand extends XDataTestCase { assertEquals(getNameNodeUri()+"/default-output-dir", actionConf.get("mixed")); } + public void testWFConfigPathVarResolve() throws Exception { + final OozieClient wfClient = LocalOozie.getClient(); + + OutputStream os1 = new FileOutputStream(getTestCaseDir() + "/config-default.xml"); + XConfiguration defaultConf = new XConfiguration(); + defaultConf.set("outputDir", "default-output-dir"); + defaultConf.set("should_resolve", "${should.resolve}"); + defaultConf.set("foo.bar", "default-foo-bar"); + defaultConf.set("foobarRef", "${foo.bar}"); + defaultConf.writeXml(os1); + os1.close(); + + OutputStream os2 = new FileOutputStream(getTestCaseDir() + "/extra-config-file1.xml"); + XConfiguration confFile1 = new XConfiguration(); + confFile1.set("outputDir", "default-output-dir_unused1"); + confFile1.set("should_resolve_file1", "${should.resolve.file1}"); + confFile1.set("key", "default_value_unused"); + confFile1.set("foobarRef1", "${foo.bar}"); + confFile1.writeXml(os2); + os2.close(); + + + OutputStream os3 = new FileOutputStream(getTestCaseDir() + "/extra-config-file2.xml"); + XConfiguration confFile2 = new XConfiguration(); + confFile2.set("outputDir", "default-output-dir_unused2"); + confFile2.set("should_resolve_file2", "${should.resolve.file2}"); + confFile2.set("key", "default_value"); + confFile2.set("foobarRef2", "${foobarRef1}"); + confFile2.writeXml(os3); + os3.close(); + + + String workflowUri = getTestCaseFileUri("workflow.xml"); + String actionXml = "<map-reduce>" + + "<job-tracker>${jobTracker}</job-tracker>" + + "<name-node>${nameNode}</name-node>" + + " <prepare>" + + " <delete path=\"${nameNode}/user/${wf:user()}/mr/${outputDir}\"/>" + + " </prepare>" + + " <configuration>" + + " <property><name>bb</name><value>BB</value></property>" + + " <property><name>cc</name><value>from_action</value></property>" + + " </configuration>" + + " </map-reduce>"; + String wfXml = "<workflow-app xmlns=\"uri:oozie:workflow:0.5\" name=\"map-reduce-wf\">" + + " <start to=\"mr-node\"/>" + + " <action name=\"mr-node\">" + + actionXml + + " <ok to=\"end\"/>" + + " <error to=\"fail\"/>" + + "</action>" + + "<kill name=\"fail\">" + + " <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + + "</kill>" + + "<end name=\"end\"/>" + + "</workflow-app>"; + + writeToFile(wfXml, workflowUri); + Configuration conf = new XConfiguration(); + conf.set("nameNode", getNameNodeUri()); + conf.set("jobTracker", getJobTrackerUri()); + conf.set("foobarRef", "foobarRef"); + conf.set(OozieClient.APP_PATH, workflowUri); + conf.set(OozieClient.USER_NAME, getTestUser()); + conf.set("should.resolve", "resolved"); + conf.set("should.resolve.file1", "resolved"); + conf.set("should.resolve.file2", "resolved"); + conf.set("oozie.default.configuration.path", getTestCaseDir() + "/extra-config-file1.xml," + + getTestCaseDir() + "/extra-config-file2.xml"); + SubmitXCommand sc = new SubmitXCommand(conf); + final String jobId = sc.call(); + new StartXCommand(jobId).call(); + waitFor(15 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + return wfClient.getJobInfo(jobId).getStatus() == WorkflowJob.Status.KILLED; + } + }); + String actionId = jobId + "@mr-node"; + WorkflowActionBean action = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQueryExecutor.WorkflowActionQuery + .GET_ACTION, actionId); + Element eAction = XmlUtils.parseXml(action.getConf()); + Element eConf = eAction.getChild("configuration", eAction.getNamespace()); + Configuration actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(eConf).toString())); + assertEquals("Variable should_resolve!='resolved' from config-default.xml", + "resolved", actionConf.get("should_resolve")); + assertEquals("Variable 'should_resolve_file1' from extra-config-file1 did not resolve correctly", + "resolved", actionConf.get("should_resolve_file1")); + assertEquals("Variable 'should_resolve_file2' from extra-config-file2 did not resolve correctly", + "resolved", actionConf.get("should_resolve_file2")); + assertEquals("Variable 'outputDir' from config-default did not resolve correctly", + "default-output-dir", actionConf.get("outputDir")); + assertEquals("Variable 'bb' from workflow did not resolve correctly", + "BB", actionConf.get("bb")); + assertEquals("Variable 'cc' from workflow did not resolve correctly", + "from_action", actionConf.get("cc")); + assertEquals("Variable 'foo.bar' from config-default did not resolve correctly", + "default-foo-bar", actionConf.get("foo.bar")); + assertEquals("Variable 'foobarRef' from config-default did not resolve correctly", + "default-foo-bar", actionConf.get("foobarRef")); + assertEquals("Variable 'foobarRef1' from extra-config-file1 did not resolve correctly", + "default-foo-bar", actionConf.get("foobarRef1")); + assertEquals("Variable 'foobarRef2' from extra-config-file2 did not resolve correctly", + "default-foo-bar", actionConf.get("foobarRef2")); + assertEquals("Variable 'key' from extra-config-file2 did not resolve correctly", + "default_value", actionConf.get("key")); + } + private void writeToFile(String appXml, String appPath) throws IOException { File wf = new File(URI.create(appPath)); diff --git a/release-log.txt b/release-log.txt index c5cbc5f..d777a38 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.2.0 release (trunk - unreleased) +OOZIE-3179 Adding a configurable config-default.xml location to a workflow (jphelps via asalamon74) OOZIE-3405 SSH action shows empty error Message and Error code (matijhs via asalamon74) OOZIE-3542 Handle better old Hdfs implementations in ECPolicyDisabler (zsombor dionusos via kmarton) OOZIE-3540 Use StringBuilder instead of StringBuffer if concurrent access is not required (zsombor via asalamon74)