Repository: oozie Updated Branches: refs/heads/master 8b247f28f -> 8aca098c2
http://git-wip-us.apache.org/repos/asf/oozie/blob/8aca098c/core/src/test/java/org/apache/oozie/service/TestSchemaService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestSchemaService.java b/core/src/test/java/org/apache/oozie/service/TestSchemaService.java index 940868a..88a10db 100644 --- a/core/src/test/java/org/apache/oozie/service/TestSchemaService.java +++ b/core/src/test/java/org/apache/oozie/service/TestSchemaService.java @@ -18,8 +18,11 @@ package org.apache.oozie.service; +import org.apache.oozie.service.Services; import org.apache.oozie.service.SchemaService.SchemaName; import org.apache.oozie.test.XTestCase; +import org.apache.oozie.util.XmlUtils; +import org.jdom.Element; import javax.xml.validation.Validator; import javax.xml.transform.stream.StreamSource; @@ -29,32 +32,26 @@ import java.io.StringReader; public class TestSchemaService extends XTestCase { private static final String APP1 = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='app'>" + - " <start to='end'/>" + - " <end name='end'/>" + + "<start to='end'/>" + + "<end name='end'/>" + "</workflow-app>"; private static final String APP_V2 = "<workflow-app xmlns='uri:oozie:workflow:0.2' name='app'>" + "<start to='end'/>" + "<end name='end'/>" + "</workflow-app>"; - private static final String APP_V25 = "<?xml version=\"1.0\"?>\n" + - "<workflow-app xmlns=\"uri:oozie:workflow:0.2.5\" name=\"app\">\n" + - " <credentials/>\n" + - " <start to=\"end\"/>\n" + - " <end name=\"end\"/>\n" + - "</workflow-app>"; + private static final String APP_V25 = "<workflow-app xmlns='uri:oozie:workflow:0.2.5' name='app'>" + "<credentials></credentials>"+"<start to='end'/>" + + "<end name='end'/>" + "</workflow-app>"; - private static final String WF_SLA_APP = "<workflow-app xmlns='uri:oozie:workflow:0.2' name='app'" - + " xmlns:sla='uri:oozie:sla:0.1'>" - + " <start to='end'/>" - + " <end name='end'/>" - + " <sla:info><sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " - + " <sla:should-start>5</sla:should-start> <sla:should-end>50</sla:should-end> " + private static final String WF_SLA_APP = "<workflow-app xmlns='uri:oozie:workflow:0.2' name='app' xmlns:sla='uri:oozie:sla:0.1'>" + + "<start to='end'/>" + + "<end name='end'/>" + + "<sla:info> <sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " + + "<sla:should-start>5</sla:should-start> <sla:should-end>50</sla:should-end> " + "<sla:alert-contact>a...@example.com</sla:alert-contact> <sla:dev-contact>a...@example.com</sla:dev-contact>" + " <sla:qa-contact>a...@example.com</sla:qa-contact> <sla:se-contact>a...@example.com</sla:se-contact>" + "</sla:info>" + "</workflow-app>"; - private static final String WF_SLA_APP_NW = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='app'" + - " xmlns:sla='uri:oozie:sla:0.1'>" + private static final String WF_SLA_APP_NW = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='app' xmlns:sla='uri:oozie:sla:0.1'>" + "<start to='end'/>" + "<end name='end'/>" + "<sla:info> <sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " @@ -63,6 +60,14 @@ public class TestSchemaService extends XTestCase { + " <sla:qa-contact>a...@example.com</sla:qa-contact> <sla:se-contact>a...@example.com</sla:se-contact>" + "</sla:info>" + "</workflow-app>"; + private static final String COORD_APP1 = "<coordinator-app name=\"NAME\" frequency=\"${coord:days(1)}\" start=\"${start}\" end=\"${end}\" timezone=\"${timezone}\" xmlns=\"uri:oozie:coordinator:0.2\">" + + "<controls> <timeout>${timeout}</timeout> <concurrency>${concurrency_level}</concurrency> <execution>${execution_order}</execution> </controls>" + + "<datasets> <include>${include_ds_files}</include> <!-- Synchronous datasets --> <dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"${start}\" timezone=\"${timezone}\"> " + + "<uri-template>${baseFsURI}/${YEAR}/${DAY}</uri-template> </dataset> </datasets> " + + "<input-events> <data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in> <data-in name=\"B\" dataset=\"b\"> <start-instance>${coord:current(-2)}</start-instance> <end-instance>${coord:current(0)}</end-instance> </data-in> </input-events> <output-events> " + + "<data-out name=\"LOCAL_A\" dataset=\"local_a\"> <instance>${coord:current(0)}</instance> </data-out> </output-events> <action> <workflow> <app-path>${app_path}</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataIn('B')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> </property> <property> <name>TESTING</name> <value>${start}</value> </property> </configuration> </workflow> </action> </coordinator-app>"; + + private static final String APP2 = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='app'>" + "<start to='a'/>" + "<action name='a'>" + @@ -96,230 +101,10 @@ public class TestSchemaService extends XTestCase { "<end name='end'/>" + "</workflow-app>"; - private static final String WF_GLOBAL_LAUNCHER_CONF = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"test-wf\">\n" + - " <global>\n" + - " <launcher>\n" + - " <memory.mb>1024</memory.mb>\n" + - " <vcores>2</vcores>\n" + - " <java-opts>dummyJavaOpts</java-opts>\n" + - " <env>dummyEnv</env>\n" + - " <queue>dummyQueue</queue>\n" + - " <sharelib>a,b,c</sharelib>\n" + - " </launcher>\n" + - " </global>\n" + - " <start to=\"a\"/>\n" + - " <action name=\"a\">\n" + - " <fs>\n" + - " <mkdir path='/tmp'/>\n" + - " </fs>\n" + - " <ok to=\"e\"/>\n" + - " <error to=\"k\"/>\n" + - " </action>\n" + - " <kill name=\"k\">\n" + - " <message>kill</message>\n" + - " </kill>\n" + - " <end name=\"e\"/>\n" + - "</workflow-app>\n"; - - private static final String HIVE_ACTION_LAUNCHER_CONF = - "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"hive-wf\">\n" + - " <start to=\"hive-node\"/>\n" + - " <action name=\"hive-node\">\n" + - " <hive xmlns=\"uri:oozie:hive-action:1.0\">\n" + - " <job-tracker>${jobTracker}</job-tracker>\n" + - " <name-node>${nameNode}</name-node>\n" + - " <prepare>\n" + - " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/hive\"/>\n" + - " <mkdir path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data\"/>\n" + - " </prepare>\n" + - " <launcher>\n" + - " <memory.mb>1024</memory.mb>\n" + - " <vcores>2</vcores>\n" + - " <java-opts>dummyJavaOpts</java-opts>\n" + - " <env>dummyEnv</env>\n" + - " <queue>dummyQueue</queue>\n" + - " <sharelib>a,b,c</sharelib>\n" + - " </launcher>\n" + - " <configuration>\n" + - " <property>\n" + - " <name>mapred.job.queue.name</name>\n" + - " <value>${queueName}</value>\n" + - " </property>\n" + - " </configuration>\n" + - " <script>script.q</script>\n" + - " <param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param>\n" + - " <param>OUTPUT=/user/${wf:user()}/${examplesRoot}/output-data/hive</param>\n" + - " </hive>\n" + - " <ok to=\"end\"/>\n" + - " <error to=\"fail\"/>\n" + - " </action>\n" + - " <kill name=\"fail\">\n" + - " <message>Hive failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>\n" + - " </kill>\n" + - " <end name=\"end\"/>\n" + - "</workflow-app>\n"; - - private static final String HIVE2_ACTION_LAUNCHER_CONF = - "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"hive2-wf\">\n" + - " <start to=\"hive2-node\"/>\n" + - " <action name=\"hive2-node\">\n" + - " <hive2 xmlns=\"uri:oozie:hive2-action:1.0\">\n" + - " <job-tracker>${jobTracker}</job-tracker>\n" + - " <name-node>${nameNode}</name-node>\n" + - " <prepare>\n" + - " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/hive2\"/>\n" + - " <mkdir path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data\"/>\n" + - " </prepare>\n" + - " <launcher>\n" + - " <memory.mb>1024</memory.mb>\n" + - " <vcores>2</vcores>\n" + - " <java-opts>dummyJavaOpts</java-opts>\n" + - " <env>dummyEnv</env>\n" + - " <queue>dummyQueue</queue>\n" + - " <sharelib>a,b,c</sharelib>\n" + - " </launcher>\n" + - " <configuration>\n" + - " <property>\n" + - " <name>mapred.job.queue.name</name>\n" + - " <value>${queueName}</value>\n" + - " </property>\n" + - " </configuration>\n" + - " <jdbc-url>${jdbcURL}</jdbc-url>\n" + - " <script>script.q</script>\n" + - " <param>INPUT=/user/${wf:user()}/${examplesRoot}/input-data/table</param>\n" + - " <param>OUTPUT=/user/${wf:user()}/${examplesRoot}/output-data/hive2</param>\n" + - " </hive2>\n" + - " <ok to=\"end\"/>\n" + - " <error to=\"fail\"/>\n" + - " </action>\n" + - " <kill name=\"fail\">\n" + - " <message>Hive2 (Beeline) action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>\n" + - " </kill>\n" + - " <end name=\"end\"/>\n" + - "</workflow-app>\n"; - - private static final String SHELL_ACTION_LAUNCHER_CONF = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" " + - " name=\"shell-wf\">\n" + - " <start to=\"shell-node\"/>\n" + - " <action name=\"shell-node\">\n" + - " <shell xmlns=\"uri:oozie:shell-action:1.0\">\n" + - " <job-tracker>${jobTracker}</job-tracker>\n" + - " <name-node>${nameNode}</name-node>\n" + - " <launcher>\n" + - " <memory.mb>1024</memory.mb>\n" + - " </launcher>\n" + - " <configuration>\n" + - " <property>\n" + - " <name>mapred.job.queue.name</name>\n" + - " <value>${queueName}</value>\n" + - " </property>\n" + - " </configuration>\n" + - " <exec>echo</exec>\n" + - " <argument>my_output=Hello Oozie</argument>\n" + - " <capture-output/>\n" + - " </shell>\n" + - " <ok to=\"check-output\"/>\n" + - " <error to=\"fail\"/>\n" + - " </action>\n" + - " <decision name=\"check-output\">\n" + - " <switch>\n" + - " <case to=\"end\">\n" + - " ${wf:actionData('shell-node')['my_output'] eq 'Hello Oozie'}\n" + - " </case>\n" + - " <default to=\"fail-output\"/>\n" + - " </switch>\n" + - " </decision>\n" + - " <kill name=\"fail\">\n" + - " <message>Shell action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>\n" + - " </kill>\n" + - " <kill name=\"fail-output\">\n" + - " <message>Incorrect output</message>\n" + - " </kill>\n" + - " <end name=\"end\"/>\n" + - "</workflow-app>"; - - private static final String SQQP_ACTION_LAUNCHER_CONF = "<workflow-app xmlns=\"uri:oozie:workflow:1.0\" name=\"sqoop-wf\">\n" + - " <start to=\"sqoop-node\"/>\n" + - " <action name=\"sqoop-node\">\n" + - " <sqoop xmlns=\"uri:oozie:sqoop-action:1.0\">\n" + - " <job-tracker>${jobTracker}</job-tracker>\n" + - " <name-node>${nameNode}</name-node>\n" + - " <prepare>\n" + - " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/sqoop\"/>\n" + - " <mkdir path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data\"/>\n" + - " </prepare>\n" + - " <launcher>\n" + - " <memory.mb>1024</memory.mb>\n" + - " </launcher>\n" + - " <configuration>\n" + - " <property>\n" + - " <name>mapred.job.queue.name</name>\n" + - " <value>${queueName}</value>\n" + - " </property>\n" + - " </configuration>\n" + - " <command>import --connect jdbc:hsqldb:file:db.hsqldb --table TT --target-dir " + - "/user/${wf:user()}/${examplesRoot}/output-data/sqoop -m 1</command>\n" + - " <file>db.hsqldb.properties#db.hsqldb.properties</file>\n" + - " <file>db.hsqldb.script#db.hsqldb.script</file>\n" + - " </sqoop>\n" + - " <ok to=\"end\"/>\n" + - " <error to=\"fail\"/>\n" + - " </action>\n" + - " <kill name=\"fail\">\n" + - " <message>Sqoop failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>\n" + - " </kill>\n" + - " <end name=\"end\"/>\n" + - "</workflow-app>"; - - private static final String SPARK_ACTION_LAUNCHER_CONF = "<workflow-app xmlns='uri:oozie:workflow:1.0' " + - "name='SparkFileCopy'>\n" + - " <start to='spark-node' />\n" + - " <action name='spark-node'>\n" + - " <spark xmlns=\"uri:oozie:spark-action:1.0\">\n" + - " <job-tracker>${jobTracker}</job-tracker>\n" + - " <name-node>${nameNode}</name-node>\n" + - " <prepare>\n" + - " <delete path=\"${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/spark\"/>\n" + - " </prepare>\n" + - " <launcher>\n" + - " <memory.mb>1024</memory.mb>\n" + - " <vcores>2</vcores>\n" + - " <java-opts>dummyJavaOpts</java-opts>\n" + - " <env>dummyEnv</env>\n" + - " <queue>dummyQueue</queue>\n" + - " <sharelib>a,b,c</sharelib>\n" + - " </launcher>\n" + - " <master>${master}</master>\n" + - " <name>Spark-FileCopy</name>\n" + - " <class>org.apache.oozie.example.SparkFileCopy</class>\n" + - " <jar>${nameNode}/user/${wf:user()}/${examplesRoot}/apps/spark/lib/oozie-examples.jar</jar>\n" + - " <arg>${nameNode}/user/${wf:user()}/${examplesRoot}/input-data/text/data.txt</arg>\n" + - " <arg>${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/spark</arg>\n" + - " </spark>\n" + - " <ok to=\"end\" />\n" + - " <error to=\"fail\" />\n" + - " </action>\n" + - " <kill name=\"fail\">\n" + - " <message>Workflow failed, error\n" + - " message[${wf:errorMessage(wf:lastErrorNode())}]\n" + - " </message>\n" + - " </kill>\n" + - " <end name='end' />\n" + - "</workflow-app>\n"; - - private SchemaService wss; - private Validator workflowValidator; - private Validator coordinatorValidator; - private Validator bundleValidator; - @Override protected void setUp() throws Exception { super.setUp(); new Services().init(); - wss = Services.get().get(SchemaService.class); - workflowValidator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); - coordinatorValidator = wss.getSchema(SchemaName.COORDINATOR).newValidator(); - bundleValidator = wss.getSchema(SchemaName.BUNDLE).newValidator(); } @Override @@ -333,19 +118,27 @@ public class TestSchemaService extends XTestCase { } public void testWfSchema() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(APP1))); + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + validator.validate(new StreamSource(new StringReader(APP1))); } public void testWfMultipleJavaOpts() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(WF_4_MULTIPLE_JAVA_OPTS))); + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + validator.validate(new StreamSource(new StringReader(WF_4_MULTIPLE_JAVA_OPTS))); } public void testWfSchemaV2() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(APP_V2))); + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + validator.validate(new StreamSource(new StringReader(APP_V2))); } public void testWfSchemaV25() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(APP_V25))); + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + validator.validate(new StreamSource(new StringReader(APP_V25))); } public void testExtSchema() throws Exception { @@ -358,12 +151,16 @@ public class TestSchemaService extends XTestCase { } public void testWfSLASchema() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(WF_SLA_APP))); + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); + validator.validate(new StreamSource(new StringReader(WF_SLA_APP))); } public void testWfSLASchemaNW() throws Exception { + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.WORKFLOW).newValidator(); try { - workflowValidator.validate(new StreamSource(new StringReader(WF_SLA_APP_NW))); + validator.validate(new StreamSource(new StringReader(WF_SLA_APP_NW))); fail("Schema service check does not work"); } catch (Exception ex) { @@ -372,199 +169,59 @@ public class TestSchemaService extends XTestCase { } public void testCoordSchema() throws Exception { - String COORD_APP1 = "<?xml version=\"1.0\"?>\n" + - "<coordinator-app xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns=\"uri:oozie:coordinator:0.1\"" + - " xmlns:sla=\"uri:oozie:sla:0.1\" name=\"NAME\" frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\"" + - " end=\"2009-02-03T23:59Z\" timezone=\"UTC\">\n" + - " <controls>\n" + - " <timeout>10</timeout>\n" + - " <concurrency>2</concurrency>\n" + - " <execution>LIFO</execution>\n" + - " </controls>\n" + - " <datasets>\n" + - " <dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + - " timezone=\"UTC\">\n" + - " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + - " </dataset>\n" + - " <dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + - " timezone=\"UTC\">\n" + - " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + - " </dataset>\n" + - " </datasets>\n" + - " <input-events>\n" + - " <data-in name=\"A\" dataset=\"a\">\n" + - " <instance>${coord:latest(0)}</instance>\n" + - " </data-in>\n" + - " </input-events>\n" + - " <output-events>\n" + - " <data-out name=\"LOCAL_A\" dataset=\"local_a\">\n" + - " <instance>${coord:current(-1)}</instance>\n" + - " </data-out>\n" + - " </output-events>\n" + - " <action>\n" + - " <workflow>\n" + - " <app-path>hdfs:///tmp/workflows/</app-path>\n" + - " <configuration>\n" + - " <property>\n" + - " <name>inputA</name>\n" + - " <value>${coord:dataIn('A')}</value>\n" + - " </property>\n" + - " <property>\n" + - " <name>inputB</name>\n" + - " <value>${coord:dataOut('LOCAL_A')}</value>\n" + - " </property>\n" + - " </configuration>\n" + - " </workflow>\n" + - " </action>\n" + - "</coordinator-app>"; - - coordinatorValidator.validate(new StreamSource(new StringReader(COORD_APP1))); + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.COORDINATOR).newValidator(); + String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.1' xmlns:sla='uri:oozie:sla:0.1'> " + + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> </controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> </p roperty></configuration> </workflow> " + + "</action> </coordinator-app>"; + + Element e = XmlUtils.parseXml(COORD_APP1); + //System.out.println("XML :"+ XmlUtils.prettyPrint(e)); + validator.validate(new StreamSource(new StringReader(COORD_APP1))); } public void testCoordSchema2() throws Exception { - String COORD_APP1 = "<coordinator-app xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"" + - " xmlns=\"uri:oozie:coordinator:0.2\" xmlns:sla=\"uri:oozie:sla:0.1\" name=\"NAME\"" + - " frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\" end=\"2009-02-03T23:59Z\" timezone=\"UTC\">\n" + - " <controls>\n" + - " <timeout>10</timeout>\n" + - " <concurrency>2</concurrency>\n" + - " <execution>LIFO</execution>\n" + - " <throttle>3</throttle>\n" + - " </controls>\n" + - " <datasets>\n" + - " <dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + - " timezone=\"UTC\">\n" + - " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + - " </dataset>\n" + - " <dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + - " timezone=\"UTC\">\n" + - " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + - " </dataset>\n" + - " </datasets>\n" + - " <input-events>\n" + - " <data-in name=\"A\" dataset=\"a\">\n" + - " <instance>${coord:latest(0)}</instance>\n" + - " </data-in>\n" + - " </input-events>\n" + - " <output-events>\n" + - " <data-out name=\"LOCAL_A\" dataset=\"local_a\">\n" + - " <instance>${coord:current(-1)}</instance>\n" + - " </data-out>\n" + - " </output-events>\n" + - " <action>\n" + - " <workflow>\n" + - " <app-path>hdfs:///tmp/workflows/</app-path>\n" + - " <configuration>\n" + - " <property>\n" + - " <name>inputA</name>\n" + - " <value>${coord:dataIn('A')}</value>\n" + - " </property>\n" + - " <property>\n" + - " <name>inputB</name>\n" + - " <value>${coord:dataOut('LOCAL_A')}</value>\n" + - " </property>\n" + - " </configuration>\n" + - " </workflow>\n" + - " </action>\n" + - "</coordinator-app>"; - - coordinatorValidator.validate(new StreamSource(new StringReader(COORD_APP1))); + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.COORDINATOR).newValidator(); + String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2' xmlns:sla='uri:oozie:sla:0.1'> " + + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> <throttle>3</throttle></controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut( 'LOCAL_A')}</value> </property></configuration> </workflow> " + + "</action> </coordinator-app>"; + + Element e = XmlUtils.parseXml(COORD_APP1); + //System.out.println("XML :"+ XmlUtils.prettyPrint(e)); + validator.validate(new StreamSource(new StringReader(COORD_APP1))); } public void testCoordSLASchema() throws Exception { - String COORD_APP1 = "<?xml version=\"1.0\"?>\n" + - "<coordinator-app xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xmlns=\"uri:oozie:coordinator:0.2\"" + - " xmlns:sla=\"uri:oozie:sla:0.1\" name=\"NAME\" frequency=\"${coord:days(1)}\" start=\"2009-02-01T01:00Z\"" + - " end=\"2009-02-03T23:59Z\" timezone=\"UTC\">\n" + - " <controls>\n" + - " <timeout>10</timeout>\n" + - " <concurrency>2</concurrency>\n" + - " <execution>LIFO</execution>\n" + - " </controls>\n" + - " <datasets>\n" + - " <dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + - " timezone=\"UTC\">\n" + - " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + - " </dataset>\n" + - " <dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\"" + - " timezone=\"UTC\">\n" + - " <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>\n" + - " </dataset>\n" + - " </datasets>\n" + - " <input-events>\n" + - " <data-in name=\"A\" dataset=\"a\">\n" + - " <instance>${coord:latest(0)}</instance>\n" + - " </data-in>\n" + - " </input-events>\n" + - " <output-events>\n" + - " <data-out name=\"LOCAL_A\" dataset=\"local_a\">\n" + - " <instance>${coord:current(-1)}</instance>\n" + - " </data-out>\n" + - " </output-events>\n" + - " <action>\n" + - " <workflow>\n" + - " <app-path>hdfs:///tmp/workflows/</app-path>\n" + - " <configuration>\n" + - " <property>\n" + - " <name>inputA</name>\n" + - " <value>${coord:dataIn('A')}</value>\n" + - " </property>\n" + - " <property>\n" + - " <name>inputB</name>\n" + - " <value>${coord:dataOut('LOCAL_A')}</value>\n" + - " </property>\n" + - " </configuration>\n" + - " </workflow>\n" + - " <sla:info>\n" + - " <sla:app-name>5</sla:app-name>\n" + - " <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time>\n" + - " <sla:should-start>5</sla:should-start>\n" + - " <sla:should-end>50</sla:should-end>\n" + - " <sla:alert-contact>a...@example.com</sla:alert-contact>\n" + - " <sla:dev-contact>a...@example.com</sla:dev-contact>\n" + - " <sla:qa-contact>a...@example.com</sla:qa-contact>\n" + - " <sla:se-contact>a...@example.com</sla:se-contact>\n" + - " </sla:info>\n" + - " </action>\n" + - "</coordinator-app>"; - - coordinatorValidator.validate(new StreamSource(new StringReader(COORD_APP1))); + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.COORDINATOR) + .newValidator(); + String COORD_APP1 = "<coordinator-app name='NAME' frequency='${coord:days(1)}' start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:coordinator:0.2' xmlns:sla='uri:oozie:sla:0.1'> " + + "<controls> <timeout>10</timeout> <concurrency>2</concurrency> <execution>LIFO</execution> </controls> <datasets> <dataset name='a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> <dataset name='local_a' frequency='${coord:days(7)}' initial-instance='2009-02-01T01:00Z' timezone='UTC'> <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> </dataset> </datasets> <input-events> <data-in name='A' dataset='a'> <instance>${coord:latest(0)}</instance> </data-in> </input-events> <output-events> <data-out name='LOCAL_A' dataset='local_a'> <instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:///tmp/workflows/</app-path> <configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> <property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> </p roperty></configuration> </workflow> " + + "<sla:info> <sla:app-name>5</sla:app-name> <sla:nominal-time>2009-03-06T010:00Z</sla:nominal-time> " + + "<sla:should-start>5</sla:should-start> <sla:should-end>50</sla:should-end> " + + "<sla:alert-contact>a...@example.com</sla:alert-contact> <sla:dev-contact>a...@example.com</sla:dev-contact>" + + " <sla:qa-contact>a...@example.com</sla:qa-contact> <sla:se-contact>a...@example.com</sla:se-contact>" + + "</sla:info>" + "</action> </coordinator-app>"; + + Element e = XmlUtils.parseXml(COORD_APP1); + // System.out.println("XML :"+ XmlUtils.prettyPrint(e)); + validator.validate(new StreamSource(new StringReader(COORD_APP1))); } public void testBundleSchema() throws Exception { - String BUNDLE_APP = "<bundle-app name='NAME' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' " - + " xmlns='uri:oozie:bundle:0.1'> " + SchemaService wss = Services.get().get(SchemaService.class); + Validator validator = wss.getSchema(SchemaName.BUNDLE).newValidator(); + String BUNDLE_APP = "<bundle-app name='NAME' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns='uri:oozie:bundle:0.1'> " + "<controls> <kick-off-time>2009-02-02T00:00Z</kick-off-time> </controls> " + "<coordinator name='c12'> " + "<app-path>hdfs://localhost:9001/tmp/bundle-apps/coordinator1.xml</app-path>" + "<configuration> " + "<property> <name>START_TIME</name> <value>2009-02-01T00:00Z</value> </property> </configuration> " + "</coordinator></bundle-app>"; - - bundleValidator.validate(new StreamSource(new StringReader(BUNDLE_APP))); + Element e = XmlUtils.parseXml(BUNDLE_APP); + // System.out.println("XML :"+ XmlUtils.prettyPrint(e)); + validator.validate(new StreamSource(new StringReader(BUNDLE_APP))); } - public void testWfLauncherConfig() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(WF_GLOBAL_LAUNCHER_CONF))); - } - - public void testHiveActionLauncherConfig() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(HIVE_ACTION_LAUNCHER_CONF))); - } - - public void testHive2ActionLauncherConfig() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(HIVE2_ACTION_LAUNCHER_CONF))); - } - - public void testShellActionLauncherConfig() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(SHELL_ACTION_LAUNCHER_CONF))); - } - - public void testSqoopActionLauncherConfig() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(SQQP_ACTION_LAUNCHER_CONF))); - } - - public void testSparkActionLauncherConfig() throws Exception { - workflowValidator.validate(new StreamSource(new StringReader(SPARK_ACTION_LAUNCHER_CONF))); - } } http://git-wip-us.apache.org/repos/asf/oozie/blob/8aca098c/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java b/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java index ab97443..a882c82 100644 --- a/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java +++ b/core/src/test/java/org/apache/oozie/util/TestMetricsInstrumentation.java @@ -228,11 +228,11 @@ public class TestMetricsInstrumentation extends XTestCase { //Setting the id of the VM unique, so we can find it. String uniqueId = UUID.randomUUID().toString(); - System.setProperty("processSettings.unique.id", uniqueId); + System.setProperty("process.unique.id", uniqueId); //Finding our own VM by the id. for(VirtualMachineDescriptor d : VirtualMachine.list()) { - String remoteUniqueId = VirtualMachine.attach(d).getSystemProperties().getProperty("processSettings.unique.id"); + String remoteUniqueId = VirtualMachine.attach(d).getSystemProperties().getProperty("process.unique.id"); if(remoteUniqueId != null && remoteUniqueId.equals(uniqueId)) { descriptor = d; http://git-wip-us.apache.org/repos/asf/oozie/blob/8aca098c/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java index a361078..2133240 100644 --- a/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java +++ b/core/src/test/java/org/apache/oozie/workflow/lite/TestLiteWorkflowAppParser.java @@ -25,20 +25,15 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.ErrorCode; -import org.apache.oozie.action.hadoop.LauncherAM; import org.apache.oozie.service.ConfigurationService; import org.apache.oozie.service.LiteWorkflowStoreService; import org.apache.oozie.service.SchemaService; import org.apache.oozie.service.Services; import org.apache.oozie.test.XTestCase; import org.apache.oozie.util.IOUtils; -import org.apache.oozie.util.XConfiguration; -import org.apache.oozie.util.XmlUtils; import org.apache.oozie.workflow.WorkflowException; import org.apache.oozie.workflow.lite.TestLiteWorkflowLib.TestActionNodeHandler; import org.apache.oozie.workflow.lite.TestLiteWorkflowLib.TestDecisionNodeHandler; -import org.jdom.Element; -import org.jdom.Namespace; public class TestLiteWorkflowAppParser extends XTestCase { public static String dummyConf = "<java></java>"; @@ -657,42 +652,6 @@ public class TestLiteWorkflowAppParser extends XTestCase { } } - public void testParserGlobalLauncherAM() throws Exception { - LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, - LiteWorkflowStoreService.LiteControlNodeHandler.class, - LiteWorkflowStoreService.LiteDecisionHandler.class, - LiteWorkflowStoreService.LiteActionHandler.class); - - LiteWorkflowApp workflowApp = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-global-launcherconf.xml", -1), new Configuration()); - - XConfiguration xconf = extractConfig(workflowApp, "action1"); - - assertEquals("Vcores", 2, xconf.getInt(LauncherAM.OOZIE_LAUNCHER_VCORES_PROPERTY, Integer.MIN_VALUE)); - assertEquals("Memory", 1024, xconf.getInt(LauncherAM.OOZIE_LAUNCHER_MEMORY_MB_PROPERTY, Integer.MIN_VALUE)); - assertEquals("Env", "dummyEnv", xconf.get(LauncherAM.OOZIE_LAUNCHER_ENV_PROPERTY)); - assertEquals("Queue", "dummyQueue", xconf.get(LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY)); - assertEquals("Java opts", "dummyJavaOpts", xconf.get(LauncherAM.OOZIE_LAUNCHER_JAVAOPTS_PROPERTY)); - assertEquals("Sharelib", "a,b,c", xconf.get(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY)); - } - - public void testParserGlobalLauncherAMOverridden() throws Exception { - LiteWorkflowAppParser parser = new LiteWorkflowAppParser(null, - LiteWorkflowStoreService.LiteControlNodeHandler.class, - LiteWorkflowStoreService.LiteDecisionHandler.class, - LiteWorkflowStoreService.LiteActionHandler.class); - - LiteWorkflowApp workflowApp = parser.validateAndParse(IOUtils.getResourceAsReader("wf-schema-global-launcherconf-override.xml", -1), new Configuration()); - - XConfiguration xconf = extractConfig(workflowApp, "a"); - - assertEquals("Vcores", 1, xconf.getInt(LauncherAM.OOZIE_LAUNCHER_VCORES_PROPERTY, Integer.MIN_VALUE)); - assertEquals("Memory", 2048, xconf.getInt(LauncherAM.OOZIE_LAUNCHER_MEMORY_MB_PROPERTY, Integer.MIN_VALUE)); - assertEquals("Java opts", "dummyJavaOpts", xconf.get(LauncherAM.OOZIE_LAUNCHER_JAVAOPTS_PROPERTY)); - assertNull("Queue", xconf.get(LauncherAM.OOZIE_LAUNCHER_QUEUE_PROPERTY)); - assertNull("Env", xconf.get(LauncherAM.OOZIE_LAUNCHER_ENV_PROPERTY)); - assertNull("Sharelib", xconf.get(LauncherAM.OOZIE_LAUNCHER_SHARELIB_PROPERTY)); - } - /* * 1->ok->2 * 2->ok->end @@ -1645,14 +1604,4 @@ public class TestLiteWorkflowAppParser extends XTestCase { assertEquals(app.getNode("retry").getUserRetryInterval(), "10"); } - private XConfiguration extractConfig(LiteWorkflowApp app, String actionNode) throws Exception { - String confXML = app.getNode(actionNode).getConf(); - Element confElement = XmlUtils.parseXml(confXML); - Namespace ns = confElement.getNamespace(); - String configSection = XmlUtils.prettyPrint(confElement.getChild("configuration", ns)).toString(); - XConfiguration xconf = new XConfiguration(new StringReader(configSection)); - - return xconf; - } - } http://git-wip-us.apache.org/repos/asf/oozie/blob/8aca098c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index efccc34..db18f30 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ <jline.version>0.9.94</jline.version> <openjpa.version>2.4.2</openjpa.version> - <xerces.version>2.11.0</xerces.version> + <xerces.version>2.10.0</xerces.version> <curator.version>2.5.0</curator.version> <jackson.version>1.9.13</jackson.version> <log4j.version>1.2.17</log4j.version> http://git-wip-us.apache.org/repos/asf/oozie/blob/8aca098c/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 0a4b600..1fb1a1d 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,6 +1,5 @@ -- Oozie 5.0.0 release (trunk - unreleased) -OOZIE-2687 Create XML schema for launcher configurations (asasvari) OOZIE-3041 TestWorkflowActionRetryInfoXCommand fails in oozie core module (andras.piros via gezapeti) OOZIE-2916 Set a job name for the MR Action's child job (asasvari) OOZIE-2858 HiveMain, ShellMain and SparkMain should not overwrite properties and config files locally (gezapeti) http://git-wip-us.apache.org/repos/asf/oozie/blob/8aca098c/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java index 3e6ffc9..6a98d6e 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java @@ -49,14 +49,6 @@ public class LauncherAM { public static final String OOZIE_ACTION_CONF_XML = "oozie.action.conf.xml"; public static final String OOZIE_LAUNCHER_JOB_ID = "oozie.launcher.job.id"; - public static final String OOZIE_LAUNCHER_VCORES_PROPERTY = "oozie.launcher.vcores"; - public static final String OOZIE_LAUNCHER_MEMORY_MB_PROPERTY = "oozie.launcher.memory.mb"; - public static final String OOZIE_LAUNCHER_PRIORITY_PROPERTY = "oozie.launcher.priority"; - public static final String OOZIE_LAUNCHER_QUEUE_PROPERTY = "oozie.launcher.queue"; - public static final String OOZIE_LAUNCHER_JAVAOPTS_PROPERTY = "oozie.launcher.javaopts"; - public static final String OOZIE_LAUNCHER_ENV_PROPERTY = "oozie.launcher.env"; - public static final String OOZIE_LAUNCHER_SHARELIB_PROPERTY = "oozie.launcher.sharelib"; - public static final String JAVA_CLASS_PATH = "java.class.path"; public static final String OOZIE_ACTION_ID = "oozie.action.id"; public static final String OOZIE_JOB_ID = "oozie.job.id"; http://git-wip-us.apache.org/repos/asf/oozie/blob/8aca098c/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java index c0e2ec3..ff999ff 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/UDFTester.java @@ -39,7 +39,7 @@ public class UDFTester extends EvalFunc<String> { return query.toLowerCase().trim(); } catch (Exception e) { - System.err.println("ToLower: failed to processSettings input; error - " + e.getMessage()); + System.err.println("ToLower: failed to process input; error - " + e.getMessage()); return null; } }