Repository: oozie Updated Branches: refs/heads/master ee275389b -> e08d696e4
OOZIE-3332 [examples] Spark examples should feature yarn client and cluster modes (daniel.becker via andras.piros) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e08d696e Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e08d696e Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e08d696e Branch: refs/heads/master Commit: e08d696e49f0af1ca69ffd1f5733e0e7ef471165 Parents: ee27538 Author: Andras Piros <andras.pi...@cloudera.com> Authored: Fri Aug 24 10:53:15 2018 +0200 Committer: Andras Piros <andras.pi...@cloudera.com> Committed: Fri Aug 24 10:53:15 2018 +0200 ---------------------------------------------------------------------- examples/src/main/apps/pyspark/job.properties | 4 ++-- examples/src/main/apps/pyspark/workflow.xml | 1 + examples/src/main/apps/spark/job.properties | 1 + examples/src/main/apps/spark/workflow.xml | 1 + .../src/main/java/org/apache/oozie/example/fluentjob/Spark.java | 1 + release-log.txt | 1 + 6 files changed, 7 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/apps/pyspark/job.properties ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/pyspark/job.properties b/examples/src/main/apps/pyspark/job.properties index b3c70ad..2078a7e 100644 --- a/examples/src/main/apps/pyspark/job.properties +++ b/examples/src/main/apps/pyspark/job.properties @@ -18,9 +18,9 @@ nameNode=hdfs://localhost:8020 resourceManager=localhost:8032 +master=yarn +mode=client queueName=default examplesRoot=examples oozie.use.system.libpath=true oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pyspark -master=yarn-client - http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/apps/pyspark/workflow.xml ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/pyspark/workflow.xml b/examples/src/main/apps/pyspark/workflow.xml index 64b7f67..f83ab9b 100644 --- a/examples/src/main/apps/pyspark/workflow.xml +++ b/examples/src/main/apps/pyspark/workflow.xml @@ -25,6 +25,7 @@ <resource-manager>${resourceManager}</resource-manager> <name-node>${nameNode}</name-node> <master>${master}</master> + <mode>${mode}</mode> <name>Python-Spark-Pi</name> <jar>pi.py</jar> </spark> http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/apps/spark/job.properties ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/spark/job.properties b/examples/src/main/apps/spark/job.properties index 698711a..9866e26 100644 --- a/examples/src/main/apps/spark/job.properties +++ b/examples/src/main/apps/spark/job.properties @@ -19,6 +19,7 @@ nameNode=hdfs://localhost:8020 resourceManager=localhost:8032 master=local[*] +mode=client queueName=default examplesRoot=examples oozie.use.system.libpath=true http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/apps/spark/workflow.xml ---------------------------------------------------------------------- diff --git a/examples/src/main/apps/spark/workflow.xml b/examples/src/main/apps/spark/workflow.xml index 2ae576b..d3b3564 100644 --- a/examples/src/main/apps/spark/workflow.xml +++ b/examples/src/main/apps/spark/workflow.xml @@ -26,6 +26,7 @@ <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/spark"/> </prepare> <master>${master}</master> + <mode>${mode}</mode> <name>Spark-FileCopy</name> <class>org.apache.oozie.example.SparkFileCopy</class> <jar>${nameNode}/user/${wf:user()}/${examplesRoot}/apps/spark/lib/oozie-examples.jar</jar> http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java ---------------------------------------------------------------------- diff --git a/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java b/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java index 6065933..9527336 100644 --- a/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java +++ b/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java @@ -42,6 +42,7 @@ public class Spark implements WorkflowFactory { .withArg("${nameNode}/user/${wf:user()}/${examplesRoot}/input-data/text/data.txt") .withArg("${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/spark") .withMaster("${master}") + .withMode("${mode}") .withActionName("Spark File Copy Example") .withActionClass("org.apache.oozie.example.SparkFileCopy") .withJar("${nameNode}/user/${wf:user()}/${examplesRoot}/apps/spark/lib/oozie-examples.jar") http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index fd06d45..3c569d0 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.1.0 release (trunk - unreleased) +OOZIE-3332 [examples] Spark examples should feature yarn client and cluster modes (daniel.becker via andras.piros) OOZIE-3318 [build] Fix Javadoc check in the pre-commit (kmarton via andras.piros) OOZIE-3317 [build] Fix false positive precommit reports (kmarton via andras.piros) OOZIE-3264 Flaky test TestCoordMaterializeTransitionXCommand#testLastOnlyMaterialization (asalamon74 via andras.piros)