Joal has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/349266 )
Change subject: Update oozie restbase to graphite spark job
......................................................................
Update oozie restbase to graphite spark job
This job has been created before spark had dynamic allocation.
This patch removes the fixed number of executor and enables
dynamic allocation for this job.
Change-Id: Icf5b764ce3e8906707ff1958bcdbe1a094841c4c
---
M oozie/restbase/README.md
M oozie/restbase/coordinator.properties
M oozie/restbase/coordinator.xml
M oozie/restbase/workflow.xml
4 files changed, 4 insertions(+), 10 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery
refs/changes/66/349266/1
diff --git a/oozie/restbase/README.md b/oozie/restbase/README.md
index 6f33e88..d8142f7 100644
--- a/oozie/restbase/README.md
+++ b/oozie/restbase/README.md
@@ -10,7 +10,7 @@
oozie job -run \
-config coordinator.properties \
-D
refinery_directory=hdfs://analytics-hadoop/user/madhuvishy/refinery \
- -D
spark_job_jar=hdfs://analytics-hadoop/user/madhuvishy/source/refinery-job-0.0.18-SNAPSHOT.jar
+ -D
spark_job_jar=hdfs://analytics-hadoop/user/madhuvishy/source/refinery-job-0.0.45-SNAPSHOT.jar
The results of this job can be viewed in Graphite (graphite.wikimedia.org)
under the restbase namespace,
in restbase.requests.varnish_requests.
\ No newline at end of file
diff --git a/oozie/restbase/coordinator.properties
b/oozie/restbase/coordinator.properties
index 9b5997e..08903a3 100644
--- a/oozie/restbase/coordinator.properties
+++ b/oozie/restbase/coordinator.properties
@@ -49,9 +49,8 @@
spark_job_jar =
${artifacts_directory}/org/wikimedia/analytics/refinery/refinery-job-0.0.41.jar
spark_job_class =
org.wikimedia.analytics.refinery.job.RESTBaseMetrics
spark_job_name = restbase_metrics
-spark_number_executors = 2
-spark_executor_memory = 1G
-spark_driver_memory = 2G
+spark_executor_memory = 2G
+spark_driver_memory = 4G
graphite_namespace = restbase.requests
graphite_host = graphite-in.eqiad.wmnet
graphite_port = 2003
diff --git a/oozie/restbase/coordinator.xml b/oozie/restbase/coordinator.xml
index d2d777b..e4b0846 100644
--- a/oozie/restbase/coordinator.xml
+++ b/oozie/restbase/coordinator.xml
@@ -24,7 +24,6 @@
<property><name>spark_job_class</name></property>
<property><name>spark_executor_memory</name></property>
<property><name>spark_driver_memory</name></property>
- <property><name>spark_number_executors</name></property>
<property><name>graphite_host</name></property>
<property><name>graphite_port</name></property>
<property><name>graphite_namespace</name></property>
diff --git a/oozie/restbase/workflow.xml b/oozie/restbase/workflow.xml
index 4f74856..7d4a5b7 100644
--- a/oozie/restbase/workflow.xml
+++ b/oozie/restbase/workflow.xml
@@ -40,10 +40,6 @@
<description>Memory to allocate for spark driver
process</description>
</property>
<property>
- <name>spark_number_executors</name>
- <description>Number of executors to run in parallel</description>
- </property>
- <property>
<name>year</name>
<description>The partition's year</description>
</property>
@@ -104,7 +100,7 @@
<name>${spark_job_name}-${year}-${month}-${day}-${hour}</name>
<class>${spark_job_class}</class>
<jar>${spark_job_jar}</jar>
- <spark-opts>--conf spark.yarn.jar=${spark_assembly_jar}
--executor-memory ${spark_executor_memory} --driver-memory
${spark_driver_memory} --num-executors ${spark_number_executors} --queue
${queue_name} </spark-opts>
+ <spark-opts>--conf spark.yarn.jar=${spark_assembly_jar}
--executor-memory ${spark_executor_memory} --driver-memory
${spark_driver_memory} --queue ${queue_name} --conf
spark.dynamicAllocation.enabled=true --conf
spark.shuffle.service.enabled=true</spark-opts>
<arg>--year</arg>
<arg>${year}</arg>
<arg>--month</arg>
--
To view, visit https://gerrit.wikimedia.org/r/349266
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Icf5b764ce3e8906707ff1958bcdbe1a094841c4c
Gerrit-PatchSet: 1
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
