Ottomata has submitted this change and it was merged.
Change subject: Improve webrequest oozie jobs emails
......................................................................
Improve webrequest oozie jobs emails
Send more information in error emails in load job.
Add error email sending (with more info) in refine and legacy_tsvs jobs.
Clean coordinator files from non needed parameters passing.
Change-Id: Iecf741552b8eee315b63d0860e8ea8c39e72c723
---
M oozie/webrequest/legacy_tsvs/bundle.properties
M oozie/webrequest/legacy_tsvs/bundle.xml
M oozie/webrequest/legacy_tsvs/coordinator_misc.xml
M oozie/webrequest/legacy_tsvs/coordinator_misc_text.xml
M oozie/webrequest/legacy_tsvs/coordinator_text.xml
M oozie/webrequest/legacy_tsvs/coordinator_text_upload.xml
M oozie/webrequest/legacy_tsvs/coordinator_upload.xml
M oozie/webrequest/legacy_tsvs/workflow.xml
M oozie/webrequest/load/workflow.xml
M oozie/webrequest/refine/bundle.properties
M oozie/webrequest/refine/bundle.xml
M oozie/webrequest/refine/coordinator.xml
M oozie/webrequest/refine/workflow.xml
13 files changed, 87 insertions(+), 231 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/oozie/webrequest/legacy_tsvs/bundle.properties
b/oozie/webrequest/legacy_tsvs/bundle.properties
index 50269ae..21600bf 100644
--- a/oozie/webrequest/legacy_tsvs/bundle.properties
+++ b/oozie/webrequest/legacy_tsvs/bundle.properties
@@ -56,6 +56,9 @@
archive_job_output_workflow_file =
${oozie_directory}/util/archive_job_output/workflow.xml
+# Workflow to send an error email
+send_error_email_workflow_file =
${oozie_directory}/util/send_error_email/workflow.xml
+
# HDFS path to hive-site.xml file. This is needed to run hive actions.
hive_site_xml =
${oozie_directory}/util/hive/hive-site.xml
diff --git a/oozie/webrequest/legacy_tsvs/bundle.xml
b/oozie/webrequest/legacy_tsvs/bundle.xml
index 654bfde..86eba3d 100644
--- a/oozie/webrequest/legacy_tsvs/bundle.xml
+++ b/oozie/webrequest/legacy_tsvs/bundle.xml
@@ -25,6 +25,7 @@
<property><name>temporary_directory</name></property>
<property><name>webrequest_archive_directory</name></property>
<property><name>archive_job_output_workflow_file</name></property>
+ <property><name>send_error_email_workflow_file</name></property>
</parameters>
<coordinator name="webrequest_legacy_tsvs-sampled-1000">
diff --git a/oozie/webrequest/legacy_tsvs/coordinator_misc.xml
b/oozie/webrequest/legacy_tsvs/coordinator_misc.xml
index f202c5d..2373fd1 100644
--- a/oozie/webrequest/legacy_tsvs/coordinator_misc.xml
+++ b/oozie/webrequest/legacy_tsvs/coordinator_misc.xml
@@ -25,6 +25,7 @@
<property><name>temporary_directory</name></property>
<property><name>aspect_tsv_archive_directory</name></property>
<property><name>archive_job_output_workflow_file</name></property>
+ <property><name>send_error_email_workflow_file</name></property>
<property><name>aspect_name</name></property>
</parameters>
@@ -78,24 +79,6 @@
<workflow>
<app-path>${workflow_file}</app-path>
<configuration>
-
- <!-- Pass these properties through to the workflow -->
-
<property><name>name_node</name><value>${name_node}</value></property>
-
<property><name>job_tracker</name><value>${job_tracker}</value></property>
-
<property><name>queue_name</name><value>${queue_name}</value></property>
-
- <property>
- <name>hive_site_xml</name>
- <value>${hive_site_xml}</value>
- </property>
- <property>
- <name>artifacts_directory</name>
- <value>${artifacts_directory}</value>
- </property>
- <property>
- <name>webrequest_table</name>
- <value>${webrequest_table}</value>
- </property>
<property>
<name>year</name>
<value>${coord:formatTime(coord:nominalTime(),
"yyyy")}</value>
@@ -107,26 +90,6 @@
<property>
<name>day</name>
<value>${coord:formatTime(coord:nominalTime(),
"dd")}</value>
- </property>
- <property>
- <name>mark_directory_done_workflow_file</name>
- <value>${mark_directory_done_workflow_file}</value>
- </property>
- <property>
- <name>temporary_directory</name>
- <value>${temporary_directory}</value>
- </property>
- <property>
- <name>aspect_name</name>
- <value>${aspect_name}</value>
- </property>
- <property>
- <name>aspect_tsv_archive_directory</name>
- <value>${aspect_tsv_archive_directory}</value>
- </property>
- <property>
- <name>archive_job_output_workflow_file</name>
- <value>${archive_job_output_workflow_file}</value>
</property>
</configuration>
</workflow>
diff --git a/oozie/webrequest/legacy_tsvs/coordinator_misc_text.xml
b/oozie/webrequest/legacy_tsvs/coordinator_misc_text.xml
index 6803d89..2b66460 100644
--- a/oozie/webrequest/legacy_tsvs/coordinator_misc_text.xml
+++ b/oozie/webrequest/legacy_tsvs/coordinator_misc_text.xml
@@ -25,6 +25,7 @@
<property><name>temporary_directory</name></property>
<property><name>aspect_tsv_archive_directory</name></property>
<property><name>archive_job_output_workflow_file</name></property>
+ <property><name>send_error_email_workflow_file</name></property>
<property><name>aspect_name</name></property>
</parameters>
@@ -84,24 +85,6 @@
<workflow>
<app-path>${workflow_file}</app-path>
<configuration>
-
- <!-- Pass these properties through to the workflow -->
-
<property><name>name_node</name><value>${name_node}</value></property>
-
<property><name>job_tracker</name><value>${job_tracker}</value></property>
-
<property><name>queue_name</name><value>${queue_name}</value></property>
-
- <property>
- <name>hive_site_xml</name>
- <value>${hive_site_xml}</value>
- </property>
- <property>
- <name>artifacts_directory</name>
- <value>${artifacts_directory}</value>
- </property>
- <property>
- <name>webrequest_table</name>
- <value>${webrequest_table}</value>
- </property>
<property>
<name>year</name>
<value>${coord:formatTime(coord:nominalTime(),
"yyyy")}</value>
@@ -113,26 +96,6 @@
<property>
<name>day</name>
<value>${coord:formatTime(coord:nominalTime(),
"dd")}</value>
- </property>
- <property>
- <name>mark_directory_done_workflow_file</name>
- <value>${mark_directory_done_workflow_file}</value>
- </property>
- <property>
- <name>temporary_directory</name>
- <value>${temporary_directory}</value>
- </property>
- <property>
- <name>aspect_name</name>
- <value>${aspect_name}</value>
- </property>
- <property>
- <name>aspect_tsv_archive_directory</name>
- <value>${aspect_tsv_archive_directory}</value>
- </property>
- <property>
- <name>archive_job_output_workflow_file</name>
- <value>${archive_job_output_workflow_file}</value>
</property>
</configuration>
</workflow>
diff --git a/oozie/webrequest/legacy_tsvs/coordinator_text.xml
b/oozie/webrequest/legacy_tsvs/coordinator_text.xml
index f0262ab..ea74b3c 100644
--- a/oozie/webrequest/legacy_tsvs/coordinator_text.xml
+++ b/oozie/webrequest/legacy_tsvs/coordinator_text.xml
@@ -25,6 +25,7 @@
<property><name>temporary_directory</name></property>
<property><name>aspect_tsv_archive_directory</name></property>
<property><name>archive_job_output_workflow_file</name></property>
+ <property><name>send_error_email_workflow_file</name></property>
<property><name>aspect_name</name></property>
</parameters>
@@ -78,24 +79,6 @@
<workflow>
<app-path>${workflow_file}</app-path>
<configuration>
-
- <!-- Pass these properties through to the workflow -->
-
<property><name>name_node</name><value>${name_node}</value></property>
-
<property><name>job_tracker</name><value>${job_tracker}</value></property>
-
<property><name>queue_name</name><value>${queue_name}</value></property>
-
- <property>
- <name>hive_site_xml</name>
- <value>${hive_site_xml}</value>
- </property>
- <property>
- <name>artifacts_directory</name>
- <value>${artifacts_directory}</value>
- </property>
- <property>
- <name>webrequest_table</name>
- <value>${webrequest_table}</value>
- </property>
<property>
<name>year</name>
<value>${coord:formatTime(coord:nominalTime(),
"yyyy")}</value>
@@ -107,26 +90,6 @@
<property>
<name>day</name>
<value>${coord:formatTime(coord:nominalTime(),
"dd")}</value>
- </property>
- <property>
- <name>mark_directory_done_workflow_file</name>
- <value>${mark_directory_done_workflow_file}</value>
- </property>
- <property>
- <name>temporary_directory</name>
- <value>${temporary_directory}</value>
- </property>
- <property>
- <name>aspect_name</name>
- <value>${aspect_name}</value>
- </property>
- <property>
- <name>aspect_tsv_archive_directory</name>
- <value>${aspect_tsv_archive_directory}</value>
- </property>
- <property>
- <name>archive_job_output_workflow_file</name>
- <value>${archive_job_output_workflow_file}</value>
</property>
</configuration>
</workflow>
diff --git a/oozie/webrequest/legacy_tsvs/coordinator_text_upload.xml
b/oozie/webrequest/legacy_tsvs/coordinator_text_upload.xml
index 988ba89..0d41ff5 100644
--- a/oozie/webrequest/legacy_tsvs/coordinator_text_upload.xml
+++ b/oozie/webrequest/legacy_tsvs/coordinator_text_upload.xml
@@ -25,6 +25,7 @@
<property><name>temporary_directory</name></property>
<property><name>aspect_tsv_archive_directory</name></property>
<property><name>archive_job_output_workflow_file</name></property>
+ <property><name>send_error_email_workflow_file</name></property>
<property><name>aspect_name</name></property>
</parameters>
@@ -84,24 +85,6 @@
<workflow>
<app-path>${workflow_file}</app-path>
<configuration>
-
- <!-- Pass these properties through to the workflow -->
-
<property><name>name_node</name><value>${name_node}</value></property>
-
<property><name>job_tracker</name><value>${job_tracker}</value></property>
-
<property><name>queue_name</name><value>${queue_name}</value></property>
-
- <property>
- <name>hive_site_xml</name>
- <value>${hive_site_xml}</value>
- </property>
- <property>
- <name>artifacts_directory</name>
- <value>${artifacts_directory}</value>
- </property>
- <property>
- <name>webrequest_table</name>
- <value>${webrequest_table}</value>
- </property>
<property>
<name>year</name>
<value>${coord:formatTime(coord:nominalTime(),
"yyyy")}</value>
@@ -113,26 +96,6 @@
<property>
<name>day</name>
<value>${coord:formatTime(coord:nominalTime(),
"dd")}</value>
- </property>
- <property>
- <name>mark_directory_done_workflow_file</name>
- <value>${mark_directory_done_workflow_file}</value>
- </property>
- <property>
- <name>temporary_directory</name>
- <value>${temporary_directory}</value>
- </property>
- <property>
- <name>aspect_name</name>
- <value>${aspect_name}</value>
- </property>
- <property>
- <name>aspect_tsv_archive_directory</name>
- <value>${aspect_tsv_archive_directory}</value>
- </property>
- <property>
- <name>archive_job_output_workflow_file</name>
- <value>${archive_job_output_workflow_file}</value>
</property>
</configuration>
</workflow>
diff --git a/oozie/webrequest/legacy_tsvs/coordinator_upload.xml
b/oozie/webrequest/legacy_tsvs/coordinator_upload.xml
index c3d7b63..43c74a0 100644
--- a/oozie/webrequest/legacy_tsvs/coordinator_upload.xml
+++ b/oozie/webrequest/legacy_tsvs/coordinator_upload.xml
@@ -25,6 +25,7 @@
<property><name>temporary_directory</name></property>
<property><name>aspect_tsv_archive_directory</name></property>
<property><name>archive_job_output_workflow_file</name></property>
+ <property><name>send_error_email_workflow_file</name></property>
<property><name>aspect_name</name></property>
</parameters>
@@ -78,24 +79,6 @@
<workflow>
<app-path>${workflow_file}</app-path>
<configuration>
-
- <!-- Pass these properties through to the workflow -->
-
<property><name>name_node</name><value>${name_node}</value></property>
-
<property><name>job_tracker</name><value>${job_tracker}</value></property>
-
<property><name>queue_name</name><value>${queue_name}</value></property>
-
- <property>
- <name>hive_site_xml</name>
- <value>${hive_site_xml}</value>
- </property>
- <property>
- <name>artifacts_directory</name>
- <value>${artifacts_directory}</value>
- </property>
- <property>
- <name>webrequest_table</name>
- <value>${webrequest_table}</value>
- </property>
<property>
<name>year</name>
<value>${coord:formatTime(coord:nominalTime(),
"yyyy")}</value>
@@ -107,26 +90,6 @@
<property>
<name>day</name>
<value>${coord:formatTime(coord:nominalTime(),
"dd")}</value>
- </property>
- <property>
- <name>mark_directory_done_workflow_file</name>
- <value>${mark_directory_done_workflow_file}</value>
- </property>
- <property>
- <name>temporary_directory</name>
- <value>${temporary_directory}</value>
- </property>
- <property>
- <name>aspect_name</name>
- <value>${aspect_name}</value>
- </property>
- <property>
- <name>aspect_tsv_archive_directory</name>
- <value>${aspect_tsv_archive_directory}</value>
- </property>
- <property>
- <name>archive_job_output_workflow_file</name>
- <value>${archive_job_output_workflow_file}</value>
</property>
</configuration>
</workflow>
diff --git a/oozie/webrequest/legacy_tsvs/workflow.xml
b/oozie/webrequest/legacy_tsvs/workflow.xml
index 5bac9f6..8d0a08d 100644
--- a/oozie/webrequest/legacy_tsvs/workflow.xml
+++ b/oozie/webrequest/legacy_tsvs/workflow.xml
@@ -71,6 +71,10 @@
<name>archive_job_output_workflow_file</name>
<description>Workflow to move a data file to the
archive</description>
</property>
+ <property>
+ <name>send_error_email_workflow_file</name>
+ <description>Workflow for sending an error email</description>
+ </property>
</parameters>
<start to="extract_data_into_single_file"/>
@@ -136,7 +140,7 @@
<param>webrequest_sources=${replaceAll(replaceAll(wf:conf('oozie.coord.application.path'),
'^.*/coordinator_([a-z_]*).xml', '\'$1\''),'_','\', \'')}</param>
</hive>
<ok to="mark_dataset_done"/>
- <error to="kill"/>
+ <error to="send_error_email"/>
</action>
<action name="mark_dataset_done">
@@ -150,7 +154,7 @@
</configuration>
</sub-workflow>
<ok to="move_data_to_archive"/>
- <error to="kill"/>
+ <error to="send_error_email"/>
</action>
<action name="move_data_to_archive">
@@ -173,6 +177,33 @@
</configuration>
</sub-workflow>
<ok to="end"/>
+ <error to="send_error_email"/>
+ </action>
+
+ <action name="send_error_email">
+ <sub-workflow>
+ <app-path>${send_error_email_workflow_file}</app-path>
+ <propagate-configuration/>
+ <configuration>
+ <property>
+ <name>parent_name</name>
+ <value>${wf:name()}</value>
+ </property>
+ <property>
+ <name>parent_failed_action</name>
+ <value>${wf:lastErrorNode()}</value>
+ </property>
+ <property>
+ <name>parent_error_code</name>
+ <value>${wf:errorCode(wf:lastErrorNode())}</value>
+ </property>
+ <property>
+ <name>parent_error_message</name>
+ <value>${wf:errorMessage(wf:lastErrorNode())}</value>
+ </property>
+ </configuration>
+ </sub-workflow>
+ <ok to="kill"/>
<error to="kill"/>
</action>
diff --git a/oozie/webrequest/load/workflow.xml
b/oozie/webrequest/load/workflow.xml
index f0582fa..4db1c60 100644
--- a/oozie/webrequest/load/workflow.xml
+++ b/oozie/webrequest/load/workflow.xml
@@ -244,6 +244,14 @@
<name>parent_failed_action</name>
<value>${wf:lastErrorNode()}</value>
</property>
+ <property>
+ <name>parent_error_code</name>
+ <value>${wf:errorCode(wf:lastErrorNode())}</value>
+ </property>
+ <property>
+ <name>parent_error_message</name>
+ <value>${wf:errorMessage(wf:lastErrorNode())}</value>
+ </property>
</configuration>
</sub-workflow>
<ok to="kill"/>
diff --git a/oozie/webrequest/refine/bundle.properties
b/oozie/webrequest/refine/bundle.properties
index c0b2475..41b012a 100644
--- a/oozie/webrequest/refine/bundle.properties
+++ b/oozie/webrequest/refine/bundle.properties
@@ -47,6 +47,9 @@
# HDFS path to workflow to mark a directory as done
mark_directory_done_workflow_file =
${oozie_directory}/util/mark_directory_done/workflow.xml
+# Workflow to send an error email
+send_error_email_workflow_file =
${oozie_directory}/util/send_error_email/workflow.xml
+
# HDFS path to hive-site.xml file. This is needed to run hive actions.
hive_site_xml = ${oozie_directory}/util/hive/hive-site.xml
diff --git a/oozie/webrequest/refine/bundle.xml
b/oozie/webrequest/refine/bundle.xml
index cbd017e..4c0c3d5 100644
--- a/oozie/webrequest/refine/bundle.xml
+++ b/oozie/webrequest/refine/bundle.xml
@@ -11,6 +11,7 @@
<property><name>job_tracker</name></property>
<property><name>workflow_file</name></property>
<property><name>mark_directory_done_workflow_file</name></property>
+ <property><name>send_error_email_workflow_file</name></property>
<property><name>start_time</name></property>
<property><name>stop_time</name></property>
diff --git a/oozie/webrequest/refine/coordinator.xml
b/oozie/webrequest/refine/coordinator.xml
index 4d64a49..67a93a4 100644
--- a/oozie/webrequest/refine/coordinator.xml
+++ b/oozie/webrequest/refine/coordinator.xml
@@ -14,6 +14,7 @@
<property><name>job_tracker</name></property>
<property><name>workflow_file</name></property>
<property><name>mark_directory_done_workflow_file</name></property>
+ <property><name>send_error_email_workflow_file</name></property>
<property><name>start_time</name></property>
<property><name>stop_time</name></property>
@@ -101,40 +102,6 @@
<workflow>
<app-path>${workflow_file}</app-path>
<configuration>
-
- <!-- Pass these properties through to the workflow -->
-
<property><name>name_node</name><value>${name_node}</value></property>
-
<property><name>job_tracker</name><value>${job_tracker}</value></property>
-
<property><name>queue_name</name><value>${queue_name}</value></property>
-
- <property>
- <name>hive_site_xml</name>
- <value>${hive_site_xml}</value>
- </property>
- <property>
- <name>refinery_jar_version</name>
- <value>${refinery_jar_version}</value>
- </property>
- <property>
- <name>artifacts_directory</name>
- <value>${artifacts_directory}</value>
- </property>
- <property>
- <name>source_table</name>
- <value>${source_table}</value>
- </property>
- <property>
- <name>destination_table</name>
- <value>${destination_table}</value>
- </property>
- <property>
- <name>webrequest_source</name>
- <value>${webrequest_source}</value>
- </property>
- <property>
- <name>record_version</name>
- <value>${record_version}</value>
- </property>
<property>
<name>year</name>
<value>${coord:formatTime(coord:nominalTime(),
"y")}</value>
@@ -150,10 +117,6 @@
<property>
<name>hour</name>
<value>${coord:formatTime(coord:nominalTime(),
"H")}</value>
- </property>
- <property>
- <name>mark_directory_done_workflow_file</name>
- <value>${mark_directory_done_workflow_file}</value>
</property>
<property>
<name>destination_dataset_directory</name>
diff --git a/oozie/webrequest/refine/workflow.xml
b/oozie/webrequest/refine/workflow.xml
index 4ab73a5..11a6ab5 100644
--- a/oozie/webrequest/refine/workflow.xml
+++ b/oozie/webrequest/refine/workflow.xml
@@ -75,6 +75,10 @@
<description>Workflow for marking a directory done</description>
</property>
<property>
+ <name>send_error_email_workflow_file</name>
+ <description>Workflow for sending an error email</description>
+ </property>
+ <property>
<name>destination_dataset_directory</name>
<description>Directory to generate the done flag in</description>
</property>
@@ -121,7 +125,7 @@
</hive>
<ok to="mark_dataset_done"/>
- <error to="kill"/>
+ <error to="send_error_email"/>
</action>
<action name="mark_dataset_done">
@@ -135,6 +139,33 @@
</configuration>
</sub-workflow>
<ok to="end"/>
+ <error to="send_error_email"/>
+ </action>
+
+ <action name="send_error_email">
+ <sub-workflow>
+ <app-path>${send_error_email_workflow_file}</app-path>
+ <propagate-configuration/>
+ <configuration>
+ <property>
+ <name>parent_name</name>
+ <value>${wf:name()}</value>
+ </property>
+ <property>
+ <name>parent_failed_action</name>
+ <value>${wf:lastErrorNode()}</value>
+ </property>
+ <property>
+ <name>parent_error_code</name>
+ <value>${wf:errorCode(wf:lastErrorNode())}</value>
+ </property>
+ <property>
+ <name>parent_error_message</name>
+ <value>${wf:errorMessage(wf:lastErrorNode())}</value>
+ </property>
+ </configuration>
+ </sub-workflow>
+ <ok to="kill"/>
<error to="kill"/>
</action>
--
To view, visit https://gerrit.wikimedia.org/r/274691
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iecf741552b8eee315b63d0860e8ea8c39e72c723
Gerrit-PatchSet: 2
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: Joal <j...@wikimedia.org>
Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org>
Gerrit-Reviewer: Madhuvishy <mviswanat...@wikimedia.org>
Gerrit-Reviewer: Nuria <nu...@wikimedia.org>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
