Ottomata has submitted this change and it was merged.
Change subject: Load and refine logs from 'misc' caches
......................................................................
Load and refine logs from 'misc' caches
Change-Id: If0cf42cae05cad26984fb3d35208b54803f72f88
---
M camus/camus.webrequest.properties
M diagrams/oozie-overview.dia
M oozie/webrequest/datasets.xml
M oozie/webrequest/datasets_raw.xml
M oozie/webrequest/load/bundle.xml
M oozie/webrequest/refine/bundle.xml
6 files changed, 50 insertions(+), 1 deletion(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/camus/camus.webrequest.properties
b/camus/camus.webrequest.properties
index 4f33104..330bb69 100644
--- a/camus/camus.webrequest.properties
+++ b/camus/camus.webrequest.properties
@@ -66,7 +66,7 @@
kafka.blacklist.topics=
# These are the kafka topics camus brings to HDFS
-kafka.whitelist.topics=webrequest_bits,webrequest_mobile,webrequest_text,webrequest_upload
+kafka.whitelist.topics=webrequest_bits,webrequest_mobile,webrequest_text,webrequest_upload,webrequest_misc
# Name of the client as seen by kafka
kafka.client.name=camus-webrequest-00
diff --git a/diagrams/oozie-overview.dia b/diagrams/oozie-overview.dia
index 360b1f5..efdf32b 100644
--- a/diagrams/oozie-overview.dia
+++ b/diagrams/oozie-overview.dia
Binary files differ
diff --git a/oozie/webrequest/datasets.xml b/oozie/webrequest/datasets.xml
index 60ea069..ba2bc70 100644
--- a/oozie/webrequest/datasets.xml
+++ b/oozie/webrequest/datasets.xml
@@ -30,6 +30,14 @@
at “${...}” as input for the second EL level. There, the variables hold
their expected values, and we can start unpadding them.
-->
+ <dataset name="webrequest_misc"
+ frequency="${coord:hours(1)}"
+ initial-instance="${start_time}"
+ timezone="Universal">
+
<uri-template>${webrequest_data_directory}/webrequest_source=misc/year=${YEAR}/month=${"$"}{MONTH
+ 0}/day=${"$"}{DAY + 0}/hour=${"$"}{HOUR + 0}</uri-template>
+ <done-flag>_SUCCESS</done-flag>
+ </dataset>
+
<dataset name="webrequest_mobile"
frequency="${coord:hours(1)}"
initial-instance="${start_time}"
diff --git a/oozie/webrequest/datasets_raw.xml
b/oozie/webrequest/datasets_raw.xml
index 944dc14..2b897a6 100644
--- a/oozie/webrequest/datasets_raw.xml
+++ b/oozie/webrequest/datasets_raw.xml
@@ -22,6 +22,13 @@
<uri-template>${webrequest_raw_data_directory}/webrequest_bits/hourly/${YEAR}/${MONTH}/${DAY}/${HOUR}</uri-template>
<done-flag></done-flag>
</dataset>
+ <dataset name="webrequest_misc_raw_unchecked"
+ frequency="${coord:hours(1)}"
+ initial-instance="${start_time}"
+ timezone="Universal">
+
<uri-template>${webrequest_raw_data_directory}/webrequest_misc/hourly/${YEAR}/${MONTH}/${DAY}/${HOUR}</uri-template>
+ <done-flag></done-flag>
+ </dataset>
<dataset name="webrequest_mobile_raw_unchecked"
frequency="${coord:hours(1)}"
initial-instance="${start_time}"
@@ -55,6 +62,13 @@
initial-instance="${start_time}"
timezone="Universal">
<uri-template>${webrequest_raw_data_directory}/webrequest_bits/hourly/${YEAR}/${MONTH}/${DAY}/${HOUR}</uri-template>
+ <done-flag>_PARTITIONED</done-flag>
+ </dataset>
+ <dataset name="webrequest_misc_raw_partitioned"
+ frequency="${coord:hours(1)}"
+ initial-instance="${start_time}"
+ timezone="Universal">
+
<uri-template>${webrequest_raw_data_directory}/webrequest_misc/hourly/${YEAR}/${MONTH}/${DAY}/${HOUR}</uri-template>
<done-flag>_PARTITIONED</done-flag>
</dataset>
<dataset name="webrequest_mobile_raw_partitioned"
@@ -95,6 +109,13 @@
<uri-template>${webrequest_raw_data_directory}/webrequest_bits/hourly/${YEAR}/${MONTH}/${DAY}/${HOUR}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
+ <dataset name="webrequest_misc_raw"
+ frequency="${coord:hours(1)}"
+ initial-instance="${start_time}"
+ timezone="Universal">
+
<uri-template>${webrequest_raw_data_directory}/webrequest_misc/hourly/${YEAR}/${MONTH}/${DAY}/${HOUR}</uri-template>
+ <done-flag>_SUCCESS</done-flag>
+ </dataset>
<dataset name="webrequest_mobile_raw"
frequency="${coord:hours(1)}"
initial-instance="${start_time}"
diff --git a/oozie/webrequest/load/bundle.xml b/oozie/webrequest/load/bundle.xml
index ac3a6be..19f1106 100644
--- a/oozie/webrequest/load/bundle.xml
+++ b/oozie/webrequest/load/bundle.xml
@@ -36,6 +36,16 @@
</configuration>
</coordinator>
+ <coordinator name='hive_webrequest_load-webrequest_misc'>
+ <app-path>${coordinator_file}</app-path>
+ <configuration>
+ <property>
+ <name>webrequest_source</name>
+ <value>misc</value>
+ </property>
+ </configuration>
+ </coordinator>
+
<coordinator name='hive_webrequest_raw_load-webrequest_mobile'>
<app-path>${coordinator_file}</app-path>
<configuration>
diff --git a/oozie/webrequest/refine/bundle.xml
b/oozie/webrequest/refine/bundle.xml
index 5c636d7..859c3d0 100644
--- a/oozie/webrequest/refine/bundle.xml
+++ b/oozie/webrequest/refine/bundle.xml
@@ -43,6 +43,16 @@
</coordinator>
-->
+ <coordinator name='refine-webrequest_misc'>
+ <app-path>${coordinator_file}</app-path>
+ <configuration>
+ <property>
+ <name>webrequest_source</name>
+ <value>misc</value>
+ </property>
+ </configuration>
+ </coordinator>
+
<coordinator name='refine-webrequest_mobile'>
<app-path>${coordinator_file}</app-path>
<configuration>
--
To view, visit https://gerrit.wikimedia.org/r/184191
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If0cf42cae05cad26984fb3d35208b54803f72f88
Gerrit-PatchSet: 4
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: QChris <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
Gerrit-Reviewer: QChris <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
