Milimetric has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/370322 )
Change subject: [WIP] Don't merge this
......................................................................
[WIP] Don't merge this
This change that allows us to run one-off _private snapshots more
easily. I'm not sure whether you can set private_suffix to an empty
string, and I haven't tested that yet. I'm just going to use this
change to run the 2017-07_private snapshot, and we can decide what to do
when Joseph's back.
Change-Id: Ib63f27430c15872de1966fa51728211dfe6b3486
---
M oozie/mediawiki/history/datasets.xml
M oozie/mediawiki/history/datasets_raw.xml
M oozie/mediawiki/history/denormalize/coordinator.properties
M oozie/mediawiki/history/denormalize/coordinator.xml
4 files changed, 18 insertions(+), 16 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery
refs/changes/22/370322/1
diff --git a/oozie/mediawiki/history/datasets.xml
b/oozie/mediawiki/history/datasets.xml
index c2232d8..08bab57 100644
--- a/oozie/mediawiki/history/datasets.xml
+++ b/oozie/mediawiki/history/datasets.xml
@@ -22,7 +22,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_directory}/user_history/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_directory}/user_history/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
@@ -30,7 +30,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_directory}/page_history/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_directory}/page_history/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
@@ -38,7 +38,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_directory}/history/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_directory}/history/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
@@ -48,7 +48,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_directory}/user_history/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_directory}/user_history/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_PARTITIONED</done-flag>
</dataset>
@@ -56,7 +56,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_directory}/page_history/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_directory}/page_history/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_PARTITIONED</done-flag>
</dataset>
@@ -64,7 +64,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_directory}/history/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_directory}/history/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_PARTITIONED</done-flag>
</dataset>
@@ -73,7 +73,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_directory}/metrics/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_directory}/metrics/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
</datasets>
diff --git a/oozie/mediawiki/history/datasets_raw.xml
b/oozie/mediawiki/history/datasets_raw.xml
index a672a0e..931d5e3 100644
--- a/oozie/mediawiki/history/datasets_raw.xml
+++ b/oozie/mediawiki/history/datasets_raw.xml
@@ -25,7 +25,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_raw_directory}/tables/archive/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_raw_directory}/tables/archive/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
@@ -33,7 +33,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_raw_directory}/tables/logging/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_raw_directory}/tables/logging/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
@@ -41,7 +41,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_raw_directory}/tables/page/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_raw_directory}/tables/page/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
@@ -49,7 +49,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_raw_directory}/tables/revision/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_raw_directory}/tables/revision/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
@@ -57,7 +57,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_raw_directory}/tables/user/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_raw_directory}/tables/user/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
@@ -65,7 +65,7 @@
frequency="${coord:months(1)}"
initial-instance="${start_time}"
timezone="Universal">
-
<uri-template>${mw_raw_directory}/tables/user_groups/snapshot=${YEAR}-${MONTH}</uri-template>
+
<uri-template>${mw_raw_directory}/tables/user_groups/snapshot=${YEAR}-${MONTH}${private_suffix}</uri-template>
<done-flag>_SUCCESS</done-flag>
</dataset>
</datasets>
diff --git a/oozie/mediawiki/history/denormalize/coordinator.properties
b/oozie/mediawiki/history/denormalize/coordinator.properties
index 8ed1900..21ad83e 100644
--- a/oozie/mediawiki/history/denormalize/coordinator.properties
+++ b/oozie/mediawiki/history/denormalize/coordinator.properties
@@ -42,6 +42,8 @@
# HDFS path to mediawiki history datasets definitions
datasets_file =
${oozie_directory}/mediawiki/history/datasets.xml
mw_directory = ${name_node}/wmf/data/wmf/mediawiki
+# If running a manual _private snapshot, set this to _private, otherwise leave
empty
+private_suffix = ''
# mw tables
mw_project_namespace_map_table = wmf_raw.mediawiki_project_namespace_map
@@ -94,4 +96,4 @@
# Coordinator to start.
oozie.coord.application.path = ${coordinator_file}
oozie.use.system.libpath = true
-oozie.action.external.stats.write = true
\ No newline at end of file
+oozie.action.external.stats.write = true
diff --git a/oozie/mediawiki/history/denormalize/coordinator.xml
b/oozie/mediawiki/history/denormalize/coordinator.xml
index 4a7144f..741a436 100644
--- a/oozie/mediawiki/history/denormalize/coordinator.xml
+++ b/oozie/mediawiki/history/denormalize/coordinator.xml
@@ -125,7 +125,7 @@
<configuration>
<property>
<name>snapshot</name>
- <value>${coord:formatTime(coord:nominalTime(),
"yyyy")}-${coord:formatTime(coord:nominalTime(), "MM")}</value>
+ <value>${coord:formatTime(coord:nominalTime(),
"yyyy")}-${coord:formatTime(coord:nominalTime(), "MM")}${private_suffix}</value>
</property>
<property>
<name>user_history_location</name>
@@ -157,4 +157,4 @@
</sla:info>
</action>
-</coordinator-app>
\ No newline at end of file
+</coordinator-app>
--
To view, visit https://gerrit.wikimedia.org/r/370322
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib63f27430c15872de1966fa51728211dfe6b3486
Gerrit-PatchSet: 1
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: Milimetric <dandree...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
