Hello Ottomata,
I'd like you to do a code review. Please visit
https://gerrit.wikimedia.org/r/187417
to review the following change.
Change subject: Add refined tables to webrequest dump script
......................................................................
Add refined tables to webrequest dump script
Change-Id: I7e95d26caa0e41b0d25fb3ad4b152e5053cf1107
---
M bin/refinery-dump-status-webrequest-partitions
1 file changed, 24 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery
refs/changes/17/187417/1
diff --git a/bin/refinery-dump-status-webrequest-partitions
b/bin/refinery-dump-status-webrequest-partitions
index 40b652f..0ca523a 100755
--- a/bin/refinery-dump-status-webrequest-partitions
+++ b/bin/refinery-dump-status-webrequest-partitions
@@ -16,7 +16,11 @@
-- Select the datasets to output data for.
The following datasets are available:
raw_webrequest -- Raw webrequest (hourly)
+ webrequest -- webrequest (refined tables)
(hourly)
all -- all of the above
+
+ By default, only "raw_webrequest" is shown.
+
--quiet -- Only produce output, if there are faulty partitions
HOURS_TO_GO_BACK -- number of hours to go back in time. (Default: 51)
@@ -53,6 +57,7 @@
}
add_dataset "raw_webrequest" " bits | misc | mobile | text | upload |"
+add_dataset "webrequest" " bits | misc | mobile | text | upload |"
DATASET_VISIBILITIES["raw_webrequest"]=yes
@@ -129,6 +134,7 @@
RAW_WEBREQUEST_DATA_DIR_ABS="$HDFS_MOUNT_DIR_ABS/wmf/data/raw/webrequest"
RAW_WEBREQUEST_STATISTICS_DIR_ABS="$HDFS_MOUNT_DIR_ABS/wmf/data/raw/webrequests_faulty_hosts"
+WEBREQUEST_DATA_DIR_ABS="$HDFS_MOUNT_DIR_ABS/wmf/data/wmf/webrequest"
log_no_lf() {
if [ -n "$QUIET" ]
@@ -249,6 +255,24 @@
done
}
+dump_dataset_webrequest() {
+ local DATE="$1"
+
+ local DATE_DIRS_REL="$(date --utc -d "$DATE"
+'year=%Y/month=%m/day=%d/hour=%H')"
+ DATE_DIRS_REL="${DATE_DIRS_REL//=0/=}"
+
+ for SOURCE in bits misc mobile text upload
+ do
+ local STATUS="X"
+
SUCCESS_FILE_ABS="$WEBREQUEST_DATA_DIR_ABS/webrequest_source=$SOURCE/$DATE_DIRS_REL/_SUCCESS"
+ if [ -e "$SUCCESS_FILE_ABS" ]
+ then
+ STATUS="."
+ fi
+ log_no_lf " $STATUS |"
+ done
+}
+
hline "first"
first_caption_line
second_caption_line
--
To view, visit https://gerrit.wikimedia.org/r/187417
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7e95d26caa0e41b0d25fb3ad4b152e5053cf1107
Gerrit-PatchSet: 1
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: QChris <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
