OliverKeyes has uploaded a new change for review.
https://gerrit.wikimedia.org/r/239858
Change subject: Switch Rainbow over to Polloi
......................................................................
Switch Rainbow over to Polloi
Switches the Rainbow dashboards over to use Polloi for common
functionality rather than relying on local versions.
Bug: T112700
Change-Id: I421532aac2c43b077a2afae476358ade3a73f551
---
D assets/css/custom.css
M server.R
M utils.R
3 files changed, 152 insertions(+), 287 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/discovery/rainbow
refs/changes/58/239858/1
diff --git a/assets/css/custom.css b/assets/css/custom.css
deleted file mode 100644
index fb0d105..0000000
--- a/assets/css/custom.css
+++ /dev/null
@@ -1,3 +0,0 @@
-.dygraph-legend {
- background-color: #ECF0F5 !important;
-}
diff --git a/server.R b/server.R
index 932ea65..8144ec3 100644
--- a/server.R
+++ b/server.R
@@ -6,30 +6,30 @@
## Read in desktop data and generate means for the value boxes, along with a
time-series appropriate form for
## dygraphs.
read_desktop <- function(){
- data <- download_set("desktop_event_counts.tsv")
+ data <- polloi::read_dataset("search/desktop_event_counts.tsv")
interim <- reshape2::dcast(data, formula = timestamp ~ action, fun.aggregate
= sum)
interim[is.na(interim)] <- 0
desktop_dygraph_set <<- interim
desktop_dygraph_means <<- round(colMeans(desktop_dygraph_set[,2:5]))
- data <- download_set("desktop_load_times.tsv")
+ data <- polloi::read_dataset("search/desktop_load_times.tsv")
desktop_load_data <<- data
return(invisible())
}
read_web <- function(){
- data <- download_set("mobile_event_counts.tsv")
+ data <- polloi::read_dataset("search/mobile_event_counts.tsv")
interim <- reshape2::dcast(data, formula = timestamp ~ action, fun.aggregate
= sum)
interim[is.na(interim)] <- 0
mobile_dygraph_set <<- interim
mobile_dygraph_means <<- round(colMeans(mobile_dygraph_set[,2:4]))
- mobile_load_data <<- download_set("mobile_load_times.tsv")
+ mobile_load_data <<- polloi::read_dataset("search/mobile_load_times.tsv")
return(invisible())
}
read_apps <- function(){
- data <- download_set("app_event_counts.tsv")
+ data <- polloi::read_dataset("search/app_event_counts.tsv")
ios <- reshape2::dcast(data[data$platform == "iOS",], formula = timestamp ~
action, fun.aggregate = sum)
android <- reshape2::dcast(data[data$platform == "Android",], formula =
timestamp ~ action, fun.aggregate = sum)
@@ -39,7 +39,7 @@
android_dygraph_set <<- android
android_dygraph_means <<- round(colMeans(android[,2:4]))
- app_load_data <- download_set("app_load_times.tsv")
+ app_load_data <- polloi::read_dataset("search/app_load_times.tsv")
ios_load_data <<- app_load_data[app_load_data$platform == "iOS",
names(app_load_data) != "platform"]
android_load_data <<- app_load_data[app_load_data$platform == "Android",
names(app_load_data) != "platform"]
@@ -47,14 +47,14 @@
}
read_api <- function(){
- data <- download_set("search_api_aggregates.tsv")
+ data <- polloi::read_dataset("search/search_api_aggregates.tsv")
data <- data[order(data$event_type),]
split_dataset <<- split(data, f = data$event_type)
return(invisible())
}
read_failures <- function(date){
- data <- download_set("cirrus_query_aggregates.tsv")
+ data <- polloi::read_dataset("search/cirrus_query_aggregates.tsv")
interim_data <- reshape2::dcast(data, formula = date ~ variable,
fun.aggregate = sum)
failure_dygraph_set <<- interim_data
@@ -66,12 +66,12 @@
daily_change = output_vector*100,
stringsAsFactors = FALSE)
- interim_breakdown_data <- download_set("cirrus_query_breakdowns.tsv")
+ interim_breakdown_data <-
polloi::read_dataset("search/cirrus_query_breakdowns.tsv")
interim_breakdown_data$value <- interim_breakdown_data$value*100
failure_breakdown_dygraph_set <<- reshape2::dcast(interim_breakdown_data,
formula = date ~ variable,
fun.aggregate = sum)
- suggestion_data <- download_set("cirrus_suggestion_breakdown.tsv")
+ suggestion_data <-
polloi::read_dataset("search/cirrus_suggestion_breakdown.tsv")
suggestion_data$variable <- "Full-Text with Suggestions"
suggestion_data$value <- suggestion_data$value*100
suggestion_data <- rbind(suggestion_data,
@@ -123,17 +123,18 @@
## The dynamic graphs of events on desktop
output$desktop_event_plot <- renderDygraph({
- smooth_level <- input$smoothing_desktop_event
- make_dygraph(desktop_dygraph_set,
- "Date", "Events", "Desktop search events, by day",
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(desktop_dygraph_set,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_desktop_event)),
+ xlab = "Date", ylab = "Events", title = "Desktop
search events, by day")
})
+
output$desktop_load_plot <- renderDygraph({
- smooth_level <- input$smoothing_desktop_load
- make_dygraph(desktop_load_data,
- "Date", "Load time (ms)", "Desktop result load times, by day",
- use_si = FALSE,
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(desktop_load_data,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_desktop_load)),
+ xlab = "Date", ylab = "Load time (ms)", title =
"Desktop load times, by day",
+ use_si = FALSE)
})
## Mobile value boxes
@@ -166,14 +167,18 @@
## Mobile plots
output$mobile_event_plot <- renderDygraph({
- smooth_level <- input$smoothing_mobile_event
- make_dygraph(mobile_dygraph_set, "Date", "Events", "Mobile search events,
by day",
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(mobile_dygraph_set,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_mobile_event)),
+ xlab = "Date", ylab = "Events", title = "Mobile
search events, by day")
})
+
output$mobile_load_plot <- renderDygraph({
- smooth_level <- input$smoothing_mobile_load
- make_dygraph(mobile_load_data, "Date", "Load time (ms)", "Mobile result
load times, by day",
- use_si = FALSE, smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(mobile_load_data,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_mobile_load)),
+ xlab = "Date", ylab = "Load time (ms)", "Mobile
result load times, by day",
+ use_si = FALSE)
})
## App value boxes
@@ -206,73 +211,104 @@
## App plots
output$android_event_plot <- renderDygraph({
- smooth_level <- input$smoothing_app_event
- make_dygraph(android_dygraph_set, "Date", "Events", "Android mobile app
search events, by day",
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(android_dygraph_set,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_app_event)),
+ xlab = "Date", ylab = "Events", "Android mobile app
search events, by day")
})
+
output$android_load_plot <- renderDygraph({
- smooth_level <- input$smoothing_app_load
- make_dygraph(android_load_data, "Date", "Load time (ms)","Android result
load times, by day", use_si = FALSE,
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(android_load_data,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_app_load)),
+ xlab = "Date", ylab = "Load time (ms)", "Android
result load times, by day",
+ use_si = FALSE)
})
+
output$ios_event_plot <- renderDygraph({
- smooth_level <- input$smoothing_app_event
- make_dygraph(ios_dygraph_set, "Date", "Events","iOS mobile app search
events, by day",
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(ios_dygraph_set,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_app_event)),
+ xlab = "Date", ylab = "Events", "iOS mobile app
search events, by day")
})
+
output$ios_load_plot <- renderDygraph({
- smooth_level <- input$smoothing_app_load
- make_dygraph(ios_load_data, "Date", "Load time (ms)","iOS result load
times, by day", use_si = FALSE,
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(ios_load_data,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_app_load)),
+ xlab = "Date", ylab = "Load time (ms)", "iOS result
load times, by day",
+ use_si = FALSE)
})
## API plots
output$cirrus_aggregate <- renderDygraph({
- smooth_level <- input$smoothing_fulltext_search
- make_dygraph(split_dataset$cirrus, "Date", "Events", "Full-text via API
usage by day", TRUE,
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
- })
- output$open_aggregate <- renderDygraph({
- smooth_level <- input$smoothing_open_search
- make_dygraph(split_dataset$open, "Date", "Events", "OpenSearch API usage
by day", TRUE,
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
- })
- output$geo_aggregate <- renderDygraph({
- smooth_level <- input$smoothing_geo_search
- make_dygraph(split_dataset$geo, "Date", "Events", "Geo Search API usage by
day", TRUE,
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
- })
- output$language_aggregate <- renderDygraph({
- smooth_level <- input$smoothing_language_search
- make_dygraph(split_dataset$language, "Date", "Events", "Language Search
API usage by day", TRUE,
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
- })
- output$prefix_aggregate <- renderDygraph({
- smooth_level <- input$smoothing_prefix_search
- make_dygraph(split_dataset$prefix, "Date", "Events", "Prefix Search API
usage by day", TRUE,
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(split_dataset$cirrus[,c(1,3)],
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_fulltext_search)),
+ xlab = "Date", ylab = "Searches", "Full-text via API
usage by day",
+ legend_name = "Searches")
})
- ## Failure plots
+ output$open_aggregate <- renderDygraph({
+ polloi::make_dygraph(data = polloi::smoother(split_dataset$open[,c(1,3)],
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_open_search)),
+ xlab = "Date", ylab = "Searches", "OpenSearch API
usage by day",
+ legend_name = "Searches")
+ })
+
+ output$geo_aggregate <- renderDygraph({
+ polloi::make_dygraph(data = polloi::smoother(split_dataset$geo[,c(1,3)],
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_geo_search)),
+ xlab = "Date", ylab = "Searches", "Geo Search API
usage by day",
+ legend_name = "Searches")
+ })
+
+ output$language_aggregate <- renderDygraph({
+ polloi::make_dygraph(data =
polloi::smoother(split_dataset$language[,c(1,3)],
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_language_search)),
+ xlab = "Date", ylab = "Searches", "Language Search
API usage by day",
+ legend_name = "Searches")
+ })
+
+ output$prefix_aggregate <- renderDygraph({
+ polloi::make_dygraph(data = polloi::smoother(split_dataset$prefix[,c(1,3)],
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_prefix_search)),
+ xlab = "Date", ylab = "Searches", "Prefix Search API
usage by day",
+ legend_name = "Searches")
+ })
+
+ # Failure plots
output$failure_rate_plot <- renderDygraph({
- smooth_level <- input$smoothing_failure_rate
- make_dygraph(failure_dygraph_set, "Date", "Queries", "Search Queries with
Zero Results, by day",
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(failure_dygraph_set,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_failure_rate)),
+ xlab = "Date", ylab = "Queries", "Search Queries with
Zero Results, by day")
})
+
output$failure_rate_change_plot <- renderDygraph({
- smooth_level <- input$smoothing_failure_rate
- make_dygraph(failure_roc_dygraph_set, "Date", "Change (%)", "Zero result
rate change, by day", TRUE, "Rate of Change",
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data =
polloi::smoother(failure_roc_dygraph_set[,c(1,3)],
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_failure_rate)),
+ xlab = "Date", ylab = "Change (%)", "Zero Results
rate change, by day",
+ legend_name = "Change")
})
+
output$failure_breakdown_plot <- renderDygraph({
- smooth_level <- input$smoothing_failure_breakdown
- make_dygraph(failure_breakdown_dygraph_set, "Date", "Zero Results Rate
(%)", "Zero result rate by search type",
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(failure_breakdown_dygraph_set,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_failure_breakdown)),
+ xlab = "Date", ylab = "Zero Results Rate (%)", "Zero
result rate by search type")
})
+
output$suggestion_dygraph_plot <- renderDygraph({
- smooth_level <- input$smoothing_failure_suggestions
- make_dygraph(suggestion_dygraph_set, "Date", "Zero Results Rate (%)",
"Zero Result Rates with Search Suggestions",
- smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ polloi::make_dygraph(data = polloi::smoother(suggestion_dygraph_set,
+ smooth_level =
polloi::smooth_switch(input$smoothing_global,
+
input$smoothing_failure_suggestions)),
+ xlab = "Date", ylab = "Zero Results Rate (%)", "Zero
Result Rates with Search Suggestions")
})
## KPI module
@@ -280,7 +316,7 @@
date_range <- input$kpi_summary_date_range_selector
switch(date_range,
daily = {
- temp <- safe_tail(desktop_load_data, 2)$timestamp %>% {
+ temp <- polloi::safe_tail(desktop_load_data, 2)$timestamp %>% {
paste0(as.character(., "%A, %b "),
sub("([a-z]{2})", "<sup>\\1</sup>",
sapply(as.numeric(as.character(., "%e")),
toOrdinal)))
@@ -288,7 +324,7 @@
},
weekly = {
date_range_index <- c(1, 7, 8, 14)
- temp <- safe_tail(desktop_load_data,
date_range_index[4])$timestamp %>% {
+ temp <- polloi::safe_tail(desktop_load_data,
date_range_index[4])$timestamp %>% {
paste0(as.character(.[date_range_index], "%b "),
sub("([a-z]{2})", "<sup>\\1</sup>",
sapply(as.numeric(as.character(.[date_range_index],
"%e")), toOrdinal)))
@@ -298,7 +334,7 @@
},
monthly = {
date_range_index <- c(1, 31, 31, 60)
- temp <- safe_tail(desktop_load_data,
date_range_index[4])$timestamp %>% {
+ temp <- polloi::safe_tail(desktop_load_data,
date_range_index[4])$timestamp %>% {
paste0(as.character(.[date_range_index], "%b "),
sub("([a-z]{2})", "<sup>\\1</sup>",
sapply(as.numeric(as.character(.[date_range_index],
"%e")), toOrdinal)))
@@ -308,7 +344,7 @@
},
quarterly = {
date_range_index <- c(1, 90)
- temp <- safe_tail(desktop_load_data,
date_range_index[2])$timestamp %>% {
+ temp <- polloi::safe_tail(desktop_load_data,
date_range_index[2])$timestamp %>% {
paste0(as.character(.[date_range_index], "%B "),
sub("([a-z]{2})", "<sup>\\1</sup>",
sapply(as.numeric(as.character(.[date_range_index],
"%e")), toOrdinal)))
@@ -323,7 +359,7 @@
daily = {
x <- lapply(list(desktop_load_data, mobile_load_data,
android_load_data, ios_load_data),
- safe_tail, n = 2) %>%
+ polloi::safe_tail, n = 2) %>%
lapply(function(data_tail) return(data_tail$Median)) %>%
do.call(cbind, .) %>%
apply(MARGIN = 1, FUN = median)
@@ -331,7 +367,7 @@
weekly = {
x <- lapply(list(desktop_load_data, mobile_load_data,
android_load_data, ios_load_data),
- safe_tail, n = 14) %>%
+ polloi::safe_tail, n = 14) %>%
lapply(function(data_tail) return(data_tail$Median)) %>%
do.call(cbind, .) %>%
apply(MARGIN = 1, FUN = median)
@@ -339,7 +375,7 @@
monthly = {
x <- lapply(list(desktop_load_data, mobile_load_data,
android_load_data, ios_load_data),
- safe_tail, n = 60) %>%
+ polloi::safe_tail, n = 60) %>%
lapply(function(data_tail) return(data_tail$Median)) %>%
do.call(cbind, .) %>%
apply(MARGIN = 1, FUN = median)
@@ -347,37 +383,37 @@
quarterly = {
x <- lapply(list(desktop_load_data, mobile_load_data,
android_load_data, ios_load_data),
- safe_tail, n = 90) %>%
+ polloi::safe_tail, n = 90) %>%
lapply(function(data_tail) return(data_tail$Median))
y <- median(apply(do.call(cbind, x), 1, median))
return(valueBox(subtitle = "Load time", value = sprintf("%.0fms",
y), color = "orange"))
})
- y1 <- median(half(x, "top")); y2 <- median(half(x, "bottom")); z <- 100 *
(y2 - y1) / y1
+ y1 <- median(polloi::half(x)); y2 <- median(polloi::half(x, FALSE)); z <-
100 * (y2 - y1) / y1
if (abs(z) > 0) {
return(valueBox(subtitle = sprintf("Load time (%.1f%%)", z),
value = sprintf("%.0fms", y2),
- color = cond_color(z>0, "red"), icon = cond_icon(z>0)))
+ color = polloi::cond_color(z > 0, "red"), icon =
polloi::cond_icon(z > 0)))
}
return(valueBox(subtitle = "Load time (no change)", value =
sprintf("%.0fms", y2), color = "orange"))
})
output$kpi_summary_box_zero_results <- renderValueBox({
date_range <- input$kpi_summary_date_range_selector
switch(date_range,
- daily = {x <- safe_tail(failure_dygraph_set, 2)},
- weekly = {x <- safe_tail(failure_dygraph_set, 14)},
- monthly = {x <- safe_tail(failure_dygraph_set, 60)},
- quarterly = {x <- safe_tail(failure_dygraph_set, 90)})
+ daily = {x <- polloi::safe_tail(failure_dygraph_set, 2)},
+ weekly = {x <- polloi::safe_tail(failure_dygraph_set, 14)},
+ monthly = {x <- polloi::safe_tail(failure_dygraph_set, 60)},
+ quarterly = {x <- polloi::safe_tail(failure_dygraph_set, 90)})
x <- transform(x, Rate = `Zero Result Queries` / `Search Queries`)$Rate
if (date_range == "quarterly") {
return(valueBox(subtitle = "Zero results rate", color = "orange",
value = sprintf("%.1f%%", median(100 * x))))
}
- y1 <- median(half(x, "top")); y2 <- median(half(x, "bottom")); z <- 100 *
(y2 - y1)/y1
+ y1 <- median(polloi::half(x)); y2 <- median(polloi::half(x, FALSE)); z <-
100 * (y2 - y1)/y1
if (abs(z) > 0) {
return(valueBox(
subtitle = sprintf("Zero results rate (%.1f%%)", z),
value = sprintf("%.1f%%", 100 * y2),
- icon = cond_icon(z > 0), color = cond_color(z > 0, "red")
+ icon = cond_icon(z > 0), color = polloi::cond_color(z > 0, "red")
))
}
return(valueBox(subtitle = "Zero results rate (no change)",
@@ -387,24 +423,24 @@
date_range <- input$kpi_summary_date_range_selector
x <- lapply(split_dataset, function(x) {
switch(date_range,
- daily = { safe_tail(x, 2)$events },
- weekly = { safe_tail(x, 14)$events },
- monthly = { safe_tail(x, 60)$events },
- quarterly = { safe_tail(x, 90)$events })
+ daily = { polloi::safe_tail(x, 2)$events },
+ weekly = { polloi::safe_tail(x, 14)$events },
+ monthly = { polloi::safe_tail(x, 60)$events },
+ quarterly = { polloi::safe_tail(x, 90)$events })
}) %>% do.call(cbind, .) %>%
transform(total = cirrus + geo + language + open + prefix) %>%
{ .$total }
if (date_range == "quarterly") {
- return(valueBox(subtitle = "API usage", value = compress(median(x), 0),
color = "orange"))
+ return(valueBox(subtitle = "API usage", value =
polloi::compress(median(x), 0), color = "orange"))
}
- y1 <- median(half(x, "top"))
- y2 <- median(half(x, "bottom"))
+ y1 <- median(polloi::half(x, FALSE))
+ y2 <- median(polloi::half(x, FALSE))
z <- 100 * (y2 - y1) / y1 # % change from t-1 to t
if (abs(z) > 0) {
return(valueBox(subtitle = sprintf("API usage (%.1f%%)", z),
- value = compress(y2, 0), color = cond_color(z>0), icon =
cond_icon(z>0)))
+ value = polloi::compress(y2, 0), color =
polloi::cond_color(z > 0), icon = polloi::cond_icon(z > 0)))
}
- return(valueBox(subtitle = "API usage (no change)", value = compress(y2,
0), color = "orange"))
+ return(valueBox(subtitle = "API usage (no change)", value =
polloi::compress(y2, 0), color = "orange"))
})
output$kpi_summary_api_usage_proportions <- renderPlot({
switch (input$kpi_summary_date_range_selector,
@@ -413,11 +449,11 @@
monthly = { n <- 30 },
quarterly = { n <- 90 }
)
- api_latest <- cbind("Full-text via API" = safe_tail(split_dataset$cirrus,
n)$events,
- "Geo Search" = safe_tail(split_dataset$geo, n)$events,
- "OpenSearch" = safe_tail(split_dataset$open, n)$events,
- "Language" = safe_tail(split_dataset$language,
n)$events,
- "Prefix" = safe_tail(split_dataset$prefix, n)$events)
%>%
+ api_latest <- cbind("Full-text via API" =
polloi::safe_tail(split_dataset$cirrus, n)$events,
+ "Geo Search" = polloi::safe_tail(split_dataset$geo,
n)$events,
+ "OpenSearch" = polloi::safe_tail(split_dataset$open,
n)$events,
+ "Language" = polloi::safe_tail(split_dataset$language,
n)$events,
+ "Prefix" = polloi::safe_tail(split_dataset$prefix,
n)$events) %>%
apply(2, median) %>% round
api_latest <- data.frame(API = names(api_latest),
Events = api_latest,
@@ -434,7 +470,7 @@
smooth_level <- input$smoothing_kpi_load_time
num_of_days_in_common <- min(sapply(list(desktop_load_data$Median,
mobile_load_data$Median, android_load_data$Median, ios_load_data$Median),
length))
load_times <- list(desktop_load_data, mobile_load_data, android_load_data,
ios_load_data) %>%
- lapply(safe_tail, num_of_days_in_common) %>%
+ lapply(polloi::safe_tail, num_of_days_in_common) %>%
lapply(function(data_tail) return(data_tail$Median)) %>%
as.data.frame %>%
{ colnames(.) <- c("Desktop", "Mobile Web", "Android", "iOS"); . } %>%
@@ -442,8 +478,8 @@
Median = apply(., 1, median)
cbind(Median = Median, .)
} %>%
- cbind(timestamp = safe_tail(desktop_load_data,
num_of_days_in_common)$timestamp, .) %>%
- smoother(smooth_level = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level), rename = FALSE) %>%
+ cbind(timestamp = polloi::safe_tail(desktop_load_data,
num_of_days_in_common)$timestamp, .) %>%
+ polloi::smoother(smooth_level = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level), rename = FALSE) %>%
{ xts(.[, -1], order.by = .[, 1]) }
return(dygraph(load_times,
main = "Load times over time",
@@ -463,7 +499,7 @@
zrr <- 100 * failure_dygraph_set$`Zero Result Queries` /
failure_dygraph_set$`Search Queries`
zrr_change <- 100 * (zrr[2:length(zrr)] -
zrr[1:(length(zrr)-1)])/zrr[1:(length(zrr)-1)]
zrr <- data.frame(date = failure_dygraph_set$date[-1], rate = zrr[-1],
change = zrr_change)
- zrr %<>% smoother(ifelse(smooth_level == "global", input$smoothing_global,
smooth_level), rename = FALSE)
+ zrr %<>% polloi::smoother(ifelse(smooth_level == "global",
input$smoothing_global, smooth_level), rename = FALSE)
zrr <- xts(zrr[, -1], zrr[, 1])
return(dygraph(zrr,
main = "Zero results rate over time",
@@ -496,7 +532,7 @@
api_usage <- transform(api_usage, all = cirrus + geo + language + prefix)
}
if ( input$kpi_api_usage_series_data == "raw" ) {
- api_usage %<>% smoother(ifelse(smooth_level == "global",
input$smoothing_global, smooth_level), rename = FALSE)
+ api_usage %<>% polloi::smoother(ifelse(smooth_level == "global",
input$smoothing_global, smooth_level), rename = FALSE)
api_usage <- xts(api_usage[, -1], api_usage[, 1])
if (!input$kpi_api_usage_series_include_open) {
colnames(api_usage)[6] <- "all except open"
@@ -513,14 +549,14 @@
dyCSS(css = "./assets/css/custom.css"))
}
api_usage_change <- transform(api_usage,
- cirrus = percent_change(cirrus),
- geo = percent_change(geo),
- language = percent_change(language),
- open = percent_change(open),
- prefix = percent_change(prefix),
- all = percent_change(all)) %>%
+ cirrus = polloi::percent_change(cirrus),
+ geo = polloi::percent_change(geo),
+ language = polloi::percent_change(language),
+ open = polloi::percent_change(open),
+ prefix = polloi::percent_change(prefix),
+ all = polloi::percent_change(all)) %>%
{ .[-1, ] }
- api_usage_change %<>% smoother(ifelse(smooth_level == "global",
input$smoothing_global, smooth_level), rename = FALSE)
+ api_usage_change %<>% polloi::smoother(ifelse(smooth_level == "global",
input$smoothing_global, smooth_level), rename = FALSE)
api_usage_change <- xts(api_usage_change[, -1], api_usage_change[, 1])
if (!input$kpi_api_usage_series_include_open)
colnames(api_usage_change)[6] <- "all except open"
return(dygraph(api_usage_change,
diff --git a/utils.R b/utils.R
index 882dad5..a20944a 100644
--- a/utils.R
+++ b/utils.R
@@ -1,142 +1,9 @@
#Dependent libs
-library(plyr)
-library(readr)
-library(xts)
library(reshape2)
-library(RColorBrewer)
library(ggplot2)
library(toOrdinal)
-library(lubridate)
library(magrittr)
-
-#Utility functions for handling particularly common tasks
-download_set <- function(location){
- location <-
paste0("http://datasets.wikimedia.org/aggregate-datasets/search/";, location,
- "?ts=", gsub(x = Sys.time(), pattern = "(-| )",
replacement = ""))
- con <- url(location)
- return(readr::read_delim(con, delim = "\t"))
-}
-
-# Takes an untidy (read: dygraph-appropriate) dataset and adds
-# columns for each variable consisting of the smoothed, averaged mean
-smoother <- function(dataset, smooth_level = "day", rename = TRUE) {
-
- # Determine the names and levels of aggregation. By default
- # a smoothing level of "day" is assumed, which is no smoothing
- # whatsoever, and so the original dataset is returned.
- switch(smooth_level,
- moving_avg = {
- df <- apply(dataset[, -1, drop = FALSE], 2, function(x) {
- y <- xts(x, dataset[, 1])
- return(as.numeric(zoo::rollmean(x, k = 17, fill = NA)))
- }) %>% as.data.frame %>% cbind(timestamp = dataset[, 1], .)
- names(df) <- names(dataset)
- if (rename) names(df)[-1] <- paste(names(df)[-1], " (Moving
average)")
- return(df)
- },
- week = {
- dataset$filter_1 <- lubridate::week(dataset[, 1])
- dataset$filter_2 <- lubridate::year(dataset[, 1])
- name_append <- ifelse(rename, " (Weekly average)", "")
- },
- month = {
- dataset$filter_1 <- lubridate::month(dataset[, 1])
- dataset$filter_2 <- lubridate::year(dataset[, 1])
- name_append <- ifelse(rename, " (Monthly average)", "")
- },
- {
- return(dataset)
- }
- )
-
- # If we're still here it was weekly or monthly. Calculate
- # the average for each unique permutation of filters
-
- result <- ddply(.data = dataset,
- .variables = c("filter_1", "filter_2"),
- .fun = function(df, name_append){
-
- # Construct output names for the averages, compute those
averages, and
- # apply said names.
- output_names <- paste0(names(df)[2:(ncol(df) - 2)],
name_append)
- holding <- apply(df[, 2:(ncol(df) - 2), drop = FALSE], 2,
FUN = median) %>%
- round %>% t %>% as.data.frame
- names(holding) <- output_names
-
- # Return the bound original values and averaged values
- return(cbind(df[, 1, drop = FALSE], holding))
- }, name_append = name_append)
-
- return(result[, !(names(result) %in% c("filter_1","filter_2"))])
-}
-
-#Create a dygraph using our standard format.
-make_dygraph <- function(data, x, y, title, is_single = FALSE, legend_name =
NULL, use_si = TRUE, smoothing = "day") {
- # cat("Making dygraph:", title, "\n"); # Debugging
- if (is_single) {
- data <- smoother(as.data.frame(data[, c(1, 3)]), smooth_level = smoothing)
- data <- xts(data[, 2], data[, 1])
- if (is.null(legend_name)) {
- names(data) <- "events"
- } else {
- names(data) <- legend_name
- }
- } else {
- data %<>% smoother(smooth_level = smoothing)
- data <- xts(data[, -1], order.by = data[, 1])
- }
- return(dygraph(data, main = title, xlab = x, ylab = y) %>%
- dyLegend(width = 400, show = "always") %>%
- dyOptions(strokeWidth = 3,
- colors = brewer.pal(max(3, ncol(data)), "Set2"),
- drawPoints = FALSE, pointSize = 3, labelsKMB = use_si,
- includeZero = TRUE) %>%
- dyCSS(css = "./assets/css/custom.css"))
-}
-
-# Computes a median absolute deviation
-mad <- function(x) {
- median(abs(x - median(x)))
-}
-
-compress <- function(x, round.by = 2) {
- # by StackOverflow user 'BondedDust' : http://stackoverflow.com/a/28160474
- div <- findInterval(as.numeric(gsub("\\,", "", x)),
- c(1, 1e3, 1e6, 1e9, 1e12) )
- paste(round( as.numeric(gsub("\\,","",x))/10^(3*(div-1)), round.by),
- c("","K","M","B","T")[div], sep = "" )
-}
-
-# Conditional icon for widget.
-# Returns arrow-up icon on true (if true_direction is 'up'), e.g. load time %
change > 0
-cond_icon <- function(condition, true_direction = "up") {
-
- if (true_direction == "up") {
- return(icon(ifelse(condition, "arrow-up", "arrow-down")))
- }
-
- return(icon(ifelse(condition, "arrow-down", "arrow-up")))
-}
-
-# Conditional color for widget
-# Returns 'green' on true, 'red' on false, e.g. api usage % change > 0
-# load time % change < 0
-cond_color <- function(condition, true_color = "green") {
- if(is.na(condition)){
- return("black")
- }
-
- colours <- c("green","red")
- return(ifelse(condition, true_color, colours[!colours == true_color]))
-}
-
-# Allows very quickly to get half a vector:
-half <- function(x, which = c("top", "bottom")) {
- if (which == "top") {
- return(head(x, n = length(x)/2))
- }
- return(tail(x, n = length(x)/2))
-}
+library(polloi)
# Uses ggplot2 to create a pie chart in bar form. (Will look up actual name)
gg_prop_bar <- function(data, cols) {
@@ -158,39 +25,4 @@
geom_text(aes_string(label = cols$label,
y = "text_position",
x = 1))
-}
-
-# Calculates percent change either in `x` or from `x` to `y`
-percent_change <- function(x, y = NULL) {
- if(is.null(y)) {
- return(100 * (x - c(NA, x[-length(x)])) / c(NA, x[-length(x)]))
- }
- return(100 * (y - x) / x)
-}
-
-# It's fairly common to need to grab the last N [whatever] of values.
-# The problem with using tail() for this comes in the case where (perhaps
-# due to backfilling) the last rows are not actually the last rows. This
-# function will provide a 'safe' tail mechanism.
-safe_tail <- function(x, n, silent = TRUE) {
- if (!is.vector(x) && !is.data.frame(x)) {
- stop("safe_trail() only works with vectors and data frames.")
- }
- # \code{silent} suppresses messages which may be used for debugging
- if (is.vector(x)) {
- return(tail(sort(x), n))
- }
- # Intelligently figure out which column is the date/timestamp column (in
case it's not the first column):
- timestamp_column <- names(x)[sapply(x, class) %in% c("Date", "POSIXt",
"POSIXlt", "POSIXct")]
- if (length(timestamp_column) == 0) {
- if (!silent) {
- message("No date/timestamp column detected for this dataset. It'd be
faster to use tail().")
- }
- return(tail(x, n))
- }
- if (length(timestamp_column) > 1) warning("More than one date/timestamp
column detected. Defaulting to the first one.")
- if (!silent) {
- message("Sorting by the date/timestamp column before returning the bottom
", n, " rows.")
- }
- return(tail(x[order(x[[timestamp_column[1]]]), ], n))
}
--
To view, visit https://gerrit.wikimedia.org/r/239858
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I421532aac2c43b077a2afae476358ade3a73f551
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/discovery/rainbow
Gerrit-Branch: master
Gerrit-Owner: OliverKeyes <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
