Bearloga has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/374669 )
Change subject: Breakdown API calls by referer class
......................................................................
Breakdown API calls by referer class
Bug: T172452
Change-Id: Ic70d7054e02569eb8545dd347026c7f77321ab2c
---
M modules/api.R
M modules/key_performance_metrics/api_usage.R
A tab_documentation/referer_breakdown.md
M ui.R
M utils.R
5 files changed, 95 insertions(+), 27 deletions(-)
Approvals:
Bearloga: Verified; Looks good to me, approved
diff --git a/modules/api.R b/modules/api.R
index dc8e332..5fd6cd1 100644
--- a/modules/api.R
+++ b/modules/api.R
@@ -3,6 +3,7 @@
tidyr::spread(referrer, calls) %>%
polloi::smoother(smooth_level =
polloi::smooth_switch(input$smoothing_global, input$smoothing_fulltext_search))
%>%
polloi::make_dygraph(xlab = "Date", ylab = "Searches", title = "Daily
Full-text search API usage by referrer", legend_name = "Searches") %>%
+ dyLegend(width = 1000, show = "always") %>%
dyRangeSelector %>%
dyEvent(as.Date("2017-01-01"), "R (reportupdater)", labelLoc = "bottom")
%>%
dyEvent(as.Date("2017-06-29"), "U (new UDF)", labelLoc = "bottom")
@@ -13,6 +14,7 @@
tidyr::spread(referrer, calls) %>%
polloi::smoother(smooth_level =
polloi::smooth_switch(input$smoothing_global, input$smoothing_morelike_search))
%>%
polloi::make_dygraph(xlab = "Date", ylab = "Searches", title = "Daily
Morelike search API usage by referrer", legend_name = "Searches") %>%
+ dyLegend(width = 1000, show = "always") %>%
dyRangeSelector
})
@@ -21,6 +23,7 @@
tidyr::spread(referrer, calls) %>%
polloi::smoother(smooth_level =
polloi::smooth_switch(input$smoothing_global, input$smoothing_open_search)) %>%
polloi::make_dygraph(xlab = "Date", ylab = "Searches", title = "Daily
OpenSearch API usage by referrer", legend_name = "Searches") %>%
+ dyLegend(width = 1000, show = "always") %>%
dyRangeSelector %>%
dyEvent(as.Date("2017-01-01"), "R (reportupdater)", labelLoc = "bottom")
%>%
dyEvent(as.Date("2017-06-29"), "U (new UDF)", labelLoc = "bottom")
@@ -30,6 +33,7 @@
split_dataset$geo %>%
tidyr::spread(referrer, calls) %>%polloi::smoother(smooth_level =
polloi::smooth_switch(input$smoothing_global, input$smoothing_geo_search)) %>%
polloi::make_dygraph(xlab = "Date", ylab = "Searches", title = "Daily Geo
Search API usage by referrer", legend_name = "Searches") %>%
+ dyLegend(width = 1000, show = "always") %>%
dyRangeSelector %>%
dyEvent(as.Date("2017-01-01"), "R (reportupdater)", labelLoc = "bottom")
%>%
dyEvent(as.Date("2017-06-29"), "U (new UDF)", labelLoc = "bottom")
@@ -39,6 +43,7 @@
split_dataset$language %>%
tidyr::spread(referrer, calls) %>%polloi::smoother(smooth_level =
polloi::smooth_switch(input$smoothing_global, input$smoothing_language_search))
%>%
polloi::make_dygraph(xlab = "Date", ylab = "Searches", title = "Daily
Language search API usage by referrer", legend_name = "Searches") %>%
+ dyLegend(width = 1000, show = "always") %>%
dyRangeSelector %>%
dyEvent(as.Date("2017-01-01"), "R (reportupdater)", labelLoc = "bottom")
%>%
dyEvent(as.Date("2017-06-29"), "U (new UDF)", labelLoc = "bottom")
@@ -49,7 +54,27 @@
tidyr::spread(referrer, calls) %>%
polloi::smoother(smooth_level =
polloi::smooth_switch(input$smoothing_global, input$smoothing_prefix_search))
%>%
polloi::make_dygraph(xlab = "Date", ylab = "Searches", title = "Daily
Prefix search API usage by referrer", legend_name = "Searches") %>%
+ dyLegend(width = 1000, show = "always") %>%
dyRangeSelector %>%
dyEvent(as.Date("2017-01-01"), "R (reportupdater)", labelLoc = "bottom")
%>%
dyEvent(as.Date("2017-06-29"), "U (new UDF)", labelLoc = "bottom")
})
+
+output$referer_breakdown_plot <- renderDygraph({
+ temp <- split_dataset %>%
+ dplyr::bind_rows(.id = "api") %>%
+ dplyr::filter(date >= "2017-06-29") %>%
+ dplyr::group_by(date, referrer) %>%
+ dplyr::summarize(calls = sum(calls, na.rm = TRUE)) %>%
+ tidyr::spread(referrer, calls)
+ if (input$referer_breakdown_prop) {
+ temp <- cbind(temp$date, purrr::map_df(temp[, -c(1, 2)], function(x)
round(100 * x / temp$All, 2)))
+ }
+ temp %>%
+ polloi::smoother(smooth_level =
polloi::smooth_switch(input$smoothing_global,
input$smoothing_referer_breakdown)) %>%
+ polloi::make_dygraph(xlab = "Date",
+ ylab = ifelse(input$referer_breakdown_prop, "API
Calls Share (%)", "API Calls"),
+ title = "Daily API usage by referrer", legend_name =
"API Calls") %>%
+ dyLegend(width = 1000, show = "always") %>%
+ dyRangeSelector
+})
diff --git a/modules/key_performance_metrics/api_usage.R
b/modules/key_performance_metrics/api_usage.R
index 3732619..1980dda 100644
--- a/modules/key_performance_metrics/api_usage.R
+++ b/modules/key_performance_metrics/api_usage.R
@@ -26,7 +26,7 @@
ylab = ifelse(input$kpi_api_usage_series_log_scale, "Calls
(log10 scale)", "Calls")) %>%
dySeries("cirrus", label = "full-text via API") %>%
dySeries("cirrus (more like)", label = "morelike via API") %>%
- dyLegend(width = 400, show = "always") %>%
+ dyLegend(width = 1000, show = "always") %>%
dyOptions(
strokeWidth = 3, colors = RColorBrewer::brewer.pal(7,
"Set2")[7:1],
drawPoints = FALSE, pointSize = 3, labelsKMB = TRUE,
@@ -37,29 +37,29 @@
dyRangeSelector %>%
dyEvent(as.Date("2017-01-01"), "R (reportupdater)", labelLoc =
"bottom") %>%
dyEvent(as.Date("2017-06-29"), "U (new UDF)", labelLoc =
"bottom"))
+ } else {
+ api_usage_change <- api_usage %>%
+ dplyr::mutate(
+ cirrus = polloi::percent_change(cirrus),
+ `cirrus (more like)` = polloi::percent_change(`cirrus (more like)`),
+ geo = polloi::percent_change(geo),
+ language = polloi::percent_change(language),
+ open = polloi::percent_change(open),
+ prefix = polloi::percent_change(prefix),
+ all = polloi::percent_change(all)
+ ) %>%
+ { .[-1, ] } %>%
+ polloi::smoother(ifelse(smooth_level == "global",
input$smoothing_global, smooth_level), rename = FALSE) %>%
+ { xts::xts(.[, -1], .$date) }
+ return(dygraph(api_usage_change, main = "Day-to-day % change over time",
xlab = "Date", ylab = "% change") %>%
+ dyLegend(width = 1000, show = "always") %>%
+ dyOptions(
+ strokeWidth = 3, colors = RColorBrewer::brewer.pal(7, "Set2"),
+ drawPoints = FALSE, pointSize = 3, labelsKMB = TRUE,
includeZero = TRUE
+ ) %>%
+ dyCSS(css = system.file("custom.css", package = "polloi")) %>%
+ dyRangeSelector %>%
+ dyEvent(as.Date("2017-01-01"), "R (reportupdater)", labelLoc =
"bottom") %>%
+ dyEvent(as.Date("2017-06-29"), "U (new UDF)", labelLoc =
"bottom"))
}
- api_usage_change <- api_usage %>%
- dplyr::mutate(
- cirrus = polloi::percent_change(cirrus),
- `cirrus (more like)` = polloi::percent_change(`cirrus (more like)`),
- geo = polloi::percent_change(geo),
- language = polloi::percent_change(language),
- open = polloi::percent_change(open),
- prefix = polloi::percent_change(prefix),
- all = polloi::percent_change(all)
- ) %>%
- { .[-1, ] } %>%
- polloi::smoother(ifelse(smooth_level == "global", input$smoothing_global,
smooth_level), rename = FALSE) %>%
- { xts::xts(.[, -1], .$date) }
- if (!input$kpi_api_usage_series_include_open) colnames(api_usage_change)[7]
<- "all except open"
- return(dygraph(api_usage_change, main = "Day-to-day % change over time",
xlab = "Date", ylab = "% change") %>%
- dyLegend(width = 400, show = "always") %>%
- dyOptions(
- strokeWidth = 3, colors = RColorBrewer::brewer.pal(7, "Set2"),
- drawPoints = FALSE, pointSize = 3, labelsKMB = TRUE, includeZero
= TRUE
- ) %>%
- dyCSS(css = system.file("custom.css", package = "polloi")) %>%
- dyRangeSelector %>%
- dyEvent(as.Date("2017-01-01"), "R (reportupdater)", labelLoc =
"bottom") %>%
- dyEvent(as.Date("2017-06-29"), "U (new UDF)", labelLoc = "bottom"))
})
diff --git a/tab_documentation/referer_breakdown.md
b/tab_documentation/referer_breakdown.md
new file mode 100644
index 0000000..61ddb1f
--- /dev/null
+++ b/tab_documentation/referer_breakdown.md
@@ -0,0 +1,23 @@
+API Calls by Referrer Class
+=======
+
+All types of API calls are aggregated by date and referrer class.
+
+**Internal** is traffic referred by Wikimedia sites, specifically:
mediawiki.org, wikibooks.org, wikidata.org, wikinews.org, wikimedia.org,
wikimediafoundation.org, wikipedia.org, wikiquote.org, wikisource.org,
wikiversity.org, wikivoyage.org, and wiktionary.org (See [Webrequest
source](https://git.wikimedia.org/blob/analytics%2Frefinery%2Fsource.git/master/refinery-core%2Fsrc%2Fmain%2Fjava%2Forg%2Fwikimedia%2Fanalytics%2Frefinery%2Fcore%2FWebrequest.java#L203)
for more information.)
+
+Outages and inaccuracies
+------
+
+On 2017-06-29 we started to break down the API calls by referer class.
+
+Questions, bug reports, and feature suggestions
+------
+For technical, non-bug questions, [email
Mikhail](mailto:[email protected]?subject=Dashboard%20Question) or
[Chelsy](mailto:[email protected]?subject=Dashboard%20Question). If you
experience a bug or notice something wrong or have a suggestion, [open a ticket
in
Phabricator](https://phabricator.wikimedia.org/maniphest/task/create/?projects=Discovery)
in the Discovery board or [email
Deb](mailto:[email protected]?subject=Dashboard%20Question).
+
+<hr style="border-color: gray;">
+<p style="font-size: small;">
+ <strong>Link to this dashboard:</strong> <a
href="https://discovery.wmflabs.org/metrics/#referer_breakdown";>https://discovery.wmflabs.org/metrics/#referer_breakdown</a>
+ | Page is available under <a
href="https://creativecommons.org/licenses/by-sa/3.0/"; title="Creative Commons
Attribution-ShareAlike License">CC-BY-SA 3.0</a>
+ | <a href="https://phabricator.wikimedia.org/diffusion/WDRN/"; title="Search
Metrics Dashboard source code repository">Code</a> is licensed under <a
href="https://phabricator.wikimedia.org/diffusion/WDRN/browse/master/LICENSE.md";
title="MIT License">MIT</a>
+ | Part of <a href="https://discovery.wmflabs.org/";>Discovery Dashboards</a>
+</p>
diff --git a/ui.R b/ui.R
index 8b20615..d6b6ce6 100644
--- a/ui.R
+++ b/ui.R
@@ -60,7 +60,8 @@
menuSubItem(text = "Open Search", tabName =
"open_search"),
menuSubItem(text = "Geo Search", tabName =
"geo_search"),
menuSubItem(text = "Prefix Search", tabName =
"prefix_search"),
- menuSubItem(text = "Language Search", tabName =
"language_search")),
+ menuSubItem(text = "Language Search", tabName =
"language_search"),
+ menuSubItem(text = "Referrer Breakdown", tabName =
"referer_breakdown")),
menuItem(text = "Zero Results",
menuSubItem(text = "Summary", tabName =
"failure_rate"),
menuSubItem(text = "Search Type Breakdown", tabName
= "failure_breakdown"),
@@ -248,6 +249,18 @@
dygraphOutput("language_aggregate"),
includeMarkdown("./tab_documentation/language_basic.md")
),
+ tabItem(tabName = "referer_breakdown",
+ fluidRow(
+ column(
+ HTML("<label class = \"control-label\" style =
\"margin-bottom:-30px;\">Type</label>"),
+ shiny::checkboxInput("referer_breakdown_prop", label =
"Use Proportion", value = FALSE),
+ width = 2
+ ),
+ column(polloi::smooth_select("smoothing_referer_breakdown"),
width = 10)
+ ),
+ dygraphOutput("referer_breakdown_plot"),
+ includeMarkdown("./tab_documentation/referer_breakdown.md")
+ ),
tabItem(tabName = "failure_rate",
polloi::smooth_select("smoothing_failure_rate"),
polloi::automata_select(input_id = "failure_rate_automata"),
diff --git a/utils.R b/utils.R
index 70db0ed..3d9ad46 100644
--- a/utils.R
+++ b/utils.R
@@ -91,7 +91,14 @@
dplyr::filter(!is.na(api), !is.na(referer_class), !is.na(calls)) %>%
dplyr::distinct(date, api, referer_class, .keep_all = TRUE) %>%
dplyr::arrange(api, date) %>%
- dplyr::mutate(referer_class =
polloi::capitalize_first_letter(referer_class)) %>%
+ dplyr::mutate(referer_class = forcats::fct_recode(
+ referer_class,
+ `None (direct)` = "none",
+ `Search engine` = "external (search engine)",
+ `External (but not search engine)` = "external",
+ Internal = "internal",
+ Unknown = "unknown"
+ )) %>%
tidyr::spread(referer_class, calls) %>%
dplyr::mutate(All = ifelse(is.na(All), rowSums(.[, -c(1, 2)], na.rm =
TRUE), All)) %>%
tidyr::gather(referrer, calls, -c(date, api)) %>%
--
To view, visit https://gerrit.wikimedia.org/r/374669
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic70d7054e02569eb8545dd347026c7f77321ab2c
Gerrit-PatchSet: 3
Gerrit-Project: wikimedia/discovery/rainbow
Gerrit-Branch: develop
Gerrit-Owner: Chelsyx <[email protected]>
Gerrit-Reviewer: Bearloga <[email protected]>
Gerrit-Reviewer: Chelsyx <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
