Joal has uploaded a new change for review. https://gerrit.wikimedia.org/r/214349
Change subject: Add get pageview_info udf and underlying functions ...................................................................... Add get pageview_info udf and underlying functions Update getProjectFromHost function in PageviewDefinition Add getDialectFromPath and getArticleFromUri in PageviewDefinition Modify test data to test new functions Adapt existing test functions to modified test data Change-Id: Ieed48b6c520c09e62d3bc05085e67f11ed3a96b7 --- M refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java M refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java M refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java M refinery-core/src/test/resources/pageview_test_data.csv A refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java D refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetProjectUDF.java A refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetPageviewInfoUDF.java D refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetProjectUDF.java M refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java M refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java M refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java 11 files changed, 530 insertions(+), 164 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery/source refs/changes/49/214349/1 diff --git a/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java b/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java index 808c857..e9df39c 100644 --- a/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java +++ b/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java @@ -16,6 +16,8 @@ package org.wikimedia.analytics.refinery.core; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; import java.util.regex.Pattern; import java.util.HashSet; import java.util.Arrays; @@ -105,6 +107,12 @@ "www", "download" )); + + public static final String UNKNOWN_PROJECT_VALUE = "-"; + public static final String UNKNOWN_DIALECT_VALUE = "-"; + public static final String UNKNOWN_ARTICLE_VALUE = "-"; + + public static final String DEFAULT_DIALECT_VALUE = "default"; /** * All API request uriPaths will contain this @@ -212,7 +220,7 @@ * @return The project identifier in format [xxx.]xxxx (en.wikipedia or wikisource for instance) */ public String getProjectFromHost(String uriHost) { - if (uriHost == null) return "-"; + if (uriHost == null) return UNKNOWN_PROJECT_VALUE; String[] uri_parts = uriHost.toLowerCase().split("\\."); switch (uri_parts.length) { // case wikixxx.org @@ -237,7 +245,134 @@ else return uri_parts[0] + "." + uri_parts[3]; default: - return "-"; + return UNKNOWN_PROJECT_VALUE; } } + + /** + * Identifies the dialect from a pageview uriPath + * NOTE: Provides correct result only if used with is_pageview = true + * + * @param uriPath The url's path + * @return The dialect name (if any) + */ + public String getDialectFromPath(String uriPath) { + if (uriPath == null) return UNKNOWN_DIALECT_VALUE; + + // In case of api, unknown dialect + if (uriPath.startsWith("/w/api.php")) + return UNKNOWN_DIALECT_VALUE; + + // Default wiki urls, default dialect + if (uriPath.equals("/") || uriPath.equals("/wiki") || uriPath.equals("/w") + || uriPath.startsWith("/wiki/") || uriPath.startsWith("/w/")) + return DEFAULT_DIALECT_VALUE; + + // Special dialect case, + // Extract dialect if it contains a "-" + // or default dialect otherwise + int startIdx = uriPath.indexOf("/"); + startIdx = (startIdx >= 0)?(startIdx + 1):startIdx; + int middleIdx = uriPath.indexOf("-", startIdx); + int endIdx = uriPath.indexOf("/", startIdx); + endIdx = (endIdx > 0)?endIdx:(uriPath.length()); + if ((startIdx >= 0) && (startIdx < endIdx)) { + if ((middleIdx > 0) && (middleIdx < endIdx)) + return uriPath.substring(startIdx, endIdx); + else + return DEFAULT_DIALECT_VALUE; + } + + // extraction failed, unknown dialect + return UNKNOWN_DIALECT_VALUE; + + } + + /** + * Extracts an article name from uriPath + * NOTE: Assumes that the page is not "index.*" + * + * @param path The url's path + * @return The article name or UNKNOWN_ARTICLE_VALUE + */ + private String getArticleFromPath(String path) { + int endIdx = path.indexOf("#"); + endIdx = (endIdx > 0)?endIdx:(path.length()); + + // If path contains /wiki/, start from there + int startIdx = path.indexOf("/wiki/") ; + startIdx = (startIdx >= 0)?(startIdx+ "/wiki/".length()):startIdx; + if ((startIdx >= 0) && (startIdx < endIdx)) + return path.substring(startIdx, endIdx); + + // If path contains /w/, start from there + startIdx = path.indexOf("/w/"); + startIdx = (startIdx >= 0)?(startIdx+ "/w/".length()):startIdx; + if ((startIdx >= 0) && (startIdx < endIdx)) + return path.substring(startIdx, endIdx); + + // Else assume we are in /dialect/article case + startIdx = path.indexOf("/", path.indexOf("/") + 1); + startIdx = (startIdx >= 0)?(startIdx + 1):startIdx; + if ((startIdx > 0) && (startIdx < endIdx)) + return path.substring(startIdx, endIdx); + + //Case not covered, return unknown value + return UNKNOWN_ARTICLE_VALUE; + } + + /** + * Identifies an article name from a pageview uriPath and uriQuery + * NOTE: Provides correct result only if used with is_pageview = true + * + * @param uriPath The url's path + * @param uriQuery The url's query + * @return The decoded article name (if any) + */ + public String getArticleFromUri(String uriPath, String uriQuery) { + if (uriPath == null) return "-"; + + // General case of article in path + boolean pathWiki = (uriPath.contains("/wiki/") || uriPath.contains("/w/index.php/")); + + String titleQueryParam = null; + String pageQueryParam = null; + + // Extract first instance of title and page query parameters + final String[] pairs = uriQuery.split("&"); + for (String pair : pairs) { + final int idx = pair.indexOf("="); + try { + final String key = idx > 0 ? URLDecoder.decode(pair.substring(0, idx), "UTF-8") : pair; + if ((key.equals("title") && (titleQueryParam == null))) + titleQueryParam = idx > 0 && pair.length() > idx + 1 ? URLDecoder.decode(pair.substring(idx + 1), "UTF-8") : null; + if ((key.equals("page")) && (pageQueryParam == null)) + pageQueryParam = idx > 0 && pair.length() > idx + 1 ? URLDecoder.decode(pair.substring(idx + 1), "UTF-8") : null; + if ((titleQueryParam != null) && (pageQueryParam != null)) + break; + } catch (UnsupportedEncodingException e) { + return UNKNOWN_ARTICLE_VALUE; + } + } + + // Depending on case, extract article from path or query parameter + if (pathWiki) + if (titleQueryParam != null) + return titleQueryParam; + else + return getArticleFromPath(uriPath); + else + if (titleQueryParam != null) + return titleQueryParam; + else + if (pageQueryParam != null) + return pageQueryParam; + else + if (! uriPath.contains("index.")) + return getArticleFromPath(uriPath); + else + return UNKNOWN_ARTICLE_VALUE; + + } + } diff --git a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java index ca5d6c7..885dfe4 100644 --- a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java +++ b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestLegacyPageviewDefinition.java @@ -34,6 +34,8 @@ public void testIsLegacyPageview( String test_description, String project, + String dialect, + String article, boolean is_pageview, boolean is_legacy_pageview, boolean is_app_pageview, diff --git a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java index 684fead..af45a20 100644 --- a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java +++ b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java @@ -33,6 +33,8 @@ public void testIsPageview( String test_description, String project, + String dialect, + String article, boolean is_pageview, boolean is_legacy_pageview, boolean is_app_pageview, @@ -68,6 +70,8 @@ public void testIsAppPageview( String test_description, String project, + String dialect, + String article, boolean is_pageview, boolean is_legacy_pageview, boolean is_app_pageview, @@ -101,6 +105,8 @@ public void testGetProjectFromHost( String test_description, String project, + String dialect, + String article, boolean is_pageview, boolean is_legacy_pageview, boolean is_app_pageview, @@ -120,4 +126,68 @@ PageviewDefinitionInstance.getProjectFromHost(uri_host) ); } + + @Test + @FileParameters( + value = "src/test/resources/pageview_test_data.csv", + mapper = CsvWithHeaderMapper.class + ) + public void testGetDialectFromPath( + String test_description, + String project, + String dialect, + String article, + boolean is_pageview, + boolean is_legacy_pageview, + boolean is_app_pageview, + String ip_address, + String x_forwarded_for, + String uri_host, + String uri_path, + String uri_query, + String http_status, + String content_type, + String user_agent + ) { + PageviewDefinition PageviewDefinitionInstance = PageviewDefinition.getInstance(); + if (is_pageview) { + assertEquals( + test_description, + dialect, + PageviewDefinitionInstance.getDialectFromPath(uri_path) + ); + } + } + + @Test + @FileParameters( + value = "src/test/resources/pageview_test_data.csv", + mapper = CsvWithHeaderMapper.class + ) + public void testGetArticleFromUri( + String test_description, + String project, + String dialect, + String article, + boolean is_pageview, + boolean is_legacy_pageview, + boolean is_app_pageview, + String ip_address, + String x_forwarded_for, + String uri_host, + String uri_path, + String uri_query, + String http_status, + String content_type, + String user_agent + ) { + PageviewDefinition PageviewDefinitionInstance = PageviewDefinition.getInstance(); + if (is_pageview) { + assertEquals( + test_description, + article, + PageviewDefinitionInstance.getArticleFromUri(uri_path, uri_query) + ); + } + } } \ No newline at end of file diff --git a/refinery-core/src/test/resources/pageview_test_data.csv b/refinery-core/src/test/resources/pageview_test_data.csv index 49f9124..be6edd4 100644 --- a/refinery-core/src/test/resources/pageview_test_data.csv +++ b/refinery-core/src/test/resources/pageview_test_data.csv @@ -1,32 +1,32 @@ -test_description,project,is_pageview,is_legacy_pageview,is_app_pageview,ip_address,x_forwarded_for,uri_host,uri_path,uri_query,http_status,content_type,user_agent -Is Pageview - Desktop,en.wikipedia,true,true,false,174.62.175.82,-,en.wikipedia.org, /wiki/Horseshoe_crab,-,200,text/html, turnip -Is Pageview – Desktop – locally cached content,en.wikipedia,true,true,false,174.62.175.82,-,en.wikipedia.org, /wiki/Horseshoe_crab,-,304,text/html, turnip -Is Pageview – App - Android,en.wikipedia,true,false,true,174.62.175.83,-,en.wikipedia.org, /w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=0,200, application/json,WikipediaApp/2.0-r-2015-01-15 (Android 4.4.2; Phone) Google Play -is Pageview – App – iOS – old version,en.wikipedia,true,false,true,174.62.175.83,-,en.wikipedia.org, /w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=0,200, application/json,WikipediaApp/4.0.6 (iPhone OS 8.2; Phone) -Is Pageview – App - iOS – new version,en.wikipedia,true,false,true,174.62.175.83,-,en.wikipedia.org, /w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=all,200, application/json,WikipediaApp/4.0.6 (iPhone OS 8.2; Phone) -Is Pageview – Mobile Web,en.wikipedia,true,true,false,174.62.175.84,-,en.m.wikipedia.org,/wiki/Bernard_Manning,-,200,text/html,rutabaga -Is Pageview – Desktop - Serbian sr-ec,sr.wikipedia,true,false,false,174.62.175.85,-,sr.wikipedia.org,/sr-ec/Историја_Срба_пре_Немањића,-,200,text/html,Three-finger salute -Is Pageview – Desktop - Serbian sr-el,sr.wikipedia,true,false,false,174.62.175.86,-,sr.wikipedia.org,/sr-el/Историја_Срба_пре_Немањића,-,200,text/html,Three-finger salute -Is Pageview – Desktop - Chinese zh-cn,zh.wikipedia,true,false,false,174.62.175.87,-,zh.wikipedia.org,/zh-cn/Wikipedia:首页,-,200,text/html,Five-test plan -Is Pageview – Desktop - Chinese zh-hans,zh.wikipedia,true,false,false,174.62.175.88,-,zh.wikipedia.org,/zh-hans/Wikipedia:首页,-,200,text/html,Five-test plan -Is Pageview – Desktop - Chinese zh-hant,zh.wikipedia,true,false,false,174.62.175.89,-,zh.wikipedia.org,/zh-hant/Wikipedia:首页,-,200,text/html,Five-test plan -Is Pageview – Desktop - Chinese zh-hk,zh.wikipedia,true,false,false,174.62.175.90,-,zh.wikipedia.org,/zh-hk/Wikipedia:foo,-,200,text/html,Five-test plan -Is Pageview – Desktop - Chinese zh-mo,zh.wikipedia,true,false,false,174.62.175.91,-,zh.wikipedia.org,/zh-mo/Wikipedia:首页,-,200,text/html,Five-test plan -Is Pageview – Desktop - Chinese zh-my,zh.wikipedia,true,false,false,174.62.175.92,-,zh.wikipedia.org,/zh-my/Wikipedia:首页,-,200,text/html,Five-test plan -Is Pageview – Desktop - Chinese zh-sg,zh.wikipedia,true,false,false,174.62.175.93,-,zh.wikipedia.org,/zh-sg/Wikipedia:首页,-,200,text/html,Five-test plan -Is Pageview – Desktop - Chinese zh-tw,zh.wikipedia,true,false,false,174.62.175.94,-,zh.wikipedia.org,/zh-tw/Wikipedia:首页,-,200,text/html,Five-test plan -Is Pageview – Wikidata,wikidata,true,true,false,174.62.175.94,-,www.wikidata.org,/wiki/Q5651758,-,200,text/html,Five-test plan -Is Pageview – MediaWiki,mediawiki,true,true,false,174.62.175.94,-,www.mediawiki.org,/wiki/Gerrit/git-review,-,200,text/html,Five-test plan -Is Pageview – iOS search,en.wikipedia,true,false,false,174.62.175.94,-,en.wikipedia.org,/,?search=afdfsdfsd,200,text/html,Five-test plan -Is Not Pageview - http_status != 200,en.wikipedia,false,true,false,174.62.175.95,-, en.wikipedia.org, /wiki/Noppperrrrs,-,400,text/html ,turnip -Is Not Pageview - content_type does not match,en.wikipedia,false,true,false,174.62.175.96,-, en.wikipedia.org, /wiki/Noppperrrrs,-,200, image/png, turnip -Is Not Pageview - API stupidity: it outputs a 200 status code and text/html as a MIME type on certain classes of error,en.wikipedia,false,false,false,174.62.175.97,-, en.wikipedia.org, /w/api.php,-,200, text/html, turnip -Is Not Pageview – App request for non-page content,en.wikipedia,false,false,false,174.62.175.98,-,en.wikipedia.org,/w/api.php,?action=query&format=json&titles=Foo&prop=pageimages&piprop=thumbnail&pithumbsize=96&pilimit=1 ,200, application/json, WikipediaApp/1.2.3 -Is Not Pageview – Non-App request for page content,en.wikipedia,false, false,false,174.62.175.99,-, en.wikipedia.org, /w/api.php,?action=mobileview§ions=0,200, application/json, TributeApp/1.2.3 -Is Not Pageview – edit attempt,en.wikipedia,false,true,false,174.62.175.82,-,en.wikipedia.org, /wiki/Horseshoe_crab,?action=edit,200,text/html, turnip -Is not pageview – non-wikidata raw domain,wikipedia,false,false,false,174.62.175.82,-,www.wikipedia.org,-,-,200,text/html, turnip -Is Not Pageview – App – Android – Refresh,en.wikipedia,false, false,false,174.62.175.82,-,en.wikipedia.org, /w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=all,200, application/json,WikipediaApp/2.0-r-2015-01-15 (Android 4.4.2; Phone) Google Play -"Is Pageview - mobile - wikimediafoundation",wikimediafoundation,true,true,false,174.62.175.82,-,m.wikimediafoundation.org,/wiki/Horseshoe_crab,-,200,text/html,turnip -"Is Pageview - Desktop - wikimediafoundation",wikimediafoundation,true,false,false,174.62.175.82,-,wikimediafoundation.org,/wiki/Horseshoe_crab,-,200,text/html,turnip -Is Not Pageview - Desktop,test2.wikipedia, true,true,false,174.62.175.82,-,test2.wikipedia.org, /wiki/Horseshoe_crab,-,200,text/html, turnip -Is Not Pageview - Desktop,test.wikimediafoundation,true,true,false,174.62.175.82,-,test.wikimediafoundation.org, /wiki/Horseshoe_crab,-,200,text/html, turnip \ No newline at end of file +test_description,project,dialect,articleis_pageview,is_legacy_pageview,is_app_pageview,ip_address,x_forwarded_for,uri_host,uri_path,uri_query,http_status,content_type,user_agent +Is Pageview - Desktop,en.wikipedia,default,Horseshoe_crab,true,true,false,174.62.175.82,-,en.wikipedia.org, /wiki/Horseshoe_crab#anchor,-,200,text/html, turnip/ +Is Pageview – Desktop – locally cached content,en.wikipedia,default,Horseshoe/crab,true,true,false,174.62.175.82,-,en.wikipedia.org, /wiki/Horseshoe/crab,-,304,text/html, turnip +Is Pageview – App - Android,en.wikipedia,-,Hachiko_–_Eine_wunderbare_Freundschaft,true,false,true,174.62.175.83,-,en.wikipedia.org, /w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=0,200, application/json,WikipediaApp/2.0-r-2015-01-15 (Android 4.4.2; Phone) Google Play +is Pageview – App – iOS – old version,en.wikipedia,-,Hachiko_–_Eine_wunderbare_Freundschaft,true,false,true,174.62.175.83,-,en.wikipedia.org, /w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=0,200, application/json,WikipediaApp/4.0.6 (iPhone OS 8.2; Phone) +Is Pageview – App - iOS – new version,en.wikipedia,-,Hachiko_–_Eine_wunderbare_Freundschaft,true,false,true,174.62.175.83,-,en.wikipedia.org, /w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=all,200, application/json,WikipediaApp/4.0.6 (iPhone OS 8.2; Phone) +Is Pageview – Mobile Web,en.wikipedia,default,Bernard_Manning,true,true,false,174.62.175.84,-,en.m.wikipedia.org,/wiki/Bernard_Manning,-,200,text/html,rutabaga +Is Pageview – Desktop - Serbian sr-ec,sr.wikipedia,sr-ec,Историја_Срба_пре_Немањића,true,false,false,174.62.175.85,-,sr.wikipedia.org,/sr-ec/Историја_Срба_пре_Немањића,-,200,text/html,Three-finger salute +Is Pageview – Desktop - Serbian sr-el,sr.wikipedia,sr-el,Историја_Срба_пре_Немањића,true,false,false,174.62.175.86,-,sr.wikipedia.org,/sr-el/Историја_Срба_пре_Немањића,-,200,text/html,Three-finger salute +Is Pageview – Desktop - Chinese zh-cn,zh.wikipedia,zh-cn,Wikipedia:首页,true,false,false,174.62.175.87,-,zh.wikipedia.org,/zh-cn/Wikipedia:首页,-,200,text/html,Five-test plan +Is Pageview – Desktop - Chinese zh-hans,zh.wikipedia,zh-hans,Wikipedia:首页,true,false,false,174.62.175.88,-,zh.wikipedia.org,/zh-hans/Wikipedia:首页,-,200,text/html,Five-test plan +Is Pageview – Desktop - Chinese zh-hant,zh.wikipedia,zh-hant,Wikipedia:首页,true,false,false,174.62.175.89,-,zh.wikipedia.org,/zh-hant/Wikipedia:首页,-,200,text/html,Five-test plan +Is Pageview – Desktop - Chinese zh-hk,zh.wikipedia,zh-hk,Wikipedia:foo,true,false,false,174.62.175.90,-,zh.wikipedia.org,/zh-hk/Wikipedia:foo,-,200,text/html,Five-test plan +Is Pageview – Desktop - Chinese zh-mo,zh.wikipedia,zh-mo,Wikipedia:首页,true,false,false,174.62.175.91,-,zh.wikipedia.org,/zh-mo/Wikipedia:首页,-,200,text/html,Five-test plan +Is Pageview – Desktop - Chinese zh-my,zh.wikipedia,zh-my,Wikipedia:首页,true,false,false,174.62.175.92,-,zh.wikipedia.org,/zh-my/Wikipedia:首页,-,200,text/html,Five-test plan +Is Pageview – Desktop - Chinese zh-sg,zh.wikipedia,zh-sg,Wikipedia:首页,true,false,false,174.62.175.93,-,zh.wikipedia.org,/zh-sg/Wikipedia:首页,-,200,text/html,Five-test plan +"Is Pageview – Desktop - Chinese zh-tw",zh.wikipedia,default,Wikipedia:首页,true,false,false,174.62.175.94,-,zh.wikipedia.org,/zh/Wikipedia:首页,-,200,text/html,"Five-test plan" +Is Pageview – Wikidata,wikidata,default,Q5651758,true,true,false,174.62.175.94,-,www.wikidata.org,/wiki/Q5651758,-,200,text/html,Five-test plan +Is Pageview – MediaWiki,mediawiki,default,Gerrit/git-review,true,true,false,174.62.175.94,-,www.mediawiki.org,/wiki/Gerrit/git-review,-,200,text/html,Five-test plan +Is Pageview – iOS search,en.wikipedia,default,-,true,false,false,174.62.175.94,-,en.wikipedia.org,/,?search=afdfsdfsd,200,text/html,Five-test plan +Is Not Pageview - http_status != 200,en.wikipedia,-,-,false,true,false,174.62.175.95,-, en.wikipedia.org, /wiki/Noppperrrrs,-,400,text/html ,turnip +Is Not Pageview - content_type does not match,en.wikipedia,-,-,false,true,false,174.62.175.96,-, en.wikipedia.org, /wiki/Noppperrrrs,-,200, image/png, turnip +Is Not Pageview - API stupidity: it outputs a 200 status code and text/html as a MIME type on certain classes of error,en.wikipedia,-,-,false,false,false,174.62.175.97,-, en.wikipedia.org, /w/api.php,-,200, text/html, turnip +Is Not Pageview – App request for non-page content,en.wikipedia,-,-,false,false,false,174.62.175.98,-,en.wikipedia.org,/w/api.php,?action=query&format=json&titles=Foo&prop=pageimages&piprop=thumbnail&pithumbsize=96&pilimit=1 ,200, application/json, WikipediaApp/1.2.3 +Is Not Pageview – Non-App request for page content,en.wikipedia,-,-,false, false,false,174.62.175.99,-,en.wikipedia.org, /w/api.php,?action=mobileview§ions=0,200, application/json, TributeApp/1.2.3 +Is Not Pageview – edit attempt,en.wikipedia,-,-,false,true,false,174.62.175.82,-,en.wikipedia.org, /wiki/Horseshoe_crab,?action=edit,200,text/html, turnip +Is not pageview – non-wikidata raw domain,wikipedia,-,-,false,false,false,174.62.175.82,-,www.wikipedia.org,-,-,200,text/html, turnip +Is Not Pageview – App – Android – Refresh,en.wikipedia,-,-,false, false,false,174.62.175.82,-,en.wikipedia.org, /w/api.php,action=mobileview&format=json&page=Hachiko_–_Eine_wunderbare_Freundschaft&prop=text§ions=all,200, application/json,WikipediaApp/2.0-r-2015-01-15 (Android 4.4.2; Phone) Google Play +"Is Pageview - mobile - wikimediafoundation",wikimediafoundation,default,Horseshoe_crab,true,true,false,174.62.175.82,-,m.wikimediafoundation.org,/wiki/Horseshoe_crab,-,200,text/html,turnip +"Is Pageview - Desktop - wikimediafoundation",wikimediafoundation,default,Horseshoe_crab,true,false,false,174.62.175.82,-,wikimediafoundation.org,/wiki/Horseshoe_crab,-,200,text/html,turnip +Is Not Pageview - Desktop,test2.wikipedia,default,Horseshoe_crab,true,true,false,174.62.175.82,-,test2.wikipedia.org, /wiki/Horseshoe_crab,-,200,text/html, turnip +Is Not Pageview - Desktop,test.wikimediafoundation,default,Horseshoe_crab,true,true,false,174.62.175.82,-,test.wikimediafoundation.org, /wiki/Horseshoe_crab,-,200,text/html, turnip \ No newline at end of file diff --git a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java new file mode 100644 index 0000000..a202968 --- /dev/null +++ b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetPageviewInfoUDF.java @@ -0,0 +1,153 @@ +/** + * Copyright (C) 2014 Wikimedia Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wikimedia.analytics.refinery.hive; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.UDFType; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector; +import org.wikimedia.analytics.refinery.core.PageviewDefinition; + +import java.util.HashMap; +import java.util.Map; + + +/** + * A Hive UDF to identify pageview data in a map + * Map fields are project, dialect, article. + * NOTE: this udf only works well if + * (uri_host, uri_path, uri_query) comes from + * a webrequest having is_pageview = true + * + * <p> + * Hive Usage: + * ADD JAR /path/to/refinery-hive.jar; + * CREATE TEMPORARY FUNCTION get_pageview_info AS + * 'org.wikimedia.analytics.refinery.hive.PageviewInfoUDF'; + * SELECT + * get_pageview_info(uri_host, uri_path, uri_query) as pageview_data, + * count(*) as cnt + * FROM + * wmf_raw.webrequest + * WHERE + * webrequest_source = 'mobile' + * AND year=2014 + * AND month=12 + * AND day=7 + * AND hour=12 + * AND is_pageview(uri_host, uri_path, uri_query, http_status, content_type, user_agent) + * GROUP BY + * get_pageview_info(uri_host, uri_path, uri_query) + * ORDER BY cnt desc + * LIMIT 10 + * ; + */ +@UDFType(deterministic = true) +@Description(name = "get_pageview_info", + value = "_FUNC_(uri_host, uri_path, uri_query) - Returns the pageview information map " + + "(project, dialect, article) for the pageview request.", + extended = "") +public class GetPageviewInfoUDF extends GenericUDF { + + public static final String PROJECT_KEY = "project"; + public static final String DIALECT_KEY = "dialect"; + public static final String ARTICLE_KEY = "article"; + + + Map<String, String> result; + private StringObjectInspector[] inputsOI = new StringObjectInspector[3]; + private PageviewDefinition pageviewDefinition; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) + throws UDFArgumentException { + // We need exactly 3 parameters + if (arguments == null || arguments.length != 3) { + throw new UDFArgumentLengthException("The function " + + "GetPageviewInfoUDF expects exactly 3 parameter"); + } + + // ... and the parameters have to be strings + for (int i = 0; i < 3; i++) { + if (!(arguments[i] instanceof StringObjectInspector)) { + throw new UDFArgumentTypeException(0, "Parameter " + + Integer.toString(i) + " to GetPageviewInfoUDF " + + "has to be a string"); + } + inputsOI[i] = (StringObjectInspector) arguments[i]; + } + + result = new HashMap<>(3); + pageviewDefinition = PageviewDefinition.getInstance(); + + return ObjectInspectorFactory.getStandardMapObjectInspector( + PrimitiveObjectInspectorFactory.javaStringObjectInspector, + PrimitiveObjectInspectorFactory.javaStringObjectInspector); + } + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + assert arguments != null : "Method 'evaluate' of GetPageviewInfoUDF " + + "called with null arguments array"; + assert arguments.length == 3 : "Method 'evaluate' of " + + "GetPageviewInfoUDF called arguments of length " + + arguments.length + " (instead of 3)"; + // arguments is an array with exactly 3 entry. + + assert result != null : "Result object has not yet been initialized, " + + "but evaluate called"; + // result map has been initialized. + assert pageviewDefinition != null : "PageviewDefinition object has not yet been initialized, " + + "but evaluate called"; + // pageviewDefinition has been initialized. + + result.clear(); + + String uriHost = inputsOI[0].getPrimitiveJavaObject(arguments[0].get()); + String uriPath = inputsOI[1].getPrimitiveJavaObject(arguments[1].get()); + String uriQuery = inputsOI[2].getPrimitiveJavaObject(arguments[2].get()); + + result.put(PROJECT_KEY, pageviewDefinition.getProjectFromHost(uriHost)); + result.put(DIALECT_KEY, pageviewDefinition.getDialectFromPath(uriPath)); + result.put(ARTICLE_KEY, pageviewDefinition.getArticleFromUri(uriPath, uriQuery)); + + return result; + + } + + @Override + public String getDisplayString(String[] arguments) { + String argument; + if (arguments == null) { + argument = "<arguments == null>"; + } else if (arguments.length == 3) { + argument = arguments[0] + ", " + + arguments[1] + ", " + + arguments[2]; + } else { + argument = "<arguments of length " + arguments.length + ">"; + } + return "get_pageview_info(" + argument +")"; + } +} \ No newline at end of file diff --git a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetProjectUDF.java b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetProjectUDF.java deleted file mode 100644 index 0552a84..0000000 --- a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GetProjectUDF.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright (C) 2014 Wikimedia Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.wikimedia.analytics.refinery.hive; - -import org.apache.hadoop.hive.ql.exec.Description; -import org.apache.hadoop.hive.ql.exec.UDF; -import org.wikimedia.analytics.refinery.core.PageviewDefinition; - - -/** - * A Hive UDF to identify a Wikimedia webrequest pageview project. - * NOTE: this udf only works well if the uri_host comes from - * a webrequest having is_pageview = true - * - * <p> - * Hive Usage: - * ADD JAR /path/to/refinery-hive.jar; - * CREATE TEMPORARY FUNCTION get_project AS - * 'org.wikimedia.analytics.refinery.hive.IdentifyProjectUDF'; - * SELECT - * get_project(uri_host) as project_qualifier, - * count(*) as cnt - * FROM - * wmf_raw.webrequest - * WHERE - * webrequest_source = 'mobile' - * AND year=2014 - * AND month=12 - * AND day=7 - * AND hour=12 - * AND is_pageview(uri_host, uri_path, uri_query, http_status, content_type, user_agent) - * GROUP BY - * get_project(uri_host) - * ORDER BY cnt desc - * LIMIT 10 - * ; - */ -@Description(name = "get_project", - value = "_FUNC_(uri_host) - Returns the project identifier for the pageview request.", - extended = "") -public class GetProjectUDF extends UDF { - public String evaluate(String uriHost) { - PageviewDefinition pageviewDefinitionInstance = PageviewDefinition.getInstance(); - return pageviewDefinitionInstance.getProjectFromHost(uriHost); - } -} \ No newline at end of file diff --git a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetPageviewInfoUDF.java b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetPageviewInfoUDF.java new file mode 100644 index 0000000..adb8289 --- /dev/null +++ b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetPageviewInfoUDF.java @@ -0,0 +1,130 @@ +/** + * Copyright (C) 2014 Wikimedia Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.wikimedia.analytics.refinery.hive; + +import junitparams.FileParameters; +import junitparams.JUnitParamsRunner; +import junitparams.mappers.CsvWithHeaderMapper; +import org.apache.hadoop.hive.ql.exec.MapredContext; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.mapred.JobConf; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.wikimedia.analytics.refinery.core.PageviewDefinition; + +import java.io.IOException; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +@RunWith(JUnitParamsRunner.class) +public class TestGetPageviewInfoUDF { + + @Test(expected = UDFArgumentLengthException.class) + public void testBadNumberOfArguments() throws HiveException { + ObjectInspector value1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector value2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] initArguments = new ObjectInspector[]{value1, value2}; + GetPageviewInfoUDF udf = new GetPageviewInfoUDF(); + udf.initialize(initArguments); + } + + @Test(expected = UDFArgumentTypeException.class) + public void testWrongTypeOfArguments() throws HiveException { + ObjectInspector value1 = PrimitiveObjectInspectorFactory.javaIntObjectInspector; + ObjectInspector value2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector value3 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] initArguments = new ObjectInspector[]{value1, value2, value3}; + GetPageviewInfoUDF udf = new GetPageviewInfoUDF(); + udf.initialize(initArguments); + } + + + @Test + @FileParameters( + value = "../refinery-core/src/test/resources/pageview_test_data.csv", + mapper = CsvWithHeaderMapper.class + ) + public void testGetProject( + String test_description, + String project, + String dialect, + String article, + boolean is_pageview, + boolean is_legacy_pageview, + boolean is_app_pageview, + String ip_address, + String x_forwarded_for, + String uri_host, + String uri_path, + String uri_query, + String http_status, + String content_type, + String user_agent + ) throws HiveException, IOException { + if (is_pageview) { + ObjectInspector value1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector value2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector value3 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] initArguments = new ObjectInspector[]{value1, value2, value3}; + GetPageviewInfoUDF udf = new GetPageviewInfoUDF(); + + udf.initialize(initArguments); + + GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] { + new GenericUDF.DeferredJavaObject(uri_host), + new GenericUDF.DeferredJavaObject(uri_path), + new GenericUDF.DeferredJavaObject(uri_query) + }; + Map<String, String> result = (Map<String, String>)udf.evaluate(args); + udf.close(); + + assertEquals("Project check -" + test_description, project, result.get(GetPageviewInfoUDF.PROJECT_KEY)); + assertEquals("Dialect check -" + test_description, dialect, result.get(GetPageviewInfoUDF.DIALECT_KEY)); + assertEquals("Article check - " + test_description, article, result.get(GetPageviewInfoUDF.ARTICLE_KEY)); + } + } + + @Test + public void testGetProjectNull() throws HiveException, IOException { + ObjectInspector value1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector value2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector value3 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; + ObjectInspector[] initArguments = new ObjectInspector[]{value1, value2, value3}; + GetPageviewInfoUDF udf = new GetPageviewInfoUDF(); + + udf.initialize(initArguments); + + GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[] { + new GenericUDF.DeferredJavaObject(null), + new GenericUDF.DeferredJavaObject(null), + new GenericUDF.DeferredJavaObject(null) + }; + Map<String, String> result = (Map<String, String>)udf.evaluate(args); + udf.close(); + + assertEquals("Empty project", PageviewDefinition.UNKNOWN_PROJECT_VALUE, result.get(GetPageviewInfoUDF.PROJECT_KEY)); + assertEquals("Empty dialect", PageviewDefinition.UNKNOWN_DIALECT_VALUE, result.get(GetPageviewInfoUDF.DIALECT_KEY)); + assertEquals("Empty article", PageviewDefinition.UNKNOWN_ARTICLE_VALUE, result.get(GetPageviewInfoUDF.ARTICLE_KEY)); + } + +} \ No newline at end of file diff --git a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetProjectUDF.java b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetProjectUDF.java deleted file mode 100644 index 951921a..0000000 --- a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestGetProjectUDF.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright (C) 2014 Wikimedia Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.wikimedia.analytics.refinery.hive; - -import junitparams.FileParameters; -import junitparams.JUnitParamsRunner; -import junitparams.mappers.CsvWithHeaderMapper; -import org.junit.Test; -import org.junit.runner.RunWith; - -import static org.junit.Assert.assertEquals; - -@RunWith(JUnitParamsRunner.class) -public class TestGetProjectUDF { - - - @Test - @FileParameters( - value = "../refinery-core/src/test/resources/pageview_test_data.csv", - mapper = CsvWithHeaderMapper.class - ) - public void testGetProject( - String test_description, - String project, - boolean is_pageview, - boolean is_legacy_pageview, - boolean is_app_pageview, - String ip_address, - String x_forwarded_for, - String uri_host, - String uri_path, - String uri_query, - String http_status, - String content_type, - String user_agent - ) { - GetProjectUDF udf = new GetProjectUDF(); - - assertEquals( - test_description, - project, - udf.evaluate(uri_host) - ); - } - - @Test - public void testGetProjectNull() { - GetProjectUDF udf = new GetProjectUDF(); - - assertEquals( - "Test null input to getProject", - "-", - udf.evaluate(null) - ); - } - -} \ No newline at end of file diff --git a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java index f36ead9..5bea46b 100644 --- a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java +++ b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java @@ -34,6 +34,8 @@ public void testIsAppPageview( String test_description, String project, + String dialect, + String article, boolean is_pageview, boolean is_legacy_pageview, boolean is_app_pageview, diff --git a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java index 5dbdb8a..5352de5 100644 --- a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java +++ b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsLegacyPageviewUDF.java @@ -35,6 +35,8 @@ public void testIsPageview( String test_description, String project, + String dialect, + String article, boolean is_pageview, boolean is_legacy_pageview, boolean is_app_pageview, diff --git a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java index 172b85f..bd451ec 100644 --- a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java +++ b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java @@ -35,6 +35,8 @@ public void testIsPageview( String test_description, String project, + String dialect, + String article, boolean is_pageview, boolean is_legacy_pageview, boolean is_app_pageview, -- To view, visit https://gerrit.wikimedia.org/r/214349 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Ieed48b6c520c09e62d3bc05085e67f11ed3a96b7 Gerrit-PatchSet: 1 Gerrit-Project: analytics/refinery/source Gerrit-Branch: master Gerrit-Owner: Joal <j...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits