Nuria has submitted this change and it was merged.
Change subject: Revert "[WIP] Make pageview definition aware of preview
parameter"
......................................................................
Revert "[WIP] Make pageview definition aware of preview parameter"
This reverts commit 36f7f872946e0fbaa7290a0de48ae771f1007a1d.
Change-Id: I9f1a7924b8ebf076f970795b562537dc003c44e4
---
M
refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
M
refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
D
refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GenericUDFHelper.java
M
refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsAppPageviewUDF.java
M
refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
M
refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
M
refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
D
refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDFArgumentHandling.java
8 files changed, 78 insertions(+), 443 deletions(-)
Approvals:
Nuria: Verified; Looks good to me, approved
diff --git
a/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
b/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
index 9afecf3..63f6bb9 100644
---
a/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
+++
b/refinery-core/src/main/java/org/wikimedia/analytics/refinery/core/PageviewDefinition.java
@@ -18,7 +18,6 @@
import java.util.Arrays;
import java.util.HashSet;
-import java.util.Map;
import java.util.regex.Pattern;
/**
@@ -130,12 +129,6 @@
* to see if the request is an app pageview, but not
* (for example) whether it actually completed.
*
- * See: https://wikitech.wikimedia.org/wiki/X-Analytics#Keys
- * for x-analytics info.
- *
- * Please note that requests tagged as 'preview' are not counted
- * as pageviews.
- *
* @param uriPath Path portion of the URI
* @param uriQuery Query portion of the URI
* @param userAgent User-Agent of the requestor
@@ -146,8 +139,7 @@
String uriPath,
String uriQuery,
String contentType,
- String userAgent,
- Map<String,String> xAnalyticsMap
+ String userAgent
) {
final String appContentType = "application/json";
@@ -155,9 +147,6 @@
final String appPageURIQuery = "sections=0";
final String iosAppPageURIQuery = "sections=all";
final String iosUserAgent = "iPhone";
-
- if (xAnalyticsMap.containsKey("preview"))
- return false;
return (
Utilities.stringContains(uriPath, uriPathAPI)
@@ -177,12 +166,6 @@
* See:
https://meta.wikimedia.org/wiki/Research:Page_view/Generalised_filters
* for information on how to classify a pageview.
*
- * See: https://wikitech.wikimedia.org/wiki/X-Analytics#Keys
- * for x-analytics info.
- *
- * Please note that requests tagged as 'preview' are not counted
- * as pageviews.
- *
* @param uriHost Hostname portion of the URI
* @param uriPath Path portion of the URI
* @param uriQuery Query portion of the URI
@@ -198,13 +181,9 @@
String uriQuery,
String httpStatus,
String contentType,
- String userAgent,
- Map<String,String> xAnalyticsMap
+ String userAgent
) {
uriHost = uriHost.toLowerCase();
-
- if (xAnalyticsMap.containsKey("preview"))
- return false;
return (
// All pageviews have a 200 or 304 HTTP status
@@ -212,7 +191,7 @@
// check for a regular pageview contentType, or a an API
contentType
&& (
(contentTypesSet.contains(contentType) &&
!Utilities.stringContains(uriPath, uriPathAPI))
- || isAppPageview(uriPath, uriQuery, contentType,
userAgent, xAnalyticsMap)
+ || isAppPageview(uriPath, uriQuery, contentType, userAgent)
)
// A pageview must be from either a wikimedia.org domain,
// or a 'project' domain, e.g. en.wikipedia.org
diff --git
a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
index b24da28..6365b20 100644
---
a/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
+++
b/refinery-core/src/test/java/org/wikimedia/analytics/refinery/core/TestPageview.java
@@ -14,16 +14,13 @@
package org.wikimedia.analytics.refinery.core;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import static org.junit.Assert.assertEquals;
+
import junitparams.FileParameters;
import junitparams.JUnitParamsRunner;
import junitparams.mappers.CsvWithHeaderMapper;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
@RunWith(JUnitParamsRunner.class)
public class TestPageview {
@@ -51,8 +48,6 @@
String user_agent
) {
PageviewDefinition PageviewDefinitionInstance =
PageviewDefinition.getInstance();
- Map<String,String> xAnalyticsMap = new HashMap<String,String>();
-
assertEquals(
test_description,
is_pageview,
@@ -62,8 +57,7 @@
uri_query,
http_status,
content_type,
- user_agent,
- xAnalyticsMap
+ user_agent
)
);
}
@@ -91,8 +85,6 @@
String user_agent
) {
PageviewDefinition PageviewDefinitionInstance =
PageviewDefinition.getInstance();
- Map<String,String> xAnalyticsMap = new HashMap<String,String>();
-
assertEquals(
test_description,
is_app_pageview,
@@ -100,8 +92,7 @@
uri_path,
uri_query,
content_type,
- user_agent,
- xAnalyticsMap
+ user_agent
)
);
}
diff --git
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GenericUDFHelper.java
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GenericUDFHelper.java
deleted file mode 100644
index b8bc6b3..0000000
---
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/GenericUDFHelper.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.wikimedia.analytics.refinery.hive;
-
-import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-
-/**
- * Created by nuria on 9/8/15.
- *
- * Encapsulates methods to check arguments for UDFs
- *
- * For some reason these are all protected in GenericUDF.java
- * See:
https://github.com/apache/hive/blob/master/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
- */
-public class GenericUDFHelper {
-
- /**
- * @return String
- */
- protected String getFuncName(){
- return getClass().getSimpleName().substring(10).toLowerCase();
- }
-
- /*
- * Checks variable argument list, throws exception if not within bounds
- *
- * @param arguments
- * @param min
- * @param max
- * @throws UDFArgumentLengthException
- */
- protected void checkArgsSize(ObjectInspector[] arguments, int min, int max)
- throws UDFArgumentLengthException{
- if (arguments.length < min || arguments.length > max) {
- StringBuilder sb = new StringBuilder();
- sb.append(getFuncName());
- sb.append(" requires ");
- if (min == max) {
- sb.append(min);
- } else {
- sb.append(min).append("..").append(max);
- }
- sb.append(" argument(s), got ");
- sb.append(arguments.length);
- throw new UDFArgumentLengthException(sb.toString());
- }
- }
-
- /**
- * Checks argument type
- *
- * @param arguments
- * @param i
- *
- * @throws UDFArgumentTypeException
- */
- protected void checkArgPrimitive(ObjectInspector[] arguments, int i)
- throws UDFArgumentTypeException{
- ObjectInspector.Category oiCat = arguments[i].getCategory();
- if (oiCat != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentTypeException(i, getFuncName() + " Argument
should be of primitive type");
- }
- }
-}
diff --git
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsAppPageviewUDF.java
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsAppPageviewUDF.java
index 815e74a..d97b021 100644
---
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsAppPageviewUDF.java
+++
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsAppPageviewUDF.java
@@ -1,12 +1,12 @@
/**
* Copyright (C) 2014 Wikimedia Foundation
- * <p/>
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- * <p/>
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,8 +18,6 @@
import org.apache.hadoop.hive.ql.exec.UDF;
import org.wikimedia.analytics.refinery.core.PageviewDefinition;
-
-import java.util.Map;
/**
@@ -55,20 +53,17 @@
*/
public class IsAppPageviewUDF extends UDF {
public boolean evaluate(
- String uriPath,
- String uriQuery,
- String contentType,
- String userAgent,
-
- Map<String, String> xAnalyticsMap
+ String uriPath,
+ String uriQuery,
+ String contentType,
+ String userAgent
) {
PageviewDefinition pageviewDefinitionInstance =
PageviewDefinition.getInstance();
return pageviewDefinitionInstance.isAppPageview(
- uriPath,
- uriQuery,
- contentType,
- userAgent,
- xAnalyticsMap
+ uriPath,
+ uriQuery,
+ contentType,
+ userAgent
);
}
}
diff --git
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
index a94c0fa..5314729 100644
---
a/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
+++
b/refinery-hive/src/main/java/org/wikimedia/analytics/refinery/hive/IsPageviewUDF.java
@@ -1,12 +1,12 @@
/**
* Copyright (C) 2014 Wikimedia Foundation
- * <p/>
+ *
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
- * <p/>
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,182 +17,58 @@
package org.wikimedia.analytics.refinery.hive;
import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
-import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.ql.exec.UDF;
import org.wikimedia.analytics.refinery.core.PageviewDefinition;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
/**
* A Hive UDF to classify a Wikimedia webrequest as a 'pageview'.
* See: https://meta.wikimedia.org/wiki/Research:Page_view/Generalised_filters
- * for information on how to classify a pageview.
- * <p/>
- * <p/>
+ * for information on how to classify a pageview.
+ *
+ * <p>
* Hive Usage:
- * ADD JAR /path/to/refinery-hive.jar;
- * CREATE TEMPORARY FUNCTION is_pageview AS
- * 'org.wikimedia.analytics.refinery.hive.IsPageviewUDF';
- * SELECT
- * LOWER(uri_host) as uri_host,
- * count(*) as cnt
- * FROM
- * wmf_raw.webrequest
- * WHERE
- * webrequest_source = 'mobile'
- * AND year=2014
- * AND month=12
- * AND day=7
- * AND hour=12
- * AND is_pageview(uri_host, uri_path, uri_query, http_status, content_type,
user_agent, <x_analytics_map></x_analytics_map>)
- * GROUP BY
- * LOWER(uri_host)
- * ORDER BY cnt desc
- * LIMIT 10
- * ;
+ * ADD JAR /path/to/refinery-hive.jar;
+ * CREATE TEMPORARY FUNCTION is_pageview AS
+ * 'org.wikimedia.analytics.refinery.hive.IsPageviewUDF';
+ * SELECT
+ * LOWER(uri_host) as uri_host,
+ * count(*) as cnt
+ * FROM
+ * wmf_raw.webrequest
+ * WHERE
+ * webrequest_source = 'mobile'
+ * AND year=2014
+ * AND month=12
+ * AND day=7
+ * AND hour=12
+ * AND is_pageview(uri_host, uri_path, uri_query, http_status,
content_type, user_agent)
+ * GROUP BY
+ * LOWER(uri_host)
+ * ORDER BY cnt desc
+ * LIMIT 10
+ * ;
*/
@Description(name = "is_pageview",
value = "_FUNC_(uri_host, uri_path, uri_query, http_status,
content_type, user_agent) - Returns true if the request is a pageview",
extended = "")
-public class IsPageviewUDF extends GenericUDF {
-
- private ObjectInspector[] argumentsOI;
- private transient MapObjectInspector mapOI;
-
- PageviewDefinition pageviewDefinitionInstance;
-
- boolean checkForXAnalytics = false;
- int maxArguments = 7;
- int minArguments = 6;
-
- /**
- * Executed once per job, checks arguments size.
- *
- * In order for changes to be backwards compatible when adding
x_analytics_map
- * accepts that this argument might not be present, thus number
- * of arguments is variable.
- *
- * @param arguments
- * @return
- * @throws UDFArgumentException
- */
- @Override
- public ObjectInspector initialize(ObjectInspector[] arguments) throws
UDFArgumentException {
-
- //at least we should have 6 arguments
- checkArgsSize(arguments, minArguments, maxArguments);
-
- if (arguments.length > minArguments){
- checkForXAnalytics = true;
- }
-
- for (int i = 0; i < minArguments - 1; i++) {
- checkArgPrimitive(arguments, i);
- }
-
-
- if (checkForXAnalytics && arguments[arguments.length -
1].getCategory() != ObjectInspector.Category.MAP) {
- throw new UDFArgumentException("Last Argument should be
x_analytics map");
- }
-
- argumentsOI = arguments;
- pageviewDefinitionInstance = PageviewDefinition.getInstance();
-
-
- return PrimitiveObjectInspectorFactory.javaBooleanObjectInspector;
- }
-
- @Override
- public Object evaluate(DeferredObject[] arguments) throws HiveException {
- String separator = PrimitiveObjectInspectorUtils.getString(
- arguments[0].get(), (PrimitiveObjectInspector) argumentsOI[0]);
-
- String uriHost = PrimitiveObjectInspectorUtils.getString(
- arguments[0].get(), (PrimitiveObjectInspector) argumentsOI[1]);
- String uriPath = PrimitiveObjectInspectorUtils.getString(
- arguments[1].get(), (PrimitiveObjectInspector) argumentsOI[2]);
- String uriQuery = PrimitiveObjectInspectorUtils.getString(
- arguments[2].get(), (PrimitiveObjectInspector) argumentsOI[3]);
- ;
- String httpStatus = PrimitiveObjectInspectorUtils.getString(
- arguments[3].get(), (PrimitiveObjectInspector) argumentsOI[4]);
- String contentType = PrimitiveObjectInspectorUtils.getString(
- arguments[4].get(), (PrimitiveObjectInspector) argumentsOI[5]);
- String userAgent = PrimitiveObjectInspectorUtils.getString(
- arguments[5].get(), (PrimitiveObjectInspector) argumentsOI[6]);
-
- Map<String, String> xAnalyticsMap = new HashMap<String, String>();
-
- if (checkForXAnalytics) {
- Map<?, ?> xAnalyticsPreMap = ((MapObjectInspector)
argumentsOI[7]).getMap(arguments[7]);
-
-
- if (xAnalyticsPreMap != null) {
-
- for (Entry<?, ?> r : xAnalyticsMap.entrySet()) {
- xAnalyticsMap.put((String) r.getKey(), (String)
r.getValue());
-
- }
- }
- }
+public class IsPageviewUDF extends UDF {
+ public boolean evaluate(
+ String uriHost,
+ String uriPath,
+ String uriQuery,
+ String httpStatus,
+ String contentType,
+ String userAgent
+ ) {
+ PageviewDefinition pageviewDefinitionInstance =
PageviewDefinition.getInstance();
return pageviewDefinitionInstance.isPageview(
- uriHost,
- uriPath,
- uriQuery,
- httpStatus,
- contentType,
- userAgent,
- xAnalyticsMap
+ uriHost,
+ uriPath,
+ uriQuery,
+ httpStatus,
+ contentType,
+ userAgent
);
- }
-
-
- @Override
- public String getDisplayString(String[] arguments) {
- return "isPageView(" + arguments.toString() + ")";
- }
-
-
- //these two are package protected in GenericUDF.java but are real useful
-
- protected void checkArgsSize(ObjectInspector[] arguments, int min, int max)
- throws UDFArgumentLengthException {
- if (arguments.length < min || arguments.length > max) {
- StringBuilder sb = new StringBuilder();
- sb.append(getFuncName());
- sb.append(" requires ");
- if (min == max) {
- sb.append(min);
- } else {
- sb.append(min).append("..").append(max);
- }
- sb.append(" argument(s), got ");
- sb.append(arguments.length);
- throw new UDFArgumentLengthException(sb.toString());
- }
- }
-
- protected void checkArgPrimitive(ObjectInspector[] arguments, int i)
- throws UDFArgumentTypeException {
- ObjectInspector.Category oiCat = arguments[i].getCategory();
- if (oiCat != ObjectInspector.Category.PRIMITIVE) {
- throw new UDFArgumentTypeException(i, getFuncName() + " Argument
should be of primitive type");
- }
- }
-
- protected String getFuncName() {
- return
- getClass().getSimpleName().substring(10).toLowerCase();
}
}
\ No newline at end of file
diff --git
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
index 8fc2e6d..3b93b0b 100644
---
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
+++
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsAppPageviewUDF.java
@@ -15,16 +15,13 @@
*/
package org.wikimedia.analytics.refinery.hive;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import static org.junit.Assert.assertEquals;
+
import junitparams.FileParameters;
import junitparams.JUnitParamsRunner;
import junitparams.mappers.CsvWithHeaderMapper;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
@RunWith(JUnitParamsRunner.class)
public class TestIsAppPageviewUDF {
@@ -34,7 +31,7 @@
value = "../refinery-core/src/test/resources/pageview_test_data.csv",
mapper = CsvWithHeaderMapper.class
)
- public void testIsAppPageviewNoxAnalytics(
+ public void testIsAppPageview(
String test_description,
String project,
String dialect,
@@ -53,17 +50,14 @@
) {
IsAppPageviewUDF udf = new IsAppPageviewUDF();
- Map<String,String> xAnalyticsMap = new HashMap<String,String>();
-
assertEquals(
test_description,
is_app_pageview,
- udf.isPageview(
+ udf.evaluate(
uri_path,
uri_query,
content_type,
- user_agent,
- xAnalyticsMap
+ user_agent
)
);
}
diff --git
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
index 5b820b2..bcefdcc 100644
---
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
+++
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDF.java
@@ -15,40 +15,23 @@
*/
package org.wikimedia.analytics.refinery.hive;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import static org.junit.Assert.assertEquals;
+
import junitparams.FileParameters;
import junitparams.JUnitParamsRunner;
import junitparams.mappers.CsvWithHeaderMapper;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.json.simple.parser.JSONParser;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
@RunWith(JUnitParamsRunner.class)
public class TestIsPageviewUDF {
- IsPageviewUDF udf = null;
- JSONParser parser = null;
- @Before
- public void setUp() throws HiveException{
- udf = new IsPageviewUDF();
- parser = new JSONParser();
-
- }
@Test
@FileParameters(
value = "../refinery-core/src/test/resources/pageview_test_data.csv",
mapper = CsvWithHeaderMapper.class
)
- // this mapper cannot deal with reading strings formed like
- // the x analytics map: "{"WMF-Last-Access":"12-Aug-2015","https":"1"}
public void testIsPageview(
String test_description,
String project,
@@ -65,52 +48,20 @@
String http_status,
String content_type,
String user_agent
- ){
-
- Map<String, String> xAnalyticsMap = new HashMap<String, String>();
+ ) {
+ IsPageviewUDF udf = new IsPageviewUDF();
assertEquals(
test_description,
is_pageview,
- udf.isPageview(
+ udf.evaluate(
uri_host,
uri_path,
uri_query,
http_status,
content_type,
- user_agent,
- xAnalyticsMap
-
+ user_agent
)
);
}
-
- @Test
- public void testIsPageviewXAnalyticsPreview(
-
- ){
- String uri_host = "en.wikipedia";
- String uri_path = "/wiki/Horseshoe%20crab#anchor"; ;
- String uri_query = "-";
- String http_status = "200";
- String content_type = "text/html";
- String user_agent = "turnip/";
-
- Map<String, String> xAnalyticsMap = new HashMap<String, String>();
-
- xAnalyticsMap.put("preview", "1");
-
- assertTrue("Preview requests are not pageviews", udf.isPageview(
- uri_host,
- uri_path,
- uri_query,
- http_status,
- content_type,
- user_agent,
- xAnalyticsMap
-
- ) == false);
- }
-
-
}
\ No newline at end of file
diff --git
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDFArgumentHandling.java
b/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDFArgumentHandling.java
deleted file mode 100644
index d5b4fe4..0000000
---
a/refinery-hive/src/test/java/org/wikimedia/analytics/refinery/hive/TestIsPageviewUDFArgumentHandling.java
+++ /dev/null
@@ -1,87 +0,0 @@
-package org.wikimedia.analytics.refinery.hive;
-
-import junitparams.JUnitParamsRunner;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
-import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.json.simple.parser.JSONParser;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-/**
- * Created by nuria on 9/8/15.
- * Tests that isPageview UDF can handle a variable number of arguments
- * and generalerrors in argument setting
- */
-@RunWith(JUnitParamsRunner.class)
-public class TestIsPageviewUDFArgumentHandling {
-
-
- IsPageviewUDF udf = null;
- JSONParser parser = null;
-
- @Before
- public void setUp() throws HiveException{
- udf = new IsPageviewUDF();
- parser = new JSONParser();
-
- }
-
- @Test(expected = UDFArgumentLengthException.class)
- public void testBadNumberOfArgumentsTooFew() throws HiveException{
-
- ObjectInspector value1 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector value2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector[] initArgumentsFew = new ObjectInspector[]{value1,
value2};
- udf.initialize(initArgumentsFew);
- }
-
-
- @Test(expected = UDFArgumentLengthException.class)
- public void testBadNumberOfArgumentsTooMany() throws HiveException{
-
- ObjectInspector value1 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector value2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector[] initArgumentsTooMany = new ObjectInspector[]{value1,
value2, value1, value2, value1, value2, value1, value2};
- udf.initialize(initArgumentsTooMany);
- }
-
- @Test(expected = UDFArgumentException.class)
- public void testBadArgumentType() throws HiveException{
-
- // the 7th argument should be a map not a string
- ObjectInspector value1 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector value2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector[] initArguments = new ObjectInspector[]{value1,
value2, value1, value2, value1, value2, value1};
- udf.initialize(initArguments);
- }
-
- // UDF should work with variable arguments
- public void testGoodNumberOfArguments() throws HiveException{
-
- ObjectInspector value1 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector value2 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector value3 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector value4 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector value5 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
- ObjectInspector value6 =
PrimitiveObjectInspectorFactory.javaStringObjectInspector;
-
- ObjectInspector[] initArguments = new ObjectInspector[]{value1,
value2, value3, value4, value5, value6};
- udf.initialize(initArguments);
-
- ObjectInspector value7 =
ObjectInspectorFactory.getStandardMapObjectInspector(
- PrimitiveObjectInspectorFactory.writableStringObjectInspector,
- PrimitiveObjectInspectorFactory.writableIntObjectInspector
- );
-
- ObjectInspector[] initArgumentsMax = new ObjectInspector[]{value1,
value2, value3, value4, value5, value6, value7};
- udf.initialize(initArguments);
-
-
- }
-
-}
--
To view, visit https://gerrit.wikimedia.org/r/237273
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I9f1a7924b8ebf076f970795b562537dc003c44e4
Gerrit-PatchSet: 1
Gerrit-Project: analytics/refinery/source
Gerrit-Branch: master
Gerrit-Owner: Nuria <[email protected]>
Gerrit-Reviewer: Nuria <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
