Ottomata has uploaded a new change for review. https://gerrit.wikimedia.org/r/132108
Change subject: Removing kraken-dclass ...................................................................... Removing kraken-dclass Also removing references to kraken-generic and kraken-pig in main pom.xml file. These sub projects have depend on dclass and need modification before they will compile again. Change-Id: I8ded3973507abf3155d23958f5d258563dd05ec8 --- M README.md D kraken-dclass/pom.xml D kraken-dclass/src/main/java/dclass/dClass.java D kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java D kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/package-info.java D kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassWrapperTest.java D kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DeviceClassificationTest.java M kraken-generic/pom.xml M kraken-pig/pom.xml M pom.xml 10 files changed, 2 insertions(+), 726 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/analytics/kraken refs/changes/08/132108/1 diff --git a/README.md b/README.md index d3bc464..38ea15a 100644 --- a/README.md +++ b/README.md @@ -34,80 +34,6 @@ git clone https://github.com/wikimedia/kraken.git -## Get libdclass - -### Debian-like systems - -If your an a Debian system, you can probably just grab the most recent .deb files of libdclass0 and libdclass-jni for your architecture from [Wikimedia's apt repository](http://apt.wikimedia.org/wikimedia/pool/main/d/dclass/) and install them using dpkg and can proceed to the next step. - -### Non-Debian-like systems - -If your not on a Debian system, you can try getting the needed files by hand or compiling dclass by yourself. - -#### Getting files by hand - -For an amd64 system with the most recent version being 2.2.0, -that would be: - - mkdir dclass - cd dclass - - mkdir libdclass0 - cd libdclass0 - wget http://apt.wikimedia.org/wikimedia/pool/main/d/dclass/libdclass0_2.2.2-1_amd64.deb - ar x libdclass0_2.2.2-1_amd64.deb - tar -xzvf data.tar.gz - sudo mv usr/lib/x86_64-linux-gnu/libdclass.so.0.0.0 /usr/lib/libdclass.so.0 - sudo chmod +x /usr/lib/libdclass.so.0 - cd .. - - mkdir libdclass-jni - cd libdclass-jni - wget http://apt.wikimedia.org/wikimedia/pool/main/d/dclass/libdclass-jni_2.2.2-1_amd64.deb - ar x libdclass-jni_2.2.2-1_amd64.deb - tar -xzvf data.tar.gz - sudo mv usr/lib/x86_64-linux-gnu/jni/libdclassjni.so.0.0.0 /usr/lib/libdclassjni.so - sudo chmod +x /usr/lib/libdclassjni.so - cd .. - - cd .. - -You can test whether or not it's working by doing - - ldd /usr/lib/libdclassjni.so /usr/lib/libdclass.so.0 - -to see if the dynamic linker caught the files. You should get an output like: - - /usr/lib/libdclassjni.so: - linux-vdso.so.1 (0x00007f43caf4b000) - libdclass.so.0 => /usr/lib64/libdclass.so.0 (0x00007f43cab05000) - libc.so.6 => /lib64/libc.so.6 (0x00007f43ca75a000) - librt.so.1 => /lib64/librt.so.1 (0x00007f43ca551000) - /lib64/ld-linux-x86-64.so.2 (0x00007f43caf4c000) - libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f43ca334000) - /usr/lib/libdclass.so.0: - linux-vdso.so.1 (0x00007fffdb3ff000) - librt.so.1 => /lib64/librt.so.1 (0x00007f6948ba3000) - libc.so.6 => /lib64/libc.so.6 (0x00007f69487f9000) - libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f69485db000) - /lib64/ld-linux-x86-64.so.2 (0x00007f6948fef000) - -(If numbers changed, that's not a problem. Only „Not Found” lines are a problem) - -We do no longer need the dclass folder. You can remove the dclass folder again. - -#### Compiling by yourself - -Instructions on how to compile dclass by yourself are at: - -https://github.com/wikimedia/dClass/blob/package/README - -## openddr.dtree file - -Get a openddr.dtree file (version 1.13) from an [Analytics Team member](http://www.mediawiki.org/wiki/Analytics), and store it as - - /usr/share/libdclass/openddr.dtree - ## GeoIP files Get the diff --git a/kraken-dclass/pom.xml b/kraken-dclass/pom.xml deleted file mode 100644 index cf755d8..0000000 --- a/kraken-dclass/pom.xml +++ /dev/null @@ -1,27 +0,0 @@ -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - - <name>dClass JNI Wrapper</name> - - <groupId>org.wikimedia.analytics.kraken</groupId> - <artifactId>kraken-dclass</artifactId> - <version>0.0.2-SNAPSHOT</version> - <packaging>jar</packaging> - - - <parent> - <groupId>org.wikimedia.analytics.kraken</groupId> - <artifactId>kraken</artifactId> - <version>0.0.2-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <dependencies> - </dependencies> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <skip.tests>false</skip.tests> - </properties> - -</project> diff --git a/kraken-dclass/src/main/java/dclass/dClass.java b/kraken-dclass/src/main/java/dclass/dClass.java deleted file mode 100644 index a2cb699..0000000 --- a/kraken-dclass/src/main/java/dclass/dClass.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Copyright (C) 2012-2013 Wikimedia Foundation - - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package dclass; - - - -import java.io.IOException; -import java.util.Map; -import java.util.HashMap; - - -public class dClass { - private String path; - - /* - * Note: - * - * the dclass_index property of this class is being set by - * the shared object native method setindex() - * - * which is being called from native method init() - * - */ - private long dclass_index; - private boolean isDClassInitialized = false; - - - private native int init(String file); - private native void free(long index); - - private native long classify(long index,String s); - - private native int kvlength(long kv); - private native String kvgetid(long kv); - private native String kvgetkey(long kv,int pos); - private native String kvgetvalue(long kv,int pos); - - - private long getDclassIndex() { - return dclass_index; - }; - - - private static final String OS = System.getProperty("os.name").toLowerCase(); - - public dClass(String s) - { - path=s; - dclass_index=0; - isDClassInitialized = ( init(s) >= 0); - } - - public Map<String,String> classify(String s) - { - Map<String,String> ret=new HashMap<String,String>(); - - - - if(isDClassInitialized && s!=null) - { - long kv; - - - kv=classify(dclass_index,s); - - if(kv!=0) - { - int len=kvlength(kv); - String id=kvgetid(kv); - for(int i=0;i<len;i++) - { - String key=kvgetkey(kv,i); - String value=kvgetvalue(kv,i); - ret.put(key,value); - } - ret.put("id",id); - } - } - - return ret; - } - - - - protected void finalize() { - if (isDClassInitialized) - free(this.getDclassIndex()); - isDClassInitialized = false; - - } - - public static void loadDclassSharedObject() throws IOException { - try { - if (OS.contains("mac")) { - System.load("/usr/local/lib/libdclassjni.0.dylib"); - } else if (OS.contains("nix") - || OS.contains("nux") - || OS.contains("aix")) { - - try { - System.loadLibrary("dclassjni"); - } catch(Exception e) { - System.err.println("Error: " + e); - System.exit(-1); - }; - - } else { - System.err.println("OS not supported."); - System.exit(-1); - } - } catch (UnsatisfiedLinkError e) { - System.err.println("Native code library failed to load.\n" + e); - e.printStackTrace(); - } - } - - static { - try { - loadDclassSharedObject(); - } catch (IOException e) { - System.err.println("Error loading dClass Shared Object: "+e.getMessage()); - e.printStackTrace(); - } - } - - - -} - diff --git a/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java b/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java deleted file mode 100644 index 86a3755..0000000 --- a/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/DeviceClassification.java +++ /dev/null @@ -1,399 +0,0 @@ -/** - * Copyright (C) 2012-2013 Wikimedia Foundation - - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.wikimedia.analytics.dclassjni; - -import java.util.Iterator; -import java.util.Map; - - -import dclass.dClass; - - -/** - * The class dClassParsedResult. This class is a mapping of the results from - * dclass_keyvalue data structure to a Java data structure :) - * - * OpenDDR attributes => vendor: 'HTC' - * model: 'A6380' - * parentId: 'genericHTC' - * inputDevices: 'touchscreen' - * displayHeight: '480' - * displayWidth: '320' - * device_os: 'Android' - * ajax_support_javascript: 'true' - * is_tablet: 'false' - * is_wireless_device: 'true' - * is_crawler: 'false' - * is_desktop: 'false' - * - */ -public class DeviceClassification { - public final static String dtree_path = "/usr/share/libdclass/openddr.dtree"; - /** - * Only initialize the dClass JNI wrapper on-demand (and only once), - * as doing so loads 2+ MB of dtree data into memory from disk. - */ - - - /** - * Wrapper subclass to release the dtree data when the singleton is - * destroyed, presumably when its classloader is released (which *can* - * happen without imminent JVM termination, such as in IOC containers, - * like Spring). - */ - final private static dClass dClass = new dClass(dtree_path); - - synchronized private static Map classifyUA(String ua) { - - return dClass.classify(ua); - } - - - - private String parentId; - private String vendor; - private String model; - private String deviceOs; - private String deviceOsVersion; - private int displayHeight; - private int displayWidth; - private String inputDevices; - private String browser; - private String browserVersion; - private boolean isTablet; - private boolean isWirelessDevice; - private boolean isCrawler; - private boolean isDesktop; - private boolean ajaxSupportJavascript; - - - public DeviceClassification() {} - - public DeviceClassification(String userAgent) { - classifyUseragent(userAgent); - } - - /** - * - * @param userAgent - */ - public final DeviceClassification classifyUseragent(final String userAgent) { - final Map result = classifyUA(userAgent); - setVendor(result.get("vendor")); - setModel(result.get("model")); - setParentId(result.get("parentId")); - setInputDevices(result.get("inputDevices")); - setDeviceOs(result.get("device_os")); - setDeviceOsVersion(result.get("device_os_version")); - setDisplayHeight(result.get("displayHeight")); - setDisplayWidth(result.get("displayWidth")); - setAjaxSupportJavascript(result.get("ajax_support_javascript")); - setIsWirelessDevice(result.get("is_wireless_device")); - setIsTablet(result.get("is_tablet")); - setIsCrawler(result.get("is_crawler")); - setIsDesktop(result.get("is_desktop")); - setBrowser(result.get("browser")); - setBrowserVersion(result.get("browser_version")); - return this; - } - - - /** - * Gets the vendor. - * - * @return the vendor - */ - public final String getVendor() { - return vendor; - } - - /** - * Sets the vendor. - * - * @param vendor the new vendor - */ - private void setVendor(final Object vendor) { - this.vendor = (String) vendor; - } - - /** - * Gets the model. - * - * @return the model - */ - public final String getModel() { - return model; - } - - /** - * Sets the model. - * - * @param model the new model - */ - private void setModel(final Object model) { - this.model = (String) model; - } - - /** - * Gets the parent id. - * - * @return the parent id - */ - public final String getParentId() { - return parentId; - } - - /** - * Sets the parent id. - * - * @param parentId the new parent id - */ - private void setParentId(final Object parentId) { - this.parentId = (String) parentId; - } - - /** - * Gets the input devices. - * - * @return the input devices - */ - public final String getInputDevices() { - return inputDevices; - } - - /** - * Sets the input devices. - * - * @param inputDevices the new input devices - */ - private void setInputDevices(final Object inputDevices) { - this.inputDevices = (String) inputDevices; - } - - /** - * Gets the display height. - * - * @return the display height - */ - public final int getDisplayHeight() { - return displayHeight; - } - - /** - * Sets the display height. - * - * @param displayHeight the new display height - */ - private void setDisplayHeight(final Object displayHeight) { - this.displayHeight = displayHeight == null ? 0 : Integer.valueOf((String) displayHeight); - } - - /** - * Gets the display width. - * - * @return the display width - */ - public final int getDisplayWidth() { - return displayWidth; - } - - /** - * Sets the display width. - * - * @param displayWidth the new display width - */ - private void setDisplayWidth(final Object displayWidth) { - this.displayWidth = displayWidth == null ? 0 : Integer.valueOf((String) displayWidth); - } - - /** - * Gets the device_os. - * - * @return the device_os - */ - public final String getDeviceOs() { - return deviceOs; - } - - /** - * Sets the device_os. - * - * @param deviceOs the new device_os - */ - private void setDeviceOs(final Object deviceOs) { - this.deviceOs = (String) deviceOs; - } - - /** - * Gets the device_os_version. - * - * @return the device_os_version - */ - public final String getDeviceOsVersion() { - return deviceOsVersion; - } - - /** - * Sets the device_os_version. - * - * @param deviceOsVersion the new device_os_version - */ - private void setDeviceOsVersion(final Object deviceOsVersion) { - this.deviceOsVersion = (String) deviceOsVersion; - } - - /** - * Gets the ajax_support_javascript. - * - * @return the ajaxSupportJavascript - */ - public final boolean getAjaxSupportJavascript() { - return ajaxSupportJavascript; - } - - /** - * Sets the ajax_support_javascript. - * - * @param ajaxSupportJavascript the new ajax_support_javascript - */ - private void setAjaxSupportJavascript(final Object ajaxSupportJavascript) { - this.ajaxSupportJavascript = Boolean.valueOf((String) ajaxSupportJavascript); - } - - /** - * Gets the is_tablet. - * - * @return the is_tablet - */ - public final boolean getIsTablet() { - return isTablet; - } - - /** - * Sets the is_tablet. - * - * @param isTablet the new is_tablet - */ - private void setIsTablet(final Object isTablet) { - this.isTablet = Boolean.valueOf((String) isTablet); - } - - /** - * Gets the checks if is wireless_device. - * - * @return the checks if is wireless_device - */ - public final boolean getIsWirelessDevice() { - return isWirelessDevice; - } - - /** - * Sets the is_wireless_device. - * - * @param isWirelessDevice the new is_wireless_device - */ - private void setIsWirelessDevice(final Object isWirelessDevice) { - this.isWirelessDevice = Boolean.valueOf((String) isWirelessDevice); - } - - /** - * Gets the is_crawler. - * - * @return the getIsCrawler - */ - public final boolean getIsCrawler() { - return isCrawler; - } - - /** - * Sets the is_crawler. - * - * @param isCrawler the new is_crawler - */ - private void setIsCrawler(final Object isCrawler) { - this.isCrawler = Boolean.valueOf((String) isCrawler); - } - - /** - * Gets the is_desktop. - * - * @return the is_desktop - */ - public final boolean getIsDesktop() { - return isDesktop; - } - - /** - * Sets the is_desktop. - * - * @param isDesktop the new is_desktop - */ - private void setIsDesktop(final Object isDesktop) { - this.isDesktop = Boolean.valueOf((String) isDesktop); - } - - public String getBrowser() { - return browser; - } - - private void setBrowser(final Object browser) { - this.browser = (String) browser; - } - - public String getBrowserVersion() { - return browserVersion; - } - - private void setBrowserVersion(final Object browserVersion) { - this.browserVersion = (String) browserVersion; - } - - - - /* * * * Debugging Utilities * * * */ - - - /** - * Prints the properties of to stdout. - */ - final void print() { - System.out.println("vendor = " + getVendor()); - System.out.println("model = " + getModel()); - System.out.println("parentId = " + getParentId()); - System.out.println("inputDevices = " + getInputDevices()); - System.out.println("device_os = " + getDeviceOs()); - System.out.println("device_os_version = " + getDeviceOsVersion()); - System.out.println("displayHeight = " + getDisplayHeight()); - System.out.println("displayWidth = " + getDisplayWidth()); - System.out.println("ajax_support_javascript = " + ajaxSupportJavascript); - System.out.println("is_tablet = " + getIsTablet()); - System.out.println("is_wireless_device = " + getIsWirelessDevice()); - System.out.println("is_crawler = " + getIsCrawler()); - System.out.println("is_desktop = " + getIsDesktop()); - System.out.println("browser = " + getBrowser()); - System.out.println("browser_version = " + getBrowserVersion()); - } - - private static void debugOutputdClass(final String userAgent) { - final Map result = classifyUA(userAgent); - final Iterator it = result.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry<String, String> entry = (Map.Entry) it.next(); - System.out.println(entry.getKey() + " : " + entry.getValue()); - } - } - -} diff --git a/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/package-info.java b/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/package-info.java deleted file mode 100644 index c616eae..0000000 --- a/kraken-dclass/src/main/java/org/wikimedia/analytics/dclassjni/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * - */ -package org.wikimedia.analytics.dclassjni; diff --git a/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassWrapperTest.java b/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassWrapperTest.java deleted file mode 100644 index 7a402f7..0000000 --- a/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DclassWrapperTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - Copyright (C) 2012 Wikimedia Foundation - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ -package org.wikimedia.analytics.dclassjni; - -import static org.junit.Assert.*; - -import java.util.Map; - -import org.junit.Test; - -import dclass.dClass; - - - -public class DclassWrapperTest { - - public final String dtree_path = "/usr/share/libdclass/openddr.dtree"; - @Test - public void testDclassWrapper() { - String userAgentSample = "Mozilla/5.0 (Linux; U; Android 2.2; en; HTC Aria A6380 Build/ERE27) AppleWebKit/540.13+ (KHTML, like Gecko) Version/3.1 Mobile Safari/524.15.0"; - dClass dw = new dClass(dtree_path); - Map<String, String> c = dw.classify(userAgentSample); - assertEquals("Vendor as expected", "HTC" , c.get("vendor")); - assertEquals("Model as expected" , "A6380", c.get("model")); - assertEquals("device_os as expected","Android",c.get("device_os")); - assertEquals("is_tablet as expected","false",c.get("is_tablet")); - - } - -} diff --git a/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DeviceClassificationTest.java b/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DeviceClassificationTest.java deleted file mode 100644 index bd84f9a..0000000 --- a/kraken-dclass/src/test/java/org/wikimedia/analytics/dclassjni/DeviceClassificationTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.wikimedia.analytics.dclassjni; - -import static org.junit.Assert.*; - -import org.junit.Test; - -import org.wikimedia.analytics.dclassjni.DeviceClassification; - -public class DeviceClassificationTest { - - @Test - public void testDeviceClassification() { - String userAgentSample = "Mozilla/5.0 (Linux; U; Android 2.2; en; HTC Aria A6380 Build/ERE27) AppleWebKit/540.13+ (KHTML, like Gecko) Version/3.1 Mobile Safari/524.15.0"; - DeviceClassification d = new DeviceClassification(); - d.classifyUseragent(userAgentSample); - - assertEquals("Vendor as expected", "HTC" , d.getVendor()); - assertEquals("Model as expected" , "A6380", d.getModel()); - } -} diff --git a/kraken-generic/pom.xml b/kraken-generic/pom.xml index a5e63ee..26ecdb0 100644 --- a/kraken-generic/pom.xml +++ b/kraken-generic/pom.xml @@ -17,12 +17,6 @@ <dependencies> <dependency> - <groupId>org.wikimedia.analytics.kraken</groupId> - <artifactId>kraken-dclass</artifactId> - <version>0.0.2-SNAPSHOT</version> - </dependency> - - <dependency> <groupId>commons-net</groupId> <artifactId>commons-net</artifactId> <version>3.1</version> diff --git a/kraken-pig/pom.xml b/kraken-pig/pom.xml index 84dd3b7..6d75ee3 100644 --- a/kraken-pig/pom.xml +++ b/kraken-pig/pom.xml @@ -22,12 +22,6 @@ <dependency> <groupId>org.wikimedia.analytics.kraken</groupId> - <artifactId>kraken-dclass</artifactId> - <version>0.0.2-SNAPSHOT</version> - </dependency> - - <dependency> - <groupId>org.wikimedia.analytics.kraken</groupId> <artifactId>kraken-generic</artifactId> <version>0.0.2-SNAPSHOT</version> </dependency> diff --git a/pom.xml b/pom.xml index 79a6040..9a189b6 100644 --- a/pom.xml +++ b/pom.xml @@ -13,9 +13,8 @@ <modules> - <module>kraken-generic</module> - <module>kraken-dclass</module> - <module>kraken-pig</module> + <!-- <module>kraken-generic</module> --> + <!-- <module>kraken-pig</module> --> <module>kraken-toolbelt</module> </modules> -- To view, visit https://gerrit.wikimedia.org/r/132108 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I8ded3973507abf3155d23958f5d258563dd05ec8 Gerrit-PatchSet: 1 Gerrit-Project: analytics/kraken Gerrit-Branch: master Gerrit-Owner: Ottomata <o...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits