Repository: mahout Updated Branches: refs/heads/master b289f4657 -> 19182a81d
MAHOUT-1906: Ensure customJars are added to the MahoutContext under certain conditions for spark 1.6+ Project: http://git-wip-us.apache.org/repos/asf/mahout/repo Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/8e0e8b55 Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/8e0e8b55 Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/8e0e8b55 Branch: refs/heads/master Commit: 8e0e8b5572e0d24c1930ed60fec6d02693b41575 Parents: 3e2d63c Author: Andrew Palumbo <[email protected]> Authored: Mon Jan 16 14:26:47 2017 -0800 Committer: Andrew Palumbo <[email protected]> Committed: Mon Jan 16 14:48:44 2017 -0800 ---------------------------------------------------------------------- .../org/apache/mahout/math/AbstractMatrix.java | 17 ++++++++++------- .../java/org/apache/mahout/math/DenseMatrix.java | 9 ++++++++- .../java/org/apache/mahout/math/MatrixTest.java | 19 +++++++++---------- .../org/apache/mahout/math/TestDenseMatrix.java | 18 ++++++++++++++++++ .../mahout/vectorizer/encoders/Dictionary.java | 9 ++++----- .../cf/taste/hadoop/item/IDReaderTest.java | 10 +++++----- .../cf/taste/impl/common/FastByIDMapTest.java | 6 +++--- .../vectorizer/collocations/llr/GramTest.java | 11 +++++------ .../apache/mahout/sparkbindings/package.scala | 1 + 9 files changed, 63 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java ---------------------------------------------------------------------- diff --git a/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java b/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java index dc9ccd9..eaaa397 100644 --- a/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java +++ b/math/src/main/java/org/apache/mahout/math/AbstractMatrix.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -19,11 +19,14 @@ package org.apache.mahout.math; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Maps; -import org.apache.mahout.math.flavor.BackEnum; import org.apache.mahout.math.flavor.MatrixFlavor; -import org.apache.mahout.math.flavor.TraversingStructureEnum; -import org.apache.mahout.math.function.*; +import org.apache.mahout.math.function.DoubleDoubleFunction; +import org.apache.mahout.math.function.DoubleFunction; +import org.apache.mahout.math.function.Functions; +import org.apache.mahout.math.function.PlusMult; +import org.apache.mahout.math.function.VectorFunction; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; @@ -127,7 +130,7 @@ public abstract class AbstractMatrix implements Matrix { @Override public void set(String rowLabel, int row, double[] rowData) { if (rowLabelBindings == null) { - rowLabelBindings = Maps.newHashMap(); + rowLabelBindings = new HashMap<>(); } rowLabelBindings.put(rowLabel, row); set(row, rowData); @@ -149,11 +152,11 @@ public abstract class AbstractMatrix implements Matrix { @Override public void set(String rowLabel, String columnLabel, int row, int column, double value) { if (rowLabelBindings == null) { - rowLabelBindings = Maps.newHashMap(); + rowLabelBindings = new HashMap<>(); } rowLabelBindings.put(rowLabel, row); if (columnLabelBindings == null) { - columnLabelBindings = Maps.newHashMap(); + columnLabelBindings = new HashMap<>(); } columnLabelBindings.put(columnLabel, column); http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/math/src/main/java/org/apache/mahout/math/DenseMatrix.java ---------------------------------------------------------------------- diff --git a/math/src/main/java/org/apache/mahout/math/DenseMatrix.java b/math/src/main/java/org/apache/mahout/math/DenseMatrix.java index 5c1ee12..eac449a 100644 --- a/math/src/main/java/org/apache/mahout/math/DenseMatrix.java +++ b/math/src/main/java/org/apache/mahout/math/DenseMatrix.java @@ -18,7 +18,6 @@ package org.apache.mahout.math; import org.apache.mahout.math.flavor.MatrixFlavor; -import org.apache.mahout.math.flavor.TraversingStructureEnum; import java.util.Arrays; @@ -66,6 +65,14 @@ public class DenseMatrix extends AbstractMatrix { this.values = new double[rows][columns]; } + /** + * Returns the backing array + * @return double[][] + */ + public double[][] getBackingStructure() { + return this.values; + } + @Override public Matrix clone() { DenseMatrix clone = (DenseMatrix) super.clone(); http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/math/src/test/java/org/apache/mahout/math/MatrixTest.java ---------------------------------------------------------------------- diff --git a/math/src/test/java/org/apache/mahout/math/MatrixTest.java b/math/src/test/java/org/apache/mahout/math/MatrixTest.java index 2fbc6b0..fe8ace2 100644 --- a/math/src/test/java/org/apache/mahout/math/MatrixTest.java +++ b/math/src/test/java/org/apache/mahout/math/MatrixTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,17 +17,16 @@ package org.apache.mahout.math; -import java.util.Iterator; -import java.util.Map; -import java.util.Random; - import org.apache.mahout.common.RandomUtils; import org.apache.mahout.math.function.Functions; import org.apache.mahout.math.function.VectorFunction; import org.junit.Before; import org.junit.Test; -import com.google.common.collect.Maps; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Random; public abstract class MatrixTest extends MahoutTestCase { @@ -590,13 +589,13 @@ public abstract class MatrixTest extends MahoutTestCase { {1, 4, 2}}); assertNull("row bindings", m.getRowLabelBindings()); assertNull("col bindings", m.getColumnLabelBindings()); - Map<String, Integer> rowBindings = Maps.newHashMap(); + Map<String, Integer> rowBindings = new HashMap<>(); rowBindings.put("Fee", 0); rowBindings.put("Fie", 1); rowBindings.put("Foe", 2); m.setRowLabelBindings(rowBindings); assertEquals("row", rowBindings, m.getRowLabelBindings()); - Map<String, Integer> colBindings = Maps.newHashMap(); + Map<String, Integer> colBindings = new HashMap<>(); colBindings.put("Foo", 0); colBindings.put("Bar", 1); colBindings.put("Baz", 2); @@ -630,13 +629,13 @@ public abstract class MatrixTest extends MahoutTestCase { {1, 4, 2}}); assertNull("row bindings", m.getRowLabelBindings()); assertNull("col bindings", m.getColumnLabelBindings()); - Map<String, Integer> rowBindings = Maps.newHashMap(); + Map<String, Integer> rowBindings = new HashMap<>(); rowBindings.put("Fee", 0); rowBindings.put("Fie", 1); rowBindings.put("Foe", 2); m.setRowLabelBindings(rowBindings); assertEquals("row", rowBindings, m.getRowLabelBindings()); - Map<String, Integer> colBindings = Maps.newHashMap(); + Map<String, Integer> colBindings = new HashMap<>(); colBindings.put("Foo", 0); colBindings.put("Bar", 1); colBindings.put("Baz", 2); http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/math/src/test/java/org/apache/mahout/math/TestDenseMatrix.java ---------------------------------------------------------------------- diff --git a/math/src/test/java/org/apache/mahout/math/TestDenseMatrix.java b/math/src/test/java/org/apache/mahout/math/TestDenseMatrix.java index ed2a65d..2b950e3 100644 --- a/math/src/test/java/org/apache/mahout/math/TestDenseMatrix.java +++ b/math/src/test/java/org/apache/mahout/math/TestDenseMatrix.java @@ -17,6 +17,9 @@ package org.apache.mahout.math; +import org.junit.Assert; +import org.junit.Test; + public final class TestDenseMatrix extends MatrixTest { @Override @@ -24,4 +27,19 @@ public final class TestDenseMatrix extends MatrixTest { return new DenseMatrix(values); } + @Test + public void testGetValues() { + DenseMatrix m = new DenseMatrix(10, 10); + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 10; j++) { + m.set(i, j, 10 * i + j); + } + } + + double[][] values = m.getBackingStructure(); + Assert.assertEquals(values.length, 10); + Assert.assertEquals(values[0].length, 10); + Assert.assertEquals(values[9][9], 99.0, 0.0); + } + } http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/mr/src/main/java/org/apache/mahout/vectorizer/encoders/Dictionary.java ---------------------------------------------------------------------- diff --git a/mr/src/main/java/org/apache/mahout/vectorizer/encoders/Dictionary.java b/mr/src/main/java/org/apache/mahout/vectorizer/encoders/Dictionary.java index 2ea9b1b..60c89f7 100644 --- a/mr/src/main/java/org/apache/mahout/vectorizer/encoders/Dictionary.java +++ b/mr/src/main/java/org/apache/mahout/vectorizer/encoders/Dictionary.java @@ -17,9 +17,8 @@ package org.apache.mahout.vectorizer.encoders; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,7 +26,7 @@ import java.util.Map; * Assigns integer codes to strings as they appear. */ public class Dictionary { - private final Map<String, Integer> dict = Maps.newLinkedHashMap(); + private final Map<String, Integer> dict = new LinkedHashMap<>(); public int intern(String s) { if (!dict.containsKey(s)) { @@ -38,7 +37,7 @@ public class Dictionary { public List<String> values() { // order of keySet is guaranteed to be insertion order - return Lists.newArrayList(dict.keySet()); + return new ArrayList<>(dict.keySet()); } public int size() { http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/mr/src/test/java/org/apache/mahout/cf/taste/hadoop/item/IDReaderTest.java ---------------------------------------------------------------------- diff --git a/mr/src/test/java/org/apache/mahout/cf/taste/hadoop/item/IDReaderTest.java b/mr/src/test/java/org/apache/mahout/cf/taste/hadoop/item/IDReaderTest.java index 650ca98..a1cc648 100644 --- a/mr/src/test/java/org/apache/mahout/cf/taste/hadoop/item/IDReaderTest.java +++ b/mr/src/test/java/org/apache/mahout/cf/taste/hadoop/item/IDReaderTest.java @@ -17,13 +17,13 @@ package org.apache.mahout.cf.taste.hadoop.item; -import java.util.Map; - -import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; import org.apache.mahout.cf.taste.impl.TasteTestCase; -import org.junit.Test; import org.apache.mahout.cf.taste.impl.common.FastIDSet; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; public class IDReaderTest extends TasteTestCase { @@ -33,7 +33,7 @@ public class IDReaderTest extends TasteTestCase { public void testUserItemFilter() throws Exception { Configuration conf = getConfiguration(); IDReader idReader = new IDReader(conf); - Map<Long, FastIDSet> userItemFilter = Maps.newHashMap(); + Map<Long, FastIDSet> userItemFilter = new HashMap<>(); long user1 = 1; long user2 = 2; http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/mr/src/test/java/org/apache/mahout/cf/taste/impl/common/FastByIDMapTest.java ---------------------------------------------------------------------- diff --git a/mr/src/test/java/org/apache/mahout/cf/taste/impl/common/FastByIDMapTest.java b/mr/src/test/java/org/apache/mahout/cf/taste/impl/common/FastByIDMapTest.java index 8a4eed6..8195a90 100644 --- a/mr/src/test/java/org/apache/mahout/cf/taste/impl/common/FastByIDMapTest.java +++ b/mr/src/test/java/org/apache/mahout/cf/taste/impl/common/FastByIDMapTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,11 +17,11 @@ package org.apache.mahout.cf.taste.impl.common; -import com.google.common.collect.Maps; import org.apache.mahout.cf.taste.impl.TasteTestCase; import org.apache.mahout.common.RandomUtils; import org.junit.Test; +import java.util.HashMap; import java.util.Map; import java.util.Random; @@ -103,7 +103,7 @@ public final class FastByIDMapTest extends TasteTestCase { @Test public void testVersusHashMap() { FastByIDMap<String> actual = new FastByIDMap<>(); - Map<Long, String> expected = Maps.newHashMapWithExpectedSize(1000000); + Map<Long, String> expected = new HashMap<>(1000000); Random r = RandomUtils.getRandom(); for (int i = 0; i < 1000000; i++) { double d = r.nextDouble(); http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/mr/src/test/java/org/apache/mahout/vectorizer/collocations/llr/GramTest.java ---------------------------------------------------------------------- diff --git a/mr/src/test/java/org/apache/mahout/vectorizer/collocations/llr/GramTest.java b/mr/src/test/java/org/apache/mahout/vectorizer/collocations/llr/GramTest.java index 5b0c681..f04d5c6 100644 --- a/mr/src/test/java/org/apache/mahout/vectorizer/collocations/llr/GramTest.java +++ b/mr/src/test/java/org/apache/mahout/vectorizer/collocations/llr/GramTest.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,6 +17,9 @@ package org.apache.mahout.vectorizer.collocations.llr; +import org.apache.mahout.common.MahoutTestCase; +import org.junit.Test; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInput; @@ -26,10 +29,6 @@ import java.io.DataOutputStream; import java.util.Arrays; import java.util.HashMap; -import com.google.common.collect.Maps; -import org.apache.mahout.common.MahoutTestCase; -import org.junit.Test; - public final class GramTest extends MahoutTestCase { @Test @@ -113,7 +112,7 @@ public final class GramTest extends MahoutTestCase { new Gram("bar", Gram.Type.UNIGRAM) }; - HashMap<Gram,Gram> map = Maps.newHashMap(); + HashMap<Gram,Gram> map = new HashMap<>(); for (Gram n : input) { Gram val = map.get(n); if (val != null) { http://git-wip-us.apache.org/repos/asf/mahout/blob/8e0e8b55/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala ---------------------------------------------------------------------- diff --git a/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala b/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala index 8064cf0..8fd77e6 100644 --- a/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala +++ b/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala @@ -75,6 +75,7 @@ package object sparkbindings { } sparkConf.setJars(jars = mcjars.toSeq ++ customJars) + if (!(customJars.size > 0)) sparkConf.setJars(customJars.toSeq) } else { // In local mode we don't care about jars, do we?
