add tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/5088ef36 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/5088ef36 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/5088ef36 Branch: refs/heads/JIRA-22/pr-385 Commit: 5088ef36367df1cd51ae62f1c044933676975e2e Parents: a882c5f Author: amaya <g...@sapphire.in.net> Authored: Wed Sep 21 16:22:09 2016 +0900 Committer: amaya <g...@sapphire.in.net> Committed: Wed Sep 21 18:00:35 2016 +0900 ---------------------------------------------------------------------- .../tools/matrix/TransposeAndDotUDAF.java | 2 +- .../ftvec/selection/ChiSquareUDFTest.java | 80 ++++++++++++++++++++ .../tools/array/SelectKBeatUDFTest.java | 65 ++++++++++++++++ .../tools/matrix/TransposeAndDotUDAFTest.java | 58 ++++++++++++++ 4 files changed, 204 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/5088ef36/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java b/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java index 9d68f93..9df9305 100644 --- a/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java +++ b/core/src/main/java/hivemall/tools/matrix/TransposeAndDotUDAF.java @@ -70,7 +70,7 @@ public final class TransposeAndDotUDAF extends AbstractGenericUDAFResolver { return new TransposeAndDotUDAFEvaluator(); } - private static final class TransposeAndDotUDAFEvaluator extends GenericUDAFEvaluator { + static final class TransposeAndDotUDAFEvaluator extends GenericUDAFEvaluator { // PARTIAL1 and COMPLETE private ListObjectInspector matrix0RowOI; private PrimitiveObjectInspector matrix0ElOI; http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/5088ef36/core/src/test/java/hivemall/ftvec/selection/ChiSquareUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/ftvec/selection/ChiSquareUDFTest.java b/core/src/test/java/hivemall/ftvec/selection/ChiSquareUDFTest.java new file mode 100644 index 0000000..38f7f57 --- /dev/null +++ b/core/src/test/java/hivemall/ftvec/selection/ChiSquareUDFTest.java @@ -0,0 +1,80 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hivemall.ftvec.selection; + +import hivemall.utils.hadoop.WritableUtils; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.io.DoubleWritable; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class ChiSquareUDFTest { + + @Test + public void test() throws Exception { + // this test is based on iris data set + final ChiSquareUDF chi2 = new ChiSquareUDF(); + final List<List<DoubleWritable>> observed = new ArrayList<List<DoubleWritable>>(); + final List<List<DoubleWritable>> expected = new ArrayList<List<DoubleWritable>>(); + final GenericUDF.DeferredObject[] dObjs = new GenericUDF.DeferredObject[] { + new GenericUDF.DeferredJavaObject(observed), + new GenericUDF.DeferredJavaObject(expected)}; + + final double[][] matrix0 = new double[][] { + {250.29999999999998, 170.90000000000003, 73.2, 12.199999999999996}, + {296.8, 138.50000000000003, 212.99999999999997, 66.3}, + {329.3999999999999, 148.7, 277.59999999999997, 101.29999999999998}}; + final double[][] matrix1 = new double[][] { + {292.1666753739119, 152.70000455081467, 187.93333893418327, 59.93333511948589}, + {292.1666753739119, 152.70000455081467, 187.93333893418327, 59.93333511948589}, + {292.1666753739119, 152.70000455081467, 187.93333893418327, 59.93333511948589}}; + + for (double[] row : matrix0) { + observed.add(WritableUtils.toWritableList(row)); + } + for (double[] row : matrix1) { + expected.add(WritableUtils.toWritableList(row)); + } + + chi2.initialize(new ObjectInspector[] { + ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)), + ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector))}); + final Object[] result = (Object[]) chi2.evaluate(dObjs); + final double[] result0 = new double[matrix0[0].length]; + final double[] result1 = new double[matrix0[0].length]; + for (int i = 0; i < result0.length; i++) { + result0[i] = Double.valueOf(((List) result[0]).get(i).toString()); + result1[i] = Double.valueOf(((List) result[1]).get(i).toString()); + } + + final double[] answer0 = new double[] {10.817820878493995, 3.5944990176817315, + 116.16984746363957, 67.24482558215503}; + final double[] answer1 = new double[] {0.004476514990225833, 0.16575416718561453, 0.d, + 2.55351295663786e-15}; + + Assert.assertArrayEquals(answer0, result0, 0.d); + Assert.assertArrayEquals(answer1, result1, 0.d); + } +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/5088ef36/core/src/test/java/hivemall/tools/array/SelectKBeatUDFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/array/SelectKBeatUDFTest.java b/core/src/test/java/hivemall/tools/array/SelectKBeatUDFTest.java new file mode 100644 index 0000000..b86db5c --- /dev/null +++ b/core/src/test/java/hivemall/tools/array/SelectKBeatUDFTest.java @@ -0,0 +1,65 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hivemall.tools.array; + +import hivemall.utils.hadoop.WritableUtils; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class SelectKBeatUDFTest { + + @Test + public void test() throws Exception { + final SelectKBestUDF selectKBest = new SelectKBestUDF(); + final int k = 2; + final double[] data = new double[] {250.29999999999998, 170.90000000000003, 73.2, + 12.199999999999996}; + final double[] importanceList = new double[] {292.1666753739119, 152.70000455081467, + 187.93333893418327, 59.93333511948589}; + + final GenericUDF.DeferredObject[] dObjs = new GenericUDF.DeferredObject[] { + new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(data)), + new GenericUDF.DeferredJavaObject(WritableUtils.toWritableList(importanceList)), + new GenericUDF.DeferredJavaObject(new IntWritable(k))}; + + selectKBest.initialize(new ObjectInspector[] { + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + PrimitiveObjectInspectorFactory.writableIntObjectInspector}); + final List resultObj = (List) selectKBest.evaluate(dObjs); + + Assert.assertEquals(resultObj.size(), k); + + final double[] result = new double[k]; + for (int i = 0; i < k; i++) { + result[i] = Double.valueOf(resultObj.get(i).toString()); + } + + final double[] answer = new double[] {250.29999999999998, 73.2}; + + Assert.assertArrayEquals(answer, result, 0.d); + } +} http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/5088ef36/core/src/test/java/hivemall/tools/matrix/TransposeAndDotUDAFTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/hivemall/tools/matrix/TransposeAndDotUDAFTest.java b/core/src/test/java/hivemall/tools/matrix/TransposeAndDotUDAFTest.java new file mode 100644 index 0000000..93c6ef1 --- /dev/null +++ b/core/src/test/java/hivemall/tools/matrix/TransposeAndDotUDAFTest.java @@ -0,0 +1,58 @@ +/* + * Hivemall: Hive scalable Machine Learning Library + * + * Copyright (C) 2016 Makoto YUI + * Copyright (C) 2013-2015 National Institute of Advanced Industrial Science and Technology (AIST) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package hivemall.tools.matrix; + +import hivemall.utils.hadoop.WritableUtils; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.SimpleGenericUDAFParameterInfo; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.junit.Assert; +import org.junit.Test; + +public class TransposeAndDotUDAFTest { + + @Test + public void test() throws Exception { + final TransposeAndDotUDAF tad = new TransposeAndDotUDAF(); + + final double[][] matrix0 = new double[][] { {1, -2}, {-1, 3}}; + final double[][] matrix1 = new double[][] { {1, 2}, {3, 4}}; + + final ObjectInspector[] OIs = new ObjectInspector[] { + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector), + ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.writableDoubleObjectInspector)}; + final GenericUDAFEvaluator evaluator = tad.getEvaluator(new SimpleGenericUDAFParameterInfo( + OIs, false, false)); + evaluator.init(GenericUDAFEvaluator.Mode.PARTIAL1, OIs); + TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer agg = (TransposeAndDotUDAF.TransposeAndDotUDAFEvaluator.TransposeAndDotAggregationBuffer) evaluator.getNewAggregationBuffer(); + evaluator.reset(agg); + for (int i = 0; i < matrix0.length; i++) { + evaluator.iterate(agg, new Object[] {WritableUtils.toWritableList(matrix0[i]), + WritableUtils.toWritableList(matrix1[i])}); + } + + final double[][] answer = new double[][] { {-2.0, -2.0}, {7.0, 8.0}}; + + for (int i = 0; i < answer.length; i++) { + Assert.assertArrayEquals(answer[i], agg.aggMatrix[i], 0.d); + } + } +}
