Merge 'master' into 'feature/feature_selection'
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/aa7d5299 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/aa7d5299 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/aa7d5299 Branch: refs/heads/JIRA-22/pr-385 Commit: aa7d5299739349b49ef4f50cc2c1969f5cb8a78f Parents: a1f8f95 bc8b015 Author: amaya <g...@sapphire.in.net> Authored: Tue Sep 27 16:02:02 2016 +0900 Committer: amaya <g...@sapphire.in.net> Committed: Tue Sep 27 16:02:02 2016 +0900 ---------------------------------------------------------------------- README.md | 7 +- core/pom.xml | 2 +- .../hivemall/ensemble/ArgminKLDistanceUDAF.java | 1 + .../main/java/hivemall/ensemble/MaxRowUDAF.java | 21 +- .../hivemall/ensemble/MaxValueLabelUDAF.java | 1 + .../hivemall/ensemble/bagging/VotedAvgUDAF.java | 1 + .../ensemble/bagging/WeightVotedAvgUDAF.java | 1 + .../main/java/hivemall/evaluation/AUCUDAF.java | 37 +- .../evaluation/BinaryResponsesMeasures.java | 31 +- .../java/hivemall/evaluation/FMeasureUDAF.java | 1 + .../evaluation/GradedResponsesMeasures.java | 7 +- .../evaluation/LogarithmicLossUDAF.java | 1 + .../main/java/hivemall/evaluation/MAPUDAF.java | 55 +-- .../main/java/hivemall/evaluation/MRRUDAF.java | 55 +-- .../evaluation/MeanAbsoluteErrorUDAF.java | 1 + .../evaluation/MeanSquaredErrorUDAF.java | 1 + .../main/java/hivemall/evaluation/NDCGUDAF.java | 45 +-- .../java/hivemall/evaluation/PrecisionUDAF.java | 55 +-- .../main/java/hivemall/evaluation/R2UDAF.java | 1 + .../java/hivemall/evaluation/RecallUDAF.java | 55 +-- .../evaluation/RootMeanSquaredErrorUDAF.java | 1 + .../java/hivemall/fm/FMPredictGenericUDAF.java | 23 +- .../hivemall/ftvec/binning/BuildBinsUDAF.java | 45 ++- .../ftvec/binning/FeatureBinningUDF.java | 26 +- .../ftvec/binning/NumericHistogram.java | 28 +- .../ftvec/conv/ConvertToDenseModelUDAF.java | 1 + .../hivemall/ftvec/text/TermFrequencyUDAF.java | 1 + .../ftvec/trans/OnehotEncodingUDAF.java | 335 +++++++++++++++++++ .../smile/tools/RandomForestEnsembleUDAF.java | 1 + .../tools/array/ArrayAvgGenericUDAF.java | 27 +- .../java/hivemall/tools/array/ArraySumUDAF.java | 1 + .../hivemall/tools/bits/BitsCollectUDAF.java | 23 +- .../main/java/hivemall/tools/map/UDAFToMap.java | 23 +- .../hivemall/tools/map/UDAFToOrderedMap.java | 6 +- .../java/hivemall/utils/hadoop/HiveUtils.java | 9 + .../hivemall/utils/hadoop/WritableUtils.java | 15 + .../java/hivemall/utils/lang/Identifier.java | 38 ++- .../hive/ql/exec/MapredContextAccessor.java | 3 + .../ftvec/trans/TestBinarizeLabelUDTF.java | 7 +- mixserv/pom.xml | 2 +- nlp/pom.xml | 2 +- .../hivemall/nlp/tokenizer/KuromojiUDFTest.java | 31 +- pom.xml | 1 + resources/ddl/define-all-as-permanent.hive | 3 + resources/ddl/define-all.hive | 3 + resources/ddl/define-udfs.td.hql | 1 + .../org/apache/spark/sql/hive/HivemallOps.scala | 5 +- .../apache/spark/sql/hive/HiveUdfSuite.scala | 36 ++ .../spark/sql/hive/HivemallOpsSuite.scala | 47 ++- .../sql/catalyst/expressions/EachTopK.scala | 108 ++++++ .../org/apache/spark/sql/hive/HivemallOps.scala | 43 ++- .../apache/spark/sql/hive/HiveUdfSuite.scala | 43 ++- .../spark/sql/hive/HivemallOpsSuite.scala | 70 ++-- .../sql/hive/benchmark/MiscBenchmark.scala | 72 ++-- spark/spark-common/pom.xml | 2 +- xgboost/pom.xml | 2 +- 56 files changed, 1125 insertions(+), 338 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/aa7d5299/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java ---------------------------------------------------------------------- diff --cc core/src/main/java/hivemall/utils/hadoop/HiveUtils.java index 9272e60,91f1dfa..c752188 --- a/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java +++ b/core/src/main/java/hivemall/utils/hadoop/HiveUtils.java @@@ -55,9 -55,9 +55,10 @@@ import org.apache.hadoop.hive.serde2.ob import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; + import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/aa7d5299/resources/ddl/define-all-as-permanent.hive ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/aa7d5299/resources/ddl/define-all.hive ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/aa7d5299/resources/ddl/define-udfs.td.hql ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/aa7d5299/spark/spark-1.6/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/aa7d5299/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/aa7d5299/spark/spark-2.0/src/main/scala/org/apache/spark/sql/hive/HivemallOps.scala ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/aa7d5299/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala ----------------------------------------------------------------------