Github user nzw0301 commented on a diff in the pull request: https://github.com/apache/incubator-hivemall/pull/111#discussion_r138523391 --- Diff: core/src/main/java/hivemall/recommend/SlimUDTF.java --- @@ -0,0 +1,636 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package hivemall.recommend; + + +import hivemall.UDTFWithOptions; +import hivemall.annotations.VisibleForTesting; +import hivemall.common.ConversionState; +import hivemall.math.matrix.sparse.DoKMatrix; +import hivemall.math.vector.VectorProcedure; +import hivemall.utils.collections.maps.Int2FloatOpenHashTable; +import hivemall.utils.hadoop.HiveUtils; +import hivemall.utils.io.FileUtils; +import hivemall.utils.io.NioStatefullSegment; +import hivemall.utils.lang.NumberUtils; +import hivemall.utils.lang.Primitives; +import hivemall.utils.lang.SizeOf; +import hivemall.utils.lang.mutable.MutableDouble; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.objectinspector.*; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.*; +import org.apache.hadoop.io.DoubleWritable; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.mapred.Counters; +import org.apache.hadoop.mapred.Reporter; + +import javax.annotation.Nonnull; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.*; + + +@Description( + name = "train_slim", + value = "_FUNC_( int i, map<int, double> r_i, map<int, map<int, double>> topKRatesOfI, int j, map<int, double> r_j [, constant string options]) " + + "- Returns row index, column index and non-zero weight value of prediction model") +public class SlimUDTF extends UDTFWithOptions { + private static final Log logger = LogFactory.getLog(SlimUDTF.class); + + private double l1; + private double l2; + private int numIterations; + private int previousItemId; + + private transient DoKMatrix weightMatrix; // item-item weight matrix + private transient DoKMatrix dataMatrix; // item-user matrix to get the number of nnz values in column + + private PrimitiveObjectInspector itemIOI; + private PrimitiveObjectInspector itemJOI; + private MapObjectInspector riOI; + private MapObjectInspector rjOI; + + private MapObjectInspector knnItemsOI; + private PrimitiveObjectInspector knnItemsKeyOI; + private MapObjectInspector knnItemsValueOI; + private PrimitiveObjectInspector knnItemsValueKeyOI; + private PrimitiveObjectInspector knnItemsValueValueOI; + + private PrimitiveObjectInspector riKeyOI; + private PrimitiveObjectInspector riValueOI; + + private PrimitiveObjectInspector rjKeyOI; + private PrimitiveObjectInspector rjValueOI; + + // used to store KNN data into temporary file for iterative training + private NioStatefullSegment fileIO; + private ByteBuffer inputBuf; + + private ConversionState cvState; + private long observedTrainingExamples; + + public SlimUDTF() {} + + @Override + public StructObjectInspector initialize(ObjectInspector[] argOIs) throws UDFArgumentException { + final int numArgs = argOIs.length; + + if (numArgs == 1 && HiveUtils.isStringOI(argOIs[0])) { --- End diff -- I add this a few line to show the slim explanation without other arguments. But it may not be conventional way to show `help` for hivemall.
---