http://git-wip-us.apache.org/repos/asf/hama/blob/33041c09/ml/src/main/java/org/apache/hama/ml/perception/SmallMLPTrainer.java ---------------------------------------------------------------------- diff --git a/ml/src/main/java/org/apache/hama/ml/perception/SmallMLPTrainer.java b/ml/src/main/java/org/apache/hama/ml/perception/SmallMLPTrainer.java deleted file mode 100644 index 8b08136..0000000 --- a/ml/src/main/java/org/apache/hama/ml/perception/SmallMLPTrainer.java +++ /dev/null @@ -1,327 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ml.perception; - -import java.io.IOException; -import java.util.Arrays; -import java.util.BitSet; - -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hama.bsp.BSPPeer; -import org.apache.hama.bsp.sync.SyncException; -import org.apache.hama.commons.io.VectorWritable; -import org.apache.hama.commons.math.DenseDoubleMatrix; -import org.apache.hama.ml.ann.NeuralNetworkTrainer; - -/** - * The perceptron trainer for small scale MLP. - */ -class SmallMLPTrainer extends NeuralNetworkTrainer { - - /* used by master only, check whether all slaves finishes reading */ - private BitSet statusSet; - - private int numTrainingInstanceRead = 0; - /* Once reader reaches the EOF, the training procedure would be terminated */ - private boolean terminateTraining = false; - - private SmallMultiLayerPerceptron inMemoryPerceptron; - - private int[] layerSizeArray; - - @Override - protected void extraSetup( - BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer) { - - // obtain parameters - this.trainingMode = conf.get("training.mode", "minibatch.gradient.descent"); - // mini-batch by default - this.batchSize = conf.getInt("training.batch.size", 100); - - this.statusSet = new BitSet(peer.getConfiguration().getInt("tasks", 1)); - - String outputModelPath = conf.get("modelPath"); - if (outputModelPath == null || outputModelPath.trim().length() == 0) { - try { - throw new Exception("Please specify output model path."); - } catch (Exception e) { - e.printStackTrace(); - } - } - - String modelPath = conf.get("existingModelPath"); - // build model from scratch - if (modelPath == null || modelPath.trim().length() == 0) { - double learningRate = Double.parseDouble(conf.get("learningRate")); - double regularization = Double.parseDouble(conf.get("regularization")); - double momentum = Double.parseDouble(conf.get("momentum")); - String squashingFunctionName = conf.get("squashingFunctionName"); - String costFunctionName = conf.get("costFunctionName"); - String[] layerSizeArrayStr = conf.get("layerSizeArray").trim().split(" "); - this.layerSizeArray = new int[layerSizeArrayStr.length]; - for (int i = 0; i < this.layerSizeArray.length; ++i) { - this.layerSizeArray[i] = Integer.parseInt(layerSizeArrayStr[i]); - } - - this.inMemoryPerceptron = new SmallMultiLayerPerceptron(learningRate, - regularization, momentum, squashingFunctionName, costFunctionName, - layerSizeArray); - LOG.info("Training model from scratch."); - } else { // read model from existing data - this.inMemoryPerceptron = new SmallMultiLayerPerceptron(modelPath); - LOG.info("Training with existing model."); - } - - } - - @Override - protected void extraCleanup( - BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer) { - LOG.info(String.format("Task %d totally read %d records.\n", - peer.getPeerIndex(), this.numTrainingInstanceRead)); - // master write learned model to disk - if (peer.getPeerIndex() == 0) { - try { - LOG.info(String.format("Master write learned model to %s\n", - conf.get("modelPath"))); - this.inMemoryPerceptron.writeModelToFile(conf.get("modelPath")); - } catch (IOException e) { - System.err.println("Please set a correct model path."); - } - } - } - - @Override - public void bsp( - BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer) - throws IOException, SyncException, InterruptedException { - LOG.info("Start training..."); - if (trainingMode.equalsIgnoreCase("minibatch.gradient.descent")) { - LOG.info("Training Mode: minibatch.gradient.descent"); - trainByMinibatch(peer); - } - - LOG.info(String.format("Task %d finished.", peer.getPeerIndex())); - } - - /** - * Train the MLP with stochastic gradient descent. - * - * @param peer - * @throws IOException - * @throws SyncException - * @throws InterruptedException - */ - private void trainByMinibatch( - BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer) - throws IOException, SyncException, InterruptedException { - - int maxIteration = conf.getInt("training.iteration", 1); - LOG.info("# of Training Iteration: " + maxIteration); - - for (int i = 0; i < maxIteration; ++i) { - if (peer.getPeerIndex() == 0) { - LOG.info(String.format("Iteration [%d] begins...", i)); - } - peer.reopenInput(); - // reset status - if (peer.getPeerIndex() == 0) { - this.statusSet = new BitSet(peer.getConfiguration().getInt("tasks", 1)); - } - this.terminateTraining = false; - peer.sync(); - while (true) { - // each slate task updates weights according to training data - boolean terminate = updateWeights(peer); - peer.sync(); - - // master merges the updates - if (peer.getPeerIndex() == 0) { - mergeUpdate(peer); - } - peer.sync(); - - if (terminate) { - break; - } - } - - } - - } - - /** - * Merge the updates from slaves task. - * - * @param peer - * @throws IOException - */ - private void mergeUpdate( - BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer) - throws IOException { - // initialize the cache - DenseDoubleMatrix[] mergedUpdates = this.getZeroWeightMatrices(); - - int numOfPartitions = peer.getNumCurrentMessages(); - - // aggregates the weights update - while (peer.getNumCurrentMessages() > 0) { - SmallMLPMessage message = (SmallMLPMessage) peer.getCurrentMessage(); - if (message.isTerminated()) { - this.statusSet.set(message.getOwner()); - } - - DenseDoubleMatrix[] weightUpdates = message.getWeightUpdatedMatrices(); - for (int m = 0; m < mergedUpdates.length; ++m) { - mergedUpdates[m] = (DenseDoubleMatrix) mergedUpdates[m] - .add(weightUpdates[m]); - } - } - - if (numOfPartitions != 0) { - // calculate the global mean (the mean of batches from all slave tasks) of - // the weight updates - for (int m = 0; m < mergedUpdates.length; ++m) { - mergedUpdates[m] = (DenseDoubleMatrix) mergedUpdates[m] - .divide(numOfPartitions); - } - - // check if all tasks finishes reading data - if (this.statusSet.cardinality() == conf.getInt("tasks", 1)) { - this.terminateTraining = true; - } - - // update the weight matrices - this.inMemoryPerceptron.updateWeightMatrices(mergedUpdates); - this.inMemoryPerceptron.setPrevWeightUpdateMatrices(mergedUpdates); - } - - // broadcast updated weight matrices - for (String peerName : peer.getAllPeerNames()) { - SmallMLPMessage msg = new SmallMLPMessage(peer.getPeerIndex(), - this.terminateTraining, this.inMemoryPerceptron.getWeightMatrices(), - this.inMemoryPerceptron.getPrevWeightUpdateMatrices()); - peer.send(peerName, msg); - } - - } - - /** - * Train the MLP with training data. - * - * @param peer - * @return Whether terminates. - * @throws IOException - */ - private boolean updateWeights( - BSPPeer<LongWritable, VectorWritable, NullWritable, NullWritable, MLPMessage> peer) - throws IOException { - // receive update message sent by master - if (peer.getNumCurrentMessages() > 0) { - SmallMLPMessage message = (SmallMLPMessage) peer.getCurrentMessage(); - this.terminateTraining = message.isTerminated(); - // each slave renew its weight matrices - this.inMemoryPerceptron.setWeightMatrices(message - .getWeightUpdatedMatrices()); - this.inMemoryPerceptron.setPrevWeightUpdateMatrices(message - .getPrevWeightsUpdatedMatrices()); - if (this.terminateTraining) { - return true; - } - } - - // update weight according to training data - DenseDoubleMatrix[] weightUpdates = this.getZeroWeightMatrices(); - - int count = 0; - LongWritable recordId = new LongWritable(); - VectorWritable trainingInstance = new VectorWritable(); - boolean hasMore = false; - while (count++ < this.batchSize) { - hasMore = peer.readNext(recordId, trainingInstance); - - try { - DenseDoubleMatrix[] singleTrainingInstanceUpdates = this.inMemoryPerceptron - .trainByInstance(trainingInstance.getVector()); - // aggregate the updates - for (int m = 0; m < weightUpdates.length; ++m) { - weightUpdates[m] = (DenseDoubleMatrix) weightUpdates[m] - .add(singleTrainingInstanceUpdates[m]); - } - } catch (Exception e) { - e.printStackTrace(); - } - - ++numTrainingInstanceRead; - if (!hasMore) { - break; - } - } - - // calculate the local mean (the mean of the local batch) of weight updates - for (int m = 0; m < weightUpdates.length; ++m) { - weightUpdates[m] = (DenseDoubleMatrix) weightUpdates[m].divide(count); - } - - LOG.info(String.format("Task %d has read %d records.", peer.getPeerIndex(), - this.numTrainingInstanceRead)); - - // send the weight updates to master task - SmallMLPMessage message = new SmallMLPMessage(peer.getPeerIndex(), - !hasMore, weightUpdates); - peer.send(peer.getPeerName(0), message); // send status to master - - return !hasMore; - } - - /** - * Initialize the weight matrices. - */ - private DenseDoubleMatrix[] getZeroWeightMatrices() { - DenseDoubleMatrix[] weightUpdateCache = new DenseDoubleMatrix[this.layerSizeArray.length - 1]; - // initialize weight matrix each layer - for (int i = 0; i < weightUpdateCache.length; ++i) { - weightUpdateCache[i] = new DenseDoubleMatrix(this.layerSizeArray[i] + 1, - this.layerSizeArray[i + 1]); - } - return weightUpdateCache; - } - - /** - * Print out the weights. - * - * @param mat - * @return - */ - protected static String weightsToString(DenseDoubleMatrix[] mat) { - StringBuilder sb = new StringBuilder(); - - for (int i = 0; i < mat.length; ++i) { - sb.append(String.format("Matrix [%d]\n", i)); - double[][] values = mat[i].getValues(); - for (double[] value : values) { - sb.append(Arrays.toString(value)); - sb.append('\n'); - } - sb.append('\n'); - } - return sb.toString(); - } - -}
http://git-wip-us.apache.org/repos/asf/hama/blob/33041c09/ml/src/main/java/org/apache/hama/ml/perception/SmallMultiLayerPerceptron.java ---------------------------------------------------------------------- diff --git a/ml/src/main/java/org/apache/hama/ml/perception/SmallMultiLayerPerceptron.java b/ml/src/main/java/org/apache/hama/ml/perception/SmallMultiLayerPerceptron.java deleted file mode 100644 index 1b6d200..0000000 --- a/ml/src/main/java/org/apache/hama/ml/perception/SmallMultiLayerPerceptron.java +++ /dev/null @@ -1,574 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ml.perception; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.apache.commons.lang.SerializationUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.io.WritableUtils; -import org.apache.hama.HamaConfiguration; -import org.apache.hama.bsp.BSPJob; -import org.apache.hama.commons.io.MatrixWritable; -import org.apache.hama.commons.io.VectorWritable; -import org.apache.hama.commons.math.DenseDoubleMatrix; -import org.apache.hama.commons.math.DenseDoubleVector; -import org.apache.hama.commons.math.DoubleFunction; -import org.apache.hama.commons.math.DoubleVector; -import org.apache.hama.commons.math.FunctionFactory; -import org.apache.hama.ml.util.FeatureTransformer; -import org.mortbay.log.Log; - -/** - * SmallMultiLayerPerceptronBSP is a kind of multilayer perceptron whose - * parameters can be fit into the memory of a single machine. This kind of model - * can be trained and used more efficiently than the BigMultiLayerPerceptronBSP, - * whose parameters are distributedly stored in multiple machines. - * - * In general, it it is a multilayer perceptron that consists of one input - * layer, multiple hidden layer and one output layer. - * - * The number of neurons in the input layer should be consistent with the number - * of features in the training instance. The number of neurons in the output - * layer - */ -public final class SmallMultiLayerPerceptron extends MultiLayerPerceptron - implements Writable { - - /* The in-memory weight matrix */ - private DenseDoubleMatrix[] weightMatrice; - - /* Previous weight updates, used for momentum */ - private DenseDoubleMatrix[] prevWeightUpdateMatrices; - - /** - * @see MultiLayerPerceptron#MultiLayerPerceptron(double, double, double, String, String, int[]) - */ - public SmallMultiLayerPerceptron(double learningRate, double regularization, - double momentum, String squashingFunctionName, String costFunctionName, - int[] layerSizeArray) { - super(learningRate, regularization, momentum, squashingFunctionName, - costFunctionName, layerSizeArray); - initializeWeightMatrix(); - this.initializePrevWeightUpdateMatrix(); - } - - /** - * @see MultiLayerPerceptron#MultiLayerPerceptron(String) - */ - public SmallMultiLayerPerceptron(String modelPath) { - super(modelPath); - if (modelPath != null) { - try { - this.readFromModel(); - this.initializePrevWeightUpdateMatrix(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - /** - * Initialize weight matrix using Gaussian distribution. Each weight is - * initialized in range (-0.5, 0.5) - */ - private void initializeWeightMatrix() { - this.weightMatrice = new DenseDoubleMatrix[this.numberOfLayers - 1]; - // each layer contains one bias neuron - for (int i = 0; i < this.numberOfLayers - 1; ++i) { - // add weights for bias - this.weightMatrice[i] = new DenseDoubleMatrix(this.layerSizeArray[i] + 1, - this.layerSizeArray[i + 1]); - - this.weightMatrice[i].applyToElements(new DoubleFunction() { - - private final Random rnd = new Random(); - - @Override - public double apply(double value) { - return rnd.nextDouble() - 0.5; - } - - @Override - public double applyDerivative(double value) { - throw new UnsupportedOperationException("Not supported"); - } - - }); - - // int rowCount = this.weightMatrice[i].getRowCount(); - // int colCount = this.weightMatrice[i].getColumnCount(); - // for (int row = 0; row < rowCount; ++row) { - // for (int col = 0; col < colCount; ++col) { - // this.weightMatrice[i].set(row, col, rnd.nextDouble() - 0.5); - // } - // } - } - } - - /** - * Initial the momentum weight matrices. - */ - private void initializePrevWeightUpdateMatrix() { - this.prevWeightUpdateMatrices = new DenseDoubleMatrix[this.numberOfLayers - 1]; - for (int i = 0; i < this.prevWeightUpdateMatrices.length; ++i) { - int row = this.layerSizeArray[i] + 1; - int col = this.layerSizeArray[i + 1]; - this.prevWeightUpdateMatrices[i] = new DenseDoubleMatrix(row, col); - } - } - - @Override - /** - * {@inheritDoc} - * The model meta-data is stored in memory. - */ - public DoubleVector outputWrapper(DoubleVector featureVector) { - List<double[]> outputCache = this.outputInternal(featureVector); - // the output of the last layer is the output of the MLP - return new DenseDoubleVector(outputCache.get(outputCache.size() - 1)); - } - - private List<double[]> outputInternal(DoubleVector featureVector) { - // store the output of the hidden layers and output layer, each array store - // one layer - List<double[]> outputCache = new ArrayList<double[]>(); - - // start from the first hidden layer - double[] intermediateResults = new double[this.layerSizeArray[0] + 1]; - if (intermediateResults.length - 1 != featureVector.getDimension()) { - throw new IllegalStateException( - "Input feature dimension incorrect! The dimension of input layer is " - + (this.layerSizeArray[0] - 1) - + ", but the dimension of input feature is " - + featureVector.getDimension()); - } - - // fill with input features - intermediateResults[0] = 1.0; // bias - - // transform the original features to another space - featureVector = this.featureTransformer.transform(featureVector); - - for (int i = 0; i < featureVector.getDimension(); ++i) { - intermediateResults[i + 1] = featureVector.get(i); - } - outputCache.add(intermediateResults); - - // forward the intermediate results to next layer - for (int fromLayer = 0; fromLayer < this.numberOfLayers - 1; ++fromLayer) { - intermediateResults = forward(fromLayer, intermediateResults); - outputCache.add(intermediateResults); - } - - return outputCache; - } - - /** - * Calculate the intermediate results of layer fromLayer + 1. - * - * @param fromLayer The index of layer that forwards the intermediate results - * from. - * @return the value of intermediate results of layer. - */ - private double[] forward(int fromLayer, double[] intermediateResult) { - int toLayer = fromLayer + 1; - double[] results = null; - int offset = 0; - - if (toLayer < this.layerSizeArray.length - 1) { // add bias if it is not - // output layer - results = new double[this.layerSizeArray[toLayer] + 1]; - offset = 1; - results[0] = 1.0; // the bias - } else { - results = new double[this.layerSizeArray[toLayer]]; // no bias - } - - for (int neuronIdx = 0; neuronIdx < this.layerSizeArray[toLayer]; ++neuronIdx) { - // aggregate the results from previous layer - for (int prevNeuronIdx = 0; prevNeuronIdx < this.layerSizeArray[fromLayer] + 1; ++prevNeuronIdx) { - results[neuronIdx + offset] += this.weightMatrice[fromLayer].get( - prevNeuronIdx, neuronIdx) * intermediateResult[prevNeuronIdx]; - } - // calculate via squashing function - results[neuronIdx + offset] = this.squashingFunction - .apply(results[neuronIdx + offset]); - } - - return results; - } - - /** - * Get the updated weights using one training instance. - * - * @param trainingInstance The trainingInstance is the concatenation of - * feature vector and class label vector. - * @return The update of each weight. - * @throws Exception - */ - DenseDoubleMatrix[] trainByInstance(DoubleVector trainingInstance) - throws Exception { - // initialize weight update matrices - DenseDoubleMatrix[] weightUpdateMatrices = new DenseDoubleMatrix[this.layerSizeArray.length - 1]; - for (int m = 0; m < weightUpdateMatrices.length; ++m) { - weightUpdateMatrices[m] = new DenseDoubleMatrix( - this.layerSizeArray[m] + 1, this.layerSizeArray[m + 1]); - } - - if (trainingInstance == null) { - return weightUpdateMatrices; - } - - // transform the features (exclude the labels) to new space - double[] trainingVec = trainingInstance.toArray(); - double[] trainingFeature = this.featureTransformer.transform( - trainingInstance.sliceUnsafe(0, this.layerSizeArray[0] - 1)).toArray(); - double[] trainingLabels = Arrays.copyOfRange(trainingVec, - this.layerSizeArray[0], trainingVec.length); - - DoubleVector trainingFeatureVec = new DenseDoubleVector(trainingFeature); - List<double[]> outputCache = this.outputInternal(trainingFeatureVec); - - // calculate the delta of output layer - double[] delta = new double[this.layerSizeArray[this.layerSizeArray.length - 1]]; - double[] outputLayerOutput = outputCache.get(outputCache.size() - 1); - double[] lastHiddenLayerOutput = outputCache.get(outputCache.size() - 2); - - DenseDoubleMatrix prevWeightUpdateMatrix = this.prevWeightUpdateMatrices[this.prevWeightUpdateMatrices.length - 1]; - for (int j = 0; j < delta.length; ++j) { - delta[j] = this.costFunction.applyDerivative(trainingLabels[j], - outputLayerOutput[j]); - // add regularization term - if (this.regularization != 0.0) { - double derivativeRegularization = 0.0; - DenseDoubleMatrix weightMatrix = this.weightMatrice[this.weightMatrice.length - 1]; - for (int k = 0; k < this.layerSizeArray[this.layerSizeArray.length - 1]; ++k) { - derivativeRegularization += weightMatrix.get(k, j); - } - derivativeRegularization /= this.layerSizeArray[this.layerSizeArray.length - 1]; - delta[j] += this.regularization * derivativeRegularization; - } - - delta[j] *= this.squashingFunction.applyDerivative(outputLayerOutput[j]); - - // calculate the weight update matrix between the last hidden layer and - // the output layer - for (int i = 0; i < this.layerSizeArray[this.layerSizeArray.length - 2] + 1; ++i) { - double updatedValue = -this.learningRate * delta[j] - * lastHiddenLayerOutput[i]; - // add momentum - updatedValue += this.momentum * prevWeightUpdateMatrix.get(i, j); - weightUpdateMatrices[weightUpdateMatrices.length - 1].set(i, j, - updatedValue); - } - } - - // calculate the delta for each hidden layer through back-propagation - for (int l = this.layerSizeArray.length - 2; l >= 1; --l) { - delta = backpropagate(l, delta, outputCache, weightUpdateMatrices); - } - - return weightUpdateMatrices; - } - - /** - * Back-propagate the errors from nextLayer to prevLayer. The weight updated - * information will be stored in the weightUpdateMatrices, and the delta of - * the prevLayer would be returned. - * - * @param curLayerIdx The layer index of the current layer. - * @param nextLayerDelta The delta of the next layer. - * @param outputCache The cache of the output of all the layers. - * @param weightUpdateMatrices The weight update matrices. - * @return The delta of the previous layer, will be used for next iteration of - * back-propagation. - */ - private double[] backpropagate(int curLayerIdx, double[] nextLayerDelta, - List<double[]> outputCache, DenseDoubleMatrix[] weightUpdateMatrices) { - int prevLayerIdx = curLayerIdx - 1; - double[] delta = new double[this.layerSizeArray[curLayerIdx]]; - double[] curLayerOutput = outputCache.get(curLayerIdx); - double[] prevLayerOutput = outputCache.get(prevLayerIdx); - - // DenseDoubleMatrix prevWeightUpdateMatrix = this.prevWeightUpdateMatrices[curLayerIdx - 1]; - // for each neuron j in nextLayer, calculate the delta - for (int j = 0; j < delta.length; ++j) { - // aggregate delta from next layer - for (int k = 0; k < nextLayerDelta.length; ++k) { - double weight = this.weightMatrice[curLayerIdx].get(j, k); - delta[j] += weight * nextLayerDelta[k]; - } - delta[j] *= this.squashingFunction.applyDerivative(curLayerOutput[j + 1]); - - // calculate the weight update matrix between the previous layer and the - // current layer - for (int i = 0; i < weightUpdateMatrices[prevLayerIdx].getRowCount(); ++i) { - double updatedValue = -this.learningRate * delta[j] - * prevLayerOutput[i]; - // add momemtum - // updatedValue += this.momentum * prevWeightUpdateMatrix.get(i, j); - weightUpdateMatrices[prevLayerIdx].set(i, j, updatedValue); - } - } - - return delta; - } - - @Override - /** - * {@inheritDoc} - */ - public void train(Path dataInputPath, Map<String, String> trainingParams) - throws IOException, InterruptedException, ClassNotFoundException { - // create the BSP training job - Configuration conf = new Configuration(); - for (Map.Entry<String, String> entry : trainingParams.entrySet()) { - conf.set(entry.getKey(), entry.getValue()); - } - - // put model related parameters - if (modelPath == null || modelPath.trim().length() == 0) { // build model - // from scratch - conf.set("MLPType", this.MLPType); - conf.set("learningRate", "" + this.learningRate); - conf.set("regularization", "" + this.regularization); - conf.set("momentum", "" + this.momentum); - conf.set("squashingFunctionName", this.squashingFunctionName); - conf.set("costFunctionName", this.costFunctionName); - StringBuilder layerSizeArraySb = new StringBuilder(); - for (int layerSize : this.layerSizeArray) { - layerSizeArraySb.append(layerSize); - layerSizeArraySb.append(' '); - } - conf.set("layerSizeArray", layerSizeArraySb.toString()); - } - - HamaConfiguration hamaConf = new HamaConfiguration(conf); - - BSPJob job = new BSPJob(hamaConf, SmallMLPTrainer.class); - job.setJobName("Small scale MLP training"); - job.setJarByClass(SmallMLPTrainer.class); - job.setBspClass(SmallMLPTrainer.class); - job.setInputPath(dataInputPath); - job.setInputFormat(org.apache.hama.bsp.SequenceFileInputFormat.class); - job.setInputKeyClass(LongWritable.class); - job.setInputValueClass(VectorWritable.class); - job.setOutputKeyClass(NullWritable.class); - job.setOutputValueClass(NullWritable.class); - job.setOutputFormat(org.apache.hama.bsp.NullOutputFormat.class); - - int numTasks = conf.getInt("tasks", 1); - job.setNumBspTask(numTasks); - job.waitForCompletion(true); - - // reload learned model - Log.info(String.format("Reload model from %s.", - trainingParams.get("modelPath"))); - this.modelPath = trainingParams.get("modelPath"); - this.readFromModel(); - } - - @SuppressWarnings("rawtypes") - @Override - public void readFields(DataInput input) throws IOException { - this.MLPType = WritableUtils.readString(input); - this.learningRate = input.readDouble(); - this.regularization = input.readDouble(); - this.momentum = input.readDouble(); - this.numberOfLayers = input.readInt(); - this.squashingFunctionName = WritableUtils.readString(input); - this.costFunctionName = WritableUtils.readString(input); - - this.squashingFunction = FunctionFactory - .createDoubleFunction(this.squashingFunctionName); - this.costFunction = FunctionFactory - .createDoubleDoubleFunction(this.costFunctionName); - - // read the number of neurons for each layer - this.layerSizeArray = new int[this.numberOfLayers]; - for (int i = 0; i < numberOfLayers; ++i) { - this.layerSizeArray[i] = input.readInt(); - } - this.weightMatrice = new DenseDoubleMatrix[this.numberOfLayers - 1]; - for (int i = 0; i < numberOfLayers - 1; ++i) { - this.weightMatrice[i] = (DenseDoubleMatrix) MatrixWritable.read(input); - } - - // read feature transformer - int bytesLen = input.readInt(); - byte[] featureTransformerBytes = new byte[bytesLen]; - for (int i = 0; i < featureTransformerBytes.length; ++i) { - featureTransformerBytes[i] = input.readByte(); - } - Class featureTransformerCls = (Class) SerializationUtils - .deserialize(featureTransformerBytes); - Constructor constructor = featureTransformerCls.getConstructors()[0]; - try { - this.featureTransformer = (FeatureTransformer) constructor - .newInstance(new Object[] {}); - } catch (InstantiationException e) { - e.printStackTrace(); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - } catch (InvocationTargetException e) { - e.printStackTrace(); - } - } - - @Override - public void write(DataOutput output) throws IOException { - WritableUtils.writeString(output, MLPType); - output.writeDouble(learningRate); - output.writeDouble(regularization); - output.writeDouble(momentum); - output.writeInt(numberOfLayers); - WritableUtils.writeString(output, squashingFunctionName); - WritableUtils.writeString(output, costFunctionName); - - // write the number of neurons for each layer - for (int i = 0; i < this.numberOfLayers; ++i) { - output.writeInt(this.layerSizeArray[i]); - } - for (int i = 0; i < numberOfLayers - 1; ++i) { - MatrixWritable matrixWritable = new MatrixWritable(this.weightMatrice[i]); - matrixWritable.write(output); - } - - // serialize the feature transformer - Class<? extends FeatureTransformer> featureTransformerCls = this.featureTransformer - .getClass(); - byte[] featureTransformerBytes = SerializationUtils - .serialize(featureTransformerCls); - output.writeInt(featureTransformerBytes.length); - output.write(featureTransformerBytes); - } - - /** - * Read the model meta-data from the specified location. - * - * @throws IOException - */ - @Override - protected void readFromModel() throws IOException { - Configuration conf = new Configuration(); - try { - URI uri = new URI(modelPath); - FileSystem fs = FileSystem.get(uri, conf); - FSDataInputStream is = new FSDataInputStream(fs.open(new Path(modelPath))); - this.readFields(is); - if (!this.MLPType.equals(this.getClass().getName())) { - throw new IllegalStateException(String.format( - "Model type incorrect, cannot load model '%s' for '%s'.", - this.MLPType, this.getClass().getName())); - } - } catch (URISyntaxException e) { - e.printStackTrace(); - } - } - - /** - * Write the model to file. - * - * @throws IOException - */ - @Override - public void writeModelToFile(String modelPath) throws IOException { - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(conf); - FSDataOutputStream stream = fs.create(new Path(modelPath), true); - this.write(stream); - stream.close(); - } - - DenseDoubleMatrix[] getWeightMatrices() { - return this.weightMatrice; - } - - DenseDoubleMatrix[] getPrevWeightUpdateMatrices() { - return this.prevWeightUpdateMatrices; - } - - void setWeightMatrices(DenseDoubleMatrix[] newMatrices) { - this.weightMatrice = newMatrices; - } - - void setPrevWeightUpdateMatrices( - DenseDoubleMatrix[] newPrevWeightUpdateMatrices) { - this.prevWeightUpdateMatrices = newPrevWeightUpdateMatrices; - } - - /** - * Update the weight matrices with given updates. - * - * @param updateMatrices The updates weights in matrix format. - */ - void updateWeightMatrices(DenseDoubleMatrix[] updateMatrices) { - for (int m = 0; m < this.weightMatrice.length; ++m) { - this.weightMatrice[m] = (DenseDoubleMatrix) this.weightMatrice[m] - .add(updateMatrices[m]); - } - } - - /** - * Print out the weights. - * - * @param mat - * @return the weights value. - */ - static String weightsToString(DenseDoubleMatrix[] mat) { - StringBuilder sb = new StringBuilder(); - - for (int i = 0; i < mat.length; ++i) { - sb.append(String.format("Matrix [%d]\n", i)); - double[][] values = mat[i].getValues(); - for (double[] value : values) { - sb.append(Arrays.toString(value)); - sb.append('\n'); - } - sb.append('\n'); - } - return sb.toString(); - } - - @Override - protected String getTypeName() { - return this.getClass().getName(); - } - -} http://git-wip-us.apache.org/repos/asf/hama/blob/33041c09/ml/src/main/java/org/apache/hama/ml/regression/LinearRegression.java ---------------------------------------------------------------------- diff --git a/ml/src/main/java/org/apache/hama/ml/regression/LinearRegression.java b/ml/src/main/java/org/apache/hama/ml/regression/LinearRegression.java deleted file mode 100644 index 50e3b08..0000000 --- a/ml/src/main/java/org/apache/hama/ml/regression/LinearRegression.java +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hama.ml.regression; - -import java.io.IOException; -import java.util.Map; - -import org.apache.hadoop.fs.Path; -import org.apache.hama.commons.math.DoubleMatrix; -import org.apache.hama.commons.math.DoubleVector; -import org.apache.hama.commons.math.FunctionFactory; -import org.apache.hama.ml.ann.SmallLayeredNeuralNetwork; -import org.apache.hama.ml.util.FeatureTransformer; - -/** - * Linear regression model. It can be used for numeric regression or prediction. - * - */ -public class LinearRegression { - - /* Internal model */ - private final SmallLayeredNeuralNetwork ann; - - public LinearRegression(int dimension) { - ann = new SmallLayeredNeuralNetwork(); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("IdentityFunction")); - ann.addLayer(1, true, - FunctionFactory.createDoubleFunction("IdentityFunction")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("SquaredError")); - } - - public LinearRegression(String modelPath) { - ann = new SmallLayeredNeuralNetwork(modelPath); - } - - /** - * Set the learning rate, recommend in range (0, 0.01]. Note that linear - * regression are easy to get diverge if the learning rate is not small - * enough. - * - * @param learningRate - */ - public LinearRegression setLearningRate(double learningRate) { - ann.setLearningRate(learningRate); - return this; - } - - /** - * Get the learning rate. - */ - public double getLearningRate() { - return ann.getLearningRate(); - } - - /** - * Set the weight of the momemtum. Recommend in range [0, 1.0]. Too large - * momemtum weight may make model hard to converge. - * - * @param momemtumWeight - */ - public LinearRegression setMomemtumWeight(double momemtumWeight) { - ann.setMomemtumWeight(momemtumWeight); - return this; - } - - /** - * Get the weight of momemtum. - * - * @return the monemtum weight value. - */ - public double getMomemtumWeight() { - return ann.getMomemtumWeight(); - } - - /** - * Set the weight of regularization, recommend in range [0, 0.1]. Too large - * regularization will mislead the model. - * - * @param regularizationWeight - */ - public LinearRegression setRegularizationWeight(double regularizationWeight) { - ann.setRegularizationWeight(regularizationWeight); - return this; - } - - /** - * Get the weight of regularization. - * - * @return the regularizatioin weight value. - */ - public double getRegularizationWeight() { - return ann.getRegularizationWeight(); - } - - /** - * Train the linear regression model with one instance. It is HIGHLY - * RECOMMENDED to normalize the data first. - * - * @param trainingInstance - */ - public void trainOnline(DoubleVector trainingInstance) { - // ann.trainOnline(trainingInstance); - DoubleMatrix[] updates = ann.trainByInstance(trainingInstance); - // System.out.printf("%s\n", updates[0]); - ann.updateWeightMatrices(updates); - } - - /** - * Train the model with given data. It is HIGHLY RECOMMENDED to normalize the - * data first. - * - * @param dataInputPath The file path that contains the training instance. - * @param trainingParams The training parameters. - * @throws IOException - * @throws InterruptedException - * @throws ClassNotFoundException - */ - public void train(Path dataInputPath, Map<String, String> trainingParams) { - ann.train(dataInputPath, trainingParams); - } - - /** - * Get the output according to given input instance. - * - * @param instance - * @return a new vector with the result of the operation. - */ - public DoubleVector getOutput(DoubleVector instance) { - return ann.getOutput(instance); - } - - /** - * Set the path to store the model. Note this is just set the path, it does - * not save the model. You should call writeModelToFile to save the model. - * - * @param modelPath - */ - public void setModelPath(String modelPath) { - ann.setModelPath(modelPath); - } - - /** - * Save the model to specified model path. - */ - public void writeModelToFile() { - try { - ann.writeModelToFile(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * Get the weights of the model. - * - * @return a new vector with the weights of the model. - */ - public DoubleVector getWeights() { - return ann.getWeightsByLayer(0).getRowVector(0); - } - - /** - * Set the feature transformer. - * @param featureTransformer - */ - public void setFeatureTransformer(FeatureTransformer featureTransformer) { - this.ann.setFeatureTransformer(featureTransformer); - } - -} http://git-wip-us.apache.org/repos/asf/hama/blob/33041c09/ml/src/main/java/org/apache/hama/ml/regression/LogisticRegression.java ---------------------------------------------------------------------- diff --git a/ml/src/main/java/org/apache/hama/ml/regression/LogisticRegression.java b/ml/src/main/java/org/apache/hama/ml/regression/LogisticRegression.java deleted file mode 100644 index dd990c7..0000000 --- a/ml/src/main/java/org/apache/hama/ml/regression/LogisticRegression.java +++ /dev/null @@ -1,180 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hama.ml.regression; - -import org.apache.hadoop.fs.Path; -import org.apache.hama.commons.math.DoubleVector; -import org.apache.hama.commons.math.FunctionFactory; -import org.apache.hama.ml.ann.SmallLayeredNeuralNetwork; -import org.apache.hama.ml.util.FeatureTransformer; - -import java.io.IOException; -import java.util.Map; - -/** - * The logistic regression model. It can be used to conduct 2-class - * classification. - * - */ -public class LogisticRegression { - - private final SmallLayeredNeuralNetwork ann; - - public LogisticRegression(int dimension) { - this.ann = new SmallLayeredNeuralNetwork(); - this.ann.addLayer(dimension, false, FunctionFactory.createDoubleFunction("Sigmoid")); - this.ann.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - this.ann.setCostFunction(FunctionFactory.createDoubleDoubleFunction("CrossEntropy")); - } - - public LogisticRegression(String modelPath) { - this.ann = new SmallLayeredNeuralNetwork(modelPath); - } - - /** - * Set the learning rate, recommend in range (0, 0.01]. Note that linear - * regression are easy to get diverge if the learning rate is not small - * enough. - * - * @param learningRate - */ - public LogisticRegression setLearningRate(double learningRate) { - ann.setLearningRate(learningRate); - return this; - } - - /** - * Get the learning rate. - */ - public double getLearningRate() { - return ann.getLearningRate(); - } - - /** - * Set the weight of the momemtum. Recommend in range [0, 1.0]. Too large - * momemtum weight may make model hard to converge. - * - * @param momemtumWeight - */ - public LogisticRegression setMomemtumWeight(double momemtumWeight) { - ann.setMomemtumWeight(momemtumWeight); - return this; - } - - /** - * Get the weight of momemtum. - * - * @return the monemtum weight value. - */ - public double getMomemtumWeight() { - return ann.getMomemtumWeight(); - } - - /** - * Set the weight of regularization, recommend in range [0, 0.1]. Too large - * regularization will mislead the model. - * - * @param regularizationWeight - */ - public LogisticRegression setRegularizationWeight(double regularizationWeight) { - ann.setRegularizationWeight(regularizationWeight); - return this; - } - - /** - * Get the weight of regularization. - * - * @return the regularizatioin weight value. - */ - public double getRegularizationWeight() { - return ann.getRegularizationWeight(); - } - - /** - * Train the linear regression model with one instance. It is HIGHLY - * RECOMMENDED to normalize the data first. - * - * @param trainingInstance - */ - public void trainOnline(DoubleVector trainingInstance) { - ann.trainOnline(trainingInstance); - } - - /** - * Train the model with given data. It is HIGHLY RECOMMENDED to normalize the - * data first. - * - * @param dataInputPath The file path that contains the training instance. - * @param trainingParams The training parameters. - * @throws IOException - * @throws InterruptedException - * @throws ClassNotFoundException - */ - public void train(Path dataInputPath, Map<String, String> trainingParams) { - ann.train(dataInputPath, trainingParams); - } - - /** - * Get the output according to given input instance. - * - * @param instance - * @return a new vector with the result of the operation. - */ - public DoubleVector getOutput(DoubleVector instance) { - return ann.getOutput(instance); - } - - /** - * Set the path to store the model. Note this is just set the path, it does - * not save the model. You should call writeModelToFile to save the model. - * - * @param modelPath - */ - public void setModelPath(String modelPath) { - ann.setModelPath(modelPath); - } - - /** - * Save the model to specified model path. - */ - public void writeModelToFile() { - try { - ann.writeModelToFile(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * Get the weights of the model. - * - * @return a new vector with the weights of the model. - */ - public DoubleVector getWeights() { - return ann.getWeightsByLayer(0).getRowVector(0); - } - - /** - * Set the feature transformer. - * @param featureTransformer - */ - public void setFeatureTransformer(FeatureTransformer featureTransformer) { - this.ann.setFeatureTransformer(featureTransformer); - } -} http://git-wip-us.apache.org/repos/asf/hama/blob/33041c09/ml/src/test/java/org/apache/hama/ml/ann/TestAutoEncoder.java ---------------------------------------------------------------------- diff --git a/ml/src/test/java/org/apache/hama/ml/ann/TestAutoEncoder.java b/ml/src/test/java/org/apache/hama/ml/ann/TestAutoEncoder.java deleted file mode 100644 index 0077cb0..0000000 --- a/ml/src/test/java/org/apache/hama/ml/ann/TestAutoEncoder.java +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ml.ann; - -import static org.junit.Assert.assertEquals; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hama.commons.io.VectorWritable; -import org.apache.hama.commons.math.DenseDoubleVector; -import org.apache.hama.commons.math.DoubleVector; -import org.apache.hama.ml.MLTestBase; -import org.junit.Test; -import org.mortbay.log.Log; - -/** - * Test the functionality of {@link AutoEncoder}. - * - */ -public class TestAutoEncoder extends MLTestBase { - - @Test - public void testAutoEncoderSimple() { - double[][] instances = { { 0, 0, 0, 1 }, { 0, 0, 1, 0 }, { 0, 1, 0, 0 }, - { 0, 0, 0, 0 } }; - AutoEncoder encoder = new AutoEncoder(4, 2); - encoder.setLearningRate(0.5); - encoder.setMomemtumWeight(0.2); - - int maxIteration = 2000; - Random rnd = new Random(); - for (int iteration = 0; iteration < maxIteration; ++iteration) { - for (int i = 0; i < instances.length; ++i) { - encoder.trainOnline(new DenseDoubleVector(instances[rnd.nextInt(instances.length)])); - } - } - - for (int i = 0; i < instances.length; ++i) { - DoubleVector encodeVec = encoder.encode(new DenseDoubleVector( - instances[i])); - DoubleVector decodeVec = encoder.decode(encodeVec); - for (int d = 0; d < instances[i].length; ++d) { - assertEquals(instances[i][d], decodeVec.get(d), 0.1); - } - } - - } - - @Test - public void testAutoEncoderSwissRollDataset() { - List<double[]> instanceList = new ArrayList<double[]>(); - try { - BufferedReader br = new BufferedReader(new FileReader("src/test/resources/dimensional_reduction.txt")); - String line = null; - while ((line = br.readLine()) != null) { - String[] tokens = line.split("\t"); - double[] instance = new double[tokens.length]; - for (int i = 0; i < instance.length; ++i) { - instance[i] = Double.parseDouble(tokens[i]); - } - instanceList.add(instance); - } - br.close(); - // normalize instances - zeroOneNormalization(instanceList, instanceList.get(0).length); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (NumberFormatException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - List<DoubleVector> vecInstanceList = new ArrayList<DoubleVector>(); - for (double[] instance : instanceList) { - vecInstanceList.add(new DenseDoubleVector(instance)); - } - AutoEncoder encoder = new AutoEncoder(3, 2); - encoder.setLearningRate(0.05); - encoder.setMomemtumWeight(0.1); - int maxIteration = 2000; - for (int iteration = 0; iteration < maxIteration; ++iteration) { - for (DoubleVector vector : vecInstanceList) { - encoder.trainOnline(vector); - } - } - - double errorInstance = 0; - for (DoubleVector vector : vecInstanceList) { - DoubleVector decoded = encoder.getOutput(vector); - DoubleVector diff = vector.subtract(decoded); - double error = diff.dot(diff); - if (error > 0.1) { - ++errorInstance; - } - } - Log.info(String.format("Autoecoder error rate: %f%%\n", errorInstance * 100 / vecInstanceList.size())); - - } - - @Test - public void testAutoEncoderSwissRollDatasetDistributed() { - String strDataPath = "/tmp/dimensional_reduction.txt"; - Path path = new Path(strDataPath); - List<double[]> instanceList = new ArrayList<double[]>(); - try { - Configuration conf = new Configuration(); - FileSystem fs = FileSystem.get(new URI(strDataPath), conf); - if (fs.exists(path)) { - fs.delete(path, true); - } - - String line = null; - BufferedReader br = new BufferedReader(new FileReader("src/test/resources/dimensional_reduction.txt")); - while ((line = br.readLine()) != null) { - String[] tokens = line.split("\t"); - double[] instance = new double[tokens.length]; - for (int i = 0; i < instance.length; ++i) { - instance[i] = Double.parseDouble(tokens[i]); - } - instanceList.add(instance); - } - br.close(); - // normalize instances - zeroOneNormalization(instanceList, instanceList.get(0).length); - - SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, LongWritable.class, VectorWritable.class); - for (int i = 0; i < instanceList.size(); ++i) { - DoubleVector vector = new DenseDoubleVector(instanceList.get(i)); - writer.append(new LongWritable(i), new VectorWritable(vector)); - } - - writer.close(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - - AutoEncoder encoder = new AutoEncoder(3, 2); - String modelPath = "/tmp/autoencoder-modelpath"; - encoder.setModelPath(modelPath); - Map<String, String> trainingParams = new HashMap<String, String>(); - encoder.setLearningRate(0.5); - trainingParams.put("tasks", "5"); - trainingParams.put("training.max.iterations", "3000"); - trainingParams.put("training.batch.size", "200"); - encoder.train(path, trainingParams); - - double errorInstance = 0; - for (double[] instance : instanceList) { - DoubleVector vector = new DenseDoubleVector(instance); - DoubleVector decoded = encoder.getOutput(vector); - DoubleVector diff = vector.subtract(decoded); - double error = diff.dot(diff); - if (error > 0.1) { - ++errorInstance; - } - } - Log.info(String.format("Autoecoder error rate: %f%%\n", errorInstance * 100 / instanceList.size())); - } - -} http://git-wip-us.apache.org/repos/asf/hama/blob/33041c09/ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetwork.java ---------------------------------------------------------------------- diff --git a/ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetwork.java b/ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetwork.java deleted file mode 100644 index 8ad88af..0000000 --- a/ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetwork.java +++ /dev/null @@ -1,643 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ml.ann; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hama.commons.io.VectorWritable; -import org.apache.hama.commons.math.DenseDoubleMatrix; -import org.apache.hama.commons.math.DenseDoubleVector; -import org.apache.hama.commons.math.DoubleMatrix; -import org.apache.hama.commons.math.DoubleVector; -import org.apache.hama.commons.math.FunctionFactory; -import org.apache.hama.ml.MLTestBase; -import org.apache.hama.ml.ann.AbstractLayeredNeuralNetwork.LearningStyle; -import org.apache.hama.ml.ann.AbstractLayeredNeuralNetwork.TrainingMethod; -import org.apache.hama.ml.util.DefaultFeatureTransformer; -import org.apache.hama.ml.util.FeatureTransformer; -import org.junit.Test; -import org.mortbay.log.Log; - -/** - * Test the functionality of SmallLayeredNeuralNetwork. - * - */ -public class TestSmallLayeredNeuralNetwork extends MLTestBase { - - @Test - public void testReadWrite() { - SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); - ann.addLayer(2, false, - FunctionFactory.createDoubleFunction("IdentityFunction")); - ann.addLayer(5, false, - FunctionFactory.createDoubleFunction("IdentityFunction")); - ann.addLayer(1, true, - FunctionFactory.createDoubleFunction("IdentityFunction")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("SquaredError")); - double learningRate = 0.2; - ann.setLearningRate(learningRate); - double momentumWeight = 0.5; - ann.setMomemtumWeight(momentumWeight); - double regularizationWeight = 0.05; - ann.setRegularizationWeight(regularizationWeight); - // intentionally initialize all weights to 0.5 - DoubleMatrix[] matrices = new DenseDoubleMatrix[2]; - matrices[0] = new DenseDoubleMatrix(5, 3, 0.2); - matrices[1] = new DenseDoubleMatrix(1, 6, 0.8); - ann.setWeightMatrices(matrices); - ann.setLearningStyle(LearningStyle.UNSUPERVISED); - - FeatureTransformer defaultFeatureTransformer = new DefaultFeatureTransformer(); - ann.setFeatureTransformer(defaultFeatureTransformer); - - - // write to file - String modelPath = "/tmp/testSmallLayeredNeuralNetworkReadWrite"; - ann.setModelPath(modelPath); - try { - ann.writeModelToFile(); - } catch (IOException e) { - e.printStackTrace(); - } - - // read from file - SmallLayeredNeuralNetwork annCopy = new SmallLayeredNeuralNetwork(modelPath); - assertEquals(annCopy.getClass().getSimpleName(), annCopy.getModelType()); - assertEquals(modelPath, annCopy.getModelPath()); - assertEquals(learningRate, annCopy.getLearningRate(), 0.000001); - assertEquals(momentumWeight, annCopy.getMomemtumWeight(), 0.000001); - assertEquals(regularizationWeight, annCopy.getRegularizationWeight(), - 0.000001); - assertEquals(TrainingMethod.GRADIENT_DESCENT, annCopy.getTrainingMethod()); - assertEquals(LearningStyle.UNSUPERVISED, annCopy.getLearningStyle()); - - // compare weights - DoubleMatrix[] weightsMatrices = annCopy.getWeightMatrices(); - for (int i = 0; i < weightsMatrices.length; ++i) { - DoubleMatrix expectMat = matrices[i]; - DoubleMatrix actualMat = weightsMatrices[i]; - for (int j = 0; j < expectMat.getRowCount(); ++j) { - for (int k = 0; k < expectMat.getColumnCount(); ++k) { - assertEquals(expectMat.get(j, k), actualMat.get(j, k), 0.000001); - } - } - } - - FeatureTransformer copyTransformer = annCopy.getFeatureTransformer(); - assertEquals(defaultFeatureTransformer.getClass().getName(), copyTransformer.getClass().getName()); - } - - @Test - /** - * Test the forward functionality. - */ - public void testOutput() { - // first network - SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); - ann.addLayer(2, false, - FunctionFactory.createDoubleFunction("IdentityFunction")); - ann.addLayer(5, false, - FunctionFactory.createDoubleFunction("IdentityFunction")); - ann.addLayer(1, true, - FunctionFactory.createDoubleFunction("IdentityFunction")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("SquaredError")); - ann.setLearningRate(0.1); - // intentionally initialize all weights to 0.5 - DoubleMatrix[] matrices = new DenseDoubleMatrix[2]; - matrices[0] = new DenseDoubleMatrix(5, 3, 0.5); - matrices[1] = new DenseDoubleMatrix(1, 6, 0.5); - ann.setWeightMatrices(matrices); - - double[] arr = new double[] { 0, 1 }; - DoubleVector training = new DenseDoubleVector(arr); - DoubleVector result = ann.getOutput(training); - assertEquals(1, result.getDimension()); - // assertEquals(3, result.get(0), 0.000001); - - // second network - SmallLayeredNeuralNetwork ann2 = new SmallLayeredNeuralNetwork(); - ann2.addLayer(2, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann2.addLayer(3, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann2.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - ann2.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("SquaredError")); - ann2.setLearningRate(0.3); - // intentionally initialize all weights to 0.5 - DoubleMatrix[] matrices2 = new DenseDoubleMatrix[2]; - matrices2[0] = new DenseDoubleMatrix(3, 3, 0.5); - matrices2[1] = new DenseDoubleMatrix(1, 4, 0.5); - ann2.setWeightMatrices(matrices2); - - double[] test = { 0, 0 }; - double[] result2 = { 0.807476 }; - - DoubleVector vec = ann2.getOutput(new DenseDoubleVector(test)); - assertArrayEquals(result2, vec.toArray(), 0.000001); - - SmallLayeredNeuralNetwork ann3 = new SmallLayeredNeuralNetwork(); - ann3.addLayer(2, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann3.addLayer(3, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann3.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - ann3.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("SquaredError")); - ann3.setLearningRate(0.3); - // intentionally initialize all weights to 0.5 - DoubleMatrix[] initMatrices = new DenseDoubleMatrix[2]; - initMatrices[0] = new DenseDoubleMatrix(3, 3, 0.5); - initMatrices[1] = new DenseDoubleMatrix(1, 4, 0.5); - ann3.setWeightMatrices(initMatrices); - - double[] instance = { 0, 1 }; - DoubleVector output = ann3.getOutput(new DenseDoubleVector(instance)); - assertEquals(0.8315410, output.get(0), 0.000001); - } - - @Test - public void testXORlocal() { - SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); - ann.addLayer(2, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(3, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("SquaredError")); - ann.setLearningRate(0.5); - ann.setMomemtumWeight(0.0); - - int iterations = 50000; // iteration should be set to a very large number - double[][] instances = { { 0, 1, 1 }, { 0, 0, 0 }, { 1, 0, 1 }, { 1, 1, 0 } }; - for (int i = 0; i < iterations; ++i) { - DoubleMatrix[] matrices = null; - for (int j = 0; j < instances.length; ++j) { - matrices = ann.trainByInstance(new DenseDoubleVector(instances[j - % instances.length])); - ann.updateWeightMatrices(matrices); - } - } - - for (int i = 0; i < instances.length; ++i) { - DoubleVector input = new DenseDoubleVector(instances[i]).slice(2); - // the expected output is the last element in array - double result = instances[i][2]; - double actual = ann.getOutput(input).get(0); - if (result < 0.5 && actual >= 0.5 || result >= 0.5 && actual < 0.5) { - Log.info("Neural network failes to lear the XOR."); - } - } - - // write model into file and read out - String modelPath = "/tmp/testSmallLayeredNeuralNetworkXORLocal"; - ann.setModelPath(modelPath); - try { - ann.writeModelToFile(); - } catch (IOException e) { - e.printStackTrace(); - } - SmallLayeredNeuralNetwork annCopy = new SmallLayeredNeuralNetwork(modelPath); - // test on instances - for (int i = 0; i < instances.length; ++i) { - DoubleVector input = new DenseDoubleVector(instances[i]).slice(2); - // the expected output is the last element in array - double result = instances[i][2]; - double actual = annCopy.getOutput(input).get(0); - if (result < 0.5 && actual >= 0.5 || result >= 0.5 && actual < 0.5) { - Log.info("Neural network failes to lear the XOR."); - } - } - } - - @Test - public void testXORWithMomentum() { - SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); - ann.addLayer(2, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(3, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("SquaredError")); - ann.setLearningRate(0.6); - ann.setMomemtumWeight(0.3); - - int iterations = 2000; // iteration should be set to a very large number - double[][] instances = { { 0, 1, 1 }, { 0, 0, 0 }, { 1, 0, 1 }, { 1, 1, 0 } }; - for (int i = 0; i < iterations; ++i) { - for (int j = 0; j < instances.length; ++j) { - ann.trainOnline(new DenseDoubleVector(instances[j % instances.length])); - } - } - - for (int i = 0; i < instances.length; ++i) { - DoubleVector input = new DenseDoubleVector(instances[i]).slice(2); - // the expected output is the last element in array - double result = instances[i][2]; - double actual = ann.getOutput(input).get(0); - if (result < 0.5 && actual >= 0.5 || result >= 0.5 && actual < 0.5) { - Log.info("Neural network failes to lear the XOR."); - } - } - - // write model into file and read out - String modelPath = "/tmp/testSmallLayeredNeuralNetworkXORLocalWithMomentum"; - ann.setModelPath(modelPath); - try { - ann.writeModelToFile(); - } catch (IOException e) { - e.printStackTrace(); - } - SmallLayeredNeuralNetwork annCopy = new SmallLayeredNeuralNetwork(modelPath); - // test on instances - for (int i = 0; i < instances.length; ++i) { - DoubleVector input = new DenseDoubleVector(instances[i]).slice(2); - // the expected output is the last element in array - double result = instances[i][2]; - double actual = annCopy.getOutput(input).get(0); - if (result < 0.5 && actual >= 0.5 || result >= 0.5 && actual < 0.5) { - Log.info("Neural network failes to lear the XOR."); - } - } - } - - @Test - public void testXORLocalWithRegularization() { - SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); - ann.addLayer(2, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(3, false, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("SquaredError")); - ann.setLearningRate(0.7); - ann.setMomemtumWeight(0.5); - ann.setRegularizationWeight(0.002); - - int iterations = 5000; // iteration should be set to a very large number - double[][] instances = { { 0, 1, 1 }, { 0, 0, 0 }, { 1, 0, 1 }, { 1, 1, 0 } }; - for (int i = 0; i < iterations; ++i) { - for (int j = 0; j < instances.length; ++j) { - ann.trainOnline(new DenseDoubleVector(instances[j % instances.length])); - } - } - - for (int i = 0; i < instances.length; ++i) { - DoubleVector input = new DenseDoubleVector(instances[i]).slice(2); - // the expected output is the last element in array - double result = instances[i][2]; - double actual = ann.getOutput(input).get(0); - if (result < 0.5 && actual >= 0.5 || result >= 0.5 && actual < 0.5) { - Log.info("Neural network failes to lear the XOR."); - } - } - - // write model into file and read out - String modelPath = "/tmp/testSmallLayeredNeuralNetworkXORLocalWithRegularization"; - ann.setModelPath(modelPath); - try { - ann.writeModelToFile(); - } catch (IOException e) { - e.printStackTrace(); - } - SmallLayeredNeuralNetwork annCopy = new SmallLayeredNeuralNetwork(modelPath); - // test on instances - for (int i = 0; i < instances.length; ++i) { - DoubleVector input = new DenseDoubleVector(instances[i]).slice(2); - // the expected output is the last element in array - double result = instances[i][2]; - double actual = annCopy.getOutput(input).get(0); - if (result < 0.5 && actual >= 0.5 || result >= 0.5 && actual < 0.5) { - Log.info("Neural network failes to lear the XOR."); - } - } - } - - @Test - public void testTwoClassClassification() { - // use logistic regression data - String filepath = "src/test/resources/logistic_regression_data.txt"; - List<double[]> instanceList = new ArrayList<double[]>(); - - try { - BufferedReader br = new BufferedReader(new FileReader(filepath)); - String line = null; - while ((line = br.readLine()) != null) { - String[] tokens = line.trim().split(","); - double[] instance = new double[tokens.length]; - for (int i = 0; i < tokens.length; ++i) { - instance[i] = Double.parseDouble(tokens[i]); - } - instanceList.add(instance); - } - br.close(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } - - zeroOneNormalization(instanceList, instanceList.get(0).length - 1); - - int dimension = instanceList.get(0).length - 1; - - // divide dataset into training and testing - List<double[]> testInstances = new ArrayList<double[]>(); - testInstances.addAll(instanceList.subList(instanceList.size() - 100, - instanceList.size())); - List<double[]> trainingInstances = instanceList.subList(0, - instanceList.size() - 100); - - SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); - ann.setLearningRate(0.001); - ann.setMomemtumWeight(0.1); - ann.setRegularizationWeight(0.01); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("CrossEntropy")); - - long start = new Date().getTime(); - int iterations = 1000; - for (int i = 0; i < iterations; ++i) { - for (double[] trainingInstance : trainingInstances) { - ann.trainOnline(new DenseDoubleVector(trainingInstance)); - } - } - long end = new Date().getTime(); - Log.info(String.format("Training time: %fs\n", - (double) (end - start) / 1000)); - - double errorRate = 0; - // calculate the error on test instance - for (double[] testInstance : testInstances) { - DoubleVector instance = new DenseDoubleVector(testInstance); - double expected = instance.get(instance.getDimension() - 1); - instance = instance.slice(instance.getDimension() - 1); - double actual = ann.getOutput(instance).get(0); - if (actual < 0.5 && expected >= 0.5 || actual >= 0.5 && expected < 0.5) { - ++errorRate; - } - } - errorRate /= testInstances.size(); - - Log.info(String.format("Relative error: %f%%\n", errorRate * 100)); - } - - @Test - public void testLogisticRegression() { - this.testLogisticRegressionDistributedVersion(); - this.testLogisticRegressionDistributedVersionWithFeatureTransformer(); - } - - public void testLogisticRegressionDistributedVersion() { - // write data into a sequence file - String tmpStrDatasetPath = "/tmp/logistic_regression_data"; - Path tmpDatasetPath = new Path(tmpStrDatasetPath); - String strDataPath = "src/test/resources/logistic_regression_data.txt"; - String modelPath = "/tmp/logistic-regression-distributed-model"; - - Configuration conf = new Configuration(); - List<double[]> instanceList = new ArrayList<double[]>(); - List<double[]> trainingInstances = null; - List<double[]> testInstances = null; - - try { - FileSystem fs = FileSystem.get(new URI(tmpStrDatasetPath), conf); - fs.delete(tmpDatasetPath, true); - if (fs.exists(tmpDatasetPath)) { - fs.createNewFile(tmpDatasetPath); - } - - BufferedReader br = new BufferedReader(new FileReader(strDataPath)); - String line = null; - int count = 0; - while ((line = br.readLine()) != null) { - String[] tokens = line.trim().split(","); - double[] instance = new double[tokens.length]; - for (int i = 0; i < tokens.length; ++i) { - instance[i] = Double.parseDouble(tokens[i]); - } - instanceList.add(instance); - } - br.close(); - - zeroOneNormalization(instanceList, instanceList.get(0).length - 1); - - // write training data to temporal sequence file - SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, - tmpDatasetPath, LongWritable.class, VectorWritable.class); - int testSize = 150; - - Collections.shuffle(instanceList); - testInstances = new ArrayList<double[]>(); - testInstances.addAll(instanceList.subList(instanceList.size() - testSize, - instanceList.size())); - trainingInstances = instanceList.subList(0, instanceList.size() - - testSize); - - for (double[] instance : trainingInstances) { - DoubleVector vec = new DenseDoubleVector(instance); - writer.append(new LongWritable(count++), new VectorWritable(vec)); - } - writer.close(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - - // create model - int dimension = 8; - SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); - ann.setLearningRate(0.7); - ann.setMomemtumWeight(0.5); - ann.setRegularizationWeight(0.1); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("CrossEntropy")); - ann.setModelPath(modelPath); - - long start = new Date().getTime(); - Map<String, String> trainingParameters = new HashMap<String, String>(); - trainingParameters.put("tasks", "5"); - trainingParameters.put("training.max.iterations", "2000"); - trainingParameters.put("training.batch.size", "300"); - trainingParameters.put("convergence.check.interval", "1000"); - ann.train(tmpDatasetPath, trainingParameters); - - long end = new Date().getTime(); - - // validate results - double errorRate = 0; - // calculate the error on test instance - for (double[] testInstance : testInstances) { - DoubleVector instance = new DenseDoubleVector(testInstance); - double expected = instance.get(instance.getDimension() - 1); - instance = instance.slice(instance.getDimension() - 1); - double actual = ann.getOutput(instance).get(0); - if (actual < 0.5 && expected >= 0.5 || actual >= 0.5 && expected < 0.5) { - ++errorRate; - } - } - errorRate /= testInstances.size(); - - Log.info(String.format("Training time: %fs\n", - (double) (end - start) / 1000)); - Log.info(String.format("Relative error: %f%%\n", errorRate * 100)); - } - - public void testLogisticRegressionDistributedVersionWithFeatureTransformer() { - // write data into a sequence file - String tmpStrDatasetPath = "/tmp/logistic_regression_data_feature_transformer"; - Path tmpDatasetPath = new Path(tmpStrDatasetPath); - String strDataPath = "src/test/resources/logistic_regression_data.txt"; - String modelPath = "/tmp/logistic-regression-distributed-model-feature-transformer"; - - Configuration conf = new Configuration(); - List<double[]> instanceList = new ArrayList<double[]>(); - List<double[]> trainingInstances = null; - List<double[]> testInstances = null; - - try { - FileSystem fs = FileSystem.get(new URI(tmpStrDatasetPath), conf); - fs.delete(tmpDatasetPath, true); - if (fs.exists(tmpDatasetPath)) { - fs.createNewFile(tmpDatasetPath); - } - - BufferedReader br = new BufferedReader(new FileReader(strDataPath)); - String line = null; - int count = 0; - while ((line = br.readLine()) != null) { - String[] tokens = line.trim().split(","); - double[] instance = new double[tokens.length]; - for (int i = 0; i < tokens.length; ++i) { - instance[i] = Double.parseDouble(tokens[i]); - } - instanceList.add(instance); - } - br.close(); - - zeroOneNormalization(instanceList, instanceList.get(0).length - 1); - - // write training data to temporal sequence file - SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, - tmpDatasetPath, LongWritable.class, VectorWritable.class); - int testSize = 150; - - Collections.shuffle(instanceList); - testInstances = new ArrayList<double[]>(); - testInstances.addAll(instanceList.subList(instanceList.size() - testSize, - instanceList.size())); - trainingInstances = instanceList.subList(0, instanceList.size() - - testSize); - - for (double[] instance : trainingInstances) { - DoubleVector vec = new DenseDoubleVector(instance); - writer.append(new LongWritable(count++), new VectorWritable(vec)); - } - writer.close(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - - // create model - int dimension = 8; - SmallLayeredNeuralNetwork ann = new SmallLayeredNeuralNetwork(); - ann.setLearningRate(0.7); - ann.setMomemtumWeight(0.5); - ann.setRegularizationWeight(0.1); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(dimension, false, - FunctionFactory.createDoubleFunction("Sigmoid")); - ann.addLayer(1, true, FunctionFactory.createDoubleFunction("Sigmoid")); - ann.setCostFunction(FunctionFactory - .createDoubleDoubleFunction("CrossEntropy")); - ann.setModelPath(modelPath); - - FeatureTransformer featureTransformer = new DefaultFeatureTransformer(); - - ann.setFeatureTransformer(featureTransformer); - - long start = new Date().getTime(); - Map<String, String> trainingParameters = new HashMap<String, String>(); - trainingParameters.put("tasks", "5"); - trainingParameters.put("training.max.iterations", "2000"); - trainingParameters.put("training.batch.size", "300"); - trainingParameters.put("convergence.check.interval", "1000"); - ann.train(tmpDatasetPath, trainingParameters); - - - long end = new Date().getTime(); - - // validate results - double errorRate = 0; - // calculate the error on test instance - for (double[] testInstance : testInstances) { - DoubleVector instance = new DenseDoubleVector(testInstance); - double expected = instance.get(instance.getDimension() - 1); - instance = instance.slice(instance.getDimension() - 1); - instance = featureTransformer.transform(instance); - double actual = ann.getOutput(instance).get(0); - if (actual < 0.5 && expected >= 0.5 || actual >= 0.5 && expected < 0.5) { - ++errorRate; - } - } - errorRate /= testInstances.size(); - - Log.info(String.format("Training time: %fs\n", - (double) (end - start) / 1000)); - Log.info(String.format("Relative error: %f%%\n", errorRate * 100)); - } - -} http://git-wip-us.apache.org/repos/asf/hama/blob/33041c09/ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetworkMessage.java ---------------------------------------------------------------------- diff --git a/ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetworkMessage.java b/ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetworkMessage.java deleted file mode 100644 index 148be6e..0000000 --- a/ml/src/test/java/org/apache/hama/ml/ann/TestSmallLayeredNeuralNetworkMessage.java +++ /dev/null @@ -1,172 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hama.ml.ann; - -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hama.commons.math.DenseDoubleMatrix; -import org.apache.hama.commons.math.DoubleMatrix; -import org.junit.Test; - -/** - * Test the functionalities of SmallLayeredNeuralNetworkMessage. - * - */ -public class TestSmallLayeredNeuralNetworkMessage { - - @Test - public void testReadWriteWithoutPrev() { - double error = 0.22; - double[][] matrix1 = new double[][] { { 0.1, 0.2, 0.8, 0.5 }, - { 0.3, 0.4, 0.6, 0.2 }, { 0.5, 0.6, 0.1, 0.5 } }; - double[][] matrix2 = new double[][] { { 0.8, 1.2, 0.5 } }; - DoubleMatrix[] matrices = new DoubleMatrix[2]; - matrices[0] = new DenseDoubleMatrix(matrix1); - matrices[1] = new DenseDoubleMatrix(matrix2); - - boolean isConverge = false; - - SmallLayeredNeuralNetworkMessage message = new SmallLayeredNeuralNetworkMessage( - error, isConverge, matrices, null); - Configuration conf = new Configuration(); - String strPath = "/tmp/testReadWriteSmallLayeredNeuralNetworkMessage"; - Path path = new Path(strPath); - try { - FileSystem fs = FileSystem.get(new URI(strPath), conf); - FSDataOutputStream out = fs.create(path); - message.write(out); - out.close(); - - FSDataInputStream in = fs.open(path); - SmallLayeredNeuralNetworkMessage readMessage = new SmallLayeredNeuralNetworkMessage( - 0, isConverge, null, null); - readMessage.readFields(in); - in.close(); - assertEquals(error, readMessage.getTrainingError(), 0.000001); - assertFalse(readMessage.isConverge()); - DoubleMatrix[] readMatrices = readMessage.getCurMatrices(); - assertEquals(2, readMatrices.length); - for (int i = 0; i < readMatrices.length; ++i) { - double[][] doubleMatrices = ((DenseDoubleMatrix) readMatrices[i]) - .getValues(); - double[][] doubleExpected = ((DenseDoubleMatrix) matrices[i]) - .getValues(); - for (int r = 0; r < doubleMatrices.length; ++r) { - assertArrayEquals(doubleExpected[r], doubleMatrices[r], 0.000001); - } - } - - DoubleMatrix[] readPrevMatrices = readMessage.getPrevMatrices(); - assertNull(readPrevMatrices); - - // delete - fs.delete(path, true); - } catch (IOException e) { - e.printStackTrace(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - } - - @Test - public void testReadWriteWithPrev() { - double error = 0.22; - boolean isConverge = true; - - double[][] matrix1 = new double[][] { { 0.1, 0.2, 0.8, 0.5 }, - { 0.3, 0.4, 0.6, 0.2 }, { 0.5, 0.6, 0.1, 0.5 } }; - double[][] matrix2 = new double[][] { { 0.8, 1.2, 0.5 } }; - DoubleMatrix[] matrices = new DoubleMatrix[2]; - matrices[0] = new DenseDoubleMatrix(matrix1); - matrices[1] = new DenseDoubleMatrix(matrix2); - - double[][] prevMatrix1 = new double[][] { { 0.1, 0.1, 0.2, 0.3 }, - { 0.2, 0.4, 0.1, 0.5 }, { 0.5, 0.1, 0.5, 0.2 } }; - double[][] prevMatrix2 = new double[][] { { 0.1, 0.2, 0.5, 0.9 }, - { 0.3, 0.5, 0.2, 0.6 }, { 0.6, 0.8, 0.7, 0.5 } }; - - DoubleMatrix[] prevMatrices = new DoubleMatrix[2]; - prevMatrices[0] = new DenseDoubleMatrix(prevMatrix1); - prevMatrices[1] = new DenseDoubleMatrix(prevMatrix2); - - SmallLayeredNeuralNetworkMessage message = new SmallLayeredNeuralNetworkMessage( - error, isConverge, matrices, prevMatrices); - Configuration conf = new Configuration(); - String strPath = "/tmp/testReadWriteSmallLayeredNeuralNetworkMessageWithPrev"; - Path path = new Path(strPath); - try { - FileSystem fs = FileSystem.get(new URI(strPath), conf); - FSDataOutputStream out = fs.create(path); - message.write(out); - out.close(); - - FSDataInputStream in = fs.open(path); - SmallLayeredNeuralNetworkMessage readMessage = new SmallLayeredNeuralNetworkMessage( - 0, isConverge, null, null); - readMessage.readFields(in); - in.close(); - - assertTrue(readMessage.isConverge()); - - DoubleMatrix[] readMatrices = readMessage.getCurMatrices(); - assertEquals(2, readMatrices.length); - for (int i = 0; i < readMatrices.length; ++i) { - double[][] doubleMatrices = ((DenseDoubleMatrix) readMatrices[i]) - .getValues(); - double[][] doubleExpected = ((DenseDoubleMatrix) matrices[i]) - .getValues(); - for (int r = 0; r < doubleMatrices.length; ++r) { - assertArrayEquals(doubleExpected[r], doubleMatrices[r], 0.000001); - } - } - - DoubleMatrix[] readPrevMatrices = readMessage.getPrevMatrices(); - assertEquals(2, readPrevMatrices.length); - for (int i = 0; i < readPrevMatrices.length; ++i) { - double[][] doubleMatrices = ((DenseDoubleMatrix) readPrevMatrices[i]) - .getValues(); - double[][] doubleExpected = ((DenseDoubleMatrix) prevMatrices[i]) - .getValues(); - for (int r = 0; r < doubleMatrices.length; ++r) { - assertArrayEquals(doubleExpected[r], doubleMatrices[r], 0.000001); - } - } - - // delete - fs.delete(path, true); - } catch (IOException e) { - e.printStackTrace(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - } - -}
