Github user jkbradley commented on a diff in the pull request: https://github.com/apache/spark/pull/2451#discussion_r17800692 --- Diff: mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala --- @@ -57,13 +250,709 @@ trait Matrix extends Serializable { * @param numCols number of columns * @param values matrix entries in column major */ -class DenseMatrix(val numRows: Int, val numCols: Int, val values: Array[Double]) extends Matrix { +class DenseMatrix(val numRows: Int, val numCols: Int, val values: Array[Double]) extends Matrix with Serializable { - require(values.length == numRows * numCols) + require(values.length == numRows * numCols, "The number of values supplied doesn't match the " + + s"size of the matrix! values.length: ${values.length}, numRows * numCols: ${numRows * numCols}") override def toArray: Array[Double] = values - private[mllib] override def toBreeze: BM[Double] = new BDM[Double](numRows, numCols, values) + private[mllib] def toBreeze: BM[Double] = new BDM[Double](numRows, numCols, values) + + private[mllib] def apply(i: Int): Double = values(i) + + private[mllib] def apply(i: Int, j: Int): Double = values(index(i, j)) + + private[mllib] def index(i: Int, j: Int): Int = i + numRows * j + + private[mllib] def update(i: Int, j: Int, v: Double): Unit = { + values(index(i, j)) = v + } + + override def copy = new DenseMatrix(numRows, numCols, values.clone()) + + private[mllib] def elementWiseOperateOnColumnsInPlace( + f: (Double, Double) => Double, + y: Matrix): DenseMatrix = { + val y_vals = y.toArray + val len = y_vals.length + require(y_vals.length == numRows) + var j = 0 + while (j < numCols){ + var i = 0 + while (i < len){ + val idx = index(i, j) + values(idx) = f(values(idx), y_vals(i)) + i += 1 + } + j += 1 + } + this + } + + private[mllib] def elementWiseOperateOnRowsInPlace( + f: (Double, Double) => Double, + y: Matrix): DenseMatrix = { + val y_vals = y.toArray + require(y_vals.length == numCols) + var j = 0 + while (j < numCols){ + var i = 0 + while (i < numRows){ + val idx = index(i, j) + values(idx) = f(values(idx), y_vals(j)) + i += 1 + } + j += 1 + } + this + } + + private[mllib] def elementWiseOperateInPlace(f: (Double, Double) => Double, y: Matrix): DenseMatrix = { + val y_val = y.toArray + val len = values.length + require(y_val.length == values.length) + var j = 0 + while (j < len){ + values(j) = f(values(j), y_val(j)) + j += 1 + } + this + } + + private[mllib] def elementWiseOperateScalarInPlace(f: (Double, Double) => Double, y: Double): DenseMatrix = { + var j = 0 + val len = values.length + while (j < len){ + values(j) = f(values(j), y) + j += 1 + } + this + } + + private[mllib] def operateInPlace(f: (Double, Double) => Double, y: Matrix): DenseMatrix = { + if (y.numCols==1 || y.numRows == 1){ + require(numCols != numRows, "Operation is ambiguous. Please use elementWiseOperateOnRows " + + "or elementWiseOperateOnColumns instead") + } + if (y.numCols == 1 && y.numRows == 1){ + elementWiseOperateScalarInPlace(f, y.toArray(0)) + } else { + if (y.numCols==1) { + elementWiseOperateOnColumnsInPlace(f, y) + }else if (y.numRows==1){ + elementWiseOperateOnRowsInPlace(f, y) + }else{ + elementWiseOperateInPlace(f, y) + } + } + } + + private[mllib] def elementWiseOperateOnColumns(f: (Double, Double) => Double, y: Matrix): DenseMatrix = { + val dup = this.copy + dup.elementWiseOperateOnColumnsInPlace(f, y) + } + + private[mllib] def elementWiseOperateOnRows(f: (Double, Double) => Double, y: Matrix): DenseMatrix = { + val dup = this.copy + dup.elementWiseOperateOnRowsInPlace(f, y) + } + + private[mllib] def elementWiseOperate(f: (Double, Double) => Double, y: Matrix): DenseMatrix = { + val dup = this.copy + dup.elementWiseOperateInPlace(f, y) + } + + private[mllib] def elementWiseOperateScalar(f: (Double, Double) => Double, y: Double): DenseMatrix = { + val dup = this.copy + dup.elementWiseOperateScalarInPlace(f, y) + } + + private[mllib] def operate(f: (Double, Double) => Double, y: Matrix): DenseMatrix = { + val dup = this.copy + dup.operateInPlace(f, y) + } + + def map(f: Double => Double) = new DenseMatrix(numRows, numCols, values.map(f)) + + def update(f: Double => Double): DenseMatrix = { + val len = values.length + var i = 0 + while (i < len) { + values(i) = f(values(i)) + i += 1 + } + this + } + + def colNorms(p: Double): DenseMatrix = { + if (p==1.0) return colSums(true) + val sums = new DenseMatrix(1, numCols, Array.fill(numCols)(0.0)) + var j = 0 + while (j < numCols){ + var i = 0 + while (i < numRows){ + val idx = index(i, j) + sums.update(0,j, sums(j) + math.pow(values(idx),p)) + i += 1 + } + j += 1 + } + j = 0 + while (j < numCols){ + sums.update(0, j, math.pow(sums(j), 1/p)) + j += 1 + } + sums + } + + private[mllib] def negInPlace: DenseMatrix = { + var j = 0 + val len = values.length + while (j < len){ + values(j) *= -1 + j += 1 + } + this + } + + private[mllib] def neg: DenseMatrix = { + val copy = new DenseMatrix(numRows, numCols, values.clone()) + copy.negInPlace + } + + private[mllib] def compareInPlace(v: Double, f: (Double, Double) => Boolean): DenseMatrix = { + var j = 0 + val len = values.length + while (j < len){ + values(j) = if (f(values(j), v)) 1.0 else 0.0 + j += 1 + } + this + } + + private[mllib] def compare(v: Double, f: (Double, Double) => Boolean): DenseMatrix = { + val copy = new DenseMatrix(numRows, numCols, values.clone()) + copy.compareInPlace(v, f) + } + + private[mllib] def multiplyInPlace(y: Matrix): DenseMatrix = { + val copy = this multiply y + BLAS.copy(Vectors.dense(copy.values), Vectors.dense(values)) + this + } +} + +object DenseMatrix { + + /** + * Generate a `DenseMatrix` consisting of zeros. + * @param numRows number of rows of the matrix + * @param numCols number of columns of the matrix + * @return `DenseMatrix` with size `numRows` x `numCols` and values of zeros + */ + def zeros(numRows: Int, numCols: Int): DenseMatrix = + new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(0.0)) + + /** + * Generate a `DenseMatrix` consisting of ones. + * @param numRows number of rows of the matrix + * @param numCols number of columns of the matrix + * @return `DenseMatrix` with size `numRows` x `numCols` and values of ones + */ + def ones(numRows: Int, numCols: Int): DenseMatrix = + new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(1.0)) + + /** + * Generate an Identity Matrix in `DenseMatrix` format. + * @param n number of rows and columns of the matrix + * @return `DenseMatrix` with size `n` x `n` and values of ones on the diagonal + */ + def eye(n: Int): DenseMatrix = { + val identity = DenseMatrix.zeros(n, n) + var i = 0 + while (i < n){ + identity.update(i, i, 1.0) + i += 1 + } + identity + } + + /** + * Generate a `DenseMatrix` consisting of i.i.d. uniform random numbers. + * @param numRows number of rows of the matrix + * @param numCols number of columns of the matrix + * @return `DenseMatrix` with size `numRows` x `numCols` and values in U(0, 1) + */ + def rand(numRows: Int, numCols: Int): DenseMatrix = { + val rand = new XORShiftRandom + new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(rand.nextDouble())) + } + + /** + * Generate a `DenseMatrix` consisting of i.i.d. gaussian random numbers. + * @param numRows number of rows of the matrix + * @param numCols number of columns of the matrix + * @return `DenseMatrix` with size `numRows` x `numCols` and values in N(0, 1) + */ + def randn(numRows: Int, numCols: Int): DenseMatrix = { + val rand = new XORShiftRandom + new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(rand.nextGaussian())) + } + + /** + * Generate a diagonal matrix in `DenseMatrix` format from the supplied values. + * @param vector a `Vector` that will form the values on the diagonal of the matrix + * @return Square `DenseMatrix` with size `values.length` x `values.length` and `values` + * on the diagonal + */ + def diag(vector: Vector): DenseMatrix = { + val n = vector.size + val matrix = DenseMatrix.eye(n) + val values = vector.toArray + var i = 0 + while (i < n) { + matrix.update(i, i, values(i)) + i += 1 + } + matrix + } +} + +/** + * Column-majored sparse matrix. + * The entry values are stored in Compressed Sparse Column (CSC) format. + * For example, the following matrix + * {{{ + * 1.0 0.0 4.0 + * 0.0 3.0 5.0 + * 2.0 0.0 6.0 + * }}} + * is stored as `values: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]`, + * `rowIndices=[0, 2, 1, 0, 1, 2]`, `colPointers=[0, 2, 3, 6]`. + * + * @param numRows number of rows + * @param numCols number of columns + * @param colPtrs the index corresponding to the start of a new column + * @param rowIndices the row index of the entry. They must be in strictly increasing order for each + * column + * @param values non-zero matrix entries in column major + */ +class SparseMatrix( + val numRows: Int, + val numCols: Int, + val colPtrs: Array[Int], + val rowIndices: Array[Int], + val values: Array[Double]) extends Matrix with Serializable { + + require(values.length == rowIndices.length, "The number of row indices and values don't match! " + + s"values.length: ${values.length}, rowIndices.length: ${rowIndices.length}") + require(colPtrs.length == numCols + 1, "The length of the column indices should be the " + + s"number of columns + 1. Currently, colPointers.length: ${colPtrs.length}, " + + s"numCols: $numCols") + + override def toArray: Array[Double] = { + val arr = new Array[Double](numRows * numCols) + var j = 0 + while (j < numCols) { + var i = colPtrs(j) + val indEnd = colPtrs(j + 1) + val offset = j * numRows + while (i < indEnd) { + val rowIndex = rowIndices(i) + arr(offset + rowIndex) = values(i) + i += 1 + } + j += 1 + } + arr + } + + private[mllib] def toBreeze: BM[Double] = + new BSM[Double](values, numRows, numCols, colPtrs, rowIndices) + + private[mllib] def apply(i: Int, j: Int): Double = { + val ind = index(i, j) + if (ind < 0) 0.0 else values(ind) + } + + private[mllib] def index(i: Int, j: Int): Int = { + Arrays.binarySearch(rowIndices, colPtrs(j), colPtrs(j + 1), i) + } + + private[mllib] def update(i: Int, j: Int, v: Double): Unit = { + val ind = index(i, j) + if (ind == -1){ + throw new NoSuchElementException("The given row and column indices correspond to a zero " + + "value. Only non-zero elements in Sparse Matrices can be updated.") + } else { + values(index(i, j)) = v + } + } + + override def copy = new SparseMatrix(numRows, numCols, colPtrs, rowIndices, values.clone()) + + private[mllib] def elementWiseOperateOnColumnsInPlace(f: (Double, Double) => Double, y: Matrix): Matrix = { + if (isMultiplication(f) || isDivision(f)) { + val y_vals = y.toArray + require(y_vals.length == numRows) + var j = 0 + while (j < numCols){ + var i = colPtrs(j) + val indEnd = colPtrs(j + 1) + while (i < indEnd){ + values(i) = f(values(i), y_vals(rowIndices(i))) + i += 1 + } + j += 1 + } + this + } else { + val dup = this.toDense + dup.elementWiseOperateOnColumnsInPlace(f, y) + } + } + + private[mllib] def elementWiseOperateOnRowsInPlace( + f: (Double, Double) => Double, + y: Matrix): Matrix = { + if (isMultiplication(f) || isDivision(f)) { + val y_vals = y.toArray + require(y_vals.length == numCols) + var j = 0 + while (j < numCols){ + var i = colPtrs(j) + val indEnd = colPtrs(j + 1) + while (i < indEnd){ + values(i) = f(values(i), y_vals(j)) + i += 1 + } + j += 1 + } + this + } else { + val dup = this.toDense + dup.elementWiseOperateOnRowsInPlace(f, y) + } + } + + private[mllib] def elementWiseOperateInPlace( + f: (Double, Double) => Double, + y: Matrix): Matrix = { + require(y.numCols == numCols) + require(y.numRows == numRows) + if (isMultiplication(f) || isDivision(f)) { + var j = 0 + while (j < numCols){ + var i = colPtrs(j) + val indEnd = colPtrs(j + 1) + while (i < indEnd) { + values(i) = f(values(i), y(rowIndices(i), j)) + i += 1 + } + j += 1 + } + this + } else { + val dup = this.toDense + dup.elementWiseOperateInPlace(f, y) + } + } + + private[mllib] def elementWiseOperateScalarInPlace( + f: (Double, Double) => Double, + y: Double): Matrix = { + if (isMultiplication(f) || isDivision(f)) { + var j = 0 + val len = values.length + while (j < len){ + values(j) = f(values(j), y) + j += 1 + } + this + } else { + val dup = this.toDense + dup.elementWiseOperateScalarInPlace(f, y) + } + } + + private def isMultiplication(f: (Double, Double) => Double): Boolean = { --- End diff -- This seems too unsafe to me. As long as these functions are all private[mllib], why not define private[mllib] static Multiply and Divide functions and check for those with match-case?
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- --------------------------------------------------------------------- To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org