This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 4a07cac [SYSTEMDS-2577,2824] Fix sparse cumsumprod operations,
functions tests
4a07cac is described below
commit 4a07cacffad03f2b31b3c650a609bb40997c6cd3
Author: Matthias Boehm <[email protected]>
AuthorDate: Wed Jun 2 23:52:52 2021 +0200
[SYSTEMDS-2577,2824] Fix sparse cumsumprod operations, functions tests
This patch fixes the missing support for sparse cumsumprod operations.
The input is always a two-column matrix and hence was assumed dense.
However, with ultra-sparse matrices and optimizations that use CSR as a
more space-efficient data structure, sparse inputs can occur.
Furthermore, this patch also adds dedicated tests for function
precendence checks (user functions, dml-bodied builtins, namespace
functions, native builtins) but these did not reveal additional issues.
---
.../apache/sysds/runtime/data/SparseBlockMCSR.java | 3 +-
.../sysds/runtime/matrix/data/LibMatrixAgg.java | 33 +++++++++++
.../test/component/matrix/CumsumprodTest.java | 65 +++++++++++++++++++++
.../sysds/test/component/matrix/SliceTest.java | 68 +++++++++++-----------
.../test/functions/misc/FunctionPotpourriTest.java | 6 ++
.../misc/FunPotpourriBuiltinPrecedence.dml | 38 ++++++++++++
.../misc/FunPotpourriBuiltinPrecedence2.dml | 26 +++++++++
7 files changed, 204 insertions(+), 35 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
index 77caaec..159e581 100644
--- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
+++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java
@@ -48,7 +48,8 @@ public class SparseBlockMCSR extends SparseBlock
SparseRow[] orows = ((SparseBlockMCSR)sblock)._rows;
_rows = new SparseRow[orows.length];
for( int i=0; i<_rows.length; i++ )
- _rows[i] = new SparseRowVector(orows[i]);
+ if( orows[i] != null )
+ _rows[i] = new
SparseRowVector(orows[i]);
}
//general case SparseBlock
else {
diff --git
a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java
b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java
index 389700b..d9578ae 100644
--- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java
+++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixAgg.java
@@ -1549,6 +1549,12 @@ public class LibMatrixAgg
s_ucumkp(a, agg, dc, m, n, kbuff, kplus, rl,
ru);
break;
}
+ case CUM_SUM_PROD: { //CUMSUMPROD
+ if( n != 2 )
+ throw new
DMLRuntimeException("Cumsumprod expects two-column input (n="+n+").");
+ s_ucumkpp(a, agg, dc, rl, ru);
+ break;
+ }
case CUM_PROD: { //CUMPROD
s_ucumm(a, agg, c, n, rl, ru);
break;
@@ -2365,6 +2371,33 @@ public class LibMatrixAgg
}
}
+
+ /**
+ * CUMSUMPROD, opcode: ucumk+*, dense input.
+ *
+ * @param a sparse block
+ * @param agg ?
+ * @param c ?
+ * @param rl row lower index
+ * @param ru row upper index
+ */
+ private static void s_ucumkpp( SparseBlock a, double[] agg, DenseBlock
c, int rl, int ru ) {
+ //init current row sum/correction arrays w/ neutral 0
+ double sum = (agg != null) ? agg[0] : 0;
+ //scan once and compute prefix sums
+ double[] cvals = c.values(0);
+ for( int i=rl; i<ru; i++ ) {
+ if( a.isEmpty(i) )
+ sum = cvals[i] = 0;
+ else if( a.size(i) == 2 ) {
+ double[] avals = a.values(i); int apos =
a.pos(i);
+ sum = cvals[i] = avals[apos] + avals[apos+1] *
sum;
+ }
+ else //fallback
+ sum = cvals[i] = a.get(i,0) + a.get(i,1) * sum;
+ }
+ }
+
/**
* CUMPROD, opcode: ucum*, sparse input.
*
diff --git
a/src/test/java/org/apache/sysds/test/component/matrix/CumsumprodTest.java
b/src/test/java/org/apache/sysds/test/component/matrix/CumsumprodTest.java
new file mode 100644
index 0000000..3ad96fe
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/component/matrix/CumsumprodTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.matrix;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.sysds.runtime.data.SparseBlock;
+import org.apache.sysds.runtime.functionobjects.Builtin;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.matrix.operators.UnaryOperator;
+import org.junit.Test;
+
+public class CumsumprodTest {
+ @Test
+ public void testCumsumprodDense() {
+ MatrixBlock A = MatrixBlock.randOperations(100, 2, 0.9, 0, 10,
"uniform", 7);
+ UnaryOperator uop = new
UnaryOperator(Builtin.getBuiltinFnObject("ucumk+*"), 1, false);
+ MatrixBlock B = A.unaryOperations(uop, new MatrixBlock());
+ assertEquals(100, B.getNumRows());
+ }
+
+ @Test
+ public void testCumsumprodSparseMCSR() {
+ MatrixBlock A = MatrixBlock.randOperations(1000, 2, 0.05, 0,
10, "uniform", 7);
+ A = new MatrixBlock(A, SparseBlock.Type.MCSR, true);
+ UnaryOperator uop = new
UnaryOperator(Builtin.getBuiltinFnObject("ucumk+*"), 1, false);
+ MatrixBlock B = A.unaryOperations(uop, new MatrixBlock());
+ assertEquals(1000, B.getNumRows());
+ }
+
+ @Test
+ public void testCumsumprodSparseCSR() {
+ MatrixBlock A = MatrixBlock.randOperations(1000, 2, 0.05, 0,
10, "uniform", 7);
+ A = new MatrixBlock(A, SparseBlock.Type.CSR, true);
+ UnaryOperator uop = new
UnaryOperator(Builtin.getBuiltinFnObject("ucumk+*"), 1, false);
+ MatrixBlock B = A.unaryOperations(uop, new MatrixBlock());
+ assertEquals(1000, B.getNumRows());
+ }
+
+ @Test
+ public void testCumsumprodSparseCOO() {
+ MatrixBlock A = MatrixBlock.randOperations(1000, 2, 0.05, 0,
10, "uniform", 7);
+ A = new MatrixBlock(A, SparseBlock.Type.COO, true);
+ UnaryOperator uop = new
UnaryOperator(Builtin.getBuiltinFnObject("ucumk+*"), 1, false);
+ MatrixBlock B = A.unaryOperations(uop, new MatrixBlock());
+ assertEquals(1000, B.getNumRows());
+ }
+}
diff --git
a/src/test/java/org/apache/sysds/test/component/matrix/SliceTest.java
b/src/test/java/org/apache/sysds/test/component/matrix/SliceTest.java
index f69df13..0fc0d7b 100644
--- a/src/test/java/org/apache/sysds/test/component/matrix/SliceTest.java
+++ b/src/test/java/org/apache/sysds/test/component/matrix/SliceTest.java
@@ -26,38 +26,38 @@ import org.apache.sysds.runtime.util.DataConverter;
import org.junit.Test;
public class SliceTest {
- MatrixBlock a = genIncMatrix(10, 10);
-
- @Test
- public void sliceTest_01() {
- MatrixBlock b = a.slice(0, 4);
- assertEquals(5, b.getNumRows());
- }
-
- @Test
- public void sliceTest_02() {
- MatrixBlock b = a.slice(0, 9);
- assertEquals(10, b.getNumRows());
- }
-
- @Test
- public void sliceTest_03() {
- MatrixBlock b = a.slice(9, 9);
- assertEquals(1, b.getNumRows());
- }
-
- private static MatrixBlock gen(int[][] v) {
- return DataConverter.convertToMatrixBlock(v);
- }
-
- private static MatrixBlock genIncMatrix(int rows, int cols) {
- int[][] ret = new int[rows][cols];
- int x = 0;
- for(int i = 0; i < rows; i++) {
- for(int j = 0; j < cols; j++) {
- ret[i][j] = x++;
- }
- }
- return gen(ret);
- }
+ MatrixBlock a = genIncMatrix(10, 10);
+
+ @Test
+ public void sliceTest_01() {
+ MatrixBlock b = a.slice(0, 4);
+ assertEquals(5, b.getNumRows());
+ }
+
+ @Test
+ public void sliceTest_02() {
+ MatrixBlock b = a.slice(0, 9);
+ assertEquals(10, b.getNumRows());
+ }
+
+ @Test
+ public void sliceTest_03() {
+ MatrixBlock b = a.slice(9, 9);
+ assertEquals(1, b.getNumRows());
+ }
+
+ private static MatrixBlock gen(int[][] v) {
+ return DataConverter.convertToMatrixBlock(v);
+ }
+
+ private static MatrixBlock genIncMatrix(int rows, int cols) {
+ int[][] ret = new int[rows][cols];
+ int x = 0;
+ for(int i = 0; i < rows; i++) {
+ for(int j = 0; j < cols; j++) {
+ ret[i][j] = x++;
+ }
+ }
+ return gen(ret);
+ }
}
diff --git
a/src/test/java/org/apache/sysds/test/functions/misc/FunctionPotpourriTest.java
b/src/test/java/org/apache/sysds/test/functions/misc/FunctionPotpourriTest.java
index b174dbb..6756272 100644
---
a/src/test/java/org/apache/sysds/test/functions/misc/FunctionPotpourriTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/misc/FunctionPotpourriTest.java
@@ -55,6 +55,7 @@ public class FunctionPotpourriTest extends AutomatedTestBase
"FunPotpourriEvalList1Arg",
"FunPotpourriEvalList2Arg",
"FunPotpourriEvalNamespace",
+ "FunPotpourriBuiltinPrecedence",
};
private final static String TEST_DIR = "functions/misc/";
@@ -192,6 +193,11 @@ public class FunctionPotpourriTest extends
AutomatedTestBase
runFunctionTest( TEST_NAMES[24], null );
}
+ @Test
+ public void testFunctionBuiltinPrecedence() {
+ runFunctionTest( TEST_NAMES[25], null );
+ }
+
private void runFunctionTest(String testName, Class<?> error) {
TestConfiguration config = getTestConfiguration(testName);
loadTestConfiguration(config);
diff --git a/src/test/scripts/functions/misc/FunPotpourriBuiltinPrecedence.dml
b/src/test/scripts/functions/misc/FunPotpourriBuiltinPrecedence.dml
new file mode 100644
index 0000000..3e557a0
--- /dev/null
+++ b/src/test/scripts/functions/misc/FunPotpourriBuiltinPrecedence.dml
@@ -0,0 +1,38 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("./src/test/scripts/functions/misc/FunPotpourriBuiltinPrecedence2.dml")
as NS
+
+sigmoid = function(Matrix[Double] X) return (Matrix[Double] Y) {
+ while(FALSE){} # no inlining
+ #Y = 1 / (1 + exp(-X));
+ Y = 1 / (1 + exp(-X)) * 2; # corrupted result
+}
+
+X = rand(rows=10, cols=10, seed=7)
+R1 = 1 / (1 + exp(-X)); # sigmoid
+R2 = sigmoid(X); #must not call builtin
+R3 = NS::sigmoid(X); #must not call builtin or local
+
+if( sum(abs(R1*2 - R2) > 1e-8) != 0 )
+ print(R2[-sum(R1),-sum(R2)]) #crash test
+if( sum(abs(R1*3 - R3) > 1e-8) != 0 )
+ print(R3[-sum(R1),-sum(R3)]) #crash test
diff --git a/src/test/scripts/functions/misc/FunPotpourriBuiltinPrecedence2.dml
b/src/test/scripts/functions/misc/FunPotpourriBuiltinPrecedence2.dml
new file mode 100644
index 0000000..a8e21a2
--- /dev/null
+++ b/src/test/scripts/functions/misc/FunPotpourriBuiltinPrecedence2.dml
@@ -0,0 +1,26 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+sigmoid = function(Matrix[Double] X) return (Matrix[Double] Y) {
+ while(FALSE){} # no inlining
+ #Y = 1 / (1 + exp(-X));
+ Y = 1 / (1 + exp(-X)) * 3; # corrupted result
+}
