This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 809d53f6236232cb31eb02668263aab2ac80116d Author: Olga <[email protected]> AuthorDate: Fri Nov 13 03:01:11 2020 +0100 [MINOR] Modifications to Federated Tests This commit change the federated tests to execute more consistantly on github. --- scripts/builtin/dist.dml | 1 - .../controlprogram/federated/FederationUtils.java | 42 ++++------------------ .../fed/AggregateUnaryFEDInstruction.java | 2 +- .../instructions/fed/FEDInstructionUtils.java | 17 ++++----- .../federated/algorithms/FederatedCorTest.java | 7 ++-- .../federated/algorithms/FederatedVarTest.java | 8 ++--- .../primitives/FederatedFullAggregateTest.java | 31 ++++++++++++---- .../primitives/FederatedRowColAggregateTest.java | 37 ++++++++++++++----- .../federated/aggregate/FederatedColVarTest.dml | 20 ++++++----- .../aggregate/FederatedColVarTestReference.dml | 14 +++----- .../federated/aggregate/FederatedRowVarTest.dml | 20 ++++++----- .../aggregate/FederatedRowVarTestReference.dml | 14 +++----- .../federated/aggregate/FederatedVarTest.dml | 21 ++++++----- .../aggregate/FederatedVarTestReference.dml | 14 +++----- 14 files changed, 124 insertions(+), 124 deletions(-) diff --git a/scripts/builtin/dist.dml b/scripts/builtin/dist.dml index e5fe930..1245087 100644 --- a/scripts/builtin/dist.dml +++ b/scripts/builtin/dist.dml @@ -24,7 +24,6 @@ m_dist = function(Matrix[Double] X) return (Matrix[Double] Y) { G = X %*% t(X); I = matrix(1, rows = nrow(G), cols = ncol(G)); Y = -2 * (G) + (diag(G) * I) + (I * t(diag(G))); -# Y = -2 * (G) + t(I %*% diag(diag(G))) + t(diag(diag(G)) %*% I); Y = sqrt(Y); Y = replace(target = Y, pattern=0/0, replacement = 0); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java index 0a24cea..22f4e69 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederationUtils.java @@ -172,30 +172,6 @@ public class FederationUtils { public static MatrixBlock aggVar(Future<FederatedResponse>[] ffr, Future<FederatedResponse>[] meanFfr, FederationMap map, boolean isRowAggregate, boolean isScalar) { try { -// else if(aop.aggOp.increOp.fn instanceof CM) { -// double var = ((ScalarObject) ffr[0].get().getData()[0]).getDoubleValue(); -// double mean = ((ScalarObject) meanFfr[0].get().getData()[0]).getDoubleValue(); -// long size = map.getFederatedRanges()[0].getSize(); -// for(int i = 0; i < ffr.length - 1; i++) { -// long l = size + map.getFederatedRanges()[i+1].getSize(); -// double k = ((size * var) + (map.getFederatedRanges()[i+1].getSize() * ((ScalarObject) ffr[i+1].get().getData()[0]).getDoubleValue())) / l; -// var = k + (size * map.getFederatedRanges()[i+1].getSize()) * Math.pow((mean - ((ScalarObject) meanFfr[i+1].get().getData()[0]).getDoubleValue()) / l, 2); -// mean = (mean * size + ((ScalarObject) meanFfr[i+1].get().getData()[0]).getDoubleValue() * (map.getFederatedRanges()[i+1].getSize())) / l; -// size = l; -// System.out.println("Olga"); -// // long l = sizes[i] + sizes[i + 1]; -// // double k = Math.pow(means[i] - means[i+1], 2) * (sizes[i] * sizes[i+1]); -// // k += ((sizes[i] * vars[i]) + (sizes[i+1] * vars[i+1])) * l; -// // vars[i+1] = k / Math.pow(l, 2); -// // -// // means[i+1] = (means[i] * sizes[i] + means[i] * sizes[i]) / l; -// // sizes[i+1] = l; -// } -// return new DoubleObject(var); -// -// } - - FederatedRange[] ranges = map.getFederatedRanges(); BinaryOperator plus = InstructionUtils.parseBinaryOperator("+"); BinaryOperator minus = InstructionUtils.parseBinaryOperator("-"); @@ -204,13 +180,13 @@ public class FederationUtils { ScalarOperator dev1 = InstructionUtils.parseScalarBinaryOperator("/", false); ScalarOperator pow = InstructionUtils.parseScalarBinaryOperator("^2", false); - long size1 = isScalar ? ranges[0].getSize() : ranges[0].getSize(isRowAggregate ? 0 : 1); + long size1 = isScalar ? ranges[0].getSize() : ranges[0].getSize(isRowAggregate ? 1 : 0); MatrixBlock var1 = (MatrixBlock)ffr[0].get().getData()[0]; MatrixBlock mean1 = (MatrixBlock)meanFfr[0].get().getData()[0]; for(int i=0; i < ffr.length - 1; i++) { MatrixBlock var2 = (MatrixBlock)ffr[i+1].get().getData()[0]; MatrixBlock mean2 = (MatrixBlock)meanFfr[i+1].get().getData()[0]; - long size2 = isScalar ? ranges[i+1].getSize() : ranges[i+1].getSize(isRowAggregate ? 0 : 1); + long size2 = isScalar ? ranges[i+1].getSize() : ranges[i+1].getSize(isRowAggregate ? 1 : 0); mult1 = mult1.setConstant(size1); var1 = var1.scalarOperations(mult1, new MatrixBlock()); @@ -219,11 +195,12 @@ public class FederationUtils { dev1 = dev1.setConstant(size1 + size2); var1 = var1.scalarOperations(dev1, new MatrixBlock()); - MatrixBlock tmp1 = (mean1.binaryOperationsInPlace(minus, mean2)).scalarOperations(dev1, new MatrixBlock()); + MatrixBlock tmp1 = new MatrixBlock(mean1); + tmp1 = tmp1.binaryOperationsInPlace(minus, mean2); + tmp1 = tmp1.scalarOperations(dev1, new MatrixBlock()); tmp1 = tmp1.scalarOperations(pow, new MatrixBlock()); mult1 = mult1.setConstant(size1*size2); tmp1 = tmp1.scalarOperations(mult1, new MatrixBlock()); - var1 = tmp1.binaryOperationsInPlace(plus, var1); // next mean @@ -272,13 +249,6 @@ public class FederationUtils { var = k + (size * map.getFederatedRanges()[i+1].getSize()) * Math.pow((mean - ((ScalarObject) meanFfr[i+1].get().getData()[0]).getDoubleValue()) / l, 2); mean = (mean * size + ((ScalarObject) meanFfr[i+1].get().getData()[0]).getDoubleValue() * (map.getFederatedRanges()[i+1].getSize())) / l; size = l; -// long l = sizes[i] + sizes[i + 1]; -// double k = Math.pow(means[i] - means[i+1], 2) * (sizes[i] * sizes[i+1]); -// k += ((sizes[i] * vars[i]) + (sizes[i+1] * vars[i+1])) * l; -// vars[i+1] = k / Math.pow(l, 2); -// -// means[i+1] = (means[i] * sizes[i] + means[i] * sizes[i]) / l; -// sizes[i+1] = l; } return new DoubleObject(var); @@ -311,7 +281,7 @@ public class FederationUtils { boolean isMin = ((Builtin) aop.aggOp.increOp.fn).getBuiltinCode() == BuiltinCode.MIN; return aggMinMax(ffr,isMin,false, Optional.of(map.getType())); } else if(aop.aggOp.increOp.fn instanceof CM) { - return aggVar(ffr, meanFfr, map, aop.isRowAggregate(), !(aop.isColAggregate() && aop.isRowAggregate())); //TODO + return aggVar(ffr, meanFfr, map, aop.isRowAggregate(), !(aop.isColAggregate() || aop.isRowAggregate())); //TODO } else throw new DMLRuntimeException("Unsupported aggregation operator: " diff --git a/src/main/java/org/apache/sysds/runtime/instructions/fed/AggregateUnaryFEDInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/fed/AggregateUnaryFEDInstruction.java index 1429dd3..b9f220b 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/fed/AggregateUnaryFEDInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/fed/AggregateUnaryFEDInstruction.java @@ -119,6 +119,6 @@ public class AggregateUnaryFEDInstruction extends UnaryFEDInstruction { if( output.isScalar() ) ec.setVariable(output.getName(), FederationUtils.aggScalar(aop, tmp, meanTmp, map)); else - ec.setMatrixOutput(output.getName(), FederationUtils.aggMatrix(aop, meanTmp, tmp, map)); + ec.setMatrixOutput(output.getName(), FederationUtils.aggMatrix(aop, tmp, meanTmp, map)); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/fed/FEDInstructionUtils.java b/src/main/java/org/apache/sysds/runtime/instructions/fed/FEDInstructionUtils.java index 1b095e1..f4b19bf 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/fed/FEDInstructionUtils.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/fed/FEDInstructionUtils.java @@ -99,23 +99,20 @@ public class FEDInstructionUtils { if((mo instanceof MatrixObject || mo instanceof FrameObject) && mo.isFederated() ) fedinst = ReorgFEDInstruction.parseInstruction(rinst.getInstructionString()); } - else if(instruction.input1 != null && instruction.input1.isMatrix() - && ec.getMatrixObject(instruction.input1).isFederated() + else if(instruction.input1 != null && instruction.input1.isMatrix() && ec.containsVariable(instruction.input1)) { MatrixObject mo1 = ec.getMatrixObject(instruction.input1); - if(instruction.getOpcode().equalsIgnoreCase("cm")) { + if(instruction.getOpcode().equalsIgnoreCase("cm") && mo1.isFederated()) { fedinst = CentralMomentFEDInstruction.parseInstruction(inst.getInstructionString()); - } - else if(inst instanceof AggregateUnaryCPInstruction && + } else if(inst.getOpcode().equalsIgnoreCase("qsort") && mo1.isFederated()) { + if(mo1.getFedMapping().getFederatedRanges().length == 1) + fedinst = QuantileSortFEDInstruction.parseInstruction(inst.getInstructionString()); + } else if(inst instanceof AggregateUnaryCPInstruction && mo1.isFederated() && ((AggregateUnaryCPInstruction) instruction).getAUType() == AggregateUnaryCPInstruction.AUType.DEFAULT) { fedinst = AggregateUnaryFEDInstruction.parseInstruction(inst.getInstructionString()); } - else if(inst.getOpcode().equalsIgnoreCase("qsort") && - mo1.getFedMapping().getFederatedRanges().length == 1) { - fedinst = QuantileSortFEDInstruction.parseInstruction(inst.getInstructionString()); - } } } else if (inst instanceof BinaryCPInstruction) { @@ -154,7 +151,7 @@ public class FEDInstructionUtils { // matrix indexing LOG.info("Federated Indexing"); MatrixIndexingCPInstruction minst = (MatrixIndexingCPInstruction) inst; - if(inst.getOpcode().equalsIgnoreCase("rightIndex") + if(inst.getOpcode().equalsIgnoreCase("rightIndex") && minst.input1.isMatrix() && ec.getCacheableData(minst.input1).isFederated()) { LOG.info("Federated Right Indexing"); fedinst = MatrixIndexingFEDInstruction.parseInstruction(minst.getInstructionString()); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedCorTest.java b/src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedCorTest.java index 1b06279..15383b2 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedCorTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedCorTest.java @@ -53,7 +53,7 @@ public class FederatedCorTest extends AutomatedTestBase { @Parameterized.Parameters public static Collection<Object[]> data() { - return Arrays.asList(new Object[][] {{1600, 8, true}}); + return Arrays.asList(new Object[][] {{1600, 40, true}}); } @Override @@ -133,9 +133,10 @@ public class FederatedCorTest extends AutomatedTestBase { runTest(true, false, null, -1); // compare via files - compareResults(1e-9); + compareResults(1e-2); - // Assert.assertTrue(heavyHittersContainsString("k+")); + Assert.assertTrue(heavyHittersContainsString("fed_uacvar")); + Assert.assertTrue(heavyHittersContainsString("fed_tsmm")); // check that federated input files are still existing Assert.assertTrue(HDFSTool.existsFileOnHDFS(input("X1"))); diff --git a/src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedVarTest.java b/src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedVarTest.java index 9579fef..280f0d3 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedVarTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/algorithms/FederatedVarTest.java @@ -58,12 +58,8 @@ public class FederatedVarTest extends AutomatedTestBase { @Parameterized.Parameters public static Collection<Object[]> data() { return Arrays.asList(new Object[][] { - // {10, 1000, false}, - {100, 4, false}, - // {36, 1000, true}, - {1000, 10, true}, - // {4, 100, true} - // {1600, 8, false}, + {1000, 40, false}, + {1000, 400, true} }); } diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedFullAggregateTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedFullAggregateTest.java index 1617ab6..ec7bda6 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedFullAggregateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedFullAggregateTest.java @@ -43,6 +43,7 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { private final static String TEST_NAME2 = "FederatedMeanTest"; private final static String TEST_NAME3 = "FederatedMaxTest"; private final static String TEST_NAME4 = "FederatedMinTest"; + private final static String TEST_NAME5 = "FederatedVarTest"; private final static String TEST_DIR = "functions/federated/aggregate/"; private static final String TEST_CLASS_DIR = TEST_DIR + FederatedFullAggregateTest.class.getSimpleName() + "/"; @@ -68,7 +69,7 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { } private enum OpType { - SUM, MEAN, MAX, MIN + SUM, MEAN, MAX, MIN, VAR } @Override @@ -78,6 +79,7 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"S.scalar"})); addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"S.scalar"})); addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"S.scalar"})); + addTestConfiguration(TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"S.scalar"})); } @Test @@ -101,7 +103,11 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { } @Test - @Ignore + public void testVarDenseMatrixCP() { + runColAggregateOperationTest(OpType.VAR, ExecType.CP); + } + + @Test public void testSumDenseMatrixSP() { runColAggregateOperationTest(OpType.SUM, ExecType.SPARK); } @@ -124,6 +130,11 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { runColAggregateOperationTest(OpType.MIN, ExecType.SPARK); } + @Test + public void testVarDenseMatrixSP() { + runColAggregateOperationTest(OpType.VAR, ExecType.SPARK); + } + private void runColAggregateOperationTest(OpType type, ExecType instType) { ExecMode platformOld = rtplatform; switch(instType) { @@ -152,6 +163,9 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { case MIN: TEST_NAME = TEST_NAME4; break; + case VAR: + TEST_NAME = TEST_NAME5; + break; } getAndLoadTestConfiguration(TEST_NAME); @@ -165,10 +179,10 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { c = cols; } - double[][] X1 = getRandomMatrix(r, c, 1, 5, 1, 3); - double[][] X2 = getRandomMatrix(r, c, 1, 5, 1, 7); - double[][] X3 = getRandomMatrix(r, c, 1, 5, 1, 8); - double[][] X4 = getRandomMatrix(r, c, 1, 5, 1, 9); + double[][] X1 = getRandomMatrix(r, c, 1, 3, 1, 3); + double[][] X2 = getRandomMatrix(r, c, 1, 3, 1, 7); + double[][] X3 = getRandomMatrix(r, c, 1, 3, 1, 8); + double[][] X4 = getRandomMatrix(r, c, 1, 3, 1, 9); MatrixCharacteristics mc = new MatrixCharacteristics(r, c, blocksize, r * c); writeInputMatrixWithMTD("X1", X1, false, mc); @@ -209,7 +223,7 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { runTest(true, false, null, -1); // compare via files - compareResults(1e-9); + compareResults(type == OpType.VAR ? 1e-2 : 1e-9); switch(type) { case SUM: @@ -224,6 +238,9 @@ public class FederatedFullAggregateTest extends AutomatedTestBase { case MIN: Assert.assertTrue(heavyHittersContainsString("fed_uamin")); break; + case VAR: + Assert.assertTrue(heavyHittersContainsString("fed_uavar")); + break; } // check that federated input files are still existing diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRowColAggregateTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRowColAggregateTest.java index 7bfbeee..31800af 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRowColAggregateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/FederatedRowColAggregateTest.java @@ -45,6 +45,8 @@ public class FederatedRowColAggregateTest extends AutomatedTestBase { private final static String TEST_NAME6 = "FederatedRowMeanTest"; private final static String TEST_NAME7 = "FederatedRowMaxTest"; private final static String TEST_NAME8 = "FederatedRowMinTest"; + private final static String TEST_NAME9 = "FederatedRowVarTest"; + private final static String TEST_NAME10 = "FederatedColVarTest"; private final static String TEST_DIR = "functions/federated/aggregate/"; private static final String TEST_CLASS_DIR = TEST_DIR + FederatedRowColAggregateTest.class.getSimpleName() + "/"; @@ -60,13 +62,14 @@ public class FederatedRowColAggregateTest extends AutomatedTestBase { @Parameterized.Parameters public static Collection<Object[]> data() { return Arrays.asList( - new Object[][] {{10, 1000, false}, - //{100, 4, false}, {36, 1000, true}, {1000, 10, true}, {4, 100, true} + new Object[][] { + {10, 1000, false}, + {1000, 40, true}, }); } private enum OpType { - SUM, MEAN, MAX, MIN + SUM, MEAN, MAX, MIN, VAR } private enum InstType { @@ -84,6 +87,8 @@ public class FederatedRowColAggregateTest extends AutomatedTestBase { addTestConfiguration(TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"S"})); addTestConfiguration(TEST_NAME7, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME7, new String[] {"S"})); addTestConfiguration(TEST_NAME8, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME8, new String[] {"S"})); + addTestConfiguration(TEST_NAME9, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME9, new String[] {"S"})); + addTestConfiguration(TEST_NAME10, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME10, new String[] {"S"})); } @Test @@ -126,6 +131,16 @@ public class FederatedRowColAggregateTest extends AutomatedTestBase { runAggregateOperationTest(OpType.MIN, InstType.COL, ExecMode.SINGLE_NODE); } + @Test + public void testRowVarDenseMatrixCP() { + runAggregateOperationTest(OpType.VAR, InstType.ROW, ExecMode.SINGLE_NODE); + } + + @Test + public void testColVarDenseMatrixCP() { + runAggregateOperationTest(OpType.VAR, InstType.COL, ExecMode.SINGLE_NODE); + } + private void runAggregateOperationTest(OpType type, InstType instr, ExecMode execMode) { boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; ExecMode platformOld = rtplatform; @@ -147,6 +162,9 @@ public class FederatedRowColAggregateTest extends AutomatedTestBase { case MIN: TEST_NAME = instr == InstType.COL ? TEST_NAME4 : TEST_NAME8; break; + case VAR: + TEST_NAME = instr == InstType.COL ? TEST_NAME10 : TEST_NAME9; + break; } getAndLoadTestConfiguration(TEST_NAME); @@ -160,10 +178,10 @@ public class FederatedRowColAggregateTest extends AutomatedTestBase { c = cols; } - double[][] X1 = getRandomMatrix(r, c, 1, 5, 1, 3); - double[][] X2 = getRandomMatrix(r, c, 1, 5, 1, 7); - double[][] X3 = getRandomMatrix(r, c, 1, 5, 1, 8); - double[][] X4 = getRandomMatrix(r, c, 1, 5, 1, 9); + double[][] X1 = getRandomMatrix(r, c, 1, 3, 1, 3); + double[][] X2 = getRandomMatrix(r, c, 1, 3, 1, 7); + double[][] X3 = getRandomMatrix(r, c, 1, 3, 1, 8); + double[][] X4 = getRandomMatrix(r, c, 1, 3, 1, 9); MatrixCharacteristics mc = new MatrixCharacteristics(r, c, blocksize, r * c); writeInputMatrixWithMTD("X1", X1, false, mc); @@ -209,7 +227,7 @@ public class FederatedRowColAggregateTest extends AutomatedTestBase { runTest(true, false, null, -1); // compare via files - compareResults(1e-9); + compareResults(type == FederatedRowColAggregateTest.OpType.VAR ? 1e-2 : 1e-9); String fedInst = instr == InstType.COL ? "fed_uac" : "fed_uar"; @@ -226,6 +244,9 @@ public class FederatedRowColAggregateTest extends AutomatedTestBase { case MIN: Assert.assertTrue(heavyHittersContainsString(fedInst.concat("min"))); break; + case VAR: + Assert.assertTrue(heavyHittersContainsString(fedInst.concat("var"))); + break; } // check that federated input files are still existing diff --git a/scripts/builtin/dist.dml b/src/test/scripts/functions/federated/aggregate/FederatedColVarTest.dml similarity index 57% copy from scripts/builtin/dist.dml copy to src/test/scripts/functions/federated/aggregate/FederatedColVarTest.dml index e5fe930..186dc1d 100644 --- a/scripts/builtin/dist.dml +++ b/src/test/scripts/functions/federated/aggregate/FederatedColVarTest.dml @@ -17,14 +17,18 @@ # specific language governing permissions and limitations # under the License. # +#------------------------------------------------------------- -# Returns Euclidian distance matrix (distances between N n-dimensional points) -m_dist = function(Matrix[Double] X) return (Matrix[Double] Y) { - G = X %*% t(X); - I = matrix(1, rows = nrow(G), cols = ncol(G)); - Y = -2 * (G) + (diag(G) * I) + (I * t(diag(G))); -# Y = -2 * (G) + t(I %*% diag(diag(G))) + t(diag(diag(G)) %*% I); - Y = sqrt(Y); - Y = replace(target = Y, pattern=0/0, replacement = 0); +if ($rP) { + A = federated(addresses=list($in_X1, $in_X2, $in_X3, $in_X4), + ranges=list(list(0, 0), list($rows/4, $cols), list($rows/4, 0), list(2*$rows/4, $cols), + list(2*$rows/4, 0), list(3*$rows/4, $cols), list(3*$rows/4, 0), list($rows, $cols))); +} else { + A = federated(addresses=list($in_X1, $in_X2, $in_X3, $in_X4), + ranges=list(list(0, 0), list($rows, $cols/4), list(0,$cols/4), list($rows, $cols/2), + list(0,$cols/2), list($rows, 3*($cols/4)), list(0, 3*($cols/4)), list($rows, $cols))); } + +s = colVars(A); +write(s, $out_S); \ No newline at end of file diff --git a/scripts/builtin/dist.dml b/src/test/scripts/functions/federated/aggregate/FederatedColVarTestReference.dml similarity index 67% copy from scripts/builtin/dist.dml copy to src/test/scripts/functions/federated/aggregate/FederatedColVarTestReference.dml index e5fe930..ec9b021 100644 --- a/scripts/builtin/dist.dml +++ b/src/test/scripts/functions/federated/aggregate/FederatedColVarTestReference.dml @@ -17,14 +17,10 @@ # specific language governing permissions and limitations # under the License. # +#------------------------------------------------------------- -# Returns Euclidian distance matrix (distances between N n-dimensional points) +if($6) { A = rbind(read($1), read($2), read($3), read($4)); } +else { A = cbind(read($1), read($2), read($3), read($4)); } -m_dist = function(Matrix[Double] X) return (Matrix[Double] Y) { - G = X %*% t(X); - I = matrix(1, rows = nrow(G), cols = ncol(G)); - Y = -2 * (G) + (diag(G) * I) + (I * t(diag(G))); -# Y = -2 * (G) + t(I %*% diag(diag(G))) + t(diag(diag(G)) %*% I); - Y = sqrt(Y); - Y = replace(target = Y, pattern=0/0, replacement = 0); -} +s = colVars(A); +write(s, $5); diff --git a/scripts/builtin/dist.dml b/src/test/scripts/functions/federated/aggregate/FederatedRowVarTest.dml similarity index 57% copy from scripts/builtin/dist.dml copy to src/test/scripts/functions/federated/aggregate/FederatedRowVarTest.dml index e5fe930..8b4a57d 100644 --- a/scripts/builtin/dist.dml +++ b/src/test/scripts/functions/federated/aggregate/FederatedRowVarTest.dml @@ -17,14 +17,18 @@ # specific language governing permissions and limitations # under the License. # +#------------------------------------------------------------- -# Returns Euclidian distance matrix (distances between N n-dimensional points) -m_dist = function(Matrix[Double] X) return (Matrix[Double] Y) { - G = X %*% t(X); - I = matrix(1, rows = nrow(G), cols = ncol(G)); - Y = -2 * (G) + (diag(G) * I) + (I * t(diag(G))); -# Y = -2 * (G) + t(I %*% diag(diag(G))) + t(diag(diag(G)) %*% I); - Y = sqrt(Y); - Y = replace(target = Y, pattern=0/0, replacement = 0); +if ($rP) { + A = federated(addresses=list($in_X1, $in_X2, $in_X3, $in_X4), + ranges=list(list(0, 0), list($rows/4, $cols), list($rows/4, 0), list(2*$rows/4, $cols), + list(2*$rows/4, 0), list(3*$rows/4, $cols), list(3*$rows/4, 0), list($rows, $cols))); +} else { + A = federated(addresses=list($in_X1, $in_X2, $in_X3, $in_X4), + ranges=list(list(0, 0), list($rows, $cols/4), list(0,$cols/4), list($rows, $cols/2), + list(0,$cols/2), list($rows, 3*($cols/4)), list(0, 3*($cols/4)), list($rows, $cols))); } + +s = rowVars(A); +write(s, $out_S); \ No newline at end of file diff --git a/scripts/builtin/dist.dml b/src/test/scripts/functions/federated/aggregate/FederatedRowVarTestReference.dml similarity index 67% copy from scripts/builtin/dist.dml copy to src/test/scripts/functions/federated/aggregate/FederatedRowVarTestReference.dml index e5fe930..e983899 100644 --- a/scripts/builtin/dist.dml +++ b/src/test/scripts/functions/federated/aggregate/FederatedRowVarTestReference.dml @@ -17,14 +17,10 @@ # specific language governing permissions and limitations # under the License. # +#------------------------------------------------------------- -# Returns Euclidian distance matrix (distances between N n-dimensional points) +if($6) { A = rbind(read($1), read($2), read($3), read($4)); } +else { A = cbind(read($1), read($2), read($3), read($4)); } -m_dist = function(Matrix[Double] X) return (Matrix[Double] Y) { - G = X %*% t(X); - I = matrix(1, rows = nrow(G), cols = ncol(G)); - Y = -2 * (G) + (diag(G) * I) + (I * t(diag(G))); -# Y = -2 * (G) + t(I %*% diag(diag(G))) + t(diag(diag(G)) %*% I); - Y = sqrt(Y); - Y = replace(target = Y, pattern=0/0, replacement = 0); -} +s = rowVars(A); +write(s, $5); diff --git a/scripts/builtin/dist.dml b/src/test/scripts/functions/federated/aggregate/FederatedVarTest.dml similarity index 57% copy from scripts/builtin/dist.dml copy to src/test/scripts/functions/federated/aggregate/FederatedVarTest.dml index e5fe930..88fd6a8 100644 --- a/scripts/builtin/dist.dml +++ b/src/test/scripts/functions/federated/aggregate/FederatedVarTest.dml @@ -17,14 +17,17 @@ # specific language governing permissions and limitations # under the License. # +#------------------------------------------------------------- -# Returns Euclidian distance matrix (distances between N n-dimensional points) - -m_dist = function(Matrix[Double] X) return (Matrix[Double] Y) { - G = X %*% t(X); - I = matrix(1, rows = nrow(G), cols = ncol(G)); - Y = -2 * (G) + (diag(G) * I) + (I * t(diag(G))); -# Y = -2 * (G) + t(I %*% diag(diag(G))) + t(diag(diag(G)) %*% I); - Y = sqrt(Y); - Y = replace(target = Y, pattern=0/0, replacement = 0); +if ($rP) { + A = federated(addresses=list($in_X1, $in_X2, $in_X3, $in_X4), + ranges=list(list(0, 0), list($rows/4, $cols), list($rows/4, 0), list(2*$rows/4, $cols), + list(2*$rows/4, 0), list(3*$rows/4, $cols), list(3*$rows/4, 0), list($rows, $cols))); +} else { + A = federated(addresses=list($in_X1, $in_X2, $in_X3, $in_X4), + ranges=list(list(0, 0), list($rows, $cols/4), list(0,$cols/4), list($rows, $cols/2), + list(0,$cols/2), list($rows, 3*($cols/4)), list(0, 3*($cols/4)), list($rows, $cols))); } + +s = var(A); +write(s, $out_S); diff --git a/scripts/builtin/dist.dml b/src/test/scripts/functions/federated/aggregate/FederatedVarTestReference.dml similarity index 67% copy from scripts/builtin/dist.dml copy to src/test/scripts/functions/federated/aggregate/FederatedVarTestReference.dml index e5fe930..af98e13 100644 --- a/scripts/builtin/dist.dml +++ b/src/test/scripts/functions/federated/aggregate/FederatedVarTestReference.dml @@ -17,14 +17,10 @@ # specific language governing permissions and limitations # under the License. # +#------------------------------------------------------------- -# Returns Euclidian distance matrix (distances between N n-dimensional points) +if($6) { A = rbind(read($1), read($2), read($3), read($4)); } +else { A = cbind(read($1), read($2), read($3), read($4)); } -m_dist = function(Matrix[Double] X) return (Matrix[Double] Y) { - G = X %*% t(X); - I = matrix(1, rows = nrow(G), cols = ncol(G)); - Y = -2 * (G) + (diag(G) * I) + (I * t(diag(G))); -# Y = -2 * (G) + t(I %*% diag(diag(G))) + t(diag(diag(G)) %*% I); - Y = sqrt(Y); - Y = replace(target = Y, pattern=0/0, replacement = 0); -} +s = var(A); +write(s, $5);
