sebwrede commented on a change in pull request #919: URL: https://github.com/apache/systemml/pull/919#discussion_r430243089
########## File path: src/main/java/org/apache/sysds/runtime/privacy/PrivacyConstraint.java ########## @@ -24,19 +24,26 @@ */ public class PrivacyConstraint { - protected boolean _privacy = false; + + public enum PrivacyLevel { Review comment: > Nested enum types are implicitly static https://docs.oracle.com/javase/specs/jls/se7/html/jls-8.html#jls-8.9 ########## File path: src/test/java/org/apache/sysds/test/functions/privacy/FederatedWorkerHandlerTest.java ########## @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.privacy; + +import java.util.Arrays; + +import org.apache.sysds.api.DMLException; +import org.apache.sysds.api.DMLScript; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.privacy.PrivacyConstraint; +import org.apache.sysds.runtime.privacy.PrivacyConstraint.PrivacyLevel; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; +import org.apache.sysds.common.Types; +import static java.lang.Thread.sleep; + +public class FederatedWorkerHandlerTest extends AutomatedTestBase { Review comment: How would you like it to be indented? ########## File path: src/test/java/org/apache/sysds/test/functions/privacy/MatrixMultiplicationPropagationTest.java ########## @@ -46,26 +47,36 @@ public void setUp() { } @Test - public void testMatrixMultiplicationPropagation() throws JSONException { - matrixMultiplicationPropagation(true, true); + public void testMatrixMultiplicationPropagationPrivate() throws JSONException { Review comment: Yes, that is on purpose. It throws a JSONException if the privacy constraints are not written to the result metadata file. ########## File path: src/test/java/org/apache/sysds/test/functions/privacy/MatrixRuntimePropagationTest.java ########## @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.privacy; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import org.apache.sysds.parser.DataExpression; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.privacy.PrivacyConstraint; +import org.apache.sysds.runtime.privacy.PrivacyConstraint.PrivacyLevel; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.apache.wink.json4j.JSONException; +import org.junit.Test; + +public class MatrixRuntimePropagationTest extends AutomatedTestBase { + + private static final String TEST_DIR = "functions/privacy/"; + private final static String TEST_CLASS_DIR = TEST_DIR + MatrixMultiplicationPropagationTest.class.getSimpleName() + "/"; + private final int m = 20; + private final int n = 20; + private final int k = 20; Review comment: Yes, I am using the same template as many other tests in the system. If we change the name of the variable in this test, then we should do the same for the other tests as well. ########## File path: src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java ########## @@ -598,192 +545,116 @@ else if ( getInput1().getDataType() == DataType.SCALAR ){ break; case RemoveVariableAndFile: - // Remove the variable from HashMap _variables, and possibly delete the data on disk. - boolean del = ( (BooleanObject) ec.getScalarInput(getInput2().getName(), getInput2().getValueType(), true) ).getBooleanValue(); - MatrixObject m = (MatrixObject) ec.removeVariable(getInput1().getName()); - - if ( !del ) { - // HDFS file should be retailed after clearData(), - // therefore data must be exported if dirty flag is set - if ( m.isDirty() ) - m.exportData(); - } - else { - //throw new DMLRuntimeException("rmfilevar w/ true is not expected! " + instString); - //cleanDataOnHDFS(pb, input1.getName()); - cleanDataOnHDFS( m ); - } - - // check if in-memory object can be cleaned up - if ( !ec.getVariables().hasReferences(m) ) { - // no other variable in the symbol table points to the same Data object as that of input1.getName() - - //remove matrix object from cache - m.clearData(); - } - + processRemoveVariableAndFileInstruction(ec); break; case CastAsScalarVariable: //castAsScalarVariable - if( getInput1().getDataType().isFrame() ) { - FrameBlock fBlock = ec.getFrameInput(getInput1().getName()); - if( fBlock.getNumRows()!=1 || fBlock.getNumColumns()!=1 ) - throw new DMLRuntimeException("Dimension mismatch - unable to cast frame '"+getInput1().getName()+"' of dimension ("+fBlock.getNumRows()+" x "+fBlock.getNumColumns()+") to scalar."); - Object value = fBlock.get(0,0); - ec.releaseFrameInput(getInput1().getName()); - ec.setScalarOutput(output.getName(), - ScalarObjectFactory.createScalarObject(fBlock.getSchema()[0], value)); - } - else if( getInput1().getDataType().isMatrix() ) { - MatrixBlock mBlock = ec.getMatrixInput(getInput1().getName()); - if( mBlock.getNumRows()!=1 || mBlock.getNumColumns()!=1 ) - throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix '"+getInput1().getName()+"' of dimension ("+mBlock.getNumRows()+" x "+mBlock.getNumColumns()+") to scalar."); - double value = mBlock.getValue(0,0); - ec.releaseMatrixInput(getInput1().getName()); - ec.setScalarOutput(output.getName(), new DoubleObject(value)); - } - else if( getInput1().getDataType().isTensor() ) { - TensorBlock tBlock = ec.getTensorInput(getInput1().getName()); - if (tBlock.getNumDims() != 2 || tBlock.getNumRows() != 1 || tBlock.getNumColumns() != 1) - throw new DMLRuntimeException("Dimension mismatch - unable to cast tensor '" + getInput1().getName() + "' to scalar."); - ValueType vt = !tBlock.isBasic() ? tBlock.getSchema()[0] : tBlock.getValueType(); - ec.setScalarOutput(output.getName(), ScalarObjectFactory - .createScalarObject(vt, tBlock.get(new int[] {0, 0}))); - ec.releaseTensorInput(getInput1().getName()); - } - else if( getInput1().getDataType().isList() ) { - //TODO handling of cleanup status, potentially new object - ListObject list = (ListObject)ec.getVariable(getInput1().getName()); - ec.setVariable(output.getName(), list.slice(0)); - } - else { - throw new DMLRuntimeException("Unsupported data type " - + "in as.scalar(): "+getInput1().getDataType().name()); - } + processCastAsScalarVariableInstruction(ec); break; - case CastAsMatrixVariable:{ - if( getInput1().getDataType().isFrame() ) { - FrameBlock fin = ec.getFrameInput(getInput1().getName()); - MatrixBlock out = DataConverter.convertToMatrixBlock(fin); - ec.releaseFrameInput(getInput1().getName()); - ec.setMatrixOutput(output.getName(), out); - } - else if( getInput1().getDataType().isScalar() ) { - ScalarObject scalarInput = ec.getScalarInput( - getInput1().getName(), getInput1().getValueType(), getInput1().isLiteral()); - MatrixBlock out = new MatrixBlock(scalarInput.getDoubleValue()); - ec.setMatrixOutput(output.getName(), out); - } - else if( getInput1().getDataType().isList() ) { - //TODO handling of cleanup status, potentially new object - ListObject list = (ListObject)ec.getVariable(getInput1().getName()); - if( list.getLength() > 1 ) { - if( !list.checkAllDataTypes(DataType.SCALAR) ) - throw new DMLRuntimeException("as.matrix over multi-entry list only allows scalars."); - MatrixBlock out = new MatrixBlock(list.getLength(), 1, false); - for( int i=0; i<list.getLength(); i++ ) - out.quickSetValue(i, 0, ((ScalarObject)list.slice(i)).getDoubleValue()); - ec.setMatrixOutput(output.getName(), out); - } - else { - //pass through matrix input or create 1x1 matrix for scalar - Data tmp = list.slice(0); - if( tmp instanceof ScalarObject && tmp.getValueType()!=ValueType.STRING ) { - MatrixBlock out = new MatrixBlock(((ScalarObject)tmp).getDoubleValue()); - ec.setMatrixOutput(output.getName(), out); - } - else { - ec.setVariable(output.getName(), tmp); - } - } - } - else { - throw new DMLRuntimeException("Unsupported data type " - + "in as.matrix(): "+getInput1().getDataType().name()); - } + + case CastAsMatrixVariable: + processCastAsMatrixVariableInstruction(ec); break; - } - case CastAsFrameVariable:{ - FrameBlock out = null; - if( getInput1().getDataType()==DataType.SCALAR ) { - ScalarObject scalarInput = ec.getScalarInput(getInput1()); - out = new FrameBlock(1, getInput1().getValueType()); - out.ensureAllocatedColumns(1); - out.set(0, 0, scalarInput.getStringValue()); - } - else { //DataType.FRAME - MatrixBlock min = ec.getMatrixInput(getInput1().getName()); - out = DataConverter.convertToFrameBlock(min); - ec.releaseMatrixInput(getInput1().getName()); - } - ec.setFrameOutput(output.getName(), out); + + case CastAsFrameVariable: + processCastAsFrameVariableInstruction(ec); break; - } - case CastAsDoubleVariable:{ - ScalarObject in = ec.getScalarInput(getInput1()); - ec.setScalarOutput(output.getName(), ScalarObjectFactory.castToDouble(in)); + + case CastAsDoubleVariable: + ScalarObject scalarDoubleInput = ec.getScalarInput(getInput1()); + ec.setScalarOutput(output.getName(), ScalarObjectFactory.castToDouble(scalarDoubleInput)); break; - } - case CastAsIntegerVariable:{ - ScalarObject in = ec.getScalarInput(getInput1()); - ec.setScalarOutput(output.getName(), ScalarObjectFactory.castToLong(in)); + + case CastAsIntegerVariable: + ScalarObject scalarLongInput = ec.getScalarInput(getInput1()); + ec.setScalarOutput(output.getName(), ScalarObjectFactory.castToLong(scalarLongInput)); break; - } - case CastAsBooleanVariable:{ - ScalarObject scalarInput = ec.getScalarInput(getInput1()); - ec.setScalarOutput(output.getName(), new BooleanObject(scalarInput.getBooleanValue())); + + case CastAsBooleanVariable: + ScalarObject scalarBooleanInput = ec.getScalarInput(getInput1()); + ec.setScalarOutput(output.getName(), new BooleanObject(scalarBooleanInput.getBooleanValue())); break; - } case Read: - ScalarObject res = null; - try { - switch(getInput1().getValueType()) { - case FP64: - res = new DoubleObject(HDFSTool.readDoubleFromHDFSFile(getInput2().getName())); - break; - case INT64: - res = new IntObject(HDFSTool.readIntegerFromHDFSFile(getInput2().getName())); - break; - case BOOLEAN: - res = new BooleanObject(HDFSTool.readBooleanFromHDFSFile(getInput2().getName())); - break; - case STRING: - res = new StringObject(HDFSTool.readStringFromHDFSFile(getInput2().getName())); - break; - default: - throw new DMLRuntimeException("Invalid value type (" - + getInput1().getValueType() + ") while processing readScalar instruction."); - } - } catch ( IOException e ) { - throw new DMLRuntimeException(e); - } - ec.setScalarOutput(getInput1().getName(), res); - + processReadInstruction(ec); break; case Write: processWriteInstruction(ec); break; case SetFileName: - Data data = ec.getVariable(getInput1().getName()); - if ( data.getDataType() == DataType.MATRIX ) { - if ( getInput3().getName().equalsIgnoreCase("remote") ) { - ((MatrixObject)data).setFileName(getInput2().getName()); - } - else { - throw new DMLRuntimeException("Invalid location (" + getInput3().getName() + ") in SetFileName instruction: " + instString); - } - } else{ - throw new DMLRuntimeException("Invalid data type (" + getInput1().getDataType() + ") in SetFileName instruction: " + instString); - } + processSetFileNameInstruction(ec); break; default: throw new DMLRuntimeException("Unknown opcode: " + opcode ); } } + + /** + * Handler for processInstruction "CreateVariable" case + * @param ec execution context of the instruction + */ + private void processCreateVariableInstruction(ExecutionContext ec){ + //PRE: for robustness we cleanup existing variables, because a setVariable + //would cause a buffer pool memory leak as these objects would never be removed + if(ec.containsVariable(getInput1())) + processRemoveVariableInstruction(ec, getInput1().getName()); + + if ( getInput1().getDataType() == DataType.MATRIX ) { + String fname = createUniqueFilename(); + MatrixObject obj = new MatrixObject(getInput1().getValueType(), fname); + setCacheableDataFields(obj); + obj.setUpdateType(_updateType); + obj.setMarkForLinCache(true); + ec.setVariable(getInput1().getName(), obj); + if(DMLScript.STATISTICS && _updateType.isInPlace()) + Statistics.incrementTotalUIPVar(); + } + else if( getInput1().getDataType() == DataType.TENSOR ) { + String fname = createUniqueFilename(); + CacheableData<?> obj = new TensorObject(getInput1().getValueType(), fname); + setCacheableDataFields(obj); + ec.setVariable(getInput1().getName(), obj); + } + else if( getInput1().getDataType() == DataType.FRAME ) { + String fname = createUniqueFilename(); + FrameObject fobj = new FrameObject(fname); + setCacheableDataFields(fobj); + if( _schema != null ) + fobj.setSchema(_schema); //after metadata + ec.setVariable(getInput1().getName(), fobj); + } + else if ( getInput1().getDataType() == DataType.SCALAR ){ + //created variable not called for scalars + ec.setScalarOutput(getInput1().getName(), null); + } + else { Review comment: Yes, it could be. The if-statement was like this before. I just changed the code inside the if-cases. Should I change it to a switch statement? ########## File path: src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java ########## @@ -843,6 +714,192 @@ else if (object instanceof FrameObject) + ((FrameObject)object).getNumColumns() + "," + ((FrameObject)object).getNumColumns() + "] to " + getInput3().getName()); } } + + /** + * Handler for RemoveVariableAndFile instruction + * + * @param ec execution context + */ + private void processRemoveVariableAndFileInstruction(ExecutionContext ec){ + // Remove the variable from HashMap _variables, and possibly delete the data on disk. + boolean del = ( (BooleanObject) ec.getScalarInput(getInput2().getName(), getInput2().getValueType(), true) ).getBooleanValue(); + MatrixObject m = (MatrixObject) ec.removeVariable(getInput1().getName()); + + if ( !del ) { + // HDFS file should be retailed after clearData(), + // therefore data must be exported if dirty flag is set + if ( m.isDirty() ) + m.exportData(); + } + else { + //throw new DMLRuntimeException("rmfilevar w/ true is not expected! " + instString); + //cleanDataOnHDFS(pb, input1.getName()); + cleanDataOnHDFS( m ); + } + + // check if in-memory object can be cleaned up + if ( !ec.getVariables().hasReferences(m) ) { + // no other variable in the symbol table points to the same Data object as that of input1.getName() + + //remove matrix object from cache + m.clearData(); + } + } + + /** + * Process CastAsScalarVariable instruction. + * @param ec execution context + */ + private void processCastAsScalarVariableInstruction(ExecutionContext ec){ + //TODO: Create privacy constraints for ScalarObject so that the privacy constraints can be propagated to scalars as well. + blockIfInputPrivacyActivated(ec.getVariable(getInput1())); + + if( getInput1().getDataType().isFrame() ) { + FrameBlock fBlock = ec.getFrameInput(getInput1().getName()); + if( fBlock.getNumRows()!=1 || fBlock.getNumColumns()!=1 ) + throw new DMLRuntimeException("Dimension mismatch - unable to cast frame '"+getInput1().getName()+"' of dimension ("+fBlock.getNumRows()+" x "+fBlock.getNumColumns()+") to scalar."); + Object value = fBlock.get(0,0); + ec.releaseFrameInput(getInput1().getName()); + ec.setScalarOutput(output.getName(), + ScalarObjectFactory.createScalarObject(fBlock.getSchema()[0], value)); + } + else if( getInput1().getDataType().isMatrix() ) { + MatrixBlock mBlock = ec.getMatrixInput(getInput1().getName()); + if( mBlock.getNumRows()!=1 || mBlock.getNumColumns()!=1 ) + throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix '"+getInput1().getName()+"' of dimension ("+mBlock.getNumRows()+" x "+mBlock.getNumColumns()+") to scalar."); + double value = mBlock.getValue(0,0); + ec.releaseMatrixInput(getInput1().getName()); + ec.setScalarOutput(output.getName(), new DoubleObject(value)); + } + else if( getInput1().getDataType().isTensor() ) { + TensorBlock tBlock = ec.getTensorInput(getInput1().getName()); + if (tBlock.getNumDims() != 2 || tBlock.getNumRows() != 1 || tBlock.getNumColumns() != 1) + throw new DMLRuntimeException("Dimension mismatch - unable to cast tensor '" + getInput1().getName() + "' to scalar."); + ValueType vt = !tBlock.isBasic() ? tBlock.getSchema()[0] : tBlock.getValueType(); + ec.setScalarOutput(output.getName(), ScalarObjectFactory + .createScalarObject(vt, tBlock.get(new int[] {0, 0}))); + ec.releaseTensorInput(getInput1().getName()); + } + else if( getInput1().getDataType().isList() ) { + //TODO handling of cleanup status, potentially new object + ListObject list = (ListObject)ec.getVariable(getInput1().getName()); + ec.setVariable(output.getName(), list.slice(0)); + } + else { Review comment: It is called in different cases. This is a cast as scalar operation. The other one was creating a variable. Both could easily be changed to switch-statements, but they have not been like this until now. I just moved the code to separate methods and tried to clean the if-cases a bit. ########## File path: src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java ########## @@ -843,6 +714,192 @@ else if (object instanceof FrameObject) + ((FrameObject)object).getNumColumns() + "," + ((FrameObject)object).getNumColumns() + "] to " + getInput3().getName()); } } + + /** + * Handler for RemoveVariableAndFile instruction + * + * @param ec execution context + */ + private void processRemoveVariableAndFileInstruction(ExecutionContext ec){ + // Remove the variable from HashMap _variables, and possibly delete the data on disk. + boolean del = ( (BooleanObject) ec.getScalarInput(getInput2().getName(), getInput2().getValueType(), true) ).getBooleanValue(); + MatrixObject m = (MatrixObject) ec.removeVariable(getInput1().getName()); + + if ( !del ) { + // HDFS file should be retailed after clearData(), + // therefore data must be exported if dirty flag is set + if ( m.isDirty() ) + m.exportData(); + } + else { + //throw new DMLRuntimeException("rmfilevar w/ true is not expected! " + instString); + //cleanDataOnHDFS(pb, input1.getName()); + cleanDataOnHDFS( m ); + } + + // check if in-memory object can be cleaned up + if ( !ec.getVariables().hasReferences(m) ) { + // no other variable in the symbol table points to the same Data object as that of input1.getName() + + //remove matrix object from cache + m.clearData(); + } + } + + /** + * Process CastAsScalarVariable instruction. + * @param ec execution context + */ + private void processCastAsScalarVariableInstruction(ExecutionContext ec){ + //TODO: Create privacy constraints for ScalarObject so that the privacy constraints can be propagated to scalars as well. + blockIfInputPrivacyActivated(ec.getVariable(getInput1())); + + if( getInput1().getDataType().isFrame() ) { + FrameBlock fBlock = ec.getFrameInput(getInput1().getName()); + if( fBlock.getNumRows()!=1 || fBlock.getNumColumns()!=1 ) + throw new DMLRuntimeException("Dimension mismatch - unable to cast frame '"+getInput1().getName()+"' of dimension ("+fBlock.getNumRows()+" x "+fBlock.getNumColumns()+") to scalar."); + Object value = fBlock.get(0,0); + ec.releaseFrameInput(getInput1().getName()); + ec.setScalarOutput(output.getName(), + ScalarObjectFactory.createScalarObject(fBlock.getSchema()[0], value)); + } + else if( getInput1().getDataType().isMatrix() ) { + MatrixBlock mBlock = ec.getMatrixInput(getInput1().getName()); + if( mBlock.getNumRows()!=1 || mBlock.getNumColumns()!=1 ) + throw new DMLRuntimeException("Dimension mismatch - unable to cast matrix '"+getInput1().getName()+"' of dimension ("+mBlock.getNumRows()+" x "+mBlock.getNumColumns()+") to scalar."); + double value = mBlock.getValue(0,0); + ec.releaseMatrixInput(getInput1().getName()); + ec.setScalarOutput(output.getName(), new DoubleObject(value)); + } + else if( getInput1().getDataType().isTensor() ) { + TensorBlock tBlock = ec.getTensorInput(getInput1().getName()); + if (tBlock.getNumDims() != 2 || tBlock.getNumRows() != 1 || tBlock.getNumColumns() != 1) + throw new DMLRuntimeException("Dimension mismatch - unable to cast tensor '" + getInput1().getName() + "' to scalar."); + ValueType vt = !tBlock.isBasic() ? tBlock.getSchema()[0] : tBlock.getValueType(); + ec.setScalarOutput(output.getName(), ScalarObjectFactory + .createScalarObject(vt, tBlock.get(new int[] {0, 0}))); + ec.releaseTensorInput(getInput1().getName()); + } + else if( getInput1().getDataType().isList() ) { + //TODO handling of cleanup status, potentially new object + ListObject list = (ListObject)ec.getVariable(getInput1().getName()); + ec.setVariable(output.getName(), list.slice(0)); + } + else { + throw new DMLRuntimeException("Unsupported data type " + + "in as.scalar(): "+getInput1().getDataType().name()); + } + } + + + /** + * Throw DMLPrivacyException if privacy is activated for the input variable + * @param input variable for which the privacy constraint is checked + */ + private void blockIfInputPrivacyActivated(Data input){ + if ( input != null && (input instanceof CacheableData<?>)){ + PrivacyConstraint privacyConstraintIn = ((CacheableData<?>) input).getPrivacyConstraint(); + if ( privacyConstraintIn != null && (privacyConstraintIn.getPrivacyLevel() == PrivacyLevel.Private) ){ + throw new DMLPrivacyException("Privacy constraint cannot be propagated to scalar for input " + getInput1().getName()); + } + } Review comment: It has only been added to CacheableData. It could be stored in other types as well, but that is future work. ########## File path: src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java ########## @@ -956,7 +1031,7 @@ private void writeCSVFile(ExecutionContext ec, String fname) { else { mo.exportData(fname, outFmt, _formatProperties); } - HDFSTool.writeMetaDataFile (fname + ".mtd", mo.getValueType(), dc, FileFormat.CSV, _formatProperties); + HDFSTool.writeMetaDataFile (fname + ".mtd", mo.getValueType(), dc, FileFormat.CSV, _formatProperties, mo.getPrivacyConstraint()); Review comment: What extra space? ########## File path: src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java ########## @@ -898,20 +955,38 @@ else if( fmt == FileFormat.CSV ) else { // Default behavior MatrixObject mo = ec.getMatrixObject(getInput1().getName()); - mo.setPrivacyConstraints(getPrivacyConstraint()); mo.exportData(fname, fmtStr, _formatProperties); } + // Set privacy constraint of write instruction to the same as that of the input + setPrivacyConstraint(ec.getMatrixObject(getInput1().getName()).getPrivacyConstraint()); } else if( getInput1().getDataType() == DataType.FRAME ) { FrameObject mo = ec.getFrameObject(getInput1().getName()); mo.exportData(fname, fmtStr, _formatProperties); + setPrivacyConstraint(mo.getPrivacyConstraint()); } else if( getInput1().getDataType() == DataType.TENSOR ) { Review comment: It should not have any influence. The privacy constraint is read from _mo_ and set in _this_. The call to exportData exports data from _mo_. ########## File path: src/test/java/org/apache/sysds/test/functions/privacy/FederatedL2SVMTest.java ########## @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.privacy; + +import org.junit.Test; +import org.apache.sysds.api.DMLException; +import org.apache.sysds.api.DMLScript; +import org.apache.sysds.common.Types; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.privacy.PrivacyConstraint; +import org.apache.sysds.runtime.privacy.PrivacyConstraint.PrivacyLevel; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.apache.wink.json4j.JSONException; + +import java.util.HashMap; +import java.util.Map; + +@net.jcip.annotations.NotThreadSafe +public class FederatedL2SVMTest extends AutomatedTestBase { + + private final static String TEST_DIR = "functions/federated/"; + private final static String TEST_NAME = "FederatedL2SVMTest"; + private final static String TEST_CLASS_DIR = TEST_DIR + FederatedL2SVMTest.class.getSimpleName() + "/"; + + private final static int blocksize = 1024; + private int rows = 100; + private int cols = 10; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"Z"})); + } + + // PrivateAggregation Single Input + + @Test + public void federatedL2SVMCPPrivateAggregationX1() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation); + } + + @Test + public void federatedL2SVMCPPrivateAggregationX2() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation); + } + + @Test + public void federatedL2SVMCPPrivateAggregationY() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation); + } + + // Private Single Input + + @Test + public void federatedL2SVMCPPrivateFederatedX1() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateFederatedX2() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateFederatedY() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private); + } + + // Setting Privacy of Matrix (Throws Exception) + + @Test + public void federatedL2SVMCPPrivateMatrixX1() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, null, privacyConstraints, PrivacyLevel.Private, true, DMLException.class, false, null); + } + + @Test + public void federatedL2SVMCPPrivateMatrixX2() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, null, privacyConstraints, PrivacyLevel.Private, true, DMLException.class, false, null); + } + + @Test + public void federatedL2SVMCPPrivateMatrixY() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, null, privacyConstraints, PrivacyLevel.Private, true, DMLException.class, false, null); + } + + @Test + public void federatedL2SVMCPPrivateFederatedAndMatrixX1() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, privacyConstraints, PrivacyLevel.Private, true, DMLException.class, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateFederatedAndMatrixX2() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, privacyConstraints, PrivacyLevel.Private, true, DMLException.class, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateFederatedAndMatrixY() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, privacyConstraints, PrivacyLevel.Private, true, DMLException.class, false, null); + } + + // Privacy Level Private Combinations + + @Test + public void federatedL2SVMCPPrivateFederatedX1X2() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateFederatedX1Y() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateFederatedX2Y() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateFederatedX1X2Y() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + // Privacy Level PrivateAggregation Combinations + @Test + public void federatedL2SVMCPPrivateAggregationFederatedX1X2() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation); + } + + @Test + public void federatedL2SVMCPPrivateAggregationFederatedX1Y() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation); + } + + @Test + public void federatedL2SVMCPPrivateAggregationFederatedX2Y() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation); + } + + @Test + public void federatedL2SVMCPPrivateAggregationFederatedX1X2Y() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation); + } + + // Privacy Level Combinations + @Test + public void federatedL2SVMCPPrivatePrivateAggregationFederatedX1X2() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivatePrivateAggregationFederatedX1Y() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivatePrivateAggregationFederatedX2Y() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivatePrivateAggregationFederatedYX1() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private); + } + + @Test + public void federatedL2SVMCPPrivatePrivateAggregationFederatedYX2() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("Y", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVMNoException(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private); + } + + @Test + public void federatedL2SVMCPPrivatePrivateAggregationFederatedX2X1() throws JSONException { + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.Private)); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + // Require Federated Workers to return matrix + + @Test + public void federatedL2SVMCPPrivateAggregationX1Exception() throws JSONException { + this.rows = 1000; + this.cols = 1; + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateAggregationX2Exception() throws JSONException { + this.rows = 1000; + this.cols = 1; + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X2", new PrivacyConstraint(PrivacyLevel.PrivateAggregation)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.PrivateAggregation, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateX1Exception() throws JSONException { + this.rows = 1000; + this.cols = 1; + Map<String, PrivacyConstraint> privacyConstraints = new HashMap<>(); + privacyConstraints.put("X1", new PrivacyConstraint(PrivacyLevel.Private)); + federatedL2SVM(Types.ExecMode.SINGLE_NODE, privacyConstraints, null, PrivacyLevel.Private, false, null, true, DMLException.class); + } + + @Test + public void federatedL2SVMCPPrivateX2Exception() throws JSONException { Review comment: Are you sure that it will crash? When I run it locally, the test will just fail and it will print the exception. This is easier to work with compared to if the exception is caught. ########## File path: src/test/java/org/apache/sysds/test/functions/privacy/FederatedWorkerHandlerTest.java ########## @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.privacy; + +import java.util.Arrays; + +import org.apache.sysds.api.DMLException; +import org.apache.sysds.api.DMLScript; +import org.apache.sysds.runtime.meta.MatrixCharacteristics; +import org.apache.sysds.runtime.privacy.PrivacyConstraint; +import org.apache.sysds.runtime.privacy.PrivacyConstraint.PrivacyLevel; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; +import org.apache.sysds.common.Types; +import static java.lang.Thread.sleep; + +public class FederatedWorkerHandlerTest extends AutomatedTestBase { Review comment: I am using the codeformatter. I will change the spaces to tabs, if that is better. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org