This is an automated email from the ASF dual-hosted git repository. mboehm7 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/systemml.git
The following commit(s) were added to refs/heads/master by this push: new 47924e6 [SYSTEMML-2538] Fix csv/text output rename in forced singlenode 47924e6 is described below commit 47924e6aced3dac0768756c7dfec932d696b6a3f Author: Matthias Boehm <mboe...@gmail.com> AuthorDate: Sat Apr 11 22:51:50 2020 +0200 [SYSTEMML-2538] Fix csv/text output rename in forced singlenode This patch fixes an issue where an input csv/text file is directly fed into a persistent write, which eventually just renames the input file because it already exist on HDFS in the right format. We now explicitly guard against persistently read inputs, which only can occur w/ forced singelnode execution mode because other (in spark and hybrid) there is a reblock (potentially in memory) that creates a new metadata object. Furthermore, this also includes a minor internal refactoring for consistently obtaining input/output infos for external format strings, as well as a slight modification of the MatrixMatrixCellwiseTest to run over smaller inputs (because R is taken quite a while for them). --- .../java/org/apache/sysds/api/jmlc/Connection.java | 4 +- .../org/apache/sysds/parser/DataExpression.java | 4 +- .../controlprogram/caching/CacheableData.java | 2 +- .../federated/FederatedWorkerHandler.java | 2 +- .../instructions/cp/VariableCPInstruction.java | 38 ++- .../sysds/runtime/io/MatrixReaderFactory.java | 2 +- .../sysds/runtime/matrix/data/InputInfo.java | 8 +- .../sysds/runtime/matrix/data/OutputInfo.java | 2 +- .../org/apache/sysds/test/AutomatedTestBase.java | 14 +- .../FullMatrixMatrixCellwiseOperationTest.java | 4 +- .../test/functions/data/misc/NoRenameTest.java | 254 +++++++++++++++++++++ .../functions/frame/FrameMatrixReblockTest.java | 5 +- .../test/functions/frame/FrameMatrixWriteTest.java | 2 +- .../transform/TransformEncodeDecodeTest.java | 2 +- src/test/scripts/functions/data/NoRenameTest1.dml | 24 ++ src/test/scripts/functions/data/NoRenameTest2.dml | 24 ++ 16 files changed, 345 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/jmlc/Connection.java b/src/main/java/org/apache/sysds/api/jmlc/Connection.java index e1557b8..a008939 100644 --- a/src/main/java/org/apache/sysds/api/jmlc/Connection.java +++ b/src/main/java/org/apache/sysds/api/jmlc/Connection.java @@ -372,7 +372,7 @@ public class Connection implements Closeable long nnz = jmtd.containsKey(DataExpression.READNNZPARAM)? jmtd.getLong(DataExpression.READNNZPARAM) : -1; String format = jmtd.getString(DataExpression.FORMAT_TYPE); - InputInfo iinfo = InputInfo.stringExternalToInputInfo(format); + InputInfo iinfo = InputInfo.fromExternalString(format); //read matrix file return readDoubleMatrix(fname, iinfo, rows, cols, blen, nnz); @@ -614,7 +614,7 @@ public class Connection implements Closeable long rows = jmtd.getLong(DataExpression.READROWPARAM); long cols = jmtd.getLong(DataExpression.READCOLPARAM); String format = jmtd.getString(DataExpression.FORMAT_TYPE); - InputInfo iinfo = InputInfo.stringExternalToInputInfo(format); + InputInfo iinfo = InputInfo.fromExternalString(format); //read frame file return readStringFrame(fname, iinfo, rows, cols); diff --git a/src/main/java/org/apache/sysds/parser/DataExpression.java b/src/main/java/org/apache/sysds/parser/DataExpression.java index 6ab1eb2..1b7ddc4 100644 --- a/src/main/java/org/apache/sysds/parser/DataExpression.java +++ b/src/main/java/org/apache/sysds/parser/DataExpression.java @@ -220,7 +220,7 @@ public class DataExpression extends DataIdentifier return null; } dataExpr.addVarParam(currName, currExpr); - } + } } else if (functionName.equalsIgnoreCase("rand")){ @@ -1178,7 +1178,7 @@ public class DataExpression extends DataIdentifier getOutput().setNnz(-1L); } - else{ + else{ raiseValidateError("Unknown Data Type " + dataTypeString + ". Valid values: " + Statement.SCALAR_DATA_TYPE +", " + Statement.MATRIX_DATA_TYPE, conditional, LanguageErrorCodes.INVALID_PARAMETERS); } diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java index edd7519..a27318a 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java @@ -734,7 +734,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data //get object from cache if( _data == null ) getCache(); - acquire( false, _data==null ); //incl. read matrix if evicted + acquire( false, _data==null ); //incl. read matrix if evicted // b) write the matrix try { diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java index c2817c5..36bd258 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/federated/FederatedWorkerHandler.java @@ -137,7 +137,7 @@ public class FederatedWorkerHandler extends ChannelInboundHandlerAdapter { mc.setRows(mtd.getLong(DataExpression.READROWPARAM)); mc.setCols(mtd.getLong(DataExpression.READCOLPARAM)); String format = mtd.getString(DataExpression.FORMAT_TYPE); - oi = OutputInfo.outputInfoFromStringExternal(format); + oi = OutputInfo.fromExternalString(format); ii = OutputInfo.getMatchingInputInfo(oi); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java index d2e65f6..9176335 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/VariableCPInstruction.java @@ -945,26 +945,23 @@ public class VariableCPInstruction extends CPInstruction implements LineageTrace try { OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo(); DataCharacteristics dc = (mo.getMetaData()).getDataCharacteristics(); - if(oi == OutputInfo.CSVOutputInfo) { + if( oi == OutputInfo.CSVOutputInfo + && !getInput1().getName().startsWith(org.apache.sysds.lops.Data.PREAD_PREFIX) ) + { WriterTextCSV writer = new WriterTextCSV((FileFormatPropertiesCSV)_formatProperties); writer.addHeaderToCSV(mo.getFileName(), fname, dc.getRows(), dc.getCols()); } - else if ( oi == OutputInfo.BinaryBlockOutputInfo || oi == OutputInfo.TextCellOutputInfo ) { - mo.exportData(fname, outFmt, _formatProperties); - } else { - throw new DMLRuntimeException("Unexpected data format (" + OutputInfo.outputInfoToString(oi) + "): can not export into CSV format."); + mo.exportData(fname, outFmt, _formatProperties); } - - // Write Metadata file HDFSTool.writeMetaDataFile (fname + ".mtd", mo.getValueType(), dc, OutputInfo.CSVOutputInfo, _formatProperties); - } catch (IOException e) { + } + catch (IOException e) { throw new DMLRuntimeException(e); } } } - /** * Helper function to write MM files to HDFS. * @@ -980,24 +977,25 @@ public class VariableCPInstruction extends CPInstruction implements LineageTrace mo.exportData(fname, outFmt); } else { - OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo(); - DataCharacteristics dc = mo.getDataCharacteristics(); - if(oi == OutputInfo.TextCellOutputInfo) { - try { + try { + OutputInfo oi = ((MetaDataFormat)mo.getMetaData()).getOutputInfo(); + DataCharacteristics dc = mo.getDataCharacteristics(); + if( oi == OutputInfo.TextCellOutputInfo + && !getInput1().getName().startsWith(org.apache.sysds.lops.Data.PREAD_PREFIX) ) + { WriterMatrixMarket.mergeTextcellToMatrixMarket(mo.getFileName(), fname, dc.getRows(), dc.getCols(), dc.getNonZeros()); - } catch (IOException e) { - throw new DMLRuntimeException(e); + } + else { + mo.exportData(fname, outFmt); } } - else if ( oi == OutputInfo.BinaryBlockOutputInfo) { - mo.exportData(fname, outFmt); - } - else { - throw new DMLRuntimeException("Unexpected data format (" + OutputInfo.outputInfoToString(oi) + "): can not export into MatrixMarket format."); + catch (IOException e) { + throw new DMLRuntimeException(e); } } } + /** * Helper function to write scalars to HDFS based on its value type. * diff --git a/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java b/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java index d64d52c..3d2af34 100644 --- a/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java +++ b/src/main/java/org/apache/sysds/runtime/io/MatrixReaderFactory.java @@ -38,7 +38,7 @@ public class MatrixReaderFactory if( ConfigurationManager.getCompilerConfigFlag(ConfigType.PARALLEL_CP_READ_TEXTFORMATS) && MatrixBlock.DEFAULT_SPARSEBLOCK == SparseBlock.Type.MCSR ) reader = new ReaderTextCellParallel( iinfo ); else - reader = new ReaderTextCell( iinfo ); + reader = new ReaderTextCell( iinfo ); } else if( iinfo == InputInfo.CSVInputInfo ) { diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/InputInfo.java b/src/main/java/org/apache/sysds/runtime/matrix/data/InputInfo.java index fa7c3ce..7786d5d 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/InputInfo.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/InputInfo.java @@ -123,17 +123,17 @@ public class InputInfo implements Serializable return null; } - public static InputInfo stringExternalToInputInfo(String str) { + public static InputInfo fromExternalString(String str) { if( DataExpression.FORMAT_TYPE_VALUE_TEXT.equals(str) ) return InputInfo.TextCellInputInfo; else if( DataExpression.FORMAT_TYPE_VALUE_MATRIXMARKET.equals(str) ) return InputInfo.MatrixMarketInputInfo; else if( DataExpression.FORMAT_TYPE_VALUE_CSV.equals(str) ) - return InputInfo.CSVInputInfo; + return InputInfo.CSVInputInfo; else if( DataExpression.FORMAT_TYPE_VALUE_LIBSVM.equals(str) ) - return InputInfo.LIBSVMInputInfo; + return InputInfo.LIBSVMInputInfo; else if( DataExpression.FORMAT_TYPE_VALUE_BINARY.equals(str) ) // TODO BinaryTensorBlockInputInfo - return InputInfo.BinaryBlockInputInfo; + return InputInfo.BinaryBlockInputInfo; return null; } diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/OutputInfo.java b/src/main/java/org/apache/sysds/runtime/matrix/data/OutputInfo.java index b8dce5a..00e4895 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/OutputInfo.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/OutputInfo.java @@ -153,7 +153,7 @@ public class OutputInfo implements Serializable return "specialized"; } - public static OutputInfo outputInfoFromStringExternal(String format) { + public static OutputInfo fromExternalString(String format) { if (format.equalsIgnoreCase(FORMAT_TYPE_VALUE_TEXT)) return OutputInfo.TextCellOutputInfo; else if (format.equalsIgnoreCase(FORMAT_TYPE_VALUE_BINARY)) diff --git a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java index dee4ad5..c3d224f 100644 --- a/src/test/java/org/apache/sysds/test/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysds/test/AutomatedTestBase.java @@ -288,15 +288,15 @@ public abstract class AutomatedTestBase { } protected ExecMode setExecMode(ExecType instType) { - ExecMode platformOld = rtplatform; switch(instType) { - case SPARK: - rtplatform = ExecMode.SPARK; - break; - default: - rtplatform = ExecMode.HYBRID; - break; + case SPARK: return setExecMode(ExecMode.SPARK); + default: return setExecMode(ExecMode.HYBRID); } + } + + protected ExecMode setExecMode(ExecMode execMode) { + ExecMode platformOld = rtplatform; + rtplatform = execMode; if(rtplatform != ExecMode.SINGLE_NODE) DMLScript.USE_LOCAL_SPARK_CONFIG = true; return platformOld; diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java index 2564ff1..b02c970 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix_full_cellwise/FullMatrixMatrixCellwiseOperationTest.java @@ -44,8 +44,8 @@ public class FullMatrixMatrixCellwiseOperationTest extends AutomatedTestBase private final static String TEST_CLASS_DIR = TEST_DIR + FullMatrixMatrixCellwiseOperationTest.class.getSimpleName() + "/"; private final static double eps = 1e-10; - private final static int rows = 1100; - private final static int cols = 900; + private final static int rows = 1010; + private final static int cols = 300; private final static double sparsity1 = 0.7; private final static double sparsity2 = 0.1; diff --git a/src/test/java/org/apache/sysds/test/functions/data/misc/NoRenameTest.java b/src/test/java/org/apache/sysds/test/functions/data/misc/NoRenameTest.java new file mode 100644 index 0000000..619a325 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/data/misc/NoRenameTest.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.test.functions.data.misc; + +import java.io.IOException; + +import org.junit.Assert; +import org.junit.Test; +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.conf.ConfigurationManager; +import org.apache.sysds.runtime.io.MatrixReader; +import org.apache.sysds.runtime.io.MatrixReaderFactory; +import org.apache.sysds.runtime.io.MatrixWriter; +import org.apache.sysds.runtime.io.MatrixWriterFactory; +import org.apache.sysds.runtime.matrix.data.InputInfo; +import org.apache.sysds.runtime.matrix.data.MatrixBlock; +import org.apache.sysds.runtime.matrix.data.OutputInfo; +import org.apache.sysds.runtime.util.DataConverter; +import org.apache.sysds.runtime.util.HDFSTool; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; + +public class NoRenameTest extends AutomatedTestBase +{ + private final static String TEST_NAME1 = "NoRenameTest1"; + private final static String TEST_NAME2 = "NoRenameTest2"; + private final static String TEST_DIR = "functions/data/"; + private final static String TEST_CLASS_DIR = TEST_DIR + NoRenameTest.class.getSimpleName() + "/"; + + private final static int rows = 100; + private final static int cols = 50; + private final static int blocksize = ConfigurationManager.getBlocksize(); + private final static double sparsity1 = 0.7; + private final static double sparsity2 = 0.3; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "C" }) ); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "C" }) ); + } + + @Test + public void testTextcellDenseSinglenode() { + runRenameTest("text", false, ExecMode.SINGLE_NODE); + } + + @Test + public void testTextcellSparseSinglenode() { + runRenameTest("text", true, ExecMode.SINGLE_NODE); + } + + @Test + public void testTextcsvDenseSinglenode() { + runRenameTest("csv", false, ExecMode.SINGLE_NODE); + } + + @Test + public void testTextcsvSparseSinglenode() { + runRenameTest("csv", true, ExecMode.SINGLE_NODE); + } + + @Test + public void testTextmmDenseSinglenode() { + runRenameTest("mm", false, ExecMode.SINGLE_NODE); + } + + @Test + public void testTextmmSparseSinglenode() { + runRenameTest("mm", true, ExecMode.SINGLE_NODE); + } +// +// @Test +// public void testTextlibsvmDenseSinglenode() { +// runRenameTest("libsvm", false, ExecMode.SINGLE_NODE); +// } +// +// @Test +// public void testTextlibsvmSparseSinglenode() { +// runRenameTest("libsvm", true, ExecMode.SINGLE_NODE); +// } +// + @Test + public void testBinaryDenseSinglenode() { + runRenameTest("binary", false, ExecMode.SINGLE_NODE); + } + + @Test + public void testBinarySparseSinglenode() { + runRenameTest("binary", true, ExecMode.SINGLE_NODE); + } + + @Test + public void testTextcellDenseHybrid() { + runRenameTest("text", false, ExecMode.HYBRID); + } + + @Test + public void testTextcellSparseHybrid() { + runRenameTest("text", true, ExecMode.HYBRID); + } + + @Test + public void testTextcsvDenseHybrid() { + runRenameTest("csv", false, ExecMode.HYBRID); + } + + @Test + public void testTextcsvSparseHybrid() { + runRenameTest("csv", true, ExecMode.HYBRID); + } + + @Test + public void testTextmmDenseHybrid() { + runRenameTest("mm", false, ExecMode.HYBRID); + } + + @Test + public void testTextmmSparseHybrid() { + runRenameTest("mm", true, ExecMode.HYBRID); + } +// +// @Test +// public void testTextlibsvmDenseHybrid() { +// runRenameTest("libsvm", false, ExecMode.HYBRID); +// } +// +// @Test +// public void testTextlibsvmSparseHybrid() { +// runRenameTest("libsvm", true, ExecMode.HYBRID); +// } + + @Test + public void testBinaryDenseHybrid() { + runRenameTest("binary", false, ExecMode.HYBRID); + } + + @Test + public void testBinarySparseHybrid() { + runRenameTest("binary", true, ExecMode.HYBRID); + } + + @Test + public void testTextcellDenseSpark() { + runRenameTest("text", false, ExecMode.SPARK); + } + + @Test + public void testTextcellSparseSpark() { + runRenameTest("text", true, ExecMode.SPARK); + } + + @Test + public void testTextcsvDenseSpark() { + runRenameTest("csv", false, ExecMode.SPARK); + } + + @Test + public void testTextcsvSparseSpark() { + runRenameTest("csv", true, ExecMode.SPARK); + } + + @Test + public void testTextmmDenseSpark() { + runRenameTest("mm", false, ExecMode.SPARK); + } + + @Test + public void testTextmmSparseSpark() { + runRenameTest("mm", true, ExecMode.SPARK); + } +// +// @Test +// public void testTextlibsvmDenseSpark() { +// runRenameTest("libsvm", false, ExecMode.SPARK); +// } +// +// @Test +// public void testTextlibsvmSparseSpark() { +// runRenameTest("libsvm", true, ExecMode.SPARK); +// } + + @Test + public void testBinaryDenseSpark() { + runRenameTest("binary", false, ExecMode.SPARK); + } + + @Test + public void testBinarySparseSpark() { + runRenameTest("binary", true, ExecMode.SPARK); + } + + private void runRenameTest(String fmt, boolean sparse, ExecMode et) + { + ExecMode platformOld = setExecMode(et); + double sparsity = (sparse) ? sparsity2 : sparsity1; + + String TEST_NAME = fmt.equals("binary") ? TEST_NAME2 : TEST_NAME1; + TestConfiguration config = getTestConfiguration(TEST_NAME); + config.addVariable("rows", rows); + config.addVariable("cols", cols); + loadTestConfiguration(config); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[]{"-explain","-args", input("A"), fmt, + String.valueOf(rows), String.valueOf(cols), output("C")}; + + try { + //write input + double[][] A = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); + MatrixBlock mbA = DataConverter.convertToMatrixBlock(A); + MatrixWriter writer = MatrixWriterFactory.createMatrixWriter( + OutputInfo.fromExternalString(fmt)); + writer.writeMatrixToHDFS(mbA, input("A"), rows, cols, blocksize, mbA.getNonZeros()); + + //execute test + runTest(true, false, null, -1); + MatrixReader reader = MatrixReaderFactory.createMatrixReader( + InputInfo.fromExternalString(fmt)); + MatrixBlock mbC = reader.readMatrixFromHDFS(output("C"), rows, cols, blocksize, -1); + double[][] C = DataConverter.convertToDoubleMatrix(mbC); + + //compare matrices and check valid input + TestUtils.compareMatrices(A, C, rows, cols, 1e-10); + Assert.assertTrue(HDFSTool.existsFileOnHDFS(input("A"))); + } + catch(IOException e) { + e.printStackTrace(); + throw new RuntimeException(e); + } + finally { + rtplatform = platformOld; + } + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixReblockTest.java b/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixReblockTest.java index bc342c2..8b76883 100644 --- a/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixReblockTest.java +++ b/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixReblockTest.java @@ -232,16 +232,15 @@ public class FrameMatrixReblockTest extends AutomatedTestBase //write input data FrameWriter writer = FrameWriterFactory.createFrameWriter( - InputInfo.getMatchingOutputInfo(InputInfo.stringExternalToInputInfo(ofmt))); + InputInfo.getMatchingOutputInfo(InputInfo.fromExternalString(ofmt))); writer.writeFrameToHDFS(fb, fname, rows, cols); } private static double[][] readMatrixOutput(String fname, String ofmt, int rows, int cols) throws IOException { - MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.stringExternalToInputInfo(ofmt)); + MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.fromExternalString(ofmt)); MatrixBlock mb = reader.readMatrixFromHDFS(fname, rows, cols, 1000, -1); - return DataConverter.convertToDoubleMatrix(mb); } } diff --git a/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixWriteTest.java b/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixWriteTest.java index 8a62dca..7252f84 100644 --- a/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/frame/FrameMatrixWriteTest.java @@ -172,7 +172,7 @@ public class FrameMatrixWriteTest extends AutomatedTestBase throws IOException { //read input data - FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.stringExternalToInputInfo(ofmt)); + FrameReader reader = FrameReaderFactory.createFrameReader(InputInfo.fromExternalString(ofmt)); FrameBlock fb = reader.readFrameFromHDFS(fname, rows, cols); MatrixBlock ret = DataConverter.convertToMatrixBlock(fb); diff --git a/src/test/java/org/apache/sysds/test/functions/transform/TransformEncodeDecodeTest.java b/src/test/java/org/apache/sysds/test/functions/transform/TransformEncodeDecodeTest.java index ff8bed4..8057b11 100644 --- a/src/test/java/org/apache/sysds/test/functions/transform/TransformEncodeDecodeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/transform/TransformEncodeDecodeTest.java @@ -100,7 +100,7 @@ public class TransformEncodeDecodeTest extends AutomatedTestBase getAndLoadTestConfiguration(TEST_NAME1); //get input/output info - InputInfo iinfo = InputInfo.stringExternalToInputInfo(fmt); + InputInfo iinfo = InputInfo.fromExternalString(fmt); OutputInfo oinfo = InputInfo.getMatchingOutputInfo(iinfo); //generate and write input data diff --git a/src/test/scripts/functions/data/NoRenameTest1.dml b/src/test/scripts/functions/data/NoRenameTest1.dml new file mode 100644 index 0000000..8a8956d --- /dev/null +++ b/src/test/scripts/functions/data/NoRenameTest1.dml @@ -0,0 +1,24 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = read($1, format=$2, rows=$3, cols=$4); +print(nrow(A)) +write(A, $5, format=$2); diff --git a/src/test/scripts/functions/data/NoRenameTest2.dml b/src/test/scripts/functions/data/NoRenameTest2.dml new file mode 100644 index 0000000..892278f --- /dev/null +++ b/src/test/scripts/functions/data/NoRenameTest2.dml @@ -0,0 +1,24 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = read($1, format=$2, rows=$3, cols=$4, rows_in_block=1000, cols_in_block=1000); +print(nrow(A)) +write(A, $5, format=$2);