This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new b23774c6fa [SYSTEMDS-3572] Async and Logging tests
b23774c6fa is described below
commit b23774c6fab2d5546021f5f2b839c8a7fbb8fae9
Author: baunsgaard <[email protected]>
AuthorDate: Mon May 22 18:17:20 2023 +0200
[SYSTEMDS-3572] Async and Logging tests
This commit adds a new testing util that allows us to capture
the LOG output from the logging framework for the tests.
This makes it easy to verify that the content printed to a user
is correct.
It is used to test the CompressedFactory that compressed MatrixBlocks.
Closes #1831
---
.../compress/CompressedMatrixBlockFactory.java | 52 +--
.../runtime/compress/CompressionStatistics.java | 6 +-
.../sysds/runtime/compress/lib/CLALibStack.java | 13 +-
.../java/org/apache/sysds/test/LoggingUtils.java | 107 +++++
src/test/java/org/apache/sysds/test/TestUtils.java | 2 +-
.../test/component/compress/AsyncCompressTest.java | 84 ++++
.../component/compress/CompressedCustomTests.java | 328 ++++++++++++++
.../component/compress/CompressedLoggingTests.java | 478 +++++++++++++++++++++
.../component/compress/CompressedSingleTests.java | 4 +-
.../component/compress/workload/WorkloadTest.java | 27 +-
.../transform/TransformCompressedTestLogger.java | 48 +--
11 files changed, 1070 insertions(+), 79 deletions(-)
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
index 423ec0c3f3..1a7241af2a 100644
---
a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
+++
b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java
@@ -35,7 +35,6 @@ import
org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty;
import org.apache.sysds.runtime.compress.colgroup.ColGroupFactory;
import org.apache.sysds.runtime.compress.colgroup.ColGroupUncompressed;
import org.apache.sysds.runtime.compress.cost.ACostEstimate;
-import org.apache.sysds.runtime.compress.cost.ComputationCostEstimator;
import org.apache.sysds.runtime.compress.cost.CostEstimatorBuilder;
import org.apache.sysds.runtime.compress.cost.CostEstimatorFactory;
import org.apache.sysds.runtime.compress.cost.InstructionTypeCounter;
@@ -162,6 +161,7 @@ public class CompressedMatrixBlockFactory {
}
public static void compressAsync(ExecutionContext ec, String varName,
InstructionTypeCounter ins) {
+ LOG.debug("Compressing Async");
CompletableFuture.runAsync(() -> {
// method call or code to be asynch.
CacheableData<?> data = ec.getCacheableData(varName);
@@ -258,15 +258,16 @@ public class CompressedMatrixBlockFactory {
AColGroup cg = ColGroupConst.create(numCols, value);
block.allocateColGroup(cg);
block.recomputeNonZeros();
- if(block.getNumRows() == 0 || block.getNumColumns() == 0)
+ if(block.getNumRows() <= 0) // NCols is already checked
throw new DMLCompressionException("Invalid size of
allocated constant compressed matrix block");
return block;
}
private Pair<MatrixBlock, CompressionStatistics> compressMatrix() {
-
- if(mb instanceof CompressedMatrixBlock) // Redundant compression
+ if(mb.getNonZeros() < 0)
+ throw new DMLCompressionException("Invalid to compress
matrices with unknown nonZeros");
+ else if(mb instanceof CompressedMatrixBlock) // Redundant
compression
return recompress((CompressedMatrixBlock) mb);
_stats.denseSize =
MatrixBlock.estimateSizeInMemory(mb.getNumRows(), mb.getNumColumns(), 1.0);
@@ -305,16 +306,18 @@ public class CompressedMatrixBlockFactory {
if(LOG.isTraceEnabled()) {
LOG.trace("Logging all individual columns estimated
cost:");
for(CompressedSizeInfoColGroup g :
compressionGroups.getInfo())
- LOG.trace(
- String.format("Cost: %8.0f Size: %16d
%15s", costEstimator.getCost(g), g.getMinSize(), g.getColumns()));
+ LOG.trace(String.format("Cost: %8.0f Size:
%16.0f %15s", costEstimator.getCost(g), g.getMinSize(),
+ g.getColumns()));
}
_stats.estimatedSizeCols = compressionGroups.memoryEstimate();
_stats.estimatedCostCols =
costEstimator.getCost(compressionGroups);
logPhase();
- final int nCols = compSettings.transposed ? mb.getNumRows() :
mb.getNumColumns();
- final double scale = (costEstimator instanceof
ComputationCostEstimator) ? ((double) nCols) / 2 : 1;
+ // final int nRows = mb.getNumRows();
+ final int nCols = mb.getNumColumns();
+ // Assume the scaling of cocoding is at maximum square root
good relative to number of columns.
+ final double scale = Math.sqrt(nCols);
final double threshold = _stats.estimatedCostCols / scale;
if(threshold < _stats.originalCost) {
@@ -389,12 +392,19 @@ public class CompressedMatrixBlockFactory {
break;
default:
if(mb.isInSparseFormat()) {
- boolean haveManyColumns =
mb.getNumColumns() > 10000;
- boolean isNnzLowAndVerySparse =
mb.getNonZeros() < 1000 && mb.getSparsity() < 0.4;
- boolean isAboveRowNumbers =
mb.getNumRows() > 500000 && mb.getSparsity() < 0.4;
- boolean isAboveThreadToColumnRatio =
compressionGroups.getNumberColGroups() > mb.getNumColumns() / 30;
- compSettings.transposed =
haveManyColumns || isNnzLowAndVerySparse ||
- (isAboveRowNumbers &&
isAboveThreadToColumnRatio);
+ if(mb.getNumColumns() > 10000)
+ // many sparse columns we have
to...
+ compSettings.transposed = true;
+ else if(mb.getNonZeros() < 1000)
+ // low nnz trivial to transpose
+ compSettings.transposed = true;
+ else {
+ // is enough rows to make it
usable
+ boolean isAboveRowNumbers =
mb.getNumRows() > 500000;
+ // Make sure that it is not
more efficient to extract the rows.
+ boolean
isAboveThreadToColumnRatio = compressionGroups.getNumberColGroups() >
mb.getNumColumns() / 30;
+ compSettings.transposed =
isAboveRowNumbers && isAboveThreadToColumnRatio;
+ }
}
else
compSettings.transposed = false;
@@ -434,27 +444,19 @@ public class CompressedMatrixBlockFactory {
if(compSettings.isInSparkInstruction)
res.clearSoftReferenceToDecompressed();
- final long oldNNZ = mb.getNonZeros();
- if(oldNNZ <= 0L)
- res.recomputeNonZeros();
- else
- res.setNonZeros(oldNNZ);
+ res.setNonZeros(mb.getNonZeros());
logPhase();
}
private Pair<MatrixBlock, CompressionStatistics> abortCompression() {
LOG.warn("Compression aborted at phase: " + phase);
-
- if(compSettings.transposed)
- LibMatrixReorg.transposeInPlace(mb, k);
-
return new ImmutablePair<>(mb, _stats);
}
private Pair<MatrixBlock, CompressionStatistics>
recompress(CompressedMatrixBlock cmb) {
LOG.debug("Recompressing an already compressed MatrixBlock");
- LOG.error("Not Implemented Recompress yet");
+ LOG.warn("Not Implemented Recompress yet");
return new ImmutablePair<>(cmb, null);
// _stats.originalSize = cmb.getInMemorySize();
// CompressedMatrixBlock combined =
CLALibCombineGroups.combine(cmb, k);
@@ -501,6 +503,7 @@ public class CompressedMatrixBlockFactory {
LOG.debug("--compressed initial
actual size:" + _stats.compressedInitialSize);
break;
case 4:
+ default:
LOG.debug("--num col groups:
" + res.getColGroups().size());
LOG.debug("--compression phase
" + phase + " Cleanup : " + getLastTimePhase());
LOG.debug("--col groups types
" + _stats.getGroupsTypesString());
@@ -541,7 +544,6 @@ public class CompressedMatrixBlockFactory {
}
}
}
- default:
}
}
}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java
b/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java
index c47712fb9c..e9abe835b4 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java
@@ -84,10 +84,6 @@ public class CompressionStatistics {
this.colGroupCounts = ret;
}
- public Map<String, int[]> getColGroups() {
- return colGroupCounts;
- }
-
public String getGroupsTypesString() {
StringBuilder sb = new StringBuilder();
@@ -122,6 +118,8 @@ public class CompressionStatistics {
sb.append("\nOriginal Size : " + originalSize);
sb.append("\nCompressed Size : " + compressedSize);
sb.append("\nCompressionRatio : " + getRatio());
+ sb.append("\nDenseCompressionRatio : " + getDenseRatio());
+
if(colGroupCounts != null) {
sb.append("\nCompressionTypes : " +
getGroupsTypesString());
sb.append("\nCompressionGroupSizes : " +
getGroupsSizesString());
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibStack.java
b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibStack.java
index a1f11573fe..adce71a24a 100644
--- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibStack.java
+++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibStack.java
@@ -100,7 +100,7 @@ public final class CLALibStack {
final int rlen, final int clen, final int blen, final int k) {
if(rlen < blen) // Shortcut, in case file only contains one
block in r length.
- return CombiningColumnGroups(m, lookup, rlen, clen,
blen, k);
+ return combineColumnGroups(m, lookup, rlen, clen, blen,
k);
final CompressionType[] colTypes = new CompressionType[clen];
// Look through the first blocks in to the top.
@@ -164,7 +164,7 @@ public final class CLALibStack {
}
}
- return CombiningColumnGroups(m, lookup, rlen, clen, blen, k);
+ return combineColumnGroups(m, lookup, rlen, clen, blen, k);
}
private static MatrixBlock combineViaDecompression(final
Map<MatrixIndexes, MatrixBlock> m, final int rlen,
@@ -186,7 +186,7 @@ public final class CLALibStack {
}
// It is known all of the matrices are Compressed and they are non
overlapping.
- private static MatrixBlock CombiningColumnGroups(final
Map<MatrixIndexes, MatrixBlock> m, final MatrixIndexes lookup,
+ private static MatrixBlock combineColumnGroups(final Map<MatrixIndexes,
MatrixBlock> m, final MatrixIndexes lookup,
final int rlen, final int clen, final int blen, int k) {
final AColGroup[][] finalCols = new AColGroup[clen][]; // temp
array for combining
@@ -202,8 +202,13 @@ public final class CLALibStack {
final int c = gc.getColIndices().get(0);
if(br == 0)
finalCols[c] = new
AColGroup[blocksInColumn];
-
+ else if(finalCols[c] == null) {
+ LOG.warn("Combining via
decompression. There was an column"
+ + " assigned not
assigned in block 1 indicating spark compression");
+ return
combineViaDecompression(m, rlen, clen, blen, k);
+ }
finalCols[c][br] = gc;
+
}
}
}
diff --git a/src/test/java/org/apache/sysds/test/LoggingUtils.java
b/src/test/java/org/apache/sysds/test/LoggingUtils.java
new file mode 100644
index 0000000000..b3e8474658
--- /dev/null
+++ b/src/test/java/org/apache/sysds/test/LoggingUtils.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.Appender;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+
+/**
+ * Logging utils, to enable catching the Log output in tests.
+ *
+ * To use simply start with overwrite and when the test is done remember to
use reinsert.
+ */
+public final class LoggingUtils {
+
+ private static Appender consoleLogger = null;
+
+ private LoggingUtils() {
+ // empty constructor
+ }
+
+ /**
+ * Overwrite the console logger to not write out to STD error.
+ *
+ * @return A TestAppender that collects the logging outputs.
+ */
+ public static TestAppender overwrite() {
+ if(consoleLogger == null)
+ findConsoleLogger();
+ final TestAppender appender = new TestAppender();
+ final Logger logger = Logger.getRootLogger();
+ logger.removeAppender(consoleLogger);
+ logger.addAppender(appender);
+
+ return appender;
+ }
+
+ /**
+ * Reinstate the console logger, usually done after the test.
+ *
+ * @param appender The appender that was used to collect the Logging
outputs
+ * @return The List of logging statements done while the Test Appender
was in use
+ */
+ public static List<LoggingEvent> reinsert(TestAppender appender) {
+ final Logger logger = Logger.getRootLogger();
+ logger.removeAppender(appender);
+ logger.addAppender(consoleLogger);
+ return appender.getLog();
+ }
+
+ private static void findConsoleLogger() {
+ final Logger logger = Logger.getRootLogger();
+ consoleLogger = (Appender)
logger.getAllAppenders().nextElement();
+ }
+
+ /**
+ * A Test Appender that collects the Logging calls into a list.
+ *
+ * To be used in connection with LoggingUtils.
+ */
+ public static class TestAppender extends AppenderSkeleton {
+ private final List<LoggingEvent> log = new
ArrayList<LoggingEvent>();
+
+ private TestAppender() {
+ // empty constructor
+ }
+
+ @Override
+ public boolean requiresLayout() {
+ return false;
+ }
+
+ @Override
+ protected void append(final LoggingEvent loggingEvent) {
+ log.add(loggingEvent);
+ }
+
+ @Override
+ public void close() {
+ }
+
+ private List<LoggingEvent> getLog() {
+ return new ArrayList<LoggingEvent>(log);
+ }
+ }
+}
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java
b/src/test/java/org/apache/sysds/test/TestUtils.java
index 23da1a8fc3..ae68201b50 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -1029,7 +1029,7 @@ public class TestUtils {
public static void compareMatricesBitAvgDistance(MatrixBlock
expectedMatrix, MatrixBlock actualMatrix,
long maxUnitsOfLeastPrecision, long maxAvgDistance) {
- compareMatricesBitAvgDistance(expectedMatrix, actualMatrix,
maxUnitsOfLeastPrecision, maxAvgDistance);
+ compareMatricesBitAvgDistance(expectedMatrix, actualMatrix,
maxUnitsOfLeastPrecision, maxAvgDistance, "");
}
public static void compareMatricesBitAvgDistance(MatrixBlock
expectedMatrix, MatrixBlock actualMatrix,
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/AsyncCompressTest.java
b/src/test/java/org/apache/sysds/test/component/compress/AsyncCompressTest.java
new file mode 100644
index 0000000000..97a93e6a3d
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/component/compress/AsyncCompressTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.compress;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.common.Types.FileFormat;
+import org.apache.sysds.common.Types.ValueType;
+import org.apache.sysds.runtime.DMLRuntimeException;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlock;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
+import org.apache.sysds.runtime.controlprogram.LocalVariableMap;
+import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
+import org.apache.sysds.runtime.controlprogram.context.ExecutionContext;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.runtime.meta.MatrixCharacteristics;
+import org.apache.sysds.runtime.meta.MetaDataFormat;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+
+public class AsyncCompressTest {
+ protected static final Log LOG =
LogFactory.getLog(AsyncCompressTest.class.getName());
+
+ @Test
+ public void empty() {
+ assertTrue(runTest(new MatrixBlock(1000, 30, 0.0)));
+ }
+
+ @Test
+ public void notCompressable() {
+ assertFalse(runTest(TestUtils.generateTestMatrixBlock(100, 100,
0, 1.0, 1.0, 13)));
+ }
+
+ public boolean runTest(MatrixBlock mb) {
+ try {
+ MatrixCharacteristics matrixCharacteristics = new
MatrixCharacteristics(mb.getNumRows(), mb.getNumColumns(),
+ -1, 0);
+ MetaDataFormat metaDataFormat = new
MetaDataFormat(matrixCharacteristics, FileFormat.TEXT);
+
+ MatrixObject mbo = new MatrixObject(ValueType.FP64,
"/dev/null", metaDataFormat, mb);
+ LocalVariableMap vars = new LocalVariableMap();
+ ExecutionContext ec = new ExecutionContext(vars);
+ ec.setVariable("mb1", mbo);
+
+ CompressedMatrixBlockFactory.compressAsync(ec, "mb1");
+
+ for(int i = 0; i < 5; i++) {
+ Thread.sleep(i * 100);
+ MatrixBlock m = mbo.acquireReadAndRelease();
+ if(m instanceof CompressedMatrixBlock)
+ return true;
+ }
+ return false;
+ }
+
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ throw new DMLRuntimeException("failed test", e);
+ }
+
+ }
+}
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/CompressedCustomTests.java
b/src/test/java/org/apache/sysds/test/component/compress/CompressedCustomTests.java
index 2816504c98..657551fc02 100644
---
a/src/test/java/org/apache/sysds/test/component/compress/CompressedCustomTests.java
+++
b/src/test/java/org/apache/sysds/test/component/compress/CompressedCustomTests.java
@@ -21,12 +21,31 @@ package org.apache.sysds.test.component.compress;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
+import org.apache.sysds.runtime.compress.CompressionSettings;
+import org.apache.sysds.runtime.compress.CompressionSettingsBuilder;
+import org.apache.sysds.runtime.compress.CompressionStatistics;
+import org.apache.sysds.runtime.compress.DMLCompressionException;
+import org.apache.sysds.runtime.compress.cost.ACostEstimate;
+import org.apache.sysds.runtime.compress.cost.CostEstimatorBuilder;
+import org.apache.sysds.runtime.compress.cost.CostEstimatorFactory;
+import org.apache.sysds.runtime.compress.cost.InstructionTypeCounter;
+import org.apache.sysds.runtime.compress.workload.WTreeRoot;
import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.test.TestUtils;
+import org.apache.sysds.test.component.compress.workload.WorkloadTest;
import org.junit.Test;
public class CompressedCustomTests {
+ protected static final Log LOG =
LogFactory.getLog(CompressedCustomTests.class.getName());
+
@Test
public void compressNaNDense() {
MatrixBlock m = new MatrixBlock(100, 100, Double.NaN);
@@ -49,4 +68,313 @@ public class CompressedCustomTests {
for(int j = 0; j < m.getNumColumns(); j++)
assertEquals(0.0, m2.quickGetValue(i, j), 0.0);
}
+
+ @Test
+ public void workloadTreeInterface() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 231);
+ Map<String, String> args = new HashMap<>();
+ args.put("$1",
"src/test/resources/component/compress/1-1.csv");
+ WTreeRoot wtr =
WorkloadTest.getWorkloadTree("functions/scale.dml", args);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, 10, wtr).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void workloadTreeInterface2() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 54);
+ Map<String, String> args = new HashMap<>();
+ args.put("$1",
"src/test/resources/component/compress/1-1.csv");
+ WTreeRoot wtr =
WorkloadTest.getWorkloadTree("functions/scale.dml", args);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, wtr).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void costEstimatorBuilder() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 1612323);
+ Map<String, String> args = new HashMap<>();
+ args.put("$1",
"src/test/resources/component/compress/1-1.csv");
+ WTreeRoot wtr =
WorkloadTest.getWorkloadTree("functions/scale.dml", args);
+ CostEstimatorBuilder csb = new
CostEstimatorBuilder(wtr);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, csb).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void costEstimatorBuilder2() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 4442);
+ Map<String, String> args = new HashMap<>();
+ args.put("$1",
"src/test/resources/component/compress/1-1.csv");
+ WTreeRoot wtr =
WorkloadTest.getWorkloadTree("functions/scale.dml", args);
+ CostEstimatorBuilder csb = new
CostEstimatorBuilder(wtr);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, 10, csb).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void instructionTypeCounter() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 4442);
+ InstructionTypeCounter ins = new
InstructionTypeCounter(0, 0, 0, 0, 24, 15, 0, 0, false);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, 10, ins).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void instructionTypeCounter2() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 521);
+ InstructionTypeCounter ins = new
InstructionTypeCounter(0, 0, 0, 0, 24, 15, 0, 0, false);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, ins).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void instructionTypeCounterNull() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 521);
+ InstructionTypeCounter ins = null;
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, ins).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void instructionTypeCounterNull2() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 521);
+ InstructionTypeCounter ins = null;
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, 14, ins).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void builder() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 521);
+ CompressionSettingsBuilder sb = new
CompressionSettingsBuilder();
+
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, sb).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void builder2() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 1313131);
+ CompressionSettingsBuilder sb = new
CompressionSettingsBuilder();
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, 16, sb).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void normal() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 42145);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void threaded() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 42145);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, 9).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void costEstimator() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 4442);
+ Map<String, String> args = new HashMap<>();
+ args.put("$1",
"src/test/resources/component/compress/1-1.csv");
+ WTreeRoot wtr =
WorkloadTest.getWorkloadTree("functions/scale.dml", args);
+ CostEstimatorBuilder csb = new
CostEstimatorBuilder(wtr);
+ CompressionSettings cs = new
CompressionSettingsBuilder().create();
+ ACostEstimate ce = CostEstimatorFactory.create(cs, csb,
m.getNumRows(), m.getNumColumns(), m.getSparsity());
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, 10, ce).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void costEstimator2() {
+ try {
+ MatrixBlock m = TestUtils.generateTestMatrixBlock(100,
4, 1, 1, 0.5, 4442);
+ Map<String, String> args = new HashMap<>();
+ args.put("$1",
"src/test/resources/component/compress/1-1.csv");
+ WTreeRoot wtr =
WorkloadTest.getWorkloadTree("functions/scale.dml", args);
+ CostEstimatorBuilder csb = new
CostEstimatorBuilder(wtr);
+ CompressionSettings cs = new
CompressionSettingsBuilder().create();
+ ACostEstimate ce = CostEstimatorFactory.create(cs, csb,
m.getNumRows(), m.getNumColumns(), m.getSparsity());
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m, ce).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m, m2, 0, 0);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void negativeCreateConstant() {
+ CompressedMatrixBlockFactory.createConstant(-1, 1, 3241);
+
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void negativeCreateConstant2() {
+ CompressedMatrixBlockFactory.createConstant(32, -1, 3241);
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void negativeCreateConstant3() {
+ CompressedMatrixBlockFactory.createConstant(32, 0, 3241);
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void negativeCreateConstant4() {
+ CompressedMatrixBlockFactory.createConstant(0, 321, 3241);
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void negativeCreateConstant5() {
+ CompressedMatrixBlockFactory.createConstant(-1, -1, 3241);
+ }
+
+ @Test
+ public void createConstant() {
+ MatrixBlock mb =
CompressedMatrixBlockFactory.createConstant(10, 10, 3241);
+ MatrixBlock mb2 = new MatrixBlock(10, 10, 3241.0);
+ TestUtils.compareMatricesBitAvgDistance(mb, mb2, 0, 0);
+ }
+
+ @Test
+ public void createUncompressedCompressedMatrixBlockTest() {
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(32, 42, 32,
123, 0.2, 2135);
+ MatrixBlock mb2 =
CompressedMatrixBlockFactory.genUncompressedCompressedMatrixBlock(mb);
+ TestUtils.compareMatricesBitAvgDistance(mb, mb2, 0, 0);
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void invalidIfNnzNotSet() {
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(32, 42, 32,
123, 0.2, 2135);
+ mb.setNonZeros(-23L);
+ CompressedMatrixBlockFactory.compress(mb);
+ }
+
+ @Test
+ public void statisticsStartInfinite() {
+ CompressionStatistics cs = new CompressionStatistics();
+ String s = cs.toString();
+ assertTrue(s.contains("Infinity"));
+ }
+
+ @Test(expected = DMLCompressionException.class)
+ public void setTransposeIncorrect() {
+ new CompressionSettingsBuilder().setTransposeInput("bb");
+ }
+
+ @Test
+ public void compressSingleCol() {
+ MatrixBlock m1 = TestUtils.generateTestMatrixBlock(1000, 1, 1,
1, 0.8, 231);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m1).getLeft();
+ TestUtils.compareMatrices(m1, m2, 0, "no");
+ }
+
+ @Test
+ public void manyColsSparse() {
+ MatrixBlock m0 = new MatrixBlock(1000, 10000, 0.0);
+ MatrixBlock m1 = TestUtils.generateTestMatrixBlock(1000, 1, 1,
1, 0.01, 231);
+ m1 = m0.append(m1);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m1).getLeft();
+
+ TestUtils.compareMatricesBitAvgDistance(m1, m2, 0, 0, "no");
+ }
+
+ @Test
+ public void manyRowsSparse() {
+ MatrixBlock m0 = new MatrixBlock(500001, 10, 0.0);
+ MatrixBlock m1 = TestUtils.generateTestMatrixBlock(500001, 1,
1, 1, 0.003, 231);
+ m1 = m0.append(m1);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m1).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m1, m2, 0, 0, "no");
+ }
+
+ @Test
+ public void manyRowsButNotQuite() {
+ MatrixBlock m0 = new MatrixBlock(10001, 10, 0.0);
+ MatrixBlock m1 = TestUtils.generateTestMatrixBlock(10001, 1, 1,
1, 0.11, 231);
+ m1 = m0.append(m1);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(m1).getLeft();
+ TestUtils.compareMatricesBitAvgDistance(m1, m2, 0, 0, "no");
+ }
}
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java
b/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java
new file mode 100644
index 0000000000..4373aa43df
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/component/compress/CompressedLoggingTests.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.compress;
+
+import static org.junit.Assert.fail;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory;
+import org.apache.sysds.runtime.compress.CompressionSettings;
+import org.apache.sysds.runtime.compress.CompressionSettingsBuilder;
+import org.apache.sysds.runtime.compress.cost.InstructionTypeCounter;
+import org.apache.sysds.runtime.matrix.data.MatrixBlock;
+import org.apache.sysds.test.LoggingUtils;
+import org.apache.sysds.test.LoggingUtils.TestAppender;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+
+public class CompressedLoggingTests {
+ protected static final Log LOG =
LogFactory.getLog(CompressedLoggingTests.class.getName());
+
+ @Test
+ public void compressedLoggingTest_Trace() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.TRACE);
+ MatrixBlock mb =
TestUtils.generateTestMatrixBlock(1000, 5, 1, 1, 0.5, 235);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+
if(l.getMessage().toString().contains("compressed colGroup dictionary sizes"))
+ return;
+ }
+ fail("Log did not contain Dictionary sizes");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_WorkloadCost() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+ MatrixBlock mb =
TestUtils.generateTestMatrixBlock(1000, 5, 1, 1, 0.5, 235);
+ InstructionTypeCounter inst = new
InstructionTypeCounter(10, 0, 0, 0, 0, 0, 0, 0, false);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb, inst).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ if(l.getMessage().toString().contains("--actual
cost:"))
+ return;
+ }
+ fail("Log did not contain Dictionary sizes");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_ManyColumns() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+ MatrixBlock mb =
TestUtils.generateTestMatrixBlock(2000, 1001, 1, 65, 0.5, 235);
+ mb = TestUtils.round(mb);
+ InstructionTypeCounter inst = new
InstructionTypeCounter(0, 0, 0, 0, 0, 1000, 0, 0, false);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb, inst).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+
if(l.getMessage().toString().contains("--CoCoded produce many columns but the
first"))
+ return;
+ }
+ fail("Log did not say Cocode many columns");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_failedCompression() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(5,
5, 0.2, 1, 0.5, 235);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ if(l.getMessage().toString().contains("Aborting
before co-code"))
+ return;
+ }
+ fail("Log did not contain abort");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_failedCompression_afterCocode() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(10,
50, 1, 1, 0.5, 235);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ if(l.getMessage().toString().contains("Aborting
after co-code"))
+ return;
+ }
+ fail("Log did not contain abort");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_WorkloadCostFail() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(10,
50, 1, 1, 0.5, 235);
+ InstructionTypeCounter inst = new
InstructionTypeCounter(10, 0, 0, 0, 0, 0, 0, 0, false);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb, inst).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ if(l.getMessage().toString().contains("Aborting
after co-code"))
+ return;
+ }
+ fail("Log did not contain abort");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_WorkloadCostFail_2() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(10,
50, 1, 1, 0.5, 235);
+ InstructionTypeCounter inst = new
InstructionTypeCounter(0, 10, 0, 0, 0, 0, 0, 0, true);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb, inst).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ if(l.getMessage().toString().contains("Aborting
before co-code"))
+ return;
+ }
+ fail("Log did not contain abort");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_SparkSettings() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(100,
2, 1, 1, 0.5, 235);
+ CompressionSettingsBuilder sb = new
CompressionSettingsBuilder();
+ sb.setIsInSparkInstruction();
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb, sb).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+
if(l.getMessage().toString().contains("Compressed Size"))
+ return;
+ }
+ fail("Log did not contain Compressed Size");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_TraceBigGroup() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.TRACE);
+ MatrixBlock mb =
TestUtils.generateTestMatrixBlock(10000, 1, 1, 128, 0.5, 235);
+ mb = TestUtils.round(mb);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ // LOG.error(l.getMessage());
+
if(l.getMessage().toString().contains("--colGroups type"))
+ return;
+ }
+ fail("Log did not contain Compressed Size");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_TraceBigGroupConst() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.TRACE);
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(10,
1000, 1, 1, 1.0, 235);
+ mb = TestUtils.round(mb);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ // LOG.error(l.getMessage());
+
if(l.getMessage().toString().contains("--colGroups type"))
+ return;
+ }
+ fail("Log did not contain Compressed Size");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTestEmpty() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.TRACE);
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(10,
1000, 1, 1, 0.0, 235);
+ mb = TestUtils.round(mb);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb).getLeft();
+ TestUtils.compareMatrices(mb, m2, 0.0);
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ // LOG.error(l.getMessage());
+ if(l.getMessage().toString().contains("Empty
input to compress"))
+ return;
+ }
+ fail("Log Did not contain Empty");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_recompress() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(10,
1000, 1, 1, 0.0, 235);
+ mb = TestUtils.round(mb);
+ MatrixBlock m2 =
CompressedMatrixBlockFactory.compress(mb).getLeft();
+ CompressedMatrixBlockFactory.compress(m2).getLeft();
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ // LOG.error(l.getMessage());
+
if(l.getMessage().toString().contains("Recompressing"))
+ return;
+ }
+ fail("Log Did not contain Recompressing");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressedLoggingTest_AbortEnd() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.DEBUG);
+ MatrixBlock mb = TestUtils.generateTestMatrixBlock(400,
600, 1, 1024, 0.18, 235);
+ mb = TestUtils.round(mb);
+ final int ss = 50;
+ CompressionSettingsBuilder sb = new
CompressionSettingsBuilder();
+ sb.setMaxSampleSize(ss);
+ sb.setMinimumSampleSize(ss);
+ CompressedMatrixBlockFactory.compress(mb, sb).getLeft();
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ // LOG.error(l.getMessage());
+ if(l.getMessage().toString().contains("Abort
block compression"))
+ return;
+ }
+ fail("Log Did not contain Recompressing");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressedMatrixBlockFactory.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressionSettings() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressionSettings.class).setLevel(Level.DEBUG);
+ new CompressionSettingsBuilder().create();
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+ // LOG.error(l.getMessage());
+
if(l.getMessage().toString().contains("CompressionSettings"))
+ return;
+ }
+ fail("failed to get Compressionsetting to log");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressionSettings.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+ @Test
+ public void compressionSettingsEstimationType() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressionSettings.class).setLevel(Level.DEBUG);
+ new
CompressionSettingsBuilder().setSamplingRatio(0.1).create();
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+
if(l.getMessage().toString().contains("Estimation Type"))
+ return;
+ }
+ fail("failed to get Compressionsetting to log");
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressionSettings.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+
+
+ @Test
+ public void compressionSettingsFull() {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressionSettings.class).setLevel(Level.DEBUG);
+ new
CompressionSettingsBuilder().setSamplingRatio(1.1).create();
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+ for(LoggingEvent l : log) {
+
if(l.getMessage().toString().contains("Estimation Type"))
+ fail("Contained estimationType");
+ }
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+
Logger.getLogger(CompressionSettings.class).setLevel(Level.WARN);
+ LoggingUtils.reinsert(appender);
+ }
+ }
+}
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/CompressedSingleTests.java
b/src/test/java/org/apache/sysds/test/component/compress/CompressedSingleTests.java
index ab966526e6..fb9233fa6d 100644
---
a/src/test/java/org/apache/sysds/test/component/compress/CompressedSingleTests.java
+++
b/src/test/java/org/apache/sysds/test/component/compress/CompressedSingleTests.java
@@ -36,6 +36,7 @@ import org.apache.sysds.runtime.compress.CompressionSettings;
import org.apache.sysds.runtime.compress.CompressionSettingsBuilder;
import org.apache.sysds.runtime.compress.cocode.CoCoderFactory;
import org.apache.sysds.runtime.compress.colgroup.AColGroup.CompressionType;
+import
org.apache.sysds.runtime.compress.colgroup.insertionsort.InsertionSorterFactory.SORT_TYPE;
import org.apache.sysds.runtime.compress.cost.CostEstimatorFactory;
import
org.apache.sysds.runtime.compress.estim.sample.SampleEstimatorFactory.EstimationType;
import org.apache.sysds.runtime.functionobjects.Multiply;
@@ -141,7 +142,8 @@ public class CompressedSingleTests {
.setAllowSharedDictionary(true).setColumnPartitioner(CoCoderFactory.PartitionerType.BIN_PACKING)
.setMaxColGroupCoCode(3).setEstimationType(EstimationType.ShlosserJackknifeEstimator).clearValidCompression()
.setSamplingRatio(0.2).setSeed(1342).setCoCodePercentage(0.22).setMinimumSampleSize(1342)
- .setCostType(CostEstimatorFactory.CostType.MEMORY);
+
.setCostType(CostEstimatorFactory.CostType.MEMORY).setMinimumCompressionRatio(2.2)
+ .setSDCSortType(SORT_TYPE.MERGE);
CompressionSettings s = b.create();
b = b.copySettings(s);
}
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/workload/WorkloadTest.java
b/src/test/java/org/apache/sysds/test/component/compress/workload/WorkloadTest.java
index fa5a3cfe8a..cf084e9ccc 100644
---
a/src/test/java/org/apache/sysds/test/component/compress/workload/WorkloadTest.java
+++
b/src/test/java/org/apache/sysds/test/component/compress/workload/WorkloadTest.java
@@ -26,6 +26,8 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.sysds.api.DMLOptions;
import org.apache.sysds.api.DMLScript;
import org.apache.sysds.parser.DMLProgram;
@@ -44,6 +46,7 @@ import org.junit.runners.Parameterized.Parameters;
@RunWith(value = Parameterized.class)
public class WorkloadTest {
+ protected static final Log LOG =
LogFactory.getLog(WorkloadTest.class.getName());
private static final String basePath =
"src/test/scripts/component/compress/workload/";
private static final String testFile =
"src/test/resources/component/compress/1-1.csv";
@@ -108,9 +111,9 @@ public class WorkloadTest {
tests.add(new Object[] {0, 0, 0, 0, 0, 0, 2, 0, false, true,
"functions/scale_onlySide.dml", args});
tests.add(new Object[] {0, 0, 0, 0, 0, 0, 6, 0, true, false,
"functions/scale_onlySide.dml", args});
-// TODO these tests are failing
-// tests.add(new Object[] {0, 0, 0, 0, 1, 1, 8, 0, true, false,
"functions/pca.dml", args});
-// tests.add(new Object[] {0, 0, 0, 0, 1, 1, 5, 0, true, true,
"functions/pca.dml", args});
+ // TODO these tests are failing
+ // tests.add(new Object[] {0, 0, 0, 0, 1, 1, 8, 0, true, false,
"functions/pca.dml", args});
+ // tests.add(new Object[] {0, 0, 0, 0, 1, 1, 5, 0, true, true,
"functions/pca.dml", args});
args = new HashMap<>();
args.put("$1", testFile);
@@ -205,7 +208,17 @@ public class WorkloadTest {
Assert.assertEquals(errorString + "Should Compresss",
shouldCompress, ceb.shouldTryToCompress());
}
- private static WTreeRoot getWorkloadTree(DMLProgram prog) {
+ public static WTreeRoot getWorkloadTree(String name, Map<String,
String> args) {
+ DMLProgram prog = parse(name, args);
+ DMLTranslator dmlt = new DMLTranslator(prog);
+ dmlt.liveVariableAnalysis(prog);
+ dmlt.validateParseTree(prog);
+ dmlt.constructHops(prog);
+
+ return getWorkloadTree(prog);
+ }
+
+ public static WTreeRoot getWorkloadTree(DMLProgram prog) {
Map<Long, WTreeRoot> c =
WorkloadAnalyzer.getAllCandidateWorkloads(prog);
Assert.assertEquals(c.size(), 1);
for(long k : c.keySet())
@@ -213,7 +226,11 @@ public class WorkloadTest {
throw new DMLRuntimeException("There was no Workload");
}
- private DMLProgram parse(String name) {
+ public DMLProgram parse(String name) {
+ return parse(name, args);
+ }
+
+ public static DMLProgram parse(String name, Map<String, String> args) {
try {
boolean isFile = true;
String filePath = basePath + name;
diff --git
a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformCompressedTestLogger.java
b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformCompressedTestLogger.java
index 59b79bfe5e..6e2d035198 100644
---
a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformCompressedTestLogger.java
+++
b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformCompressedTestLogger.java
@@ -22,13 +22,10 @@ package org.apache.sysds.test.component.frame.transform;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.Appender;
-import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
@@ -38,6 +35,8 @@ import org.apache.sysds.runtime.matrix.data.MatrixBlock;
import org.apache.sysds.runtime.transform.encode.CompressedEncode;
import org.apache.sysds.runtime.transform.encode.EncoderFactory;
import org.apache.sysds.runtime.transform.encode.MultiColumnEncoder;
+import org.apache.sysds.test.LoggingUtils;
+import org.apache.sysds.test.LoggingUtils.TestAppender;
import org.apache.sysds.test.TestUtils;
import org.junit.Test;
@@ -65,28 +64,22 @@ public class TransformCompressedTestLogger {
}
public void test(String spec) {
- final TestAppender appender = new TestAppender();
- final Logger logger = Logger.getRootLogger();
- Appender consoleLogger = (Appender)
logger.getAllAppenders().nextElement();
+ final TestAppender appender = LoggingUtils.overwrite();
+
try {
Logger.getLogger(CompressedEncode.class).setLevel(Level.DEBUG);
-
-
+
FrameBlock meta = null;
- logger.removeAppender(consoleLogger);
- logger.addAppender(appender);
MultiColumnEncoder encoderCompressed =
EncoderFactory.createEncoder(spec, data.getColumnNames(),
- data.getNumColumns(), meta);
+ data.getNumColumns(), meta);
MatrixBlock outCompressed =
encoderCompressed.encode(data, true);
FrameBlock outCompressedMD =
encoderCompressed.getMetaData(null);
MultiColumnEncoder encoderNormal =
EncoderFactory.createEncoder(spec, data.getColumnNames(),
- data.getNumColumns(), meta);
+ data.getNumColumns(), meta);
MatrixBlock outNormal = encoderNormal.encode(data);
FrameBlock outNormalMD =
encoderNormal.getMetaData(null);
- logger.removeAppender(appender);
- logger.addAppender(consoleLogger);
- final List<LoggingEvent> log = appender.getLog();
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
assertTrue(log.get(3).getMessage().toString().contains("Compression ratio"));
TestUtils.compareMatrices(outNormal, outCompressed, 0,
"Not Equal after apply");
TestUtils.compareFrames(outNormalMD, outCompressedMD,
true);
@@ -96,32 +89,9 @@ public class TransformCompressedTestLogger {
fail(e.getMessage());
}
finally {
- logger.removeAppender(appender);
- logger.addAppender(consoleLogger);
+ LoggingUtils.reinsert(appender);
}
}
- class TestAppender extends AppenderSkeleton {
- private final List<LoggingEvent> log = new
ArrayList<LoggingEvent>();
-
- @Override
- public boolean requiresLayout() {
- return false;
- }
-
- @Override
- protected void append(final LoggingEvent loggingEvent) {
- log.add(loggingEvent);
- }
-
- @Override
- public void close() {
- }
-
- public List<LoggingEvent> getLog() {
- return new ArrayList<LoggingEvent>(log);
- }
-
- }
}
