This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new ceb28dfed6 [SYSTEMDS-3596] Fix invalid stats reset during restore from
lineage
ceb28dfed6 is described below
commit ceb28dfed650bde7dde1e2eb775a606185a711b7
Author: Matthias Boehm <[email protected]>
AuthorDate: Sat Jul 15 15:16:33 2023 +0200
[SYSTEMDS-3596] Fix invalid stats reset during restore from lineage
This patch fixes an issue of invalid reset of the global statistics
(heavy hitters, cache statistics) on every bufferpool restore from
lineage. Likely, this never showed up before because restores are very
rare, and restores from lineage even more so.
Additionally, this patch includes a few minor fixes to remove warnings.
---
src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java | 1 +
.../java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java | 1 -
.../java/org/apache/sysds/runtime/lineage/LineageRecomputeUtils.java | 5 ++---
.../org/apache/sysds/runtime/transform/encode/EncoderFactory.java | 3 ++-
.../test/functions/lineage/LineageExploitationBufferPoolTest.java | 5 +++--
.../scripts/functions/lineage/LineageExploitationBufferPool2.dml | 2 +-
6 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
index dbac8d470c..777aff0b23 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/FrameBlock.java
@@ -1455,6 +1455,7 @@ public class FrameBlock implements
CacheBlock<FrameBlock>, Externalizable {
return out;
}
+ @SuppressWarnings("deprecation")
public FrameBlock valueSwap(FrameBlock schema) {
String[] schemaString =
IteratorFactory.getStringRowIterator(schema).next();
String dataValue2 = null;
diff --git
a/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java
b/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java
index 1cc953075f..dbaef2b51a 100644
--- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java
+++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibAppend.java
@@ -100,7 +100,6 @@ public class FrameLibAppend {
return new FrameBlock(retSchema, retColNames, retColMeta,
retCols);
}
- @SuppressWarnings("unchecked")
private static <T> T[] addAll(T[] a, T[] b) {
return (T[]) ArrayUtils.addAll(a, b);
}
diff --git
a/src/main/java/org/apache/sysds/runtime/lineage/LineageRecomputeUtils.java
b/src/main/java/org/apache/sysds/runtime/lineage/LineageRecomputeUtils.java
index 747bea28ef..c5fceed547 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageRecomputeUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageRecomputeUtils.java
@@ -99,9 +99,6 @@ public class LineageRecomputeUtils {
GPUenabled = true;
DMLScript.USE_ACCELERATOR = false;
}
- // Reset statistics
- if (DMLScript.STATISTICS)
- Statistics.reset();
Data ret = computeByLineage(root);
@@ -133,7 +130,9 @@ public class LineageRecomputeUtils {
constructBasicBlock(partDagRoots, varname, prog);
// Reset cache to avoid erroneous reuse
+ //FIXME: any lineage-based restore in the bufferpool flushes
the lineage cache
LineageCache.resetCache();
+
// Execute instructions and get result
if (DEBUG) {
DMLScript.STATISTICS = true;
diff --git
a/src/main/java/org/apache/sysds/runtime/transform/encode/EncoderFactory.java
b/src/main/java/org/apache/sysds/runtime/transform/encode/EncoderFactory.java
index 1fd52ee829..4357b35307 100644
---
a/src/main/java/org/apache/sysds/runtime/transform/encode/EncoderFactory.java
+++
b/src/main/java/org/apache/sysds/runtime/transform/encode/EncoderFactory.java
@@ -28,6 +28,7 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
+import java.util.Objects;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.logging.Log;
@@ -193,7 +194,7 @@ public interface EncoderFactory {
String[] colnames2 = meta.getColumnNames();
if(!TfMetaUtils.isIDSpec(jSpec) && colnames !=
null && colnames2 != null &&
- !ArrayUtils.isEquals(colnames,
colnames2)) {
+ !Objects.deepEquals(colnames,
colnames2)) {
HashMap<String, Integer> colPos =
getColumnPositions(colnames2);
// create temporary meta frame block w/
shallow column copy
FrameBlock meta2 = new
FrameBlock(meta.getSchema(), colnames2);
diff --git
a/src/test/java/org/apache/sysds/test/functions/lineage/LineageExploitationBufferPoolTest.java
b/src/test/java/org/apache/sysds/test/functions/lineage/LineageExploitationBufferPoolTest.java
index 52feeb2edb..f25c3592ad 100644
---
a/src/test/java/org/apache/sysds/test/functions/lineage/LineageExploitationBufferPoolTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/lineage/LineageExploitationBufferPoolTest.java
@@ -74,8 +74,9 @@ public class LineageExploitationBufferPoolTest extends
LineageBase
Assert.assertEquals(5,
Long.parseLong(writes[1])); // writes WB
}
else {
-
Assert.assertTrue(CacheStatistics.getLinWrites()==2);
-
Assert.assertTrue(CacheStatistics.getLinHits()>=1);
+
Assert.assertTrue(heavyHittersContainsString("+", 1, 22));
+
Assert.assertTrue(CacheStatistics.getLinWrites()==5); //2 (+ 3 for 2x restore)
+
Assert.assertTrue(CacheStatistics.getLinHits()==2);
}
} finally {
OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION =
old_simplification;
diff --git
a/src/test/scripts/functions/lineage/LineageExploitationBufferPool2.dml
b/src/test/scripts/functions/lineage/LineageExploitationBufferPool2.dml
index e1838667b9..b397a5da81 100644
--- a/src/test/scripts/functions/lineage/LineageExploitationBufferPool2.dml
+++ b/src/test/scripts/functions/lineage/LineageExploitationBufferPool2.dml
@@ -31,6 +31,6 @@ for(i in 1:20) {
}
#restore X from lineage
-R = X + Y0 + as.matrix(Z[1]);
+R = X + Y0 + as.matrix(Z[10]);
write(R, $1, format="text");
