This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new b85d1b45a7 [SYSTEMDS-3834] Dependency tasks enable via Log4j
b85d1b45a7 is described below
commit b85d1b45a7a41f0e3ee94e0c2357a8892e0d3148
Author: Sebastian Baunsgaard <[email protected]>
AuthorDate: Fri Feb 7 17:50:48 2025 +0100
[SYSTEMDS-3834] Dependency tasks enable via Log4j
This commit reenable dependency graph printing for parallel transform
encode.
Previously one would have to recompile the system with modified code, while
now
we can enable the task graph via logging configurations.
Closes #2222
---
.../apache/sysds/runtime/util/DependencyTask.java | 4 +-
.../sysds/runtime/util/DependencyThreadPool.java | 10 ++-
.../component/frame/transform/TransformLogger.java | 100 +++++++++++++++++++++
3 files changed, 108 insertions(+), 6 deletions(-)
diff --git a/src/main/java/org/apache/sysds/runtime/util/DependencyTask.java
b/src/main/java/org/apache/sysds/runtime/util/DependencyTask.java
index 943b344502..e66b90d62c 100644
--- a/src/main/java/org/apache/sysds/runtime/util/DependencyTask.java
+++ b/src/main/java/org/apache/sysds/runtime/util/DependencyTask.java
@@ -30,12 +30,12 @@ import org.apache.commons.logging.LogFactory;
import org.apache.sysds.runtime.DMLRuntimeException;
public class DependencyTask<E> implements Comparable<DependencyTask<?>>,
Callable<E> {
- public static final boolean ENABLE_DEBUG_DATA = false; // explain task
graph
protected static final Log LOG =
LogFactory.getLog(DependencyTask.class.getName());
+ /** debugging dependency tasks only used if LOG.isDebugEnabled */
+ public List<DependencyTask<?>> _dependencyTasks = null;
private final Callable<E> _task;
protected final List<DependencyTask<?>> _dependantTasks;
- public List<DependencyTask<?>> _dependencyTasks = null; // only for
debugging
private CompletableFuture<Future<?>> _future;
private int _rdy = 0;
private Integer _priority = 0;
diff --git
a/src/main/java/org/apache/sysds/runtime/util/DependencyThreadPool.java
b/src/main/java/org/apache/sysds/runtime/util/DependencyThreadPool.java
index 1e8f99b655..699e1eed11 100644
--- a/src/main/java/org/apache/sysds/runtime/util/DependencyThreadPool.java
+++ b/src/main/java/org/apache/sysds/runtime/util/DependencyThreadPool.java
@@ -92,7 +92,7 @@ public class DependencyThreadPool {
public List<Object> submitAllAndWait(List<DependencyTask<?>> dtasks)
throws ExecutionException, InterruptedException {
List<Object> res = new ArrayList<>();
- if(DependencyTask.ENABLE_DEBUG_DATA) {
+ if(LOG.isDebugEnabled()) {
if (dtasks != null && dtasks.size() > 0)
explainTaskGraph(dtasks);
}
@@ -172,7 +172,7 @@ public class DependencyThreadPool {
DependencyTask<?> t = ret.get(i);
for(Callable<?> dep : deps) {
DependencyTask<?> dt = map.get(dep);
- if(DependencyTask.ENABLE_DEBUG_DATA) {
+ if(LOG.isDebugEnabled()) {
t._dependencyTasks = t._dependencyTasks
== null ? new ArrayList<>() : t._dependencyTasks;
t._dependencyTasks.add(dt);
}
@@ -226,10 +226,12 @@ public class DependencyThreadPool {
sbs[level].append(offsets[level]);
sbs[level].append(entry.getKey().toString()+"\n");
}
- System.out.println("EXPlAIN (TASK-GRAPH):");
+ StringBuilder sb = new StringBuilder("\n");
+ sb.append("EXPlAIN (TASK-GRAPH):");
for (int i=0; i<sbs.length; i++) {
- System.out.println(sbs[i].toString());
+ sb.append(sbs[i].toString());
}
+ LOG.debug(sb.toString());
}
}
diff --git
a/src/test/java/org/apache/sysds/test/component/frame/transform/TransformLogger.java
b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformLogger.java
new file mode 100644
index 0000000000..6b00783d98
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/component/frame/transform/TransformLogger.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.frame.transform;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.sysds.common.Types.ValueType;
+import org.apache.sysds.runtime.frame.data.FrameBlock;
+import org.apache.sysds.runtime.transform.encode.CompressedEncode;
+import org.apache.sysds.runtime.transform.encode.EncoderFactory;
+import org.apache.sysds.runtime.transform.encode.MultiColumnEncoder;
+import org.apache.sysds.runtime.util.DependencyTask;
+import org.apache.sysds.runtime.util.DependencyThreadPool;
+import org.apache.sysds.test.LoggingUtils;
+import org.apache.sysds.test.LoggingUtils.TestAppender;
+import org.apache.sysds.test.TestUtils;
+import org.junit.Test;
+
+public class TransformLogger {
+ protected static final Log LOG =
LogFactory.getLog(TransformLogger.class.getName());
+
+ private final FrameBlock data;
+
+ public TransformLogger() {
+ try {
+
+ data = TestUtils.generateRandomFrameBlock(100, new
ValueType[] {ValueType.UINT4}, 231);
+ data.setSchema(new ValueType[] {ValueType.INT32});
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ throw e;
+ }
+ }
+
+ @Test
+ public void testDummyCode() {
+ test("{dummycode:[C1]}");
+ }
+
+ public void test(String spec) {
+ final TestAppender appender = LoggingUtils.overwrite();
+
+ try {
+
Logger.getLogger(CompressedEncode.class).setLevel(Level.DEBUG);
+
Logger.getLogger(DependencyThreadPool.class).setLevel(Level.DEBUG);
+
Logger.getLogger(DependencyTask.class).setLevel(Level.DEBUG);
+
+ FrameBlock meta = null;
+ MultiColumnEncoder encoderNormal =
EncoderFactory.createEncoder(spec, data.getColumnNames(),
+ data.getNumColumns(), meta);
+ encoderNormal.encode(data, 10);
+
+ final List<LoggingEvent> log =
LoggingUtils.reinsert(appender);
+
+ boolean containsMessage = false;
+ for(LoggingEvent l : log) {
+ containsMessage |=
l.getMessage().toString().contains("EXPlAIN (TASK-GRAPH):");
+ }
+
+ assertTrue(containsMessage);
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ finally {
+ LoggingUtils.reinsert(appender);
+ }
+
+ }
+
+}
