This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemml.git
The following commit(s) were added to refs/heads/master by this push:
new 498aef6 [SYSTEMDS-339] Fix robustness lineage tracing/parsing, part I
498aef6 is described below
commit 498aef6c5915c8256da44a041049cc4a59a84d41
Author: Matthias Boehm <mboe...@gmail.com>
AuthorDate: Wed May 20 23:37:12 2020 +0200
[SYSTEMDS-339] Fix robustness lineage tracing/parsing, part I
This patch adds steplm as a new parfor lineage tracing/parsing test case
and fixes many related lineage tracing/parsing issues:
1) Parfor Lineage Merge: robustness against empty first worker (no
lineage for result variable, e.g., due to conditional control flow)
2) Lineage Tracing: support for replace, rexpand
3) Lineage Parsing: support for seq, ifelse, log, log_nz, groupedagg,
rmempty, replace, rexpand
4) Steplm: Improved initialization of parfor result variables to avoid
cycles and stackoverflow errors in overwrite scenarios.
5) Minor: parsing of ternary operator codes
However, additional fixes are required for lineage tracing wrt default
handling in functions, literal replacement during dynamic recompilation,
and better overwrite support in matrix indexing (currently this creates
lineage cycles).
---
docs/Tasks.txt | 1 +
scripts/builtin/steplm.dml | 1 +
src/main/java/org/apache/sysds/common/Types.java | 2 +-
.../apache/sysds/hops/rewrite/HopRewriteUtils.java | 9 ++-
.../runtime/controlprogram/ParForProgramBlock.java | 6 +-
.../instructions/cp/DataGenCPInstruction.java | 12 ++++
.../cp/ParameterizedBuiltinCPInstruction.java | 57 +++++++++++++---
.../sysds/runtime/lineage/LineageItemUtils.java | 75 ++++++++++++++++++----
.../functions/lineage/LineageTraceParforTest.java | 19 +++++-
.../functions/lineage/LineageTraceParfor1.dml | 4 +-
.../functions/lineage/LineageTraceParfor2.dml | 4 +-
.../functions/lineage/LineageTraceParfor3.dml | 4 +-
...ageTraceParfor3.dml => LineageTraceParfor4.dml} | 13 ++--
13 files changed, 161 insertions(+), 46 deletions(-)
diff --git a/docs/Tasks.txt b/docs/Tasks.txt
index 6d5ff80..66d0901 100644
--- a/docs/Tasks.txt
+++ b/docs/Tasks.txt
@@ -272,6 +272,7 @@ SYSTEMDS-330 Lineage Tracing, Reuse and Integration
* 336 Better use of cache status to handle multithreading
* 337 Adjust disk I/O speed by recording actual time taken OK
* 338 Extended lineage tracing (rmEmpty, lists), partial rewrites OK
+ * 339 Lineage tracing robustness (indexed updates, algorithms)
SYSTEMDS-340 Compiler Assisted Lineage Caching and Reuse
* 341 Finalize unmarking of loop dependent operations
diff --git a/scripts/builtin/steplm.dml b/scripts/builtin/steplm.dml
index fd0018d..28208c8 100644
--- a/scripts/builtin/steplm.dml
+++ b/scripts/builtin/steplm.dml
@@ -126,6 +126,7 @@ m_steplm = function(Matrix[Double] X, Matrix[Double] y,
Integer icpt = 0,
while (continue) {
# Subsequent passes over the features
beta_out_all_2 = matrix(0, boa_ncol, m_orig * 1);
+ AICs = matrix(0, 1, m_orig); # full overwrite
parfor (i in 1:m_orig, check = 0) {
if (as.scalar(columns_fixed[1, i]) == 0) {
# Construct the feature matrix
diff --git a/src/main/java/org/apache/sysds/common/Types.java
b/src/main/java/org/apache/sysds/common/Types.java
index 11e597e..d693b7f 100644
--- a/src/main/java/org/apache/sysds/common/Types.java
+++ b/src/main/java/org/apache/sysds/common/Types.java
@@ -359,7 +359,7 @@ public class Types
case "cm": return OpOp3.MOMENT;
case "+*": return OpOp3.PLUS_MULT;
case "-*": return OpOp3.MINUS_MULT;
- default: return OpOp3.valueOf(code);
+ default: return
OpOp3.valueOf(code.toUpperCase());
}
}
}
diff --git a/src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java
b/src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java
index 9d86b4d..9e73fcc 100644
--- a/src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java
@@ -779,8 +779,13 @@ public class HopRewriteUtils
}
public static TernaryOp createTernaryOp(Hop mleft, Hop smid, Hop
mright, OpOp3 op) {
- TernaryOp ternOp = new TernaryOp("tmp", DataType.MATRIX,
ValueType.FP64, op, mleft, smid, mright);
- ternOp.setBlocksize(mleft.getBlocksize());
+ //NOTe: for ifelse it's sufficient to check mright as
smid==mright
+ System.out.println(mleft.getDataType()+" "+smid.getDataType()+"
"+mright.getDataType());
+ DataType dt = (op == OpOp3.IFELSE) ? mright.getDataType() :
DataType.MATRIX;
+ ValueType vt = (op == OpOp3.IFELSE) ? mright.getValueType() :
ValueType.FP64;
+ TernaryOp ternOp = new TernaryOp("tmp", dt, vt, op, mleft,
smid, mright);
+ if( dt == DataType.MATRIX )
+ ternOp.setBlocksize(mleft.getBlocksize());
copyLineNumbers(mleft, ternOp);
ternOp.refreshSizeInformation();
return ternOp;
diff --git
a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
index 812cf2c..2981be2 100644
---
a/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
+++
b/src/main/java/org/apache/sysds/runtime/controlprogram/ParForProgramBlock.java
@@ -1351,8 +1351,10 @@ public class ParForProgramBlock extends ForProgramBlock
LineageItem current = lineages[0].get(var._name);
for( int i=1; i<lineages.length; i++ ) {
LineageItem next = lineages[i].get(var._name);
- if( next != null ) //robustness for cond.
control flow
- current =
LineageItemUtils.replace(next, retIn, current);
+ if( next != null ) { //robustness for cond.
control flow
+ current = (current == null) ? next :
+ LineageItemUtils.replace(next,
retIn, current);
+ }
}
ec.getLineage().set(var._name, current);
}
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java
index cdea819..11f4e8e 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java
@@ -170,6 +170,18 @@ public class DataGenCPInstruction extends
UnaryCPInstruction {
public boolean isOnesCol() {
return minValue == maxValue && minValue == 1 && sparsity == 1
&& getCols() == 1;
}
+
+ public long getFrom() {
+ return seq_from.isLiteral() ?
Long.parseLong(seq_from.getName()) : -1;
+ }
+
+ public long getTo() {
+ return seq_to.isLiteral() ? Long.parseLong(seq_to.getName()) :
-1;
+ }
+
+ public long getIncr() {
+ return seq_incr.isLiteral() ?
Long.parseLong(seq_incr.getName()) : -1;
+ }
public static DataGenCPInstruction parseInstruction(String str)
{
diff --git
a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
index 9c92ddf..f284240 100644
---
a/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
+++
b/src/main/java/org/apache/sysds/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java
@@ -221,8 +221,8 @@ public class ParameterizedBuiltinCPInstruction extends
ComputationCPInstruction
}
else if ( opcode.equalsIgnoreCase("replace") ) {
MatrixBlock target =
ec.getMatrixInput(params.get("target"));
- double pattern = Double.parseDouble(
params.get("pattern") );
- double replacement = Double.parseDouble(
params.get("replacement") );
+ double pattern =
Double.parseDouble(params.get("pattern"));
+ double replacement =
Double.parseDouble(params.get("replacement"));
MatrixBlock ret = target.replaceOperations(new
MatrixBlock(), pattern, replacement);
ec.setMatrixOutput(output.getName(), ret);
ec.releaseMatrixInput(params.get("target"));
@@ -241,7 +241,7 @@ public class ParameterizedBuiltinCPInstruction extends
ComputationCPInstruction
MatrixBlock target =
ec.getMatrixInput(params.get("target"));
// compute the result
- double maxVal = Double.parseDouble( params.get("max") );
+ double maxVal = Double.parseDouble(params.get("max"));
boolean dirVal = params.get("dir").equals("rows");
boolean cast = Boolean.parseBoolean(params.get("cast"));
boolean ignore =
Boolean.parseBoolean(params.get("ignore"));
@@ -401,27 +401,64 @@ public class ParameterizedBuiltinCPInstruction extends
ComputationCPInstruction
public Pair<String, LineageItem> getLineageItem(ExecutionContext ec) {
String opcode = getOpcode();
if (opcode.equalsIgnoreCase("groupedagg")) {
- CPOperand target = new
CPOperand(params.get(Statement.GAGG_TARGET), ValueType.FP64, DataType.MATRIX);
+ CPOperand target = getTargetOperand();
CPOperand groups = new
CPOperand(params.get(Statement.GAGG_GROUPS), ValueType.FP64, DataType.MATRIX);
String wt = params.containsKey(Statement.GAGG_WEIGHTS)
? params.get(Statement.GAGG_WEIGHTS) : String.valueOf(-1);
CPOperand weights = new CPOperand(wt, ValueType.FP64,
DataType.MATRIX);
- CPOperand fn = new
CPOperand(params.get(Statement.GAGG_FN), ValueType.STRING, DataType.SCALAR,
true);
+ CPOperand fn = getStringLiteral(Statement.GAGG_FN);
String ng =
params.containsKey(Statement.GAGG_NUM_GROUPS) ?
params.get(Statement.GAGG_NUM_GROUPS) : String.valueOf(-1);
CPOperand ngroups = new CPOperand(ng , ValueType.INT64,
DataType.SCALAR, true);
return Pair.of(output.getName(), new
LineageItem(getOpcode(),
LineageItemUtils.getLineage(ec, target, groups,
weights, fn, ngroups)));
}
else if (opcode.equalsIgnoreCase("rmempty")) {
- CPOperand target = new CPOperand(params.get("target"),
ValueType.FP64, DataType.MATRIX);
- CPOperand margin = new CPOperand(params.get("margin"),
ValueType.STRING, DataType.SCALAR, true);
+ CPOperand target = getTargetOperand();
+ CPOperand margin = getStringLiteral("margin");
String sl = params.containsKey("select") ?
params.get("select") : String.valueOf(-1);
CPOperand select = new CPOperand(sl, ValueType.FP64,
DataType.MATRIX);
return Pair.of(output.getName(), new
LineageItem(getOpcode(),
LineageItemUtils.getLineage(ec, target, margin,
select)));
}
- //TODO: generic interface to support all the ops
- else
+ else if(opcode.equalsIgnoreCase("replace")) {
+ CPOperand target = getTargetOperand();
+ CPOperand pattern = getFP64Literal("pattern");
+ CPOperand replace = getFP64Literal("replacement");
return Pair.of(output.getName(), new
LineageItem(getOpcode(),
- LineageItemUtils.getLineage(ec,
input1,input2,input3)));
+ LineageItemUtils.getLineage(ec, target,
pattern, replace)));
+ }
+ else if(opcode.equalsIgnoreCase("rexpand")) {
+ CPOperand target = getTargetOperand();
+ CPOperand max = getFP64Literal("max");
+ CPOperand dir = getStringLiteral("dir");
+ CPOperand cast = getBoolLiteral("cast");
+ CPOperand ignore = getBoolLiteral("ignore");
+ return Pair.of(output.getName(), new
LineageItem(getOpcode(),
+ LineageItemUtils.getLineage(ec, target, max,
dir, cast, ignore)));
+ }
+ else {
+ //NOTE: for now, we cannot have a generic fall through
path, because the
+ //data and value types of parmeters are not compiled
into the instruction
+ throw new DMLRuntimeException("Unsupported lineage
tracing for: "+opcode);
+ }
+ }
+
+ private CPOperand getTargetOperand() {
+ return new CPOperand(params.get("target"), ValueType.FP64,
DataType.MATRIX);
+ }
+
+ private CPOperand getFP64Literal(String name) {
+ return getLiteral(name, ValueType.FP64);
+ }
+
+ private CPOperand getStringLiteral(String name) {
+ return getLiteral(name, ValueType.STRING);
+ }
+
+ private CPOperand getBoolLiteral(String name) {
+ return getLiteral(name, ValueType.BOOLEAN);
+ }
+
+ private CPOperand getLiteral(String name, ValueType vt) {
+ return new CPOperand(params.get(name), vt, DataType.SCALAR,
true);
}
}
diff --git
a/src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java
b/src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java
index 1c787a2..39a8c2a 100644
--- a/src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java
+++ b/src/main/java/org/apache/sysds/runtime/lineage/LineageItemUtils.java
@@ -34,6 +34,7 @@ import org.apache.sysds.common.Types.Direction;
import org.apache.sysds.common.Types.OpOpDG;
import org.apache.sysds.common.Types.OpOpData;
import org.apache.sysds.common.Types.OpOpN;
+import org.apache.sysds.common.Types.ParamBuiltinOp;
import org.apache.sysds.common.Types.ReOrgOp;
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.conf.ConfigurationManager;
@@ -56,6 +57,7 @@ import org.apache.sysds.lops.UnaryCP;
import org.apache.sysds.lops.compile.Dag;
import org.apache.sysds.parser.DataExpression;
import org.apache.sysds.parser.DataIdentifier;
+import org.apache.sysds.parser.Statement;
import org.apache.sysds.runtime.DMLRuntimeException;
import org.apache.sysds.runtime.controlprogram.BasicProgramBlock;
import org.apache.sysds.runtime.controlprogram.Program;
@@ -79,6 +81,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.stream.Collectors;
@@ -197,19 +200,27 @@ public class LineageItemUtils {
if (inst instanceof DataGenCPInstruction) {
DataGenCPInstruction rand =
(DataGenCPInstruction) inst;
HashMap<String, Hop> params = new
HashMap<>();
- if( rand.output.getDataType() ==
DataType.TENSOR)
-
params.put(DataExpression.RAND_DIMS, new LiteralOp(rand.getDims()));
- else {
-
params.put(DataExpression.RAND_ROWS, new LiteralOp(rand.getRows()));
-
params.put(DataExpression.RAND_COLS, new LiteralOp(rand.getCols()));
+ if( rand.getOpcode().equals("rand") ) {
+ if( rand.output.getDataType()
== DataType.TENSOR)
+
params.put(DataExpression.RAND_DIMS, new LiteralOp(rand.getDims()));
+ else {
+
params.put(DataExpression.RAND_ROWS, new LiteralOp(rand.getRows()));
+
params.put(DataExpression.RAND_COLS, new LiteralOp(rand.getCols()));
+ }
+
params.put(DataExpression.RAND_MIN, new LiteralOp(rand.getMinValue()));
+
params.put(DataExpression.RAND_MAX, new LiteralOp(rand.getMaxValue()));
+
params.put(DataExpression.RAND_PDF, new LiteralOp(rand.getPdf()));
+
params.put(DataExpression.RAND_LAMBDA, new LiteralOp(rand.getPdfParams()));
+
params.put(DataExpression.RAND_SPARSITY, new LiteralOp(rand.getSparsity()));
+
params.put(DataExpression.RAND_SEED, new LiteralOp(rand.getSeed()));
}
- params.put(DataExpression.RAND_MIN, new
LiteralOp(rand.getMinValue()));
- params.put(DataExpression.RAND_MAX, new
LiteralOp(rand.getMaxValue()));
- params.put(DataExpression.RAND_PDF, new
LiteralOp(rand.getPdf()));
- params.put(DataExpression.RAND_LAMBDA,
new LiteralOp(rand.getPdfParams()));
-
params.put(DataExpression.RAND_SPARSITY, new LiteralOp(rand.getSparsity()));
- params.put(DataExpression.RAND_SEED,
new LiteralOp(rand.getSeed()));
- Hop datagen = new
DataGenOp(OpOpDG.RAND, new DataIdentifier("tmp"), params);
+ else if( rand.getOpcode().equals("seq")
) {
+ params.put(Statement.SEQ_FROM,
new LiteralOp(rand.getFrom()));
+ params.put(Statement.SEQ_TO,
new LiteralOp(rand.getTo()));
+ params.put(Statement.SEQ_INCR,
new LiteralOp(rand.getIncr()));
+ }
+ Hop datagen = new
DataGenOp(OpOpDG.valueOf(rand.getOpcode().toUpperCase()),
+ new DataIdentifier("tmp"),
params);
datagen.setBlocksize(rand.getBlocksize());
operands.put(item.getId(), datagen);
} else if (inst instanceof VariableCPInstruction
@@ -260,7 +271,8 @@ public class LineageItemUtils {
operands.put(item.getId(), aggunary);
break;
}
- case Unary: {
+ case Unary:
+ case Builtin: {
Hop input =
operands.get(item.getInputs()[0].getId());
Hop unary =
HopRewriteUtils.createUnary(input, item.getOpcode());
operands.put(item.getId(), unary);
@@ -303,6 +315,10 @@ public class LineageItemUtils {
OpOpN.valueOf(opcode.toUpperCase()), createNaryInputs(item, operands)));
break;
}
+ case ParameterizedBuiltin: {
+
operands.put(item.getId(), constructParameterizedBuiltinOp(item, operands));
+ break;
+ }
case MatrixIndexing: {
operands.put(item.getId(), constructIndexingOp(item, operands));
break;
@@ -486,6 +502,39 @@ public class LineageItemUtils {
throw new DMLRuntimeException("Unsupported opcode:
"+item.getOpcode());
}
+ private static Hop constructParameterizedBuiltinOp(LineageItem item,
Map<Long, Hop> operands) {
+ String opcode = item.getOpcode();
+ Hop target = operands.get(item.getInputs()[0].getId());
+ LinkedHashMap<String,Hop> args = new LinkedHashMap<>();
+ if( opcode.equals("groupedagg") ) {
+ args.put("target", target);
+ args.put(Statement.GAGG_GROUPS,
operands.get(item.getInputs()[1].getId()));
+ args.put(Statement.GAGG_WEIGHTS,
operands.get(item.getInputs()[2].getId()));
+ args.put(Statement.GAGG_FN,
operands.get(item.getInputs()[3].getId()));
+ args.put(Statement.GAGG_NUM_GROUPS,
operands.get(item.getInputs()[4].getId()));
+ }
+ else if (opcode.equalsIgnoreCase("rmempty")) {
+ args.put("target", target);
+ args.put("margin",
operands.get(item.getInputs()[1].getId()));
+ args.put("select",
operands.get(item.getInputs()[2].getId()));
+ }
+ else if(opcode.equalsIgnoreCase("replace")) {
+ args.put("target", target);
+ args.put("pattern",
operands.get(item.getInputs()[1].getId()));
+ args.put("replacement",
operands.get(item.getInputs()[2].getId()));
+ }
+ else if(opcode.equalsIgnoreCase("rexpand")) {
+ args.put("target", target);
+ args.put("max",
operands.get(item.getInputs()[1].getId()));
+ args.put("dir",
operands.get(item.getInputs()[2].getId()));
+ args.put("cast",
operands.get(item.getInputs()[3].getId()));
+ args.put("ignore",
operands.get(item.getInputs()[4].getId()));
+ }
+
+ return HopRewriteUtils.createParameterizedBuiltinOp(
+ target, args,
ParamBuiltinOp.valueOf(opcode.toUpperCase()));
+ }
+
public static LineageItem rDecompress(LineageItem item) {
if (item.getType() == LineageItemType.Dedup) {
LineageItem dedupInput =
rDecompress(item.getInputs()[0]);
diff --git
a/src/test/java/org/apache/sysds/test/functions/lineage/LineageTraceParforTest.java
b/src/test/java/org/apache/sysds/test/functions/lineage/LineageTraceParforTest.java
index af4239f..50443c1 100644
---
a/src/test/java/org/apache/sysds/test/functions/lineage/LineageTraceParforTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/lineage/LineageTraceParforTest.java
@@ -44,6 +44,7 @@ public class LineageTraceParforTest extends AutomatedTestBase
{
protected static final String TEST_NAME1 = "LineageTraceParfor1";
//rand - matrix result - local parfor
protected static final String TEST_NAME2 = "LineageTraceParfor2";
//rand - matrix result - remote spark parfor
protected static final String TEST_NAME3 = "LineageTraceParfor3";
//rand - matrix result - remote spark parfor
+ protected static final String TEST_NAME4 = "LineageTraceParfor4";
//rand - steplm (stackoverflow error)
protected String TEST_CLASS_DIR = TEST_DIR +
LineageTraceParforTest.class.getSimpleName() + "/";
@@ -59,6 +60,7 @@ public class LineageTraceParforTest extends AutomatedTestBase
{
addTestConfiguration( TEST_NAME1, new
TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"}) );
addTestConfiguration( TEST_NAME2, new
TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"}) );
addTestConfiguration( TEST_NAME3, new
TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"}) );
+ addTestConfiguration( TEST_NAME4, new
TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"}) );
}
@Test
@@ -98,7 +100,7 @@ public class LineageTraceParforTest extends
AutomatedTestBase {
@Test
public void testLineageTraceParFor2_32() {
- testLineageTraceParFor(8, TEST_NAME2);
+ testLineageTraceParFor(32, TEST_NAME2);
}
@Test
@@ -108,9 +110,20 @@ public class LineageTraceParforTest extends
AutomatedTestBase {
@Test
public void testLineageTraceParFor3_32() {
- testLineageTraceParFor(8, TEST_NAME3);
+ testLineageTraceParFor(32, TEST_NAME3);
}
+// TODO additional fixes needed for steplm
+// @Test
+// public void testLineageTraceParFor4_8() {
+// testLineageTraceParFor(8, TEST_NAME4);
+// }
+//
+// @Test
+// public void testLineageTraceParFor4_32() {
+// testLineageTraceParFor(32, TEST_NAME4);
+// }
+
private void testLineageTraceParFor(int ncol, String testname) {
try {
System.out.println("------------ BEGIN " + testname +
"------------");
@@ -121,7 +134,6 @@ public class LineageTraceParforTest extends
AutomatedTestBase {
proArgs.add("-explain");
proArgs.add("-lineage");
proArgs.add("-args");
- proArgs.add(input("X"));
proArgs.add(output("R"));
proArgs.add(String.valueOf(numRecords));
proArgs.add(String.valueOf(ncol));
@@ -134,6 +146,7 @@ public class LineageTraceParforTest extends
AutomatedTestBase {
//get lineage and generate program
String Rtrace = readDMLLineageFromHDFS("R");
+ System.out.println(Rtrace);
LineageItem R = LineageParser.parseLineageTrace(Rtrace);
Data ret = LineageItemUtils.computeByLineage(R);
diff --git a/src/test/scripts/functions/lineage/LineageTraceParfor1.dml
b/src/test/scripts/functions/lineage/LineageTraceParfor1.dml
index e6f48bb..dc82d38 100644
--- a/src/test/scripts/functions/lineage/LineageTraceParfor1.dml
+++ b/src/test/scripts/functions/lineage/LineageTraceParfor1.dml
@@ -19,7 +19,7 @@
#
#-------------------------------------------------------------
-X = rand(rows=$3, cols=$4, seed=7);
+X = rand(rows=$2, cols=$3, seed=7);
R = matrix(0, nrow(X), ncol(X));
parfor(i in 1:ncol(X)) {
@@ -29,4 +29,4 @@ parfor(i in 1:ncol(X)) {
}
print(lineage(R));
-write(R, $2);
+write(R, $1);
diff --git a/src/test/scripts/functions/lineage/LineageTraceParfor2.dml
b/src/test/scripts/functions/lineage/LineageTraceParfor2.dml
index de04f8a..a1fd11f 100644
--- a/src/test/scripts/functions/lineage/LineageTraceParfor2.dml
+++ b/src/test/scripts/functions/lineage/LineageTraceParfor2.dml
@@ -19,7 +19,7 @@
#
#-------------------------------------------------------------
-X = rand(rows=$3, cols=$4, seed=7);
+X = rand(rows=$2, cols=$3, seed=7);
R = matrix(0, nrow(X), ncol(X));
parfor(i in 1:ncol(X), check=0, opt=None, mode=REMOTE_SPARK) {
@@ -29,4 +29,4 @@ parfor(i in 1:ncol(X), check=0, opt=None, mode=REMOTE_SPARK) {
}
print(lineage(R));
-write(R, $2);
+write(R, $1);
diff --git a/src/test/scripts/functions/lineage/LineageTraceParfor3.dml
b/src/test/scripts/functions/lineage/LineageTraceParfor3.dml
index b6e450f..7b8ad18 100644
--- a/src/test/scripts/functions/lineage/LineageTraceParfor3.dml
+++ b/src/test/scripts/functions/lineage/LineageTraceParfor3.dml
@@ -19,7 +19,7 @@
#
#-------------------------------------------------------------
-X = rand(rows=$3, cols=$4, seed=7);
+X = rand(rows=$2, cols=$3, seed=7);
R = matrix(0, 1, ncol(X));
parfor(i in 1:ncol(X), check=0, opt=None, mode=REMOTE_SPARK) {
@@ -28,4 +28,4 @@ parfor(i in 1:ncol(X), check=0, opt=None, mode=REMOTE_SPARK) {
}
print(lineage(R));
-write(R, $2);
+write(R, $1);
diff --git a/src/test/scripts/functions/lineage/LineageTraceParfor3.dml
b/src/test/scripts/functions/lineage/LineageTraceParfor4.dml
similarity index 81%
copy from src/test/scripts/functions/lineage/LineageTraceParfor3.dml
copy to src/test/scripts/functions/lineage/LineageTraceParfor4.dml
index b6e450f..576182b 100644
--- a/src/test/scripts/functions/lineage/LineageTraceParfor3.dml
+++ b/src/test/scripts/functions/lineage/LineageTraceParfor4.dml
@@ -19,13 +19,8 @@
#
#-------------------------------------------------------------
-X = rand(rows=$3, cols=$4, seed=7);
+X = rand(rows=$2, cols=$3, seed=7);
+Y = rand(rows=nrow(X), cols=1, seed=2)
+X = steplm(X=X, y=Y)
-R = matrix(0, 1, ncol(X));
-parfor(i in 1:ncol(X), check=0, opt=None, mode=REMOTE_SPARK) {
- Xi = matrix(i, rows=2, cols=4)
- R[1, i] = sum(Xi)
-}
-
-print(lineage(R));
-write(R, $2);
+write(X, $1);
