http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java index 4b9c2e0..3ecb91f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/AppendGSPInstruction.java @@ -41,7 +41,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.matrix.operators.ReorgOperator; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public class AppendGSPInstruction extends BinarySPInstruction { private boolean _cbind = true; @@ -165,8 +164,8 @@ public class AppendGSPInstruction extends BinarySPInstruction { _cbind = cbind; _startIx = cbind ? UtilFunctions.computeBlockIndex(mc1.getCols(), mc1.getColsPerBlock()) : UtilFunctions.computeBlockIndex(mc1.getRows(), mc1.getRowsPerBlock()); - _blen = IntUtils.toInt(cbind ? mc1.getColsPerBlock() : mc1.getRowsPerBlock()); - _shiftBy = IntUtils.toInt(cbind ? mc1.getCols()%_blen : mc1.getRows()%_blen); + _blen = (int) (cbind ? mc1.getColsPerBlock() : mc1.getRowsPerBlock()); + _shiftBy = (int) (cbind ? mc1.getCols()%_blen : mc1.getRows()%_blen); _outlen = cbind ? mc1.getCols()+mc2.getCols() : mc1.getRows()+mc2.getRows(); }
http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java index 6917a7c..3ff878a 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CastSPInstruction.java @@ -36,7 +36,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public class CastSPInstruction extends UnarySPInstruction { @@ -87,7 +86,7 @@ public class CastSPInstruction extends UnarySPInstruction { //update schema information for output frame if( opcode.equals(UnaryCP.CAST_AS_FRAME_OPCODE) ) { sec.getFrameObject(output.getName()).setSchema( - UtilFunctions.nCopies(IntUtils.toInt(mcIn.getCols()), ValueType.DOUBLE)); + UtilFunctions.nCopies((int)mcIn.getCols(), ValueType.DOUBLE)); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java index 2b6575a..f25899f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CentralMomentSPInstruction.java @@ -40,7 +40,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.operators.CMOperator; import org.apache.sysml.runtime.matrix.operators.CMOperator.AggregateOperationTypes; -import org.apache.sysml.utils.IntUtils; public class CentralMomentSPInstruction extends UnarySPInstruction { @@ -105,7 +104,7 @@ public class CentralMomentSPInstruction extends UnarySPInstruction { ScalarObject order = ec.getScalarInput(scalarInput.getName(), scalarInput.getValueType(), scalarInput.isLiteral()); CMOperator cop = ((CMOperator)_optr); if ( cop.getAggOpType() == AggregateOperationTypes.INVALID ) { - cop.setCMAggOp(IntUtils.toInt(order.getLongValue())); + cop.setCMAggOp((int)order.getLongValue()); } //get input http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java index d83a6ed..308e60f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CpmmSPInstruction.java @@ -49,7 +49,6 @@ import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.matrix.operators.ReorgOperator; -import org.apache.sysml.utils.IntUtils; /** * Cpmm: cross-product matrix multiplication operation (distributed matrix multiply @@ -166,9 +165,9 @@ public class CpmmSPInstruction extends BinarySPInstruction { } private static int getMaxParJoin(MatrixCharacteristics mc1, MatrixCharacteristics mc2) { - return IntUtils.toInt(mc1.colsKnown() ? mc1.getNumColBlocks() : - mc2.rowsKnown() ? mc2.getNumRowBlocks() : - Integer.MAX_VALUE); + return mc1.colsKnown() ? (int)mc1.getNumColBlocks() : + mc2.rowsKnown() ? (int)mc2.getNumRowBlocks() : + Integer.MAX_VALUE; } private static class CpmmIndexFunction implements PairFunction<Tuple2<MatrixIndexes, MatrixBlock>, Long, IndexedMatrixValue> http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java index e8696da..1da6428 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeAggregateSPInstruction.java @@ -39,7 +39,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator; import org.apache.sysml.runtime.matrix.operators.UnaryOperator; -import org.apache.sysml.utils.IntUtils; public class CumulativeAggregateSPInstruction extends AggregateUnarySPInstruction { @@ -77,7 +76,7 @@ public class CumulativeAggregateSPInstruction extends AggregateUnarySPInstructio //merge partial aggregates, adjusting for correct number of partitions //as size can significant shrink (1K) but also grow (sparse-dense) int numParts = SparkUtils.getNumPreferredPartitions(mcOut); - int minPar = IntUtils.toInt(Math.min(SparkExecutionContext.getDefaultParallelism(true), mcOut.getNumBlocks())); + int minPar = (int)Math.min(SparkExecutionContext.getDefaultParallelism(true), mcOut.getNumBlocks()); out = RDDAggregateUtils.mergeByKey(out, Math.max(numParts, minPar), false); //put output handle in symbol table @@ -134,9 +133,9 @@ public class CumulativeAggregateSPInstruction extends AggregateUnarySPInstructio //cumsum expand partial aggregates long rlenOut = (long)Math.ceil((double)_rlen/_brlen); long rixOut = (long)Math.ceil((double)ixIn.getRowIndex()/_brlen); - int rlenBlk = IntUtils.toInt( Math.min(rlenOut-(rixOut-1)*_brlen, _brlen)); + int rlenBlk = (int) Math.min(rlenOut-(rixOut-1)*_brlen, _brlen); int clenBlk = blkOut.getNumColumns(); - int posBlk = IntUtils.toInt((ixIn.getRowIndex()-1) % _brlen); + int posBlk = (int) ((ixIn.getRowIndex()-1) % _brlen); //construct sparse output blocks (single row in target block size) MatrixBlock blkOut2 = new MatrixBlock(rlenBlk, clenBlk, true); http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java index 8befc5a..1b26060 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/CumulativeOffsetSPInstruction.java @@ -44,7 +44,6 @@ import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.matrix.operators.UnaryOperator; import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public class CumulativeOffsetSPInstruction extends BinarySPInstruction { private UnaryOperator _uop = null; @@ -201,7 +200,7 @@ public class CumulativeOffsetSPInstruction extends BinarySPInstruction { //lookup offset row and return joined output MatrixBlock off = (ixIn.getRowIndex() == 1) ? new MatrixBlock(1, blkIn.getNumColumns(), _initValue) : - _pbc.getBlock(IntUtils.toInt(brix), IntUtils.toInt(ixIn.getColumnIndex())).slice(rix, rix); + _pbc.getBlock((int)brix, (int)ixIn.getColumnIndex()).slice(rix, rix); return new Tuple2<MatrixIndexes, Tuple2<MatrixBlock,MatrixBlock>>(ixIn, new Tuple2<>(blkIn,off)); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/DnnSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/DnnSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/DnnSPInstruction.java index e734fb9..fbb214e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/DnnSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/DnnSPInstruction.java @@ -47,7 +47,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.matrix.operators.ReorgOperator; import org.apache.sysml.runtime.util.DnnUtils; -import org.apache.sysml.utils.IntUtils; import org.apache.sysml.utils.NativeHelper; import scala.Tuple2; @@ -215,7 +214,7 @@ public class DnnSPInstruction extends UnarySPInstruction { MatrixCharacteristics mcRdd = sec.getMatrixCharacteristics(name); if(mcRdd.getColsPerBlock() < mcRdd.getCols() || mcRdd.getRowsPerBlock() != 1) { MatrixCharacteristics mcOut = new MatrixCharacteristics(mcRdd); - mcOut.setColsPerBlock(IntUtils.toInt(mcRdd.getCols())); + mcOut.setColsPerBlock((int)mcRdd.getCols()); mcOut.setRowsPerBlock(numRowsPerBlock); in1 = RDDAggregateUtils.mergeByKey(in1.flatMapToPair(new ExtractBlockForBinaryReblock(mcRdd, mcOut))); // TODO: Inject checkpoint to avoid doing this repeated for validation set @@ -267,8 +266,8 @@ public class DnnSPInstruction extends UnarySPInstruction { int K = getScalarInput(ec, _filter_shape, 0); int R = getScalarInput(ec, _filter_shape, 2); int S = getScalarInput(ec, _filter_shape, 3); - int P = IntUtils.toInt(DnnUtils.getP(H, R, stride_h, pad_h)); - int Q = IntUtils.toInt(DnnUtils.getQ(W, S, stride_w, pad_w)); + int P = (int) DnnUtils.getP(H, R, stride_h, pad_h); + int Q = (int) DnnUtils.getQ(W, S, stride_w, pad_w); DnnParameters params = new DnnParameters(numRowsPerBlock, C, H, W, K, R, S, stride_h, stride_w, pad_h, pad_w, 1); boolean enableNativeBLAS = NativeHelper.isNativeLibraryLoaded(); @@ -287,7 +286,7 @@ public class DnnSPInstruction extends UnarySPInstruction { throw new DMLRuntimeException("The current operator doesnot support large outputs."); } sec.setMetaData(output.getName(), - new MetaDataFormat(new MatrixCharacteristics(mcRdd.getRows(), numCols, numRowsPerBlock, IntUtils.toInt(numCols), nnz), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo)); + new MetaDataFormat(new MatrixCharacteristics(mcRdd.getRows(), numCols, numRowsPerBlock, (int)numCols, nnz), OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo)); } else { throw new DMLRuntimeException("Not implemented: " + instOpcode); @@ -295,8 +294,8 @@ public class DnnSPInstruction extends UnarySPInstruction { } private static int getScalarInput(ExecutionContext ec, ArrayList<CPOperand> aL, int index) { - return IntUtils.toInt( ec.getScalarInput(aL.get(index).getName(), - aL.get(index).getValueType(), aL.get(index).isLiteral()).getLongValue()); + return (int) ec.getScalarInput(aL.get(index).getName(), + aL.get(index).getValueType(), aL.get(index).isLiteral()).getLongValue(); } private static class RDDConv2dMapMMFunction implements PairFlatMapFunction<Iterator<Tuple2<MatrixIndexes, MatrixBlock>>, MatrixIndexes, MatrixBlock> { http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java index 64567d1..f74f293 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java @@ -47,7 +47,6 @@ import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.data.Pair; import org.apache.sysml.runtime.util.IndexRange; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; /** * This class implements the frame indexing functionality inside Spark. @@ -104,7 +103,7 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction { //update schema of output with subset of input schema sec.getFrameObject(output.getName()).setSchema( - sec.getFrameObject(input1.getName()).getSchema(IntUtils.toInt(cl), IntUtils.toInt(cu))); + sec.getFrameObject(input1.getName()).getSchema((int)cl, (int)cu)); } //left indexing else if ( opcode.equalsIgnoreCase(LeftIndex.OPCODE) || opcode.equalsIgnoreCase("mapLeftIndex")) @@ -187,8 +186,8 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction { _ixrange = ixrange; _rlen = mcLeft.getRows(); _clen = mcLeft.getCols(); - _brlen = IntUtils.toInt( Math.min(OptimizerUtils.getDefaultFrameSize(), _rlen)); - _bclen = IntUtils.toInt(mcLeft.getCols()); + _brlen = (int) Math.min(OptimizerUtils.getDefaultFrameSize(), _rlen); + _bclen = (int) mcLeft.getCols(); } @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java index f5321f2..c9566a2 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmChainSPInstruction.java @@ -38,7 +38,6 @@ import org.apache.sysml.runtime.instructions.spark.utils.RDDAggregateUtils; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.operators.Operator; -import org.apache.sysml.utils.IntUtils; public class MapmmChainSPInstruction extends SPInstruction { private ChainType _chainType = null; @@ -169,7 +168,7 @@ public class MapmmChainSPInstruction extends SPInstruction { MatrixBlock pmV = _pmV.getBlock(1, 1); MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); - int rowIx = IntUtils.toInt(ixIn.getRowIndex()); + int rowIx = (int)ixIn.getRowIndex(); //execute mapmmchain operation return blkIn.chainMatrixMultOperations(pmV, _pmW.getBlock(rowIx,1), new MatrixBlock(), _chainType); http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java index 4f6fd8d..7c8d606 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MapmmSPInstruction.java @@ -54,7 +54,6 @@ import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.Operator; -import org.apache.sysml.utils.IntUtils; public class MapmmSPInstruction extends BinarySPInstruction { private CacheType _type = null; @@ -239,7 +238,7 @@ public class MapmmSPInstruction extends BinarySPInstruction { isLeft?mcRdd.getCols():mcBc.getCols(), isLeft?mcBc.getRowsPerBlock():mcRdd.getRowsPerBlock(), isLeft?mcRdd.getColsPerBlock():mcBc.getColsPerBlock(), 1.0)); long numParts = sizeOutput / InfrastructureAnalyzer.getHDFSBlockSize(); - return IntUtils.toInt(Math.min(numParts, (isLeft?mcRdd.getNumColBlocks():mcRdd.getNumRowBlocks()))); + return (int)Math.min(numParts, (isLeft?mcRdd.getNumColBlocks():mcRdd.getNumRowBlocks())); } private static class RDDMapMMFunction implements PairFunction<Tuple2<MatrixIndexes, MatrixBlock>, MatrixIndexes, MatrixBlock> @@ -273,7 +272,7 @@ public class MapmmSPInstruction extends BinarySPInstruction { if( _type == CacheType.LEFT ) { //get the right hand side matrix - MatrixBlock left = _pbc.getBlock(1, IntUtils.toInt(ixIn.getRowIndex())); + MatrixBlock left = _pbc.getBlock(1, (int)ixIn.getRowIndex()); //execute matrix-vector mult OperationsOnMatrixValues.matMult(new MatrixIndexes(1,ixIn.getRowIndex()), @@ -282,7 +281,7 @@ public class MapmmSPInstruction extends BinarySPInstruction { else //if( _type == CacheType.RIGHT ) { //get the right hand side matrix - MatrixBlock right = _pbc.getBlock(IntUtils.toInt(ixIn.getColumnIndex()), 1); + MatrixBlock right = _pbc.getBlock((int)ixIn.getColumnIndex(), 1); //execute matrix-vector mult OperationsOnMatrixValues.matMult(ixIn, blkIn, @@ -325,7 +324,7 @@ public class MapmmSPInstruction extends BinarySPInstruction { if( _type == CacheType.LEFT ) { //get the right hand side matrix - MatrixBlock left = _pbc.getBlock(1, IntUtils.toInt(ixIn.getRowIndex())); + MatrixBlock left = _pbc.getBlock(1, (int)ixIn.getRowIndex()); //execute matrix-vector mult return OperationsOnMatrixValues.matMult( @@ -334,7 +333,7 @@ public class MapmmSPInstruction extends BinarySPInstruction { else //if( _type == CacheType.RIGHT ) { //get the right hand side matrix - MatrixBlock right = _pbc.getBlock(IntUtils.toInt(ixIn.getColumnIndex()), 1); + MatrixBlock right = _pbc.getBlock((int)ixIn.getColumnIndex(), 1); //execute matrix-vector mult return OperationsOnMatrixValues.matMult( @@ -390,7 +389,7 @@ public class MapmmSPInstruction extends BinarySPInstruction { if( _type == CacheType.LEFT ) { //get the right hand side matrix - MatrixBlock left = _pbc.getBlock(1, IntUtils.toInt(ixIn.getRowIndex())); + MatrixBlock left = _pbc.getBlock(1, (int)ixIn.getRowIndex()); //execute index preserving matrix multiplication OperationsOnMatrixValues.matMult(left, blkIn, blkOut, _op); @@ -398,7 +397,7 @@ public class MapmmSPInstruction extends BinarySPInstruction { else //if( _type == CacheType.RIGHT ) { //get the right hand side matrix - MatrixBlock right = _pbc.getBlock(IntUtils.toInt(ixIn.getColumnIndex()), 1); + MatrixBlock right = _pbc.getBlock((int)ixIn.getColumnIndex(), 1); //execute index preserving matrix multiplication OperationsOnMatrixValues.matMult(blkIn, right, blkOut, _op); @@ -438,14 +437,14 @@ public class MapmmSPInstruction extends BinarySPInstruction { //for all matching left-hand-side blocks, returned as lazy iterator return IntStream.range(1, _pbc.getNumRowBlocks()+1).mapToObj(i -> new Tuple2<>(new MatrixIndexes(i, ixIn.getColumnIndex()), - OperationsOnMatrixValues.matMult(_pbc.getBlock(i, IntUtils.toInt(ixIn.getRowIndex())), blkIn, + OperationsOnMatrixValues.matMult(_pbc.getBlock(i, (int)ixIn.getRowIndex()), blkIn, new MatrixBlock(), _op))).iterator(); } else { //RIGHT //for all matching right-hand-side blocks, returned as lazy iterator return IntStream.range(1, _pbc.getNumColumnBlocks()+1).mapToObj(j -> new Tuple2<>(new MatrixIndexes(ixIn.getRowIndex(), j), - OperationsOnMatrixValues.matMult(blkIn, _pbc.getBlock(IntUtils.toInt(ixIn.getColumnIndex()), j), + OperationsOnMatrixValues.matMult(blkIn, _pbc.getBlock((int)ixIn.getColumnIndex(), j), new MatrixBlock(), _op))).iterator(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java index da1b975..3019a78 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java @@ -39,7 +39,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue; import org.apache.sysml.runtime.matrix.operators.Operator; -import org.apache.sysml.utils.IntUtils; public class MatrixAppendMSPInstruction extends AppendMSPInstruction { @@ -139,11 +138,11 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction { //output shallow copy of rhs block if( _cbind ) { ret.add( new Tuple2<>(new MatrixIndexes(ix.getRowIndex(), ix.getColumnIndex()+1), - _pm.getBlock(IntUtils.toInt(ix.getRowIndex()), 1)) ); + _pm.getBlock((int)ix.getRowIndex(), 1)) ); } else { //rbind ret.add( new Tuple2<>(new MatrixIndexes(ix.getRowIndex()+1, ix.getColumnIndex()), - _pm.getBlock(1, IntUtils.toInt(ix.getColumnIndex()))) ); + _pm.getBlock(1, (int)ix.getColumnIndex())) ); } } //case 3: append operation on boundary block @@ -156,7 +155,7 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction { MatrixBlock value_in2 = null; if( _cbind ) { - value_in2 = _pm.getBlock(IntUtils.toInt(ix.getRowIndex()), 1); + value_in2 = _pm.getBlock((int)ix.getRowIndex(), 1); if(in1.getValue().getNumColumns()+value_in2.getNumColumns()>_bclen) { IndexedMatrixValue second=new IndexedMatrixValue(new MatrixIndexes(), new MatrixBlock()); second.getIndexes().setIndexes(ix.getRowIndex(), ix.getColumnIndex()+1); @@ -164,7 +163,7 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction { } } else { //rbind - value_in2 = _pm.getBlock(1, IntUtils.toInt(ix.getColumnIndex())); + value_in2 = _pm.getBlock(1, (int)ix.getColumnIndex()); if(in1.getValue().getNumRows()+value_in2.getNumRows()>_brlen) { IndexedMatrixValue second=new IndexedMatrixValue(new MatrixIndexes(), new MatrixBlock()); second.getIndexes().setIndexes(ix.getRowIndex()+1, ix.getColumnIndex()); @@ -229,8 +228,8 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction { } //case 3: append operation on boundary block else { - int rowix = _cbind ? IntUtils.toInt(ix.getRowIndex()) : 1; - int colix = _cbind ? 1 : IntUtils.toInt(ix.getColumnIndex()); + int rowix = _cbind ? (int)ix.getRowIndex() : 1; + int colix = _cbind ? 1 : (int)ix.getColumnIndex(); MatrixBlock in2 = _pm.getBlock(rowix, colix); MatrixBlock out = in1.append(in2, new MatrixBlock(), _cbind); return new Tuple2<>(ix, out); http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java index 2c4a01e..d30f330 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixIndexingSPInstruction.java @@ -58,7 +58,6 @@ import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue; import org.apache.sysml.runtime.util.IndexRange; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; /** * This class implements the matrix indexing functionality inside CP. @@ -204,8 +203,8 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction { .mapToPair(new SliceBlock2(ixrange, mcOut)); //slice relevant blocks //collect output without shuffle to avoid side-effects with custom PartitionPruningRDD - MatrixBlock mbout = SparkExecutionContext.toMatrixBlock(out, IntUtils.toInt(mcOut.getRows()), - IntUtils.toInt(mcOut.getCols()), mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), -1); + MatrixBlock mbout = SparkExecutionContext.toMatrixBlock(out, (int)mcOut.getRows(), + (int)mcOut.getCols(), mcOut.getRowsPerBlock(), mcOut.getColsPerBlock(), -1); return mbout; } @@ -411,11 +410,11 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction { MatrixBlock right = arg._2(); int rl = UtilFunctions.computeCellInBlock(_ixrange.rowStart, _brlen); - int ru = IntUtils.toInt(Math.min(_ixrange.rowEnd, rl+right.getNumRows())-1); + int ru = (int)Math.min(_ixrange.rowEnd, rl+right.getNumRows())-1; int cl = UtilFunctions.computeCellInBlock(_ixrange.colStart, _brlen); - int cu = IntUtils.toInt(Math.min(_ixrange.colEnd, cl+right.getNumColumns())-1); + int cu = (int)Math.min(_ixrange.colEnd, cl+right.getNumColumns())-1; - MatrixBlock left = _binput.getBlock(IntUtils.toInt(ix.getRowIndex()), IntUtils.toInt(ix.getColumnIndex())); + MatrixBlock left = _binput.getBlock((int)ix.getRowIndex(), (int)ix.getColumnIndex()); MatrixBlock tmp = left.leftIndexingOperations(right, rl, ru, cl, cu, new MatrixBlock(), UpdateType.COPY); @@ -475,10 +474,10 @@ public class MatrixIndexingSPInstruction extends IndexingSPInstruction { //compute local index range long grix = UtilFunctions.computeCellIndex(ix.getRowIndex(), _brlen, 0); long gcix = UtilFunctions.computeCellIndex(ix.getColumnIndex(), _bclen, 0); - int lrl = IntUtils.toInt((_ixrange.rowStart<grix) ? 0 : _ixrange.rowStart - grix); - int lcl = IntUtils.toInt((_ixrange.colStart<gcix) ? 0 : _ixrange.colStart - gcix); - int lru = IntUtils.toInt(Math.min(block.getNumRows()-1, _ixrange.rowEnd - grix)); - int lcu = IntUtils.toInt(Math.min(block.getNumColumns()-1, _ixrange.colEnd - gcix)); + int lrl = (int)((_ixrange.rowStart<grix) ? 0 : _ixrange.rowStart - grix); + int lcl = (int)((_ixrange.colStart<gcix) ? 0 : _ixrange.colStart - gcix); + int lru = (int)Math.min(block.getNumRows()-1, _ixrange.rowEnd - grix); + int lcu = (int)Math.min(block.getNumColumns()-1, _ixrange.colEnd - gcix); //compute output index MatrixIndexes ixOut = new MatrixIndexes( http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java index 3d66b20..cd5e2ba 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java @@ -64,7 +64,6 @@ import org.apache.sysml.runtime.transform.encode.EncoderRecode; import org.apache.sysml.runtime.transform.encode.EncoderMVImpute.MVMethod; import org.apache.sysml.runtime.transform.meta.TfMetaUtils; import org.apache.sysml.runtime.transform.meta.TfOffsetMap; -import org.apache.sysml.utils.IntUtils; import scala.Tuple2; @@ -116,7 +115,7 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI //step 1: build transform meta data Encoder encoderBuild = EncoderFactory.createEncoder(spec, colnames, - fo.getSchema(), IntUtils.toInt(fo.getNumColumns()), null); + fo.getSchema(), (int)fo.getNumColumns(), null); MaxLongAccumulator accMax = registerMaxLongAccumulator(sec.getSparkContext()); JavaRDD<String> rcMaps = in @@ -147,7 +146,7 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI //create encoder broadcast (avoiding replication per task) Encoder encoder = EncoderFactory.createEncoder(spec, colnames, - fo.getSchema(), IntUtils.toInt(fo.getNumColumns()), meta); + fo.getSchema(), (int)fo.getNumColumns(), meta); mcOut.setDimension(mcIn.getRows()-((omap!=null)?omap.getNumRmRows():0), encoder.getNumCols()); Broadcast<Encoder> bmeta = sec.getSparkContext().broadcast(encoder); Broadcast<TfOffsetMap> bomap = (omap!=null) ? sec.getSparkContext().broadcast(omap) : null; http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java index 4ddafcc..1b6435b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/PMapmmSPInstruction.java @@ -49,7 +49,6 @@ import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.Operator; -import org.apache.sysml.utils.IntUtils; /** * This pmapmm matrix multiplication instruction is still experimental @@ -106,8 +105,8 @@ public class PMapmmSPInstruction extends BinarySPInstruction { .filter(new IsBlockInRange(i+1, i+NUM_ROWBLOCKS*mc1.getRowsPerBlock(), 1, mc1.getCols(), mc1)) .mapToPair(new PMapMMRebaseBlocksFunction(i/mc1.getRowsPerBlock())); - int rlen = IntUtils.toInt(Math.min(mc1.getRows()-i, NUM_ROWBLOCKS*mc1.getRowsPerBlock())); - PartitionedBlock<MatrixBlock> pmb = SparkExecutionContext.toPartitionedMatrixBlock(rdd, rlen, IntUtils.toInt(mc1.getCols()), mc1.getRowsPerBlock(), mc1.getColsPerBlock(), -1L); + int rlen = (int)Math.min(mc1.getRows()-i, NUM_ROWBLOCKS*mc1.getRowsPerBlock()); + PartitionedBlock<MatrixBlock> pmb = SparkExecutionContext.toPartitionedMatrixBlock(rdd, rlen, (int)mc1.getCols(), mc1.getRowsPerBlock(), mc1.getColsPerBlock(), -1L); Broadcast<PartitionedBlock<MatrixBlock>> bpmb = sec.getSparkContext().broadcast(pmb); //matrix multiplication @@ -191,7 +190,7 @@ public class PMapmmSPInstruction extends BinarySPInstruction { //get the right hand side matrix for( int i=1; i<=pm.getNumRowBlocks(); i++ ) { - MatrixBlock left = pm.getBlock(i, IntUtils.toInt(ixIn.getRowIndex())); + MatrixBlock left = pm.getBlock(i, (int)ixIn.getRowIndex()); //execute matrix-vector mult OperationsOnMatrixValues.matMult(new MatrixIndexes(i,ixIn.getRowIndex()), http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java index bd0344a..4a1c710 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java @@ -83,7 +83,6 @@ import org.apache.sysml.runtime.transform.meta.TfMetaUtils; import org.apache.sysml.runtime.transform.meta.TfOffsetMap; import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction { protected HashMap<String, String> params; @@ -241,7 +240,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction MatrixCharacteristics mc1 = sec.getMatrixCharacteristics( targetVar ); MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName()); - int ngroups = IntUtils.toInt(getLongParam(ec, Statement.GAGG_NUM_GROUPS)); + int ngroups = (int) getLongParam(ec, Statement.GAGG_NUM_GROUPS); //single-block aggregation if( ngroups <= mc1.getRowsPerBlock() && mc1.getCols() <= mc1.getColsPerBlock() ) { @@ -402,12 +401,12 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction //update output statistics (required for correctness) MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName()); - mcOut.set(rows?maxDim:mcIn.getRows(), rows?mcIn.getCols():maxDim, IntUtils.toInt(brlen), IntUtils.toInt(bclen), mcIn.getNonZeros()); + mcOut.set(rows?maxDim:mcIn.getRows(), rows?mcIn.getCols():maxDim, (int)brlen, (int)bclen, mcIn.getNonZeros()); } else //special case: empty output (ensure valid dims) { int n = emptyReturn ? 1 : 0; - MatrixBlock out = new MatrixBlock(rows?n:IntUtils.toInt(mcIn.getRows()), rows?IntUtils.toInt(mcIn.getCols()):n, true); + MatrixBlock out = new MatrixBlock(rows?n:(int)mcIn.getRows(), rows?(int)mcIn.getCols():n, true); sec.setMatrixOutput(output.getName(), out, getExtendedOpcode()); } } @@ -467,8 +466,8 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction //repartition input vector for higher degree of parallelism //(avoid scenarios where few input partitions create huge outputs) MatrixCharacteristics mcTmp = new MatrixCharacteristics(dirRows?lmaxVal:mcIn.getRows(), - dirRows?mcIn.getRows():lmaxVal, IntUtils.toInt(brlen), IntUtils.toInt(bclen), mcIn.getRows()); - int numParts = IntUtils.toInt(Math.min(SparkUtils.getNumPreferredPartitions(mcTmp, in), mcIn.getNumBlocks())); + dirRows?mcIn.getRows():lmaxVal, (int)brlen, (int)bclen, mcIn.getRows()); + int numParts = (int)Math.min(SparkUtils.getNumPreferredPartitions(mcTmp, in), mcIn.getNumBlocks()); if( numParts > in.getNumPartitions()*2 ) in = in.repartition(numParts); @@ -483,7 +482,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction //update output statistics (required for correctness) MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName()); - mcOut.set(dirRows?lmaxVal:mcIn.getRows(), dirRows?mcIn.getRows():lmaxVal, IntUtils.toInt(brlen), IntUtils.toInt(bclen), -1); + mcOut.set(dirRows?lmaxVal:mcIn.getRows(), dirRows?mcIn.getRows():lmaxVal, (int)brlen, (int)bclen, -1); } else if ( opcode.equalsIgnoreCase("transformapply") ) { @@ -506,7 +505,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction //create encoder broadcast (avoiding replication per task) Encoder encoder = EncoderFactory.createEncoder(params.get("spec"), colnames, - fo.getSchema(), IntUtils.toInt(fo.getNumColumns()), meta); + fo.getSchema(), (int)fo.getNumColumns(), meta); mcOut.setDimension(mcIn.getRows()-((omap!=null)?omap.getNumRmRows():0), encoder.getNumCols()); Broadcast<Encoder> bmeta = sec.getSparkContext().broadcast(encoder); Broadcast<TfOffsetMap> bomap = (omap!=null) ? sec.getSparkContext().broadcast(omap) : null; @@ -533,7 +532,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction //reblock if necessary (clen > bclen) if( mc.getCols() > mc.getNumColBlocks() ) { in = in.mapToPair(new RDDTransformDecodeExpandFunction( - IntUtils.toInt(mc.getCols()), mc.getColsPerBlock())); + (int)mc.getCols(), mc.getColsPerBlock())); in = RDDAggregateUtils.mergeByKey(in, false); } @@ -680,8 +679,8 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction //prepare inputs (for internal api compatibility) IndexedMatrixValue data = SparkUtils.toIndexedMatrixBlock(arg0._1(),arg0._2()); IndexedMatrixValue offsets = _rmRows ? - SparkUtils.toIndexedMatrixBlock(arg0._1(), _off.getBlock(IntUtils.toInt(arg0._1().getRowIndex()), 1)) : - SparkUtils.toIndexedMatrixBlock(arg0._1(), _off.getBlock(1, IntUtils.toInt(arg0._1().getColumnIndex()))); + SparkUtils.toIndexedMatrixBlock(arg0._1(), _off.getBlock((int)arg0._1().getRowIndex(), 1)) : + SparkUtils.toIndexedMatrixBlock(arg0._1(), _off.getBlock(1, (int)arg0._1().getColumnIndex())); //execute remove empty operations ArrayList<IndexedMatrixValue> out = new ArrayList<>(); @@ -755,7 +754,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction //get all inputs MatrixIndexes ix = arg0._1(); MatrixBlock target = arg0._2(); - MatrixBlock groups = _pbm.getBlock(IntUtils.toInt(ix.getRowIndex()), 1); + MatrixBlock groups = _pbm.getBlock((int)ix.getRowIndex(), 1); //execute map grouped aggregate operations IndexedMatrixValue in1 = SparkUtils.toIndexedMatrixBlock(ix, target); @@ -791,7 +790,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction //get all inputs MatrixIndexes ix = arg0._1(); MatrixBlock target = arg0._2(); - MatrixBlock groups = _pbm.getBlock(IntUtils.toInt(ix.getRowIndex()), 1); + MatrixBlock groups = _pbm.getBlock((int)ix.getRowIndex(), 1); //execute map grouped aggregate operations return groups.groupedAggOperations(target, null, new MatrixBlock(), _ngroups, _op); @@ -966,8 +965,8 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction MatrixBlock inBlk = in._2(); //construct expanded block via leftindexing - int cl = IntUtils.toInt(UtilFunctions.computeCellIndex(inIx.getColumnIndex(), _bclen, 0)-1); - int cu = IntUtils.toInt(UtilFunctions.computeCellIndex(inIx.getColumnIndex(), _bclen, inBlk.getNumColumns()-1)-1); + int cl = (int)UtilFunctions.computeCellIndex(inIx.getColumnIndex(), _bclen, 0)-1; + int cu = (int)UtilFunctions.computeCellIndex(inIx.getColumnIndex(), _bclen, inBlk.getNumColumns()-1)-1; MatrixBlock out = new MatrixBlock(inBlk.getNumRows(), _clen, false); out = out.leftIndexingOperations(inBlk, 0, inBlk.getNumRows()-1, cl, cu, null, UpdateType.INPLACE_PINNED); @@ -982,7 +981,7 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction } if ( params.get(Statement.GAGG_NUM_GROUPS) != null) { - int ngroups = IntUtils.toInt( Double.parseDouble(params.get(Statement.GAGG_NUM_GROUPS))); + int ngroups = (int) Double.parseDouble(params.get(Statement.GAGG_NUM_GROUPS)); mcOut.set(ngroups, mc1.getCols(), -1, -1); //grouped aggregate with cell output } else { http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java index e9928ad..3914c55 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java @@ -47,7 +47,6 @@ import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public class PmmSPInstruction extends BinarySPInstruction { private CacheType _type = null; @@ -126,7 +125,7 @@ public class PmmSPInstruction extends BinarySPInstruction { MatrixBlock mb2 = arg0._2(); //get the right hand side matrix - MatrixBlock mb1 = _pmV.getBlock(IntUtils.toInt(ixIn.getRowIndex()), 1); + MatrixBlock mb1 = _pmV.getBlock((int)ixIn.getRowIndex(), 1); //compute target block indexes long minPos = UtilFunctions.toLong( mb1.minNonZero() ); http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java index 196f270..18f0bef 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -47,7 +47,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public class QuantilePickSPInstruction extends BinarySPInstruction { private OperationTypes _type = null; @@ -221,7 +220,7 @@ public class QuantilePickSPInstruction extends BinarySPInstruction { if( tmp.getNumRows() <= pos ) throw new DMLRuntimeException("Invalid key lookup for " + pos + " in block of size " + tmp.getNumRows()+"x"+tmp.getNumColumns()); - return val.get(0).quickGetValue(IntUtils.toInt(pos), 0); + return val.get(0).quickGetValue((int)pos, 0); } private static class FilterFunction implements Function<Tuple2<MatrixIndexes,MatrixBlock>, Boolean> @@ -338,7 +337,7 @@ public class QuantilePickSPInstruction extends BinarySPInstruction { return Collections.emptyIterator(); //determine which quantiles are active - int qlen = IntUtils.toInt(Arrays.stream(_qPIDs).filter(i -> i==v1).count()); + int qlen = (int)Arrays.stream(_qPIDs).filter(i -> i==v1).count(); int[] qix = new int[qlen]; for(int i=0, pos=0; i<_qPIDs.length; i++) if( _qPIDs[i]==v1 ) http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java index d717a8e..a5a5d94 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java @@ -58,7 +58,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.matrix.operators.QuaternaryOperator; -import org.apache.sysml.utils.IntUtils; public class QuaternarySPInstruction extends ComputationSPInstruction { private CPOperand _input4 = null; @@ -375,8 +374,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { MatrixIndexes ixIn = arg._1(); MatrixBlock blkIn = arg._2(); MatrixBlock blkOut = new MatrixBlock(); - MatrixBlock mbU = _pmU.getBlock(IntUtils.toInt(ixIn.getRowIndex()), 1); - MatrixBlock mbV = _pmV.getBlock(IntUtils.toInt(ixIn.getColumnIndex()), 1); + MatrixBlock mbU = _pmU.getBlock((int)ixIn.getRowIndex(), 1); + MatrixBlock mbV = _pmV.getBlock((int)ixIn.getColumnIndex(), 1); //execute core operation blkIn.quaternaryOperations(_qop, mbU, mbV, null, blkOut); @@ -403,8 +402,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { MatrixBlock blkIn1 = arg0._2()._1(); MatrixBlock blkIn2 = arg0._2()._2(); MatrixBlock blkOut = new MatrixBlock(); - MatrixBlock mbU = (_pmU!=null)?_pmU.getBlock(IntUtils.toInt(ixIn.getRowIndex()), 1) : blkIn2; - MatrixBlock mbV = (_pmV!=null)?_pmV.getBlock(IntUtils.toInt(ixIn.getColumnIndex()), 1) : blkIn2; + MatrixBlock mbU = (_pmU!=null)?_pmU.getBlock((int)ixIn.getRowIndex(), 1) : blkIn2; + MatrixBlock mbV = (_pmV!=null)?_pmV.getBlock((int)ixIn.getColumnIndex(), 1) : blkIn2; MatrixBlock mbW = (_qop.hasFourInputs()) ? blkIn2 : null; //execute core operation @@ -432,8 +431,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { MatrixBlock blkIn2 = arg0._2()._1()._2(); MatrixBlock blkIn3 = arg0._2()._2(); MatrixBlock blkOut = new MatrixBlock(); - MatrixBlock mbU = (_pmU!=null)?_pmU.getBlock(IntUtils.toInt(ixIn.getRowIndex()), 1) : blkIn2; - MatrixBlock mbV = (_pmV!=null)?_pmV.getBlock(IntUtils.toInt(ixIn.getColumnIndex()), 1) : + MatrixBlock mbU = (_pmU!=null)?_pmU.getBlock((int)ixIn.getRowIndex(), 1) : blkIn2; + MatrixBlock mbV = (_pmV!=null)?_pmV.getBlock((int)ixIn.getColumnIndex(), 1) : (_pmU!=null)? blkIn2 : blkIn3; MatrixBlock mbW = (_qop.hasFourInputs())? blkIn3 : null; http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java index 8386e2c..2b9adcb 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java @@ -65,7 +65,6 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.data.RandomMatrixGenerator; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; import org.apache.sysml.utils.Statistics; public class RandSPInstruction extends UnarySPInstruction { @@ -259,7 +258,7 @@ public class RandSPInstruction extends UnarySPInstruction { && ConfigurationManager.getExecutionMode() != RUNTIME_PLATFORM.SPARK ) { RandomMatrixGenerator rgen = LibMatrixDatagen.createRandomMatrixGenerator( - pdf, IntUtils.toInt(lrows), IntUtils.toInt(lcols), rowsInBlock, colsInBlock, + pdf, (int)lrows, (int)lcols, rowsInBlock, colsInBlock, sparsity, minValue, maxValue, pdfParams); MatrixBlock mb = MatrixBlock.randOperations(rgen, lSeed); @@ -291,7 +290,7 @@ public class RandSPInstruction extends UnarySPInstruction { } //for load balancing: degree of parallelism such that ~128MB per partition - int numPartitions = IntUtils.toInt( Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1)); + int numPartitions = (int) Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1); //create seeds rdd seedsRDD = sec.getSparkContext().parallelizePairs(seeds, numPartitions); @@ -324,7 +323,7 @@ public class RandSPInstruction extends UnarySPInstruction { } //for load balancing: degree of parallelism such that ~128MB per partition - int numPartitions = IntUtils.toInt( Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1)); + int numPartitions = (int) Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1); //create seeds rdd seedsRDD = sec.getSparkContext() @@ -382,7 +381,7 @@ public class RandSPInstruction extends UnarySPInstruction { } //for load balancing: degree of parallelism such that ~128MB per partition - int numPartitions = IntUtils.toInt( Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1)); + int numPartitions = (int) Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1); //create offset rdd offsetsRDD = sec.getSparkContext().parallelize(offsets, numPartitions); @@ -409,7 +408,7 @@ public class RandSPInstruction extends UnarySPInstruction { } //for load balancing: degree of parallelism such that ~128MB per partition - int numPartitions = IntUtils.toInt( Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1)); + int numPartitions = (int) Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1); //create seeds rdd offsetsRDD = sec.getSparkContext() @@ -443,14 +442,14 @@ public class RandSPInstruction extends UnarySPInstruction { LOG.trace("Process RandSPInstruction sample with range="+ maxValue +", size="+ lrows +", replace="+ replace + ", seed=" + seed); // sampling rate that guarantees a sample of size >= sampleSizeLowerBound 99.99% of the time. - double fraction = SamplingUtils.computeFractionForSampleSize(IntUtils.toInt(lrows), UtilFunctions.toLong(maxValue), replace); + double fraction = SamplingUtils.computeFractionForSampleSize((int)lrows, UtilFunctions.toLong(maxValue), replace); Well1024a bigrand = LibMatrixDatagen.setupSeedsForRand(seed); // divide the population range across numPartitions by creating SampleTasks double hdfsBlockSize = InfrastructureAnalyzer.getHDFSBlockSize(); long outputSize = MatrixBlock.estimateSizeDenseInMemory(lrows,1); - int numPartitions = IntUtils.toInt( Math.ceil((double)outputSize/hdfsBlockSize)); + int numPartitions = (int) Math.ceil((double)outputSize/hdfsBlockSize); long partitionSize = (long) Math.ceil(maxValue/numPartitions); ArrayList<SampleTask> offsets = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java index d725883..3e09d17 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java @@ -58,7 +58,6 @@ import org.apache.sysml.runtime.matrix.operators.ReorgOperator; import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.runtime.util.IndexRange; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public class ReorgSPInstruction extends UnarySPInstruction { // sort-specific attributes (to enable variable attributes) @@ -170,7 +169,7 @@ public class ReorgSPInstruction extends UnarySPInstruction { // extract column (if necessary) and sort if( !singleCol ) out = out.filter(new IsBlockInRange(1, mcIn.getRows(), cols[0], cols[0], mcIn)) - .mapValues(new ExtractColumn(IntUtils.toInt(UtilFunctions.computeCellInBlock(cols[0], mcIn.getColsPerBlock())))); + .mapValues(new ExtractColumn((int)UtilFunctions.computeCellInBlock(cols[0], mcIn.getColsPerBlock()))); //actual index/data sort operation if( ixret ) //sort indexes @@ -269,7 +268,7 @@ public class ReorgSPInstruction extends UnarySPInstruction { ret.add(new Tuple2<>(ixOut,blkOut)); // insert newly created empty blocks for entire row - int numBlocks = IntUtils.toInt( Math.ceil((double)_mcIn.getRows()/_mcIn.getRowsPerBlock())); + int numBlocks = (int) Math.ceil((double)_mcIn.getRows()/_mcIn.getRowsPerBlock()); for(int i = 1; i <= numBlocks; i++) { if(i != ixOut.getColumnIndex()) { int lrlen = UtilFunctions.computeBlockSize(_mcIn.getRows(), rix, _mcIn.getRowsPerBlock()); http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java index 6be3e99..90e5396 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java @@ -48,7 +48,6 @@ import org.apache.sysml.runtime.matrix.data.TripleIndexes; import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.Operator; -import org.apache.sysml.utils.IntUtils; public class RmmSPInstruction extends BinarySPInstruction { @@ -115,7 +114,7 @@ public class RmmSPInstruction extends BinarySPInstruction { * ((long) Math.ceil((double)mc2.getCols()/mc2.getColsPerBlock())); double matrix2PSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(mc2) * ((long) Math.ceil((double)mc1.getRows()/mc1.getRowsPerBlock())); - return IntUtils.toInt( Math.max(Math.ceil((matrix1PSize+matrix2PSize)/hdfsBlockSize), 1)); + return (int) Math.max(Math.ceil((matrix1PSize+matrix2PSize)/hdfsBlockSize), 1); } private static class RmmReplicateFunction implements PairFlatMapFunction<Tuple2<MatrixIndexes, MatrixBlock>, TripleIndexes, MatrixBlock> http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java index dd5c062..8f63427 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java @@ -65,7 +65,6 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; -import org.apache.sysml.utils.IntUtils; import scala.Tuple2; @@ -145,7 +144,7 @@ public class SpoofSPInstruction extends SPInstruction { long numBlocks = (op.getCellType()==CellType.ROW_AGG ) ? mcIn.getNumRowBlocks() : mcIn.getNumColBlocks(); out = RDDAggregateUtils.aggByKeyStable(out, aggop, - IntUtils.toInt(Math.min(out.getNumPartitions(), numBlocks)), false); + (int)Math.min(out.getNumPartitions(), numBlocks), false); } sec.setRDDHandleForVariable(_out.getName(), out); @@ -184,7 +183,7 @@ public class SpoofSPInstruction extends SPInstruction { if(type == OutProdType.LEFT_OUTER_PRODUCT || type == OutProdType.RIGHT_OUTER_PRODUCT ) { long numBlocks = mcOut.getNumRowBlocks() * mcOut.getNumColBlocks(); out = RDDAggregateUtils.sumByKeyStable(out, - IntUtils.toInt(Math.min(out.getNumPartitions(), numBlocks)), false); + (int)Math.min(out.getNumPartitions(), numBlocks), false); } sec.setRDDHandleForVariable(_out.getName(), out); @@ -207,7 +206,7 @@ public class SpoofSPInstruction extends SPInstruction { long clen2 = op.getRowType().isConstDim2(op.getConstDim2()) ? op.getConstDim2() : op.getRowType().isRowTypeB1() ? sec.getMatrixCharacteristics(_in[1].getName()).getCols() : -1; RowwiseFunction fmmc = new RowwiseFunction(_class.getName(), _classBytes, bcVect2, - bcMatrices, scalars, mcIn.getRowsPerBlock(), IntUtils.toInt(mcIn.getCols()), IntUtils.toInt(clen2)); + bcMatrices, scalars, mcIn.getRowsPerBlock(), (int)mcIn.getCols(), (int)clen2); out = in.mapPartitionsToPair(fmmc, op.getRowType()==RowType.ROW_AGG || op.getRowType() == RowType.NO_AGG); @@ -223,7 +222,7 @@ public class SpoofSPInstruction extends SPInstruction { { if( op.getRowType()==RowType.ROW_AGG && mcIn.getCols() > mcIn.getColsPerBlock() ) { out = RDDAggregateUtils.sumByKeyStable(out, - IntUtils.toInt(Math.min(out.getNumPartitions(), mcIn.getNumRowBlocks())), false); + (int)Math.min(out.getNumPartitions(), mcIn.getNumRowBlocks()), false); } sec.setRDDHandleForVariable(_out.getName(), out); @@ -264,8 +263,8 @@ public class SpoofSPInstruction extends SPInstruction { } private static boolean[] getMatrixBroadcastVector(SparkExecutionContext sec, CPOperand[] inputs, boolean[] bcVect) { - int numMtx = IntUtils.toInt(Arrays.stream(inputs) - .filter(in -> in.getDataType().isMatrix()).count()); + int numMtx = (int) Arrays.stream(inputs) + .filter(in -> in.getDataType().isMatrix()).count(); boolean[] ret = new boolean[numMtx]; for(int i=0, pos=0; i<inputs.length; i++) if( inputs[i].getDataType().isMatrix() ) @@ -381,9 +380,9 @@ public class SpoofSPInstruction extends SPInstruction { for( int i=0, posRdd=0, posBc=0; i<_bcInd.length; i++ ) { if( _bcInd[i] ) { PartitionedBroadcast<MatrixBlock> pb = _inputs.get(posBc++); - int rowIndex = IntUtils.toInt((outer && i==2) ? ixIn.getColumnIndex() : + int rowIndex = (int) ((outer && i==2) ? ixIn.getColumnIndex() : (pb.getNumRowBlocks()>=ixIn.getRowIndex())?ixIn.getRowIndex():1); - int colIndex = IntUtils.toInt((outer && i==2) ? 1 : + int colIndex = (int) ((outer && i==2) ? 1 : (pb.getNumColumnBlocks()>=ixIn.getColumnIndex())?ixIn.getColumnIndex():1); ret.add(pb.getBlock(rowIndex, colIndex)); } http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java index 16ed0dd..5bb686b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java @@ -50,7 +50,6 @@ import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.Operator; -import org.apache.sysml.utils.IntUtils; import scala.Tuple2; @@ -89,17 +88,17 @@ public class Tsmm2SPInstruction extends UnarySPInstruction { _type.isLeft() ? mc.getColsPerBlock()+1 : 1, mc.getCols(), mc)) .mapToPair(new ShiftTSMMIndexesFunction(_type)); PartitionedBlock<MatrixBlock> pmb = SparkExecutionContext.toPartitionedMatrixBlock(tmp1, - IntUtils.toInt(_type.isLeft() ? mc.getRows() : mc.getRows() - mc.getRowsPerBlock()), - IntUtils.toInt(_type.isLeft() ? mc.getCols()-mc.getColsPerBlock() : mc.getCols()), + (int)(_type.isLeft() ? mc.getRows() : mc.getRows() - mc.getRowsPerBlock()), + (int)(_type.isLeft() ? mc.getCols()-mc.getColsPerBlock() : mc.getCols()), mc.getRowsPerBlock(), mc.getColsPerBlock(), -1L); Broadcast<PartitionedBlock<MatrixBlock>> bpmb = sec.getSparkContext().broadcast(pmb); //step 2: second pass of X, compute tsmm/mapmm and aggregate result blocks - int outputDim = IntUtils.toInt(_type.isLeft() ? mc.getCols() : mc.getRows()); + int outputDim = (int) (_type.isLeft() ? mc.getCols() : mc.getRows()); if( OptimizerUtils.estimateSize(outputDim, outputDim) <= 32*1024*1024 ) { //default: <=32MB //output large blocks and reduceAll to avoid skew on combineByKey JavaRDD<MatrixBlock> tmp2 = in.map( - new RDDTSMM2ExtFunction(bpmb, _type, outputDim, IntUtils.toInt(mc.getRowsPerBlock()))); + new RDDTSMM2ExtFunction(bpmb, _type, outputDim, (int)mc.getRowsPerBlock())); MatrixBlock out = RDDAggregateUtils.sumStable(tmp2); //put output block into symbol table (no lineage because single block) @@ -151,8 +150,8 @@ public class Tsmm2SPInstruction extends UnarySPInstruction { if( _type.isLeft() ? ixin.getColumnIndex() == 1 : ixin.getRowIndex() == 1 ) { //execute block mapmm operation for full block only (output two blocks, due to symmetry) MatrixBlock mbin2 = _pb.getValue().getBlock( //lookup broadcast block - IntUtils.toInt(_type.isLeft()?ixin.getRowIndex():1), - IntUtils.toInt(_type.isLeft()?1:ixin.getColumnIndex())); + (int)(_type.isLeft()?ixin.getRowIndex():1), + (int)(_type.isLeft()?1:ixin.getColumnIndex())); MatrixBlock mbin2t = transpose(mbin2, new MatrixBlock()); //prep for transpose rewrite mm MatrixBlock out2 = (MatrixBlock) OperationsOnMatrixValues.matMult( //mm @@ -206,25 +205,25 @@ public class Tsmm2SPInstruction extends UnarySPInstruction { //execute block tsmm operation MatrixBlock out1 = mbin.transposeSelfMatrixMultOperations(new MatrixBlock(), _type); - int ix = IntUtils.toInt((_type.isLeft() ? ixin.getColumnIndex() : ixin.getRowIndex())-1) * _blen; + int ix = (int) ((_type.isLeft() ? ixin.getColumnIndex() : ixin.getRowIndex())-1) * _blen; out.copy(ix, ix+out1.getNumRows()-1, ix, ix+out1.getNumColumns()-1, out1, true); if( fullBlock ) { //execute block mapmm operation for full block only (output two blocks, due to symmetry) MatrixBlock mbin2 = _pb.getValue().getBlock( //lookup broadcast block - IntUtils.toInt(_type.isLeft()?ixin.getRowIndex():1), - IntUtils.toInt(_type.isLeft()?1:ixin.getColumnIndex())); + (int)(_type.isLeft()?ixin.getRowIndex():1), + (int)(_type.isLeft()?1:ixin.getColumnIndex())); MatrixBlock mbin2t = transpose(mbin2, new MatrixBlock()); //prep for transpose rewrite mm MatrixBlock out2 = OperationsOnMatrixValues.matMult( //mm _type.isLeft() ? mbin2t : mbin, _type.isLeft() ? mbin : mbin2t, new MatrixBlock(), _op); MatrixIndexes ixout2 = _type.isLeft() ? new MatrixIndexes(2,1) : new MatrixIndexes(1,2); - out.copy(IntUtils.toInt(ixout2.getRowIndex()-1)*_blen, IntUtils.toInt(ixout2.getRowIndex()-1)*_blen+out2.getNumRows()-1, - IntUtils.toInt(ixout2.getColumnIndex()-1)*_blen, IntUtils.toInt(ixout2.getColumnIndex()-1)*_blen+out2.getNumColumns()-1, out2, true); + out.copy((int)(ixout2.getRowIndex()-1)*_blen, (int)(ixout2.getRowIndex()-1)*_blen+out2.getNumRows()-1, + (int)(ixout2.getColumnIndex()-1)*_blen, (int)(ixout2.getColumnIndex()-1)*_blen+out2.getNumColumns()-1, out2, true); MatrixBlock out3 = transpose(out2, new MatrixBlock()); - out.copy(IntUtils.toInt(ixout2.getColumnIndex()-1)*_blen, IntUtils.toInt(ixout2.getColumnIndex()-1)*_blen+out3.getNumRows()-1, - IntUtils.toInt(ixout2.getRowIndex()-1)*_blen, IntUtils.toInt(ixout2.getRowIndex()-1)*_blen+out3.getNumColumns()-1, out3, true); + out.copy((int)(ixout2.getColumnIndex()-1)*_blen, (int)(ixout2.getColumnIndex()-1)*_blen+out3.getNumRows()-1, + (int)(ixout2.getRowIndex()-1)*_blen, (int)(ixout2.getRowIndex()-1)*_blen+out3.getNumColumns()-1, out3, true); } return out; http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java index 4428e2d..677fbbf 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BlockPartitioner.java @@ -25,7 +25,6 @@ import org.apache.spark.Partitioner; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; -import org.apache.sysml.utils.IntUtils; /** * Default partitioner used for all binary block rdd operations in order @@ -76,7 +75,7 @@ public class BlockPartitioner extends Partitioner } //compute meta data for runtime - _ncparts = IntUtils.toInt(Math.ceil((double)ncblks/_cbPerPart)); + _ncparts = (int)Math.ceil((double)ncblks/_cbPerPart); _numParts = numParts; } @@ -91,8 +90,8 @@ public class BlockPartitioner extends Partitioner //get partition id MatrixIndexes ix = (MatrixIndexes) arg0; - int ixr = IntUtils.toInt((ix.getRowIndex()-1)/_rbPerPart); - int ixc = IntUtils.toInt((ix.getColumnIndex()-1)/_cbPerPart); + int ixr = (int)((ix.getRowIndex()-1)/_rbPerPart); + int ixc = (int)((ix.getColumnIndex()-1)/_cbPerPart); int id = ixr * _ncparts + ixc; //ensure valid range http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java index 62501e7..8a4999b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java @@ -34,7 +34,6 @@ import org.apache.sysml.runtime.controlprogram.caching.CacheBlock; import org.apache.sysml.runtime.controlprogram.caching.CacheBlockFactory; import org.apache.sysml.runtime.util.FastBufferedDataInputStream; import org.apache.sysml.runtime.util.FastBufferedDataOutputStream; -import org.apache.sysml.utils.IntUtils; /** * This class is for partitioned matrix/frame blocks, to be used as broadcasts. @@ -133,11 +132,11 @@ public class PartitionedBlock<T extends CacheBlock> implements Externalizable } public int getNumRowBlocks() { - return IntUtils.toInt(Math.ceil((double)_rlen/_brlen)); + return (int)Math.ceil((double)_rlen/_brlen); } public int getNumColumnBlocks() { - return IntUtils.toInt(Math.ceil((double)_clen/_bclen)); + return (int)Math.ceil((double)_clen/_bclen); } @SuppressWarnings("unchecked") http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java index 6eec144..ab29c4c 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBroadcast.java @@ -30,7 +30,6 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.OperationsOnMatrixValues; import org.apache.sysml.runtime.matrix.data.Pair; import org.apache.sysml.runtime.util.IndexRange; -import org.apache.sysml.utils.IntUtils; /** * This class is a wrapper around an array of broadcasts of partitioned matrix/frame blocks, @@ -72,11 +71,11 @@ public class PartitionedBroadcast<T extends CacheBlock> implements Serializable } public int getNumRowBlocks() { - return IntUtils.toInt(_mc.getNumRowBlocks()); + return (int)_mc.getNumRowBlocks(); } public int getNumColumnBlocks() { - return IntUtils.toInt(_mc.getNumColBlocks()); + return (int)_mc.getNumColBlocks(); } public static int computeBlocksPerPartition(long rlen, long clen, long brlen, long bclen) { http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java index a2414bc..0ce654b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java @@ -31,7 +31,6 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public class ExtractBlockForBinaryReblock implements PairFlatMapFunction<Tuple2<MatrixIndexes,MatrixBlock>, MatrixIndexes, MatrixBlock> { @@ -97,8 +96,8 @@ public class ExtractBlockForBinaryReblock implements PairFlatMapFunction<Tuple2< blk.setNonZeros(in.getNonZeros()); } else { //general case - for(int i2 = 0; i2 <= IntUtils.toInt(rowUpper-rowLower); i2++) - for(int j2 = 0; j2 <= IntUtils.toInt(colUpper-colLower); j2++) + for(int i2 = 0; i2 <= (int)(rowUpper-rowLower); i2++) + for(int j2 = 0; j2 <= (int)(colUpper-colLower); j2++) blk.appendValue(cixi+i2, cixj+j2, in.quickGetValue(aixi+i2, aixj+j2)); } retVal.add(new Tuple2<>(indx, blk)); http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java index b05c0bf..3e6908b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java @@ -35,7 +35,6 @@ import org.apache.sysml.runtime.matrix.data.WeightedCell; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; public abstract class ExtractGroup implements Serializable { @@ -66,7 +65,7 @@ public abstract class ExtractGroup implements Serializable if(_op instanceof AggregateOperator && _ngroups > 0 && OptimizerUtils.isValidCPDimensions(_ngroups, target.getNumColumns()) ) { - MatrixBlock tmp = group.groupedAggOperations(target, null, new MatrixBlock(), IntUtils.toInt(_ngroups), _op); + MatrixBlock tmp = group.groupedAggOperations(target, null, new MatrixBlock(), (int)_ngroups, _op); for(int i=0; i<tmp.getNumRows(); i++) { for( int j=0; j<tmp.getNumColumns(); j++ ) { @@ -140,7 +139,7 @@ public abstract class ExtractGroup implements Serializable throws Exception { MatrixIndexes ix = arg._1; - MatrixBlock group = _pbm.getBlock(IntUtils.toInt(ix.getRowIndex()), 1); + MatrixBlock group = _pbm.getBlock((int)ix.getRowIndex(), 1); MatrixBlock target = arg._2; return execute(ix, group, target).iterator(); http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java index 84d161a..afd2e1a 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java @@ -31,7 +31,6 @@ import org.apache.sysml.runtime.instructions.spark.data.PartitionedBroadcast; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.operators.BinaryOperator; -import org.apache.sysml.utils.IntUtils; public class MatrixVectorBinaryOpPartitionFunction implements PairFlatMapFunction<Iterator<Tuple2<MatrixIndexes,MatrixBlock>>, MatrixIndexes,MatrixBlock> { @@ -75,8 +74,8 @@ public class MatrixVectorBinaryOpPartitionFunction implements PairFlatMapFunctio MatrixBlock in1 = arg._2(); //get the rhs block - int rix= IntUtils.toInt((_vtype==VectorType.COL_VECTOR) ? ix.getRowIndex() : 1); - int cix= IntUtils.toInt((_vtype==VectorType.COL_VECTOR) ? 1 : ix.getColumnIndex()); + int rix= (int)((_vtype==VectorType.COL_VECTOR) ? ix.getRowIndex() : 1); + int cix= (int)((_vtype==VectorType.COL_VECTOR) ? 1 : ix.getColumnIndex()); MatrixBlock in2 = _pmV.getBlock(rix, cix); //execute the binary operation http://git-wip-us.apache.org/repos/asf/systemml/blob/c3fdbb4d/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java index 69be559..c0fc34a 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java @@ -67,7 +67,6 @@ import org.apache.sysml.runtime.transform.TfUtils; import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.runtime.util.FastStringTokenizer; import org.apache.sysml.runtime.util.UtilFunctions; -import org.apache.sysml.utils.IntUtils; import scala.Tuple2; @@ -102,7 +101,7 @@ public class FrameRDDConverterUtils //prepare default schema if needed if( schema == null || schema.length==1 ) - schema = UtilFunctions.nCopies(IntUtils.toInt(mc.getCols()), ValueType.STRING); + schema = UtilFunctions.nCopies((int)mc.getCols(), ValueType.STRING); //convert csv rdd to binary block rdd (w/ partial blocks) JavaPairRDD<Long, FrameBlock> out = prepinput.mapPartitionsToPair( @@ -154,7 +153,7 @@ public class FrameRDDConverterUtils JavaPairRDD<Long, Text> input, MatrixCharacteristics mc, ValueType[] schema ) { //prepare default schema if needed if( schema == null || schema.length==1 ) { - schema = UtilFunctions.nCopies(IntUtils.toInt(mc.getCols()), + schema = UtilFunctions.nCopies((int)mc.getCols(), (schema!=null) ? schema[0] : ValueType.STRING); } @@ -190,7 +189,7 @@ public class FrameRDDConverterUtils if(mcIn.getCols() > mcIn.getColsPerBlock()) { //split matrix blocks into extended matrix blocks in = in.flatMapToPair(new MatrixFrameReblockFunction(mcIn)); - mc.setBlockSize(MatrixFrameReblockFunction.computeBlockSize(mc), IntUtils.toInt(mc.getCols())); + mc.setBlockSize(MatrixFrameReblockFunction.computeBlockSize(mc), (int)mc.getCols()); //shuffle matrix blocks (instead of frame blocks) in order to exploit //sparse formats (for sparse or wide matrices) during shuffle @@ -239,8 +238,8 @@ public class FrameRDDConverterUtils df.javaRDD().zipWithIndex(); //zip row index //convert data frame to frame schema (prepare once) - String[] colnames = new String[IntUtils.toInt(mc.getCols())]; - ValueType[] fschema = new ValueType[IntUtils.toInt(mc.getCols())]; + String[] colnames = new String[(int)mc.getCols()]; + ValueType[] fschema = new ValueType[(int)mc.getCols()]; int colVect = convertDFSchemaToFrameSchema(df.schema(), colnames, fschema, containsID); out.set(colnames, fschema); //make schema available @@ -261,7 +260,7 @@ public class FrameRDDConverterUtils //create data frame schema if( schema == null ) - schema = UtilFunctions.nCopies(IntUtils.toInt(mc.getCols()), ValueType.STRING); + schema = UtilFunctions.nCopies((int)mc.getCols(), ValueType.STRING); StructType dfSchema = convertFrameSchemaToDFSchema(schema, true); //rdd to data frame conversion @@ -553,7 +552,7 @@ public class FrameRDDConverterUtils _schema = schema; _hasHeader = hasHeader; _delim = delim; - _maxRowsPerBlock = Math.max(IntUtils.toInt(FrameBlock.BUFFER_SIZE/_clen), 1); + _maxRowsPerBlock = Math.max((int) (FrameBlock.BUFFER_SIZE/_clen), 1); } @Override @@ -564,7 +563,7 @@ public class FrameRDDConverterUtils long ix = -1; FrameBlock fb = null; - String[] tmprow = new String[IntUtils.toInt(_clen)]; + String[] tmprow = new String[(int)_clen]; while( arg0.hasNext() ) { @@ -576,11 +575,11 @@ public class FrameRDDConverterUtils continue; } if( row.startsWith(TfUtils.TXMTD_MVPREFIX) ) { - _mvMeta = Arrays.asList(Arrays.copyOfRange(IOUtilFunctions.splitCSV(row, _delim), 1, IntUtils.toInt(_clen+1))); + _mvMeta = Arrays.asList(Arrays.copyOfRange(IOUtilFunctions.splitCSV(row, _delim), 1, (int)_clen+1)); continue; } else if( row.startsWith(TfUtils.TXMTD_NDPREFIX) ) { - _ndMeta = Arrays.asList(Arrays.copyOfRange(IOUtilFunctions.splitCSV(row, _delim), 1, IntUtils.toInt(_clen+1))); + _ndMeta = Arrays.asList(Arrays.copyOfRange(IOUtilFunctions.splitCSV(row, _delim), 1, (int)_clen+1)); continue; } @@ -717,7 +716,7 @@ public class FrameRDDConverterUtils _schema = schema; _containsID = containsID; _colVect = colVect; - _maxRowsPerBlock = Math.max(IntUtils.toInt(FrameBlock.BUFFER_SIZE/_clen), 1); + _maxRowsPerBlock = Math.max((int) (FrameBlock.BUFFER_SIZE/_clen), 1); } @Override @@ -728,7 +727,7 @@ public class FrameRDDConverterUtils long ix = -1; FrameBlock fb = null; - Object[] tmprow = new Object[IntUtils.toInt(_clen)]; + Object[] tmprow = new Object[(int)_clen]; while( arg0.hasNext() ) { @@ -816,7 +815,7 @@ public class FrameRDDConverterUtils _clen = mc.getCols(); //determine upper bounded buffer len - _bufflen = IntUtils.toInt(Math.min(_rlen*_clen, FrameBlock.BUFFER_SIZE)); + _bufflen = (int) Math.min(_rlen*_clen, FrameBlock.BUFFER_SIZE); } protected void flushBufferToList( FrameReblockBuffer rbuff, ArrayList<Tuple2<Long,FrameBlock>> ret ) @@ -858,7 +857,7 @@ public class FrameRDDConverterUtils st.reset( strVal ); long row = st.nextLong(); long col = st.nextLong(); - Object val = UtilFunctions.stringToObject(_schema[IntUtils.toInt(col-1)], st.nextToken()); + Object val = UtilFunctions.stringToObject(_schema[(int)col-1], st.nextToken()); //flush buffer if necessary if( rbuff.getSize() >= rbuff.getCapacity() ) @@ -909,7 +908,7 @@ public class FrameRDDConverterUtils long rowix = (ix.getRowIndex()-1)*_brlen+1; //global index within frame block (0-based) - long cl = IntUtils.toInt((ix.getColumnIndex()-1)*_bclen); + long cl = (int)((ix.getColumnIndex()-1)*_bclen); long cu = Math.min(cl+mb.getNumColumns()-1, _clen); //prepare output frame blocks @@ -917,8 +916,8 @@ public class FrameRDDConverterUtils int ru = Math.min(i+_maxRowsPerBlock, mb.getNumRows())-1; long rix = UtilFunctions.computeBlockIndex(rowix+i, _maxRowsPerBlock); MatrixIndexes ixout = new MatrixIndexes(rix, 1); - MatrixBlock out = new MatrixBlock(ru-i+1, IntUtils.toInt(_clen), sparse); - out.copy(0, out.getNumRows()-1, IntUtils.toInt(cl), IntUtils.toInt(cu), + MatrixBlock out = new MatrixBlock(ru-i+1, (int)_clen, sparse); + out.copy(0, out.getNumRows()-1, (int)cl, (int)cu, mb.slice(i, ru, 0, mb.getNumColumns()-1, mbreuse), true); out.examSparsity(); ret.add(new Tuple2<>(ixout,out)); @@ -936,8 +935,8 @@ public class FrameRDDConverterUtils */ public static int computeBlockSize(MatrixCharacteristics mc) { int brlen = mc.getRowsPerBlock(); - int basic = Math.max(IntUtils.toInt(FrameBlock.BUFFER_SIZE/mc.getCols()), 1); - int div = IntUtils.toInt(Math.ceil((double)brlen/basic)); + int basic = Math.max((int)(FrameBlock.BUFFER_SIZE/mc.getCols()), 1); + int div = (int)Math.ceil((double)brlen/basic); while( brlen % div != 0 ) div++; return brlen / div; @@ -996,8 +995,8 @@ public class FrameRDDConverterUtils for( long rix=rstartix; rix<=rendix; rix++ ) { //for all row blocks long rpos = UtilFunctions.computeCellIndex(rix, brlen, 0); int lrlen = UtilFunctions.computeBlockSize(rlen, rix, brlen); - int fix = IntUtils.toInt((rpos-rowIndex>=0) ? rpos-rowIndex : 0); - int fix2 = IntUtils.toInt(Math.min(rpos+lrlen-rowIndex-1,blk.getNumRows()-1)); + int fix = (int)((rpos-rowIndex>=0) ? rpos-rowIndex : 0); + int fix2 = (int)Math.min(rpos+lrlen-rowIndex-1,blk.getNumRows()-1); int mix = UtilFunctions.computeCellInBlock(rowIndex+fix, brlen); int mix2 = mix + (fix2-fix); for( long cix=1; cix<=cendix; cix++ ) { //for all column blocks @@ -1005,7 +1004,7 @@ public class FrameRDDConverterUtils int lclen = UtilFunctions.computeBlockSize(clen, cix, bclen); MatrixBlock matrix = new MatrixBlock(lrlen, lclen, false); FrameBlock frame = blk.slice(fix, fix2, - IntUtils.toInt(cpos-1), IntUtils.toInt(cpos+lclen-2), new FrameBlock()); + (int)cpos-1, (int)cpos+lclen-2, new FrameBlock()); MatrixBlock mframe = DataConverter.convertToMatrixBlock(frame); ret.add(new Tuple2<>(new MatrixIndexes(rix, cix), matrix.leftIndexingOperations(mframe, mix, mix2, 0, lclen-1,