Repository: systemml Updated Branches: refs/heads/master bc6e941ce -> 95cbbd656
http://git-wip-us.apache.org/repos/asf/systemml/blob/95cbbd65/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java index f775e92..c8a0d3e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java @@ -71,6 +71,7 @@ import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.runtime.util.FastStringTokenizer; import org.apache.sysml.runtime.util.MapReduceTool; import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.sysml.utils.IntUtils; import scala.Tuple2; @@ -271,7 +272,7 @@ public class RDDConverterUtils //slice blocks into rows, align and convert into data frame rows JavaRDD<Row> rowsRDD = in .flatMapToPair(new SliceBinaryBlockToRowsFunction(mc.getRowsPerBlock())) - .groupByKey().map(new ConvertRowBlocksToRows((int)mc.getCols(), mc.getColsPerBlock(), toVector)); + .groupByKey().map(new ConvertRowBlocksToRows(IntUtils.toInt(mc.getCols()), mc.getColsPerBlock(), toVector)); //create data frame schema List<StructField> fields = new ArrayList<>(); @@ -322,7 +323,7 @@ public class RDDConverterUtils MapReduceTool.deleteFileIfExistOnHDFS(pathY); //convert libsvm to labeled points - int numFeatures = (int) mcOutX.getCols(); + int numFeatures = IntUtils.toInt( mcOutX.getCols() ); int numPartitions = SparkUtils.getNumPreferredPartitions(mcOutX, null); JavaRDD<org.apache.spark.mllib.regression.LabeledPoint> lpoints = MLUtils.loadLibSVMFile(sc.sc(), pathIn, numFeatures, numPartitions).toJavaRDD(); @@ -485,7 +486,7 @@ public class RDDConverterUtils _bclen = mc.getColsPerBlock(); //determine upper bounded buffer len - _bufflen = (int) Math.min(_rlen*_clen, BUFFER_SIZE); + _bufflen = IntUtils.toInt( Math.min(_rlen*_clen, BUFFER_SIZE) ); } protected void flushBufferToList( ReblockBuffer rbuff, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) @@ -702,7 +703,7 @@ public class RDDConverterUtils { ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); - int ncblks = (int)Math.ceil((double)_clen/_bclen); + int ncblks = IntUtils.toInt(Math.ceil((double)_clen/_bclen)); MatrixIndexes[] ix = new MatrixIndexes[ncblks]; MatrixBlock[] mb = new MatrixBlock[ncblks]; @@ -724,7 +725,7 @@ public class RDDConverterUtils if( ix[0] !=null ) flushBlocksToList(ix, mb, ret); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); - createBlocks(rowix, (int)len, ix, mb); + createBlocks(rowix, IntUtils.toInt(len), ix, mb); } //process row data @@ -732,7 +733,7 @@ public class RDDConverterUtils boolean emptyFound = false; for( int cix=1, pix=0; cix<=ncblks; cix++ ) { - int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); + int lclen = IntUtils.toInt(UtilFunctions.computeBlockSize(_clen, cix, _bclen)); if( mb[cix-1].isInSparseFormat() ) { //allocate row once (avoid re-allocations) int lnnz = IOUtilFunctions.countNnz(parts, pix, lclen); @@ -762,13 +763,13 @@ public class RDDConverterUtils { //compute row block index and number of column blocks long rix = UtilFunctions.computeBlockIndex(rowix, _brlen); - int ncblks = (int)Math.ceil((double)_clen/_bclen); + int ncblks = IntUtils.toInt(Math.ceil((double)_clen/_bclen)); //create all column blocks (assume dense since csv is dense text format) for( int cix=1; cix<=ncblks; cix++ ) { - int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); + int lclen = IntUtils.toInt(UtilFunctions.computeBlockSize(_clen, cix, _bclen)); ix[cix-1] = new MatrixIndexes(rix, cix); - mb[cix-1] = new MatrixBlock(lrlen, lclen, _sparse, (int)(lrlen*lclen*_sparsity)); + mb[cix-1] = new MatrixBlock(lrlen, lclen, _sparse, IntUtils.toInt(lrlen*lclen*_sparsity)); mb[cix-1].allocateBlock(); } } @@ -813,7 +814,7 @@ public class RDDConverterUtils { ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); - int ncblks = (int)Math.ceil((double)_clen/_bclen); + int ncblks = IntUtils.toInt(Math.ceil((double)_clen/_bclen)); MatrixIndexes[] ix = new MatrixIndexes[ncblks]; MatrixBlock[] mb = new MatrixBlock[ncblks]; @@ -833,7 +834,7 @@ public class RDDConverterUtils if( ix[0] !=null ) flushBlocksToList(ix, mb, ret); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); - createBlocks(rowix, (int)len, ix, mb, lsparse); + createBlocks(rowix, IntUtils.toInt(len), ix, mb, lsparse); } //process row data @@ -850,14 +851,14 @@ public class RDDConverterUtils (org.apache.spark.mllib.linalg.SparseVector) row.features(); for( int k=0; k<lnnz; k++ ) { int gix = srow.indices()[k]+1; - int cix = (int)UtilFunctions.computeBlockIndex(gix, _bclen); + int cix = IntUtils.toInt(UtilFunctions.computeBlockIndex(gix, _bclen)); int j = UtilFunctions.computeCellInBlock(gix, _bclen); mb[cix-1].appendValue(pos, j, srow.values()[k]); } } else { //dense for( int cix=1, pix=0; cix<=ncblks; cix++ ) { - int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); + int lclen = IntUtils.toInt(UtilFunctions.computeBlockSize(_clen, cix, _bclen)); for( int j=0; j<lclen; j++ ) mb[cix-1].appendValue(pos, j, row.features().apply(pix++)); } @@ -877,11 +878,11 @@ public class RDDConverterUtils { //compute row block index and number of column blocks long rix = UtilFunctions.computeBlockIndex(rowix, _brlen); - int ncblks = (int)Math.ceil((double)_clen/_bclen); + int ncblks = IntUtils.toInt(Math.ceil((double)_clen/_bclen)); //create all column blocks (assume dense since csv is dense text format) for( int cix=1; cix<=ncblks; cix++ ) { - int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); + int lclen = IntUtils.toInt(UtilFunctions.computeBlockSize(_clen, cix, _bclen)); ix[cix-1] = new MatrixIndexes(rix, cix); mb[cix-1] = new MatrixBlock(lrlen, lclen, lsparse); mb[cix-1].allocateBlock(); @@ -985,7 +986,7 @@ public class RDDConverterUtils public ConcatenateBlocksFunction(long clen, int bclen) { _clen = clen; _bclen = bclen; - _ncblks = (int)Math.ceil((double)clen/bclen); + _ncblks = IntUtils.toInt(Math.ceil((double)clen/bclen)); } @Override @@ -1001,9 +1002,9 @@ public class RDDConverterUtils tmpBlks[entry._1().intValue()-1] = entry._2(); } //concatenate blocks - MatrixBlock out = new MatrixBlock(1,(int)_clen, tmpBlks[0].isInSparseFormat()); + MatrixBlock out = new MatrixBlock(1,IntUtils.toInt(_clen), tmpBlks[0].isInSparseFormat()); for( int i=0; i<_ncblks; i++ ) - out.copy(0, 0, i*_bclen, (int)Math.min((i+1)*_bclen, _clen)-1, tmpBlks[i], false); + out.copy(0, 0, i*_bclen, IntUtils.toInt(Math.min((i+1)*_bclen, _clen)-1), tmpBlks[i], false); out.recomputeNonZeros(); //output row block return new Tuple2<>(new MatrixIndexes(rowIndex, 1),out); @@ -1043,7 +1044,7 @@ public class RDDConverterUtils { ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); - int ncblks = (int)Math.ceil((double)_clen/_bclen); + int ncblks = IntUtils.toInt(Math.ceil((double)_clen/_bclen)); MatrixIndexes[] ix = new MatrixIndexes[ncblks]; MatrixBlock[] mb = new MatrixBlock[ncblks]; @@ -1060,15 +1061,15 @@ public class RDDConverterUtils if( ix[0] !=null ) flushBlocksToList(ix, mb, ret); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); - createBlocks(rowix, (int)len, ix, mb); + createBlocks(rowix, IntUtils.toInt(len), ix, mb); } //process row data int off = _containsID ? 1 : 0; Object obj = _isVector ? tmp._1().get(off) : tmp._1(); for( int cix=1, pix=_isVector?0:off; cix<=ncblks; cix++ ) { - int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); - int cu = (int) Math.min(_clen, cix*_bclen) + (_isVector?0:off); + int lclen = IntUtils.toInt(UtilFunctions.computeBlockSize(_clen, cix, _bclen)); + int cu = IntUtils.toInt( Math.min(_clen, cix*_bclen) + (_isVector?0:off) ); //allocate sparse row once (avoid re-allocations) if( mb[cix-1].isInSparseFormat() ) { int lnnz = countNnz(obj, _isVector, pix, cu); @@ -1109,13 +1110,13 @@ public class RDDConverterUtils { //compute row block index and number of column blocks long rix = UtilFunctions.computeBlockIndex(rowix, _brlen); - int ncblks = (int)Math.ceil((double)_clen/_bclen); + int ncblks = IntUtils.toInt(Math.ceil((double)_clen/_bclen)); //create all column blocks (assume dense since csv is dense text format) for( int cix=1; cix<=ncblks; cix++ ) { - int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); + int lclen = IntUtils.toInt(UtilFunctions.computeBlockSize(_clen, cix, _bclen)); ix[cix-1] = new MatrixIndexes(rix, cix); - mb[cix-1] = new MatrixBlock(lrlen, lclen, _sparse,(int)(lrlen*lclen*_sparsity)); + mb[cix-1] = new MatrixBlock(lrlen, lclen, _sparse,IntUtils.toInt(lrlen*lclen*_sparsity)); mb[cix-1].allocateBlock(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/95cbbd65/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java index 4871aee..e350e77 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java @@ -54,6 +54,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue; import org.apache.sysml.runtime.matrix.mapred.ReblockBuffer; import org.apache.sysml.runtime.util.FastStringTokenizer; +import org.apache.sysml.utils.IntUtils; import scala.Tuple2; @@ -127,7 +128,7 @@ public class RDDConverterUtilsExt } public static MatrixBlock convertPy4JArrayToMB(byte [] data, long rlen, long clen) { - return convertPy4JArrayToMB(data, (int)rlen, (int)clen, false); + return convertPy4JArrayToMB(data, IntUtils.toInt(rlen), IntUtils.toInt(clen), false); } public static MatrixBlock convertPy4JArrayToMB(byte [] data, int rlen, int clen) { @@ -135,7 +136,7 @@ public class RDDConverterUtilsExt } public static MatrixBlock convertSciPyCOOToMB(byte [] data, byte [] row, byte [] col, long rlen, long clen, long nnz) { - return convertSciPyCOOToMB(data, row, col, (int)rlen, (int)clen, (int)nnz); + return convertSciPyCOOToMB(data, row, col, IntUtils.toInt(rlen), IntUtils.toInt(clen), IntUtils.toInt(nnz)); } public static MatrixBlock convertSciPyCOOToMB(byte [] data, byte [] row, byte [] col, int rlen, int clen, int nnz) { @@ -159,7 +160,7 @@ public class RDDConverterUtilsExt } public static MatrixBlock convertPy4JArrayToMB(byte [] data, long rlen, long clen, boolean isSparse) { - return convertPy4JArrayToMB(data, (int) rlen, (int) clen, isSparse); + return convertPy4JArrayToMB(data, IntUtils.toInt( rlen), IntUtils.toInt(clen), isSparse); } public static MatrixBlock allocateDenseOrSparse(int rlen, int clen, boolean isSparse) { @@ -186,7 +187,7 @@ public class RDDConverterUtilsExt public static void copyRowBlocks(MatrixBlock mb, long rowIndex, MatrixBlock ret, long numRowsPerBlock, long rlen, long clen) { // TODO: Double-check if synchronization is required here. // synchronized (RDDConverterUtilsExt.class) { - ret.copy((int)(rowIndex*numRowsPerBlock), (int)Math.min((rowIndex+1)*numRowsPerBlock-1, rlen-1), 0, (int)(clen-1), mb, false); + ret.copy(IntUtils.toInt(rowIndex*numRowsPerBlock), IntUtils.toInt(Math.min((rowIndex+1)*numRowsPerBlock-1, rlen-1)), 0, IntUtils.toInt(clen-1), mb, false); // } } @@ -204,7 +205,7 @@ public class RDDConverterUtilsExt long limit = rlen*clen; if( limit > Integer.MAX_VALUE ) throw new DMLRuntimeException("Dense NumPy array of size " + limit + " cannot be converted to MatrixBlock"); - double [] denseBlock = new double[(int) limit]; + double [] denseBlock = new double[IntUtils.toInt(limit)]; ByteBuffer buf = ByteBuffer.wrap(data); buf.order(ByteOrder.nativeOrder()); for(int i = 0; i < rlen*clen; i++) { @@ -227,7 +228,7 @@ public class RDDConverterUtilsExt int times = Double.SIZE / Byte.SIZE; if( limit > Integer.MAX_VALUE / times ) throw new DMLRuntimeException("MatrixBlock of size " + limit + " cannot be converted to dense numpy array"); - ret = new byte[(int) (limit * times)]; + ret = new byte[IntUtils.toInt(limit * times)]; double [] denseBlock = mb.getDenseBlockValues(); if(mb.isEmptyBlock()) { @@ -318,7 +319,7 @@ public class RDDConverterUtilsExt _brlen = mc.getRowsPerBlock(); _bclen = mc.getColsPerBlock(); //determine upper bounded buffer len - _bufflen = (int) Math.min(_rlen*_clen, BUFFER_SIZE); + _bufflen = IntUtils.toInt( Math.min(_rlen*_clen, BUFFER_SIZE)); } // ---------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/95cbbd65/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java index 01bdef8..d5abefc 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java @@ -48,6 +48,7 @@ import org.apache.sysml.runtime.matrix.operators.ReorgOperator; import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.runtime.util.SortUtils; import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.sysml.utils.IntUtils; public class RDDSortUtils { @@ -60,7 +61,7 @@ public class RDDSortUtils //sort (creates sorted range per partition) long hdfsBlocksize = InfrastructureAnalyzer.getHDFSBlockSize(); - int numPartitions = (int)Math.ceil(((double)rlen*8)/hdfsBlocksize); + int numPartitions = IntUtils.toInt(Math.ceil(((double)rlen*8)/hdfsBlocksize)); JavaRDD<Double> sdvals = dvals .sortBy(new CreateDoubleKeyFunction(), true, numPartitions); @@ -82,7 +83,7 @@ public class RDDSortUtils //sort (creates sorted range per partition) long hdfsBlocksize = InfrastructureAnalyzer.getHDFSBlockSize(); - int numPartitions = (int)Math.ceil(((double)rlen*8)/hdfsBlocksize); + int numPartitions = IntUtils.toInt(Math.ceil(((double)rlen*8)/hdfsBlocksize)); JavaRDD<DoublePair> sdvals = dvals .sortBy(new CreateDoubleKeyFunction2(), true, numPartitions); @@ -126,7 +127,7 @@ public class RDDSortUtils //sort (creates sorted range per partition) long hdfsBlocksize = InfrastructureAnalyzer.getHDFSBlockSize(); - int numPartitions = (int)Math.ceil(((double)rlen*16)/hdfsBlocksize); + int numPartitions = IntUtils.toInt(Math.ceil(((double)rlen*16)/hdfsBlocksize)); JavaRDD<ValueIndexPair> sdvals = dvals .sortByKey(new IndexComparator(asc), true, numPartitions) .keys(); //workaround for index comparator @@ -172,7 +173,7 @@ public class RDDSortUtils //sort (creates sorted range per partition) long hdfsBlocksize = InfrastructureAnalyzer.getHDFSBlockSize(); - int numPartitions = (int)Math.ceil(((double)rlen*16)/hdfsBlocksize); + int numPartitions = IntUtils.toInt(Math.ceil(((double)rlen*16)/hdfsBlocksize)); JavaRDD<ValueIndexPair> sdvals = dvals .sortByKey(new IndexComparator(asc), true, numPartitions) .keys(); //workaround for index comparator @@ -249,7 +250,7 @@ public class RDDSortUtils { //collect orderby column for in-memory sorting MatrixBlock inMatBlock = SparkExecutionContext - .toMatrixBlock(val, (int)rlen, 1, brlen, bclen, -1); + .toMatrixBlock(val, IntUtils.toInt(rlen), 1, brlen, bclen, -1); //in-memory sort operation (w/ index return: source index in target position) ReorgOperator lrop = new ReorgOperator(new SortIndex(1, !asc, true)); @@ -259,7 +260,7 @@ public class RDDSortUtils //flip sort indices from <source ix in target pos> to <target ix in source pos> MatrixBlock sortedIxSrc = new MatrixBlock(sortedIx.getNumRows(), 1, false); for (int i=0; i < sortedIx.getNumRows(); i++) - sortedIxSrc.quickSetValue((int)sortedIx.quickGetValue(i,0)-1, 0, i+1); + sortedIxSrc.quickSetValue(IntUtils.toInt(sortedIx.quickGetValue(i,0)-1), 0, i+1); //broadcast index vector PartitionedBlock<MatrixBlock> pmb = new PartitionedBlock<>(sortedIxSrc, brlen, bclen); @@ -461,7 +462,7 @@ public class RDDSortUtils ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); - mb = new MatrixBlock((int)len, 1, false); + mb = new MatrixBlock(IntUtils.toInt(len), 1, false); } mb.quickSetValue(pos, 0, val._1); @@ -508,7 +509,7 @@ public class RDDSortUtils ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); - mb = new MatrixBlock((int)len, 2, false); + mb = new MatrixBlock(IntUtils.toInt(len), 2, false); } mb.quickSetValue(pos, 0, val._1.val1); @@ -557,7 +558,7 @@ public class RDDSortUtils ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); - mb = new MatrixBlock((int)len, 1, false); + mb = new MatrixBlock(IntUtils.toInt(len), 1, false); } mb.quickSetValue(pos, 0, val._1.ix); @@ -605,7 +606,7 @@ public class RDDSortUtils ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); - mb = new MatrixBlock((int)len, 1, false); + mb = new MatrixBlock(IntUtils.toInt(len), 1, false); } mb.quickSetValue(pos, 0, val._2+1); @@ -651,7 +652,7 @@ public class RDDSortUtils ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); - mb = new MatrixBlock((int)len, val._1.getNumColumns(), false); + mb = new MatrixBlock(IntUtils.toInt(len), val._1.getNumColumns(), false); } mb.leftIndexingOperations(val._1, pos, pos, 0, val._1.getNumColumns()-1, mb, UpdateType.INPLACE); @@ -697,7 +698,7 @@ public class RDDSortUtils ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); - mb = new MatrixBlock((int)len, 1, false); + mb = new MatrixBlock(IntUtils.toInt(len), 1, false); } mb.quickSetValue(pos, 0, val._1.ix); @@ -857,7 +858,7 @@ public class RDDSortUtils //produce next output tuple MatrixIndexes ixmap = _currBlk._1(); MatrixBlock data = _currBlk._2(); - MatrixBlock mbTargetIndex = _pmb.value().getBlock((int)ixmap.getRowIndex(), 1); + MatrixBlock mbTargetIndex = _pmb.value().getBlock(IntUtils.toInt(ixmap.getRowIndex()), 1); long valix = (long) mbTargetIndex.getValue(_currPos, 0); long rix = UtilFunctions.computeBlockIndex(valix, _brlen); http://git-wip-us.apache.org/repos/asf/systemml/blob/95cbbd65/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java index 952135e..42eca7b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java @@ -49,6 +49,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.data.Pair; import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue; import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.sysml.utils.IntUtils; import scala.Tuple2; @@ -120,7 +121,7 @@ public class SparkUtils return SparkExecutionContext.getDefaultParallelism(true); double hdfsBlockSize = InfrastructureAnalyzer.getHDFSBlockSize(); double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(mc); - return (int) Math.max(Math.ceil(matrixPSize/hdfsBlockSize), 1); + return IntUtils.toInt( Math.max(Math.ceil(matrixPSize/hdfsBlockSize), 1)); } /** @@ -187,8 +188,8 @@ public class SparkUtils //compute degree of parallelism and block ranges long size = mc.getNumBlocks() * OptimizerUtils.estimateSizeEmptyBlock(Math.min( Math.max(mc.getRows(),1), mc.getRowsPerBlock()), Math.min(Math.max(mc.getCols(),1), mc.getColsPerBlock())); - int par = (int) Math.min(4*Math.max(SparkExecutionContext.getDefaultParallelism(true), - Math.ceil(size/InfrastructureAnalyzer.getHDFSBlockSize())), mc.getNumBlocks()); + int par = IntUtils.toInt(Math.min(4*Math.max(SparkExecutionContext.getDefaultParallelism(true), + Math.ceil(size/InfrastructureAnalyzer.getHDFSBlockSize())), mc.getNumBlocks())); long pNumBlocks = (long)Math.ceil((double)mc.getNumBlocks()/par); //generate block offsets per partition http://git-wip-us.apache.org/repos/asf/systemml/blob/95cbbd65/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java index fd06578..848aaa0 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java @@ -79,6 +79,7 @@ import org.apache.sysml.runtime.matrix.operators.RightScalarOperator; import org.apache.sysml.runtime.matrix.operators.ScalarOperator; import org.apache.sysml.runtime.util.IndexRange; import org.apache.sysml.utils.GPUStatistics; +import org.apache.sysml.utils.IntUtils; import org.apache.sysml.utils.Statistics; import jcuda.Pointer; @@ -1877,10 +1878,7 @@ public class LibMatrixCUDA { //********************************************************************/ public static int toInt(long num) { - if(num >= Integer.MAX_VALUE || num <= Integer.MIN_VALUE) { - throw new DMLRuntimeException("GPU : Exceeded supported size " + num); - } - return (int)num; + return IntUtils.toInt(num); } //********************************************************************/ http://git-wip-us.apache.org/repos/asf/systemml/blob/95cbbd65/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index 4e2a148..ad59e20 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -93,6 +93,7 @@ import org.apache.sysml.runtime.util.FastBufferedDataInputStream; import org.apache.sysml.runtime.util.FastBufferedDataOutputStream; import org.apache.sysml.runtime.util.IndexRange; import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.sysml.utils.IntUtils; import org.apache.sysml.utils.NativeHelper; @@ -251,7 +252,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab sparse = (val == 0) ? sp : false; nonZeros = (val == 0) ? 0 : (long)rl*cl; estimatedNNzsPerRow = (estnnz < 0 || !sparse) ? -1 : - (int)Math.ceil((double)estnnz/(double)rlen); + IntUtils.toInt(Math.ceil((double)estnnz/(double)rlen)); //reset sparse/dense blocks if( sparse ) @@ -1080,7 +1081,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab if( allowCSR && nonZeros <= Integer.MAX_VALUE ) { //allocate target in memory-efficient CSR format - int lnnz = (int) nonZeros; + int lnnz = IntUtils.toInt(nonZeros); int[] rptr = new int[m+1]; int[] indexes = new int[lnnz]; double[] values = new double[lnnz]; @@ -1281,7 +1282,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab this.rlen=that.rlen; this.clen=that.clen; this.sparse=sp; - estimatedNNzsPerRow=(int)Math.ceil((double)thatValue.getNonZeros()/(double)rlen); + estimatedNNzsPerRow=IntUtils.toInt(Math.ceil((double)thatValue.getNonZeros()/(double)rlen)); if(this.sparse && that.sparse) copySparseToSparse(that); else if(this.sparse && !that.sparse) @@ -1922,7 +1923,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab //block: read ijv-triples (ordered by row and column) via custom //init to avoid repeated updates of row pointers per append SparseBlockCSR sblockCSR = (SparseBlockCSR) sparseBlock; - sblockCSR.initUltraSparse((int)nonZeros, in); + sblockCSR.initUltraSparse(IntUtils.toInt(nonZeros), in); } else { //ULTRA-SPARSE COL //col: read iv-pairs (should never happen since always dense) @@ -2250,7 +2251,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab out.writeLong( nonZeros ); } else { - out.writeInt( (int)nonZeros ); + out.writeInt( IntUtils.toInt(nonZeros) ); } } @@ -3461,7 +3462,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab SparseBlock sblock = result.getSparseBlock(); for( int i=0; i<result.rlen; i++ ) { final int row = i; //workaround for lambda compile issue - int lnnz = (int) (this.recomputeNonZeros(i, i, 0, this.clen-1) + Arrays.stream(that) + int lnnz = IntUtils.toInt(this.recomputeNonZeros(i, i, 0, this.clen-1) + Arrays.stream(that) .mapToLong(mb -> mb.recomputeNonZeros(row, row, 0, mb.clen-1)).sum()); sblock.allocate(i, lnnz); } @@ -3676,8 +3677,8 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab public final MatrixBlock leftIndexingOperations(MatrixBlock rhsMatrix, IndexRange ixrange, MatrixBlock ret, UpdateType update) { - return leftIndexingOperations(rhsMatrix, (int)ixrange.rowStart, - (int)ixrange.rowEnd, (int)ixrange.colStart, (int)ixrange.colEnd, ret, update); + return leftIndexingOperations(rhsMatrix, IntUtils.toInt(ixrange.rowStart), + IntUtils.toInt(ixrange.rowEnd), IntUtils.toInt(ixrange.colStart), IntUtils.toInt(ixrange.colEnd), ret, update); } public MatrixBlock leftIndexingOperations(MatrixBlock rhsMatrix, @@ -3803,8 +3804,8 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab public MatrixBlock slice(IndexRange ixrange, MatrixBlock ret) { return slice( - (int)ixrange.rowStart, (int)ixrange.rowEnd, - (int)ixrange.colStart, (int)ixrange.colEnd, true, ret); + IntUtils.toInt(ixrange.rowStart), IntUtils.toInt(ixrange.rowEnd), + IntUtils.toInt(ixrange.colStart), IntUtils.toInt(ixrange.colEnd), true, ret); } public MatrixBlock slice(int rl, int ru) { @@ -3955,44 +3956,44 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab if(colCut>range.colEnd) blockColFactor=boundaryClen; - int minrowcut=(int)Math.min(rowCut,range.rowEnd); - int mincolcut=(int)Math.min(colCut, range.colEnd); - int maxrowcut=(int)Math.max(rowCut, range.rowStart); - int maxcolcut=(int)Math.max(colCut, range.colStart); + int minrowcut=IntUtils.toInt(Math.min(rowCut,range.rowEnd)); + int mincolcut=IntUtils.toInt(Math.min(colCut, range.colEnd)); + int maxrowcut=IntUtils.toInt(Math.max(rowCut, range.rowStart)); + int maxcolcut=IntUtils.toInt(Math.max(colCut, range.colStart)); if(range.rowStart<rowCut && range.colStart<colCut) { topleft=(MatrixBlock) p.next().getValue(); //topleft.reset(blockRowFactor, blockColFactor, - // checkSparcityOnSlide(rowCut-(int)range.rowStart, colCut-(int)range.colStart, blockRowFactor, blockColFactor)); + // checkSparcityOnSlide(rowCut-IntUtils.toInt(range.rowStart), colCut-IntUtils.toInt(range.colStart), blockRowFactor, blockColFactor)); topleft.reset(blockRowFactor, blockColFactor, - estimateSparsityOnSlice(minrowcut-(int)range.rowStart, mincolcut-(int)range.colStart, blockRowFactor, blockColFactor)); + estimateSparsityOnSlice(minrowcut-IntUtils.toInt(range.rowStart), mincolcut-IntUtils.toInt(range.colStart), blockRowFactor, blockColFactor)); } if(range.rowStart<rowCut && range.colEnd>=colCut) { topright=(MatrixBlock) p.next().getValue(); topright.reset(blockRowFactor, boundaryClen, - estimateSparsityOnSlice(minrowcut-(int)range.rowStart, (int)range.colEnd-maxcolcut+1, blockRowFactor, boundaryClen)); + estimateSparsityOnSlice(minrowcut-IntUtils.toInt(range.rowStart), IntUtils.toInt(range.colEnd)-maxcolcut+1, blockRowFactor, boundaryClen)); } if(range.rowEnd>=rowCut && range.colStart<colCut) { bottomleft=(MatrixBlock) p.next().getValue(); bottomleft.reset(boundaryRlen, blockColFactor, - estimateSparsityOnSlice((int)range.rowEnd-maxrowcut+1, mincolcut-(int)range.colStart, boundaryRlen, blockColFactor)); + estimateSparsityOnSlice(IntUtils.toInt(range.rowEnd)-maxrowcut+1, mincolcut-IntUtils.toInt(range.colStart), boundaryRlen, blockColFactor)); } if(range.rowEnd>=rowCut && range.colEnd>=colCut) { bottomright=(MatrixBlock) p.next().getValue(); bottomright.reset(boundaryRlen, boundaryClen, - estimateSparsityOnSlice((int)range.rowEnd-maxrowcut+1, (int)range.colEnd-maxcolcut+1, boundaryRlen, boundaryClen)); + estimateSparsityOnSlice(IntUtils.toInt(range.rowEnd)-maxrowcut+1, IntUtils.toInt(range.colEnd)-maxcolcut+1, boundaryRlen, boundaryClen)); } if(sparse) { if(sparseBlock!=null) { - int r=(int)range.rowStart; + int r=IntUtils.toInt(range.rowStart); for(; r<Math.min(Math.min(rowCut, sparseBlock.numRows()), range.rowEnd+1); r++) sliceHelp(r, range, colCut, topleft, topright, normalBlockRowFactor-rowCut, normalBlockRowFactor, normalBlockColFactor); @@ -4004,11 +4005,11 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab if(denseBlock!=null) { double[] a = getDenseBlockValues(); - int i=((int)range.rowStart)*clen; - int r=(int) range.rowStart; + int i=(IntUtils.toInt(range.rowStart))*clen; + int r=IntUtils.toInt(range.rowStart); for(; r<Math.min(rowCut, range.rowEnd+1); r++) { - int c=(int) range.colStart; + int c=IntUtils.toInt(range.colStart); for(; c<Math.min(colCut, range.colEnd+1); c++) topleft.appendValue(r+normalBlockRowFactor-rowCut, c+normalBlockColFactor-colCut, a[i+c]); for(; c<=range.colEnd; c++) @@ -4018,7 +4019,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab for(; r<=range.rowEnd; r++) { - int c=(int) range.colStart; + int c=IntUtils.toInt(range.colStart); for(; c<Math.min(colCut, range.colEnd+1); c++) bottomleft.appendValue(r-rowCut, c+normalBlockColFactor-colCut, a[i+c]); for(; c<=range.colEnd; c++) @@ -4036,10 +4037,10 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab int[] cols=sparseBlock.indexes(r); double[] values=sparseBlock.values(r); - int start=sparseBlock.posFIndexGTE(r, (int)range.colStart); + int start=sparseBlock.posFIndexGTE(r, IntUtils.toInt(range.colStart)); if(start<0) return; - int end=sparseBlock.posFIndexLTE(r, (int)range.colEnd); + int end=sparseBlock.posFIndexLTE(r, IntUtils.toInt(range.colEnd)); if(end<0 || start>end) return; @@ -4109,9 +4110,9 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab boolean lsparse = evalSparseFormatInMemory(rlen, clen, (long)(estimatedSps*rlen*clen)); if(result==null) - result=new MatrixBlock(rlen, clen, lsparse, (int)(estimatedSps*rlen*clen)); + result=new MatrixBlock(rlen, clen, lsparse, IntUtils.toInt(estimatedSps*rlen*clen)); else - result.reset(rlen, clen, lsparse, (int)(estimatedSps*rlen*clen)); + result.reset(rlen, clen, lsparse, IntUtils.toInt(estimatedSps*rlen*clen)); if(sparse) @@ -4120,12 +4121,12 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab { if(!complementary)//if zero out { - for(int r=0; r<Math.min((int)range.rowStart, sparseBlock.numRows()); r++) + for(int r=0; r<Math.min(IntUtils.toInt(range.rowStart), sparseBlock.numRows()); r++) ((MatrixBlock) result).appendRow(r, sparseBlock.get(r)); - for(int r=Math.min((int)range.rowEnd+1, sparseBlock.numRows()); r<Math.min(rlen, sparseBlock.numRows()); r++) + for(int r=Math.min(IntUtils.toInt(range.rowEnd+1), sparseBlock.numRows()); r<Math.min(rlen, sparseBlock.numRows()); r++) ((MatrixBlock) result).appendRow(r, sparseBlock.get(r)); } - for(int r=(int)range.rowStart; r<=Math.min(range.rowEnd, sparseBlock.numRows()-1); r++) + for(int r=IntUtils.toInt(range.rowStart); r<=Math.min(range.rowEnd, sparseBlock.numRows()-1); r++) { if(sparseBlock.isEmpty(r)) continue; @@ -4134,9 +4135,9 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab if(complementary)//if selection { - int start=sparseBlock.posFIndexGTE(r,(int)range.colStart); + int start=sparseBlock.posFIndexGTE(r,IntUtils.toInt(range.colStart)); if(start<0) continue; - int end=sparseBlock.posFIndexGT(r,(int)range.colEnd); + int end=sparseBlock.posFIndexGT(r,IntUtils.toInt(range.colEnd)); if(end<0 || start>end) continue; @@ -4148,9 +4149,9 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab { int pos = sparseBlock.pos(r); int len = sparseBlock.size(r); - int start=sparseBlock.posFIndexGTE(r,(int)range.colStart); + int start=sparseBlock.posFIndexGTE(r,IntUtils.toInt(range.colStart)); if(start<0) start=pos+len; - int end=sparseBlock.posFIndexGT(r,(int)range.colEnd); + int end=sparseBlock.posFIndexGT(r,IntUtils.toInt(range.colEnd)); if(end<0) end=pos+len; for(int i=pos; i<start; i++) @@ -4171,10 +4172,10 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab double[] a = getDenseBlockValues(); if(complementary)//if selection { - int offset=((int)range.rowStart)*clen; - for(int r=(int) range.rowStart; r<=range.rowEnd; r++) + int offset=(IntUtils.toInt(range.rowStart))*clen; + for(int r=IntUtils.toInt(range.rowStart); r<=range.rowEnd; r++) { - for(int c=(int) range.colStart; c<=range.colEnd; c++) + for(int c=IntUtils.toInt(range.colStart); c<=range.colEnd; c++) ((MatrixBlock) result).appendValue(r, c, a[offset+c]); offset+=clen; } @@ -4182,15 +4183,15 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab { int offset=0; int r=0; - for(; r<(int)range.rowStart; r++) + for(; r<IntUtils.toInt(range.rowStart); r++) for(int c=0; c<clen; c++, offset++) ((MatrixBlock) result).appendValue(r, c, a[offset]); - for(; r<=(int)range.rowEnd; r++) + for(; r<=IntUtils.toInt(range.rowEnd); r++) { - for(int c=0; c<(int)range.colStart; c++) + for(int c=0; c<IntUtils.toInt(range.colStart); c++) ((MatrixBlock) result).appendValue(r, c, a[offset+c]); - for(int c=(int)range.colEnd+1; c<clen; c++) + for(int c=IntUtils.toInt(range.colEnd)+1; c<clen; c++) ((MatrixBlock) result).appendValue(r, c, a[offset+c]); offset+=clen; } @@ -4613,7 +4614,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab // prepare result, currently always dense // #rows in temp matrix = 1 + #nnz in the input ( 1 is for the "zero" value) - int dim1 = (int) (1+this.getNonZeros()); + int dim1 = IntUtils.toInt(1+this.getNonZeros()); if(result==null) result=new MatrixBlock(dim1, 2, false); else @@ -4929,7 +4930,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab throw new DMLRuntimeException("Invalid value (" + min + ") encountered in 'groups' while computing groupedAggregate"); if ( max <= 0 ) throw new DMLRuntimeException("Invalid value (" + max + ") encountered in 'groups' while computing groupedAggregate."); - ngroups = (int) max; + ngroups = IntUtils.toInt( max ); } // Allocate result matrix @@ -5177,7 +5178,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab { CTable ctable = CTable.getCTableFnObject(); double w = scalarThat; - int offset = (int) ((ix1.getRowIndex()-1)*brlen); + int offset = IntUtils.toInt((ix1.getRowIndex()-1)*brlen); //sparse-unsafe ctable execution //(because input values of 0 are invalid and have to result in errors) @@ -5362,7 +5363,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab R.reset(rlen, clen, sparse); else if( qop.wtype3 != null ) { MatrixCharacteristics mc = qop.wtype3.computeOutputCharacteristics(X.rlen, X.clen, U.clen); - R.reset( (int)mc.getRows(), (int)mc.getCols(), qop.wtype3.isBasic()?X.isInSparseFormat():false); + R.reset( IntUtils.toInt(mc.getRows()), IntUtils.toInt(mc.getCols()), qop.wtype3.isBasic()?X.isInSparseFormat():false); } //core block operation http://git-wip-us.apache.org/repos/asf/systemml/blob/95cbbd65/src/main/java/org/apache/sysml/utils/IntUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/IntUtils.java b/src/main/java/org/apache/sysml/utils/IntUtils.java new file mode 100644 index 0000000..aa1f963 --- /dev/null +++ b/src/main/java/org/apache/sysml/utils/IntUtils.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysml.utils; +import static java.lang.Math.toIntExact; + + +public class IntUtils { + + public static int toInt(long val) { + return toIntExact(val); + } + + public static int toInt(double val) { + long val1 = (long) val; + return toIntExact(val1); + } + +}