IGNITE-5791 Block matrix introduction This closes #2326
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0d2b989d Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0d2b989d Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0d2b989d Branch: refs/heads/ignite-5569-debug Commit: 0d2b989d2be62533a36061940497a734463b5f10 Parents: db43b0c Author: Yury Babak <yba...@gridgain.com> Authored: Fri Jul 21 15:28:21 2017 +0300 Committer: Pavel Tupitsyn <ptupit...@apache.org> Committed: Fri Jul 21 15:28:21 2017 +0300 ---------------------------------------------------------------------- .../apache/ignite/ml/math/DistanceMeasure.java | 2 +- .../ignite/ml/math/EuclideanDistance.java | 3 +- .../math/decompositions/EigenDecomposition.java | 2 +- .../apache/ignite/ml/math/impls/CacheUtils.java | 198 +++++++-- .../ml/math/impls/matrix/AbstractMatrix.java | 4 +- .../ignite/ml/math/impls/matrix/BlockEntry.java | 50 +++ .../ml/math/impls/matrix/CacheMatrix.java | 9 +- .../matrix/SparseBlockDistributedMatrix.java | 208 +++++++++ .../impls/matrix/SparseDistributedMatrix.java | 26 +- .../storage/matrix/BaseBlockMatrixKey.java | 41 ++ .../impls/storage/matrix/BlockMatrixKey.java | 144 ++++++ .../storage/matrix/BlockMatrixStorage.java | 435 +++++++++++++++++++ .../vector/SparseLocalOnHeapVectorStorage.java | 4 +- .../ignite/ml/math/statistics/Variance.java | 1 + .../ignite/ml/math/statistics/package-info.java | 22 + .../org/apache/ignite/ml/math/util/MapUtil.java | 2 +- .../ignite/ml/math/util/package-info.java | 22 + .../java/org/apache/ignite/ml/package-info.java | 22 + .../ml/math/MathImplDistributedTestSuite.java | 2 + .../SparseDistributedBlockMatrixTest.java | 379 ++++++++++++++++ .../matrix/SparseDistributedMatrixTest.java | 32 +- 21 files changed, 1528 insertions(+), 80 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java index 09be0c3..df235a7 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/DistanceMeasure.java @@ -34,5 +34,5 @@ public interface DistanceMeasure extends Externalizable { * @return the distance between the two vectors * @throws CardinalityException if the array lengths differ. */ - double compute(Vector a, Vector b) throws CardinalityException; + public double compute(Vector a, Vector b) throws CardinalityException; } http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java index 5f962ce..edc11dc 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/EuclideanDistance.java @@ -30,8 +30,7 @@ public class EuclideanDistance implements DistanceMeasure { private static final long serialVersionUID = 1717556319784040040L; /** {@inheritDoc} */ - @Override - public double compute(Vector a, Vector b) + @Override public double compute(Vector a, Vector b) throws CardinalityException { return MatrixUtil.localCopyOf(a).minus(b).kNorm(2.0); } http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java index d0e91a5..a5c92e6 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/EigenDecomposition.java @@ -446,7 +446,7 @@ public class EigenDecomposition implements Destroyable { // Store roots isolated by balanc and compute matrix norm - double norm = h.foldMap(Functions.PLUS, Functions.ABS, 0.0); + double norm = h.foldMap(Functions.PLUS, Functions.ABS, 0.0d); // Outer loop over eigenvalue index http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java index 1bda5e6..369840b 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/CacheUtils.java @@ -39,11 +39,16 @@ import org.apache.ignite.ml.math.KeyMapper; import org.apache.ignite.ml.math.ValueMapper; import org.apache.ignite.ml.math.functions.IgniteBiFunction; import org.apache.ignite.ml.math.functions.IgniteConsumer; +import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; import org.apache.ignite.ml.math.functions.IgniteFunction; -import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage; +import org.apache.ignite.ml.math.impls.matrix.BlockEntry; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey; +import org.apache.ignite.internal.util.typedef.internal.A; /** * Distribution-related misc. support. + * + * TODO: IGNITE-5102, fix sparse key filters */ public class CacheUtils { /** @@ -127,19 +132,38 @@ public class CacheUtils { * @param matrixUuid Matrix UUID. * @return Sum obtained using sparse logic. */ - public static <K, V> double sparseSum(IgniteUuid matrixUuid) { - Collection<Double> subSums = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> ce, Double acc) -> { - Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> entry = ce.entry(); - if (entry.getKey().get2().equals(matrixUuid)) { - Map<Integer, Double> map = entry.getValue(); + @SuppressWarnings("unchecked") + public static <K, V> double sparseSum(IgniteUuid matrixUuid, String cacheName) { + A.notNull(matrixUuid, "matrixUuid"); + A.notNull(cacheName, "cacheName"); + + Collection<Double> subSums = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> { + V v = ce.entry().getValue(); + + double sum = 0.0; - double sum = sum(map.values()); + if (v instanceof Map) { + Map<Integer, Double> map = (Map<Integer, Double>)v; - return acc == null ? sum : acc + sum; + sum = sum(map.values()); + } + else if (v instanceof BlockEntry) { + BlockEntry be = (BlockEntry)v; + + sum = be.sum(); } else - return acc; - }, key -> key.get2().equals(matrixUuid)); + throw new UnsupportedOperationException(); + + return acc == null ? sum : acc + sum; + }, key -> { + if (key instanceof BlockMatrixKey) + return ((BlockMatrixKey)key).matrixId().equals(matrixUuid); + else if (key instanceof IgniteBiTuple) + return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid); + else + throw new UnsupportedOperationException(); + }); return sum(subSums); } @@ -186,23 +210,42 @@ public class CacheUtils { * @param matrixUuid Matrix UUID. * @return Minimum value obtained using sparse logic. */ - public static <K, V> double sparseMin(IgniteUuid matrixUuid) { - Collection<Double> mins = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> ce, Double acc) -> { - Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> entry = ce.entry(); + @SuppressWarnings("unchecked") + public static <K, V> double sparseMin(IgniteUuid matrixUuid, String cacheName) { + A.notNull(matrixUuid, "matrixUuid"); + A.notNull(cacheName, "cacheName"); - if (entry.getKey().get2().equals(matrixUuid)) { - Map<Integer, Double> map = entry.getValue(); + Collection<Double> mins = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> { + V v = ce.entry().getValue(); - double min = Collections.min(map.values()); + double min; - if (acc == null) - return min; - else - return Math.min(acc, min); + if (v instanceof Map) { + Map<Integer, Double> map = (Map<Integer, Double>)v; + + min = Collections.min(map.values()); + } + else if (v instanceof BlockEntry) { + BlockEntry be = (BlockEntry)v; + + min = be.minValue(); } else - return acc; - }, key -> key.get2().equals(matrixUuid)); + throw new UnsupportedOperationException(); + + if (acc == null) + return min; + else + return Math.min(acc, min); + + }, key -> { + if (key instanceof BlockMatrixKey) + return ((BlockMatrixKey)key).matrixId().equals(matrixUuid); + else if (key instanceof IgniteBiTuple) + return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid); + else + throw new UnsupportedOperationException(); + }); return Collections.min(mins); } @@ -211,22 +254,42 @@ public class CacheUtils { * @param matrixUuid Matrix UUID. * @return Maximum value obtained using sparse logic. */ - public static <K, V> double sparseMax(IgniteUuid matrixUuid) { - Collection<Double> maxes = fold(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> ce, Double acc) -> { - Cache.Entry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> entry = ce.entry(); - if (entry.getKey().get2().equals(matrixUuid)) { - Map<Integer, Double> map = entry.getValue(); + @SuppressWarnings("unchecked") + public static <K, V> double sparseMax(IgniteUuid matrixUuid, String cacheName) { + A.notNull(matrixUuid, "matrixUuid"); + A.notNull(cacheName, "cacheName"); + + Collection<Double> maxes = fold(cacheName, (CacheEntry<K, V> ce, Double acc) -> { + V v = ce.entry().getValue(); - double max = Collections.max(map.values()); + double max; - if (acc == null) - return max; - else - return Math.max(acc, max); + if (v instanceof Map) { + Map<Integer, Double> map = (Map<Integer, Double>)v; + + max = Collections.max(map.values()); + } + else if (v instanceof BlockEntry) { + BlockEntry be = (BlockEntry)v; + + max = be.maxValue(); } else - return acc; - }, key -> key.get2().equals(matrixUuid)); + throw new UnsupportedOperationException(); + + if (acc == null) + return max; + else + return Math.max(acc, max); + + }, key -> { + if (key instanceof BlockMatrixKey) + return ((BlockMatrixKey)key).matrixId().equals(matrixUuid); + else if (key instanceof IgniteBiTuple) + return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid); + else + throw new UnsupportedOperationException(); + }); return Collections.max(maxes); } @@ -279,17 +342,41 @@ public class CacheUtils { * @param matrixUuid Matrix UUID. * @param mapper Mapping {@link IgniteFunction}. */ - public static <K, V> void sparseMap(IgniteUuid matrixUuid, IgniteFunction<Double, Double> mapper) { - foreach(SparseDistributedMatrixStorage.ML_CACHE_NAME, (CacheEntry<IgniteBiTuple<Integer, IgniteUuid>, Map<Integer, Double>> ce) -> { - IgniteBiTuple k = ce.entry().getKey(); + @SuppressWarnings("unchecked") + public static <K, V> void sparseMap(IgniteUuid matrixUuid, IgniteDoubleFunction<Double> mapper, String cacheName) { + A.notNull(matrixUuid, "matrixUuid"); + A.notNull(cacheName, "cacheName"); + A.notNull(mapper, "mapper"); + + foreach(cacheName, (CacheEntry<K, V> ce) -> { + K k = ce.entry().getKey(); + + V v = ce.entry().getValue(); - Map<Integer, Double> v = ce.entry().getValue(); + if (v instanceof Map) { + Map<Integer, Double> map = (Map<Integer, Double>)v; - for (Map.Entry<Integer, Double> e : v.entrySet()) - e.setValue(mapper.apply(e.getValue())); + for (Map.Entry<Integer, Double> e : (map.entrySet())) + e.setValue(mapper.apply(e.getValue())); + + } + else if (v instanceof BlockEntry) { + BlockEntry be = (BlockEntry)v; + + be.map(mapper); + } + else + throw new UnsupportedOperationException(); ce.cache().put(k, v); - }, key -> key.get2().equals(matrixUuid)); + }, key -> { + if (key instanceof BlockMatrixKey) + return ((BlockMatrixKey)key).matrixId().equals(matrixUuid); + else if (key instanceof IgniteBiTuple) + return ((IgniteBiTuple<Integer, IgniteUuid>)key).get2().equals(matrixUuid); + else + throw new UnsupportedOperationException(); + }); } /** @@ -327,8 +414,7 @@ public class CacheUtils { // Iterate over given partition. // Query returns an empty cursor if this partition is not stored on this node. - for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, - (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) + for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) fun.accept(new CacheEntry<>(entry, cache)); } }); @@ -387,12 +473,34 @@ public class CacheUtils { }); } + /** + * Distributed version of fold operation. + * + * @param cacheName Cache name. + * @param folder Folder. + * @param keyFilter Key filter. + * @param accumulator Accumulator. + * @param zeroVal Zero value. + */ public static <K, V, A> A distributedFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder, IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal) { return sparseFold(cacheName, folder, keyFilter, accumulator, zeroVal, null, null, 0, false); } + /** + * Sparse version of fold. This method also applicable to sparse zeroes. + * + * @param cacheName Cache name. + * @param folder Folder. + * @param keyFilter Key filter. + * @param accumulator Accumulator. + * @param zeroVal Zero value. + * @param defVal Def value. + * @param defKey Def key. + * @param defValCnt Def value count. + * @param isNilpotent Is nilpotent. + */ private static <K, V, A> A sparseFold(String cacheName, IgniteBiFunction<Cache.Entry<K, V>, A, A> folder, IgnitePredicate<K> keyFilter, BinaryOperator<A> accumulator, A zeroVal, V defVal, K defKey, long defValCnt, boolean isNilpotent) { @@ -411,7 +519,7 @@ public class CacheUtils { // Use affinity in filter for ScanQuery. Otherwise we accept consumer in each node which is wrong. Affinity affinity = ignite.affinity(cacheName); - ClusterNode localNode = ignite.cluster().localNode(); + ClusterNode locNode = ignite.cluster().localNode(); A a = zeroVal; @@ -422,7 +530,7 @@ public class CacheUtils { // Iterate over given partition. // Query returns an empty cursor if this partition is not stored on this node. for (Cache.Entry<K, V> entry : cache.query(new ScanQuery<K, V>(part, - (k, v) -> affinity.mapPartitionToNode(p) == localNode && (keyFilter == null || keyFilter.apply(k))))) + (k, v) -> affinity.mapPartitionToNode(p) == locNode && (keyFilter == null || keyFilter.apply(k))))) a = folder.apply(entry, a); } http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java index d1d3904..3dc9b43 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/AbstractMatrix.java @@ -503,7 +503,7 @@ public abstract class AbstractMatrix implements Matrix { /** {@inheritDoc} */ @Override public double determinant() { - //TODO: This decomposition should be cached + //TODO: IGNITE-5799, This decomposition should be cached LUDecomposition dec = new LUDecomposition(this); double res = dec.determinant(); dec.destroy(); @@ -515,7 +515,7 @@ public abstract class AbstractMatrix implements Matrix { if (rowSize() != columnSize()) throw new CardinalityException(rowSize(), columnSize()); - //TODO: This decomposition should be cached + //TODO: IGNITE-5799, This decomposition should be cached LUDecomposition dec = new LUDecomposition(this); Matrix res = dec.solve(likeIdentity()); http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java new file mode 100644 index 0000000..47f07ce --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/BlockEntry.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.matrix; + +import org.apache.ignite.ml.math.Matrix; + +/** + * Block for {@link SparseBlockDistributedMatrix}. + */ +public final class BlockEntry extends SparseLocalOnHeapMatrix { + /** Max block size. */ + public static final int MAX_BLOCK_SIZE = 32; + + /** */ + public BlockEntry() { + // No-op. + } + + /** */ + public BlockEntry(int row, int col) { + super(row, col); + + assert col <= MAX_BLOCK_SIZE; + assert row <= MAX_BLOCK_SIZE; + } + + /** */ + public BlockEntry(Matrix mtx) { + assert mtx.columnSize() <= MAX_BLOCK_SIZE; + assert mtx.rowSize() <= MAX_BLOCK_SIZE; + + setStorage(mtx.getStorage()); + } + +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java index a7f0afc..7f00bcb 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/CacheMatrix.java @@ -65,7 +65,6 @@ public class CacheMatrix<K, V> extends AbstractMatrix { /** * - * */ @SuppressWarnings({"unchecked"}) private CacheMatrixStorage<K, V> storage() { @@ -93,7 +92,7 @@ public class CacheMatrix<K, V> extends AbstractMatrix { * @param d Value to divide to. */ @Override public Matrix divide(double d) { - return mapOverValues((Double v) -> v / d); + return mapOverValues(v -> v / d); } /** @@ -102,7 +101,7 @@ public class CacheMatrix<K, V> extends AbstractMatrix { * @param x Value to add. */ @Override public Matrix plus(double x) { - return mapOverValues((Double v) -> v + x); + return mapOverValues(v -> v + x); } /** @@ -111,12 +110,12 @@ public class CacheMatrix<K, V> extends AbstractMatrix { * @param x Value to multiply to. */ @Override public Matrix times(double x) { - return mapOverValues((Double v) -> v * x); + return mapOverValues(v -> v * x); } /** {@inheritDoc} */ @Override public Matrix assign(double val) { - return mapOverValues((Double v) -> val); + return mapOverValues(v -> val); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java new file mode 100644 index 0000000..b3481f9 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.matrix; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.Vector; +import org.apache.ignite.ml.math.exceptions.CardinalityException; +import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; +import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; +import org.apache.ignite.ml.math.impls.CacheUtils; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage; + +/** + * Sparse block distributed matrix. This matrix represented by blocks 32x32 {@link BlockEntry}. + * + * Using separate cache with keys {@link BlockMatrixKey} and values {@link BlockEntry}. + */ +public class SparseBlockDistributedMatrix extends AbstractMatrix implements StorageConstants { + /** + * + */ + public SparseBlockDistributedMatrix() { + // No-op. + } + + /** + * @param rows Amount of rows in the matrix. + * @param cols Amount of columns in the matrix. + */ + public SparseBlockDistributedMatrix(int rows, int cols) { + assert rows > 0; + assert cols > 0; + + setStorage(new BlockMatrixStorage(rows, cols)); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param d Value to divide to. + */ + @Override public Matrix divide(double d) { + return mapOverValues(v -> v / d); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param x Value to add. + */ + @Override public Matrix plus(double x) { + return mapOverValues(v -> v + x); + } + + /** + * Return the same matrix with updates values (broken contract). + * + * @param x Value to multiply. + */ + @Override public Matrix times(double x) { + return mapOverValues(v -> v * x); + } + + /** + * {@inheritDoc} + */ + @SuppressWarnings({"unchecked"}) + @Override public Matrix times(final Matrix mtx) { + if (mtx == null) + throw new IllegalArgumentException("The matrix should be not null."); + + if (columnSize() != mtx.rowSize()) + throw new CardinalityException(columnSize(), mtx.rowSize()); + + SparseBlockDistributedMatrix matrixA = this; + SparseBlockDistributedMatrix matrixB = (SparseBlockDistributedMatrix)mtx; + + String cacheName = BlockMatrixStorage.ML_BLOCK_CACHE_NAME; + SparseBlockDistributedMatrix matrixC = new SparseBlockDistributedMatrix(matrixA.rowSize(), matrixB.columnSize()); + + CacheUtils.bcast(BlockMatrixStorage.ML_BLOCK_CACHE_NAME, () -> { + Ignite ignite = Ignition.localIgnite(); + Affinity affinity = ignite.affinity(cacheName); + + IgniteCache<BlockMatrixKey, BlockEntry> cache = ignite.getOrCreateCache(cacheName); + ClusterNode locNode = ignite.cluster().localNode(); + + BlockMatrixStorage storageC = matrixC.storage(); + + Map<ClusterNode, Collection<BlockMatrixKey>> keysCToNodes = affinity.mapKeysToNodes(storageC.getAllKeys()); + Collection<BlockMatrixKey> locKeys = keysCToNodes.get(locNode); + + if (locKeys == null) + return; + + // compute Cij locally on each node + // TODO: IGNITE:5114, exec in parallel + locKeys.forEach(key -> { + long newBlockId = key.blockId(); + BlockEntry blockC = null; + + List<BlockEntry> aRow = matrixA.storage().getRowForBlock(newBlockId, storageC); + List<BlockEntry> bCol = matrixB.storage().getColForBlock(newBlockId, storageC); + + for (int i = 0; i < aRow.size(); i++) { + BlockEntry blockA = aRow.get(i); + BlockEntry blockB = bCol.get(i); + + BlockEntry tmpBlock = new BlockEntry(blockA.times(blockB)); + + blockC = blockC == null ? tmpBlock : new BlockEntry(blockC.plus(tmpBlock)); + } + + cache.put(storageC.getCacheKey(newBlockId), blockC); + }); + }); + + return matrixC; + } + + /** {@inheritDoc} */ + @Override public Matrix assign(double val) { + return mapOverValues(v -> val); + } + + /** {@inheritDoc} */ + @Override public Matrix map(IgniteDoubleFunction<Double> fun) { + return mapOverValues(fun); + } + + /** {@inheritDoc} */ + @Override public double sum() { + return CacheUtils.sparseSum(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + } + + /** {@inheritDoc} */ + @Override public double maxValue() { + return CacheUtils.sparseMax(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + } + + /** {@inheritDoc} */ + @Override public double minValue() { + return CacheUtils.sparseMin(getUUID(), BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + } + + /** {@inheritDoc} */ + @Override public Matrix copy() { + throw new UnsupportedOperationException(); + } + + /** {@inheritDoc} */ + @Override public Matrix like(int rows, int cols) { + return new SparseBlockDistributedMatrix(rows, cols); + } + + /** {@inheritDoc} */ + @Override public Vector likeVector(int crd) { + throw new UnsupportedOperationException(); + } + + /** */ + private IgniteUuid getUUID() { + return ((BlockMatrixStorage)getStorage()).getUUID(); + } + + /** + * @param mapper Mapping function. + * @return Matrix with mapped values. + */ + private Matrix mapOverValues(IgniteDoubleFunction<Double> mapper) { + CacheUtils.sparseMap(getUUID(), mapper, BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + + return this; + } + + /** + * + */ + private BlockMatrixStorage storage() { + return (BlockMatrixStorage)getStorage(); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java index df2ddc4..a86db95 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java @@ -23,7 +23,6 @@ import org.apache.ignite.ml.math.StorageConstants; import org.apache.ignite.ml.math.Vector; import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; import org.apache.ignite.ml.math.functions.IgniteDoubleFunction; -import org.apache.ignite.ml.math.functions.IgniteFunction; import org.apache.ignite.ml.math.impls.CacheUtils; import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage; @@ -61,10 +60,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo setStorage(new SparseDistributedMatrixStorage(rows, cols, stoMode, acsMode)); } - /** - * - * - */ + /** */ private SparseDistributedMatrixStorage storage() { return (SparseDistributedMatrixStorage)getStorage(); } @@ -75,7 +71,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo * @param d Value to divide to. */ @Override public Matrix divide(double d) { - return mapOverValues((Double v) -> v / d); + return mapOverValues(v -> v / d); } /** @@ -84,7 +80,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo * @param x Value to add. */ @Override public Matrix plus(double x) { - return mapOverValues((Double v) -> v + x); + return mapOverValues(v -> v + x); } /** @@ -93,42 +89,42 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo * @param x Value to multiply. */ @Override public Matrix times(double x) { - return mapOverValues((Double v) -> v * x); + return mapOverValues(v -> v * x); } /** {@inheritDoc} */ @Override public Matrix assign(double val) { - return mapOverValues((Double v) -> val); + return mapOverValues(v -> val); } /** {@inheritDoc} */ @Override public Matrix map(IgniteDoubleFunction<Double> fun) { - return mapOverValues(fun::apply); + return mapOverValues(fun); } /** * @param mapper Mapping function. * @return Matrix with mapped values. */ - private Matrix mapOverValues(IgniteFunction<Double, Double> mapper) { - CacheUtils.sparseMap(getUUID(), mapper); + private Matrix mapOverValues(IgniteDoubleFunction<Double> mapper) { + CacheUtils.sparseMap(getUUID(), mapper, SparseDistributedMatrixStorage.ML_CACHE_NAME); return this; } /** {@inheritDoc} */ @Override public double sum() { - return CacheUtils.sparseSum(getUUID()); + return CacheUtils.sparseSum(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME); } /** {@inheritDoc} */ @Override public double maxValue() { - return CacheUtils.sparseMax(getUUID()); + return CacheUtils.sparseMax(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME); } /** {@inheritDoc} */ @Override public double minValue() { - return CacheUtils.sparseMin(getUUID()); + return CacheUtils.sparseMin(getUUID(), SparseDistributedMatrixStorage.ML_CACHE_NAME); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java new file mode 100644 index 0000000..74ddfe5 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BaseBlockMatrixKey.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.matrix; + +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix; + +/** + * Cache key for blocks in {@link SparseBlockDistributedMatrix}. + */ +public interface BaseBlockMatrixKey { + /** + * @return block id. + */ + public long blockId(); + + /** + * @return matrix id. + */ + public IgniteUuid matrixId(); + + /** + * @return key affinity key. + */ + public IgniteUuid affinityKey(); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java new file mode 100644 index 0000000..3749f44 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixKey.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.matrix; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryRawReader; +import org.apache.ignite.binary.BinaryRawWriter; +import org.apache.ignite.binary.BinaryReader; +import org.apache.ignite.binary.BinaryWriter; +import org.apache.ignite.binary.Binarylizable; +import org.apache.ignite.internal.binary.BinaryUtils; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.impls.matrix.BlockEntry; +import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix; +import org.jetbrains.annotations.Nullable; + +/** + * Key implementation for {@link BlockEntry} using for {@link SparseBlockDistributedMatrix}. + */ +public class BlockMatrixKey implements BaseBlockMatrixKey, Externalizable, Binarylizable { + /** */ + private static final long serialVersionUID = 0L; + /** Block ID */ + private long blockId; + /** Matrix ID */ + private IgniteUuid matrixUuid; + /** Block affinity key. */ + private IgniteUuid affinityKey; + + /** + * Empty constructor required for {@link Externalizable}. + */ + public BlockMatrixKey() { + // No-op. + } + + /** + * Construct matrix block key. + * + * @param blockId Block id. + * @param matrixUuid Matrix uuid. + * @param affinityKey Affinity key. + */ + public BlockMatrixKey(long blockId, IgniteUuid matrixUuid, @Nullable IgniteUuid affinityKey) { + assert blockId >= 0; + assert matrixUuid != null; + + this.blockId = blockId; + this.matrixUuid = matrixUuid; + this.affinityKey = affinityKey; + } + + /** {@inheritDoc} */ + @Override public long blockId() { + return blockId; + } + + /** {@inheritDoc} */ + @Override public IgniteUuid matrixId() { + return matrixUuid; + } + + /** {@inheritDoc} */ + @Override public IgniteUuid affinityKey() { + return affinityKey; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + U.writeGridUuid(out, matrixUuid); + U.writeGridUuid(out, affinityKey); + out.writeLong(blockId); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + matrixUuid = U.readGridUuid(in); + affinityKey = U.readGridUuid(in); + blockId = in.readLong(); + } + + /** {@inheritDoc} */ + @Override public void writeBinary(BinaryWriter writer) throws BinaryObjectException { + BinaryRawWriter out = writer.rawWriter(); + + BinaryUtils.writeIgniteUuid(out, matrixUuid); + BinaryUtils.writeIgniteUuid(out, affinityKey); + out.writeLong(blockId); + } + + /** {@inheritDoc} */ + @Override public void readBinary(BinaryReader reader) throws BinaryObjectException { + BinaryRawReader in = reader.rawReader(); + + matrixUuid = BinaryUtils.readIgniteUuid(in); + affinityKey = BinaryUtils.readIgniteUuid(in); + blockId = in.readLong(); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return matrixUuid.hashCode() + (int)(blockId ^ (blockId >>> 32)); + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (obj == this) + return true; + + if (obj == null || obj.getClass() != getClass()) + return false; + + BlockMatrixKey that = (BlockMatrixKey)obj; + + return blockId == that.blockId && matrixUuid.equals(that.matrixUuid) && F.eq(affinityKey, that.affinityKey); + } + + /** {@inheritDoc} */ + @Override public String toString() { + return S.toString(BlockMatrixKey.class, this); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java new file mode 100644 index 0000000..6640e5a --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java @@ -0,0 +1,435 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.storage.matrix; + +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.LongStream; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.MatrixStorage; +import org.apache.ignite.ml.math.StorageConstants; +import org.apache.ignite.ml.math.impls.CacheUtils; +import org.apache.ignite.ml.math.impls.matrix.BlockEntry; +import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix; + +import static org.apache.ignite.ml.math.impls.matrix.BlockEntry.MAX_BLOCK_SIZE; + +/** + * Storage for {@link SparseBlockDistributedMatrix}. + */ +public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, StorageConstants { + /** Cache name used for all instances of {@link BlockMatrixStorage}. */ + public static final String ML_BLOCK_CACHE_NAME = "ML_BLOCK_SPARSE_MATRICES_CONTAINER"; + /** */ + private int blocksInCol; + /** */ + private int blocksInRow; + /** Amount of rows in the matrix. */ + private int rows; + /** Amount of columns in the matrix. */ + private int cols; + /** Matrix uuid. */ + private IgniteUuid uuid; + /** Block size about 8 KB of data. */ + private int maxBlockEdge = MAX_BLOCK_SIZE; + + /** Actual distributed storage. */ + private IgniteCache< + BlockMatrixKey /* Matrix block number with uuid. */, + BlockEntry /* Block of matrix, local sparse matrix. */ + > cache = null; + + /** + * + */ + public BlockMatrixStorage() { + // No-op. + } + + /** + * @param rows Amount of rows in the matrix. + * @param cols Amount of columns in the matrix. + */ + public BlockMatrixStorage(int rows, int cols) { + assert rows > 0; + assert cols > 0; + + this.rows = rows; + this.cols = cols; + + //cols % maxBlockEdge > 0 ? 1 : 0 + + this.blocksInRow = cols % maxBlockEdge == 0 ? cols / maxBlockEdge : cols / maxBlockEdge + 1; + this.blocksInCol = rows % maxBlockEdge == 0 ? rows / maxBlockEdge : rows / maxBlockEdge + 1; + + cache = newCache(); + + uuid = IgniteUuid.randomUuid(); + } + + /** + * + */ + public IgniteCache<BlockMatrixKey, BlockEntry> cache() { + return cache; + } + + /** {@inheritDoc} */ + @Override public double get(int x, int y) { + return matrixGet(x, y); + } + + /** {@inheritDoc} */ + @Override public void set(int x, int y, double v) { + matrixSet(x, y, v); + } + + /** {@inheritDoc} */ + @Override public int columnSize() { + return cols; + } + + /** {@inheritDoc} */ + @Override public int rowSize() { + return rows; + } + + /** {@inheritDoc} */ + @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeInt(rows); + out.writeInt(cols); + out.writeInt(blocksInRow); + out.writeInt(blocksInCol); + U.writeGridUuid(out, uuid); + out.writeUTF(cache.getName()); + } + + /** {@inheritDoc} */ + @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + rows = in.readInt(); + cols = in.readInt(); + blocksInRow = in.readInt(); + blocksInCol = in.readInt(); + uuid = U.readGridUuid(in); + cache = ignite().getOrCreateCache(in.readUTF()); + } + + /** {@inheritDoc} */ + @Override public boolean isSequentialAccess() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean isDense() { + return false; + } + + /** {@inheritDoc} */ + @Override public boolean isRandomAccess() { + return true; + } + + /** {@inheritDoc} */ + @Override public boolean isDistributed() { + return true; + } + + /** {@inheritDoc} */ + @Override public boolean isArrayBased() { + return false; + } + + /** Delete all data from cache. */ + @Override public void destroy() { + long maxBlockId = getBlockId(cols, rows); + + Set<BlockMatrixKey> keyset = LongStream.rangeClosed(0, maxBlockId).mapToObj(this::getCacheKey).collect(Collectors.toSet()); + + cache.clearAll(keyset); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = 1; + + res = res * 37 + cols; + res = res * 37 + rows; + res = res * 37 + uuid.hashCode(); + res = res * 37 + cache.hashCode(); + + return res; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object obj) { + if (this == obj) + return true; + + if (obj == null || getClass() != obj.getClass()) + return false; + + BlockMatrixStorage that = (BlockMatrixStorage)obj; + + return rows == that.rows && cols == that.cols && uuid.equals(that.uuid) + && (cache != null ? cache.equals(that.cache) : that.cache == null); + } + + /** + * Get storage UUID. + * + * @return storage UUID. + */ + public IgniteUuid getUUID() { + return uuid; + } + + /** + * Build the cache key for the given block id + */ + public BlockMatrixKey getCacheKey(long blockId) { + return new BlockMatrixKey(blockId, uuid, getAffinityKey(blockId)); + } + + /** + * Get rows for current block. + * + * @param blockId block id. + * @param storageC result storage. + * @return The list of block entries. + */ + public List<BlockEntry> getRowForBlock(long blockId, BlockMatrixStorage storageC) { + long blockRow = blockId / storageC.blocksInCol; + long blockCol = blockId % storageC.blocksInRow; + + long locBlock = this.blocksInRow * (blockRow) + (blockCol >= this.blocksInRow ? (blocksInRow - 1) : blockCol); + + return getRowForBlock(locBlock); + } + + /** + * Get cols for current block. + * + * @param blockId block id. + * @param storageC result storage. + * @return The list of block entries. + */ + public List<BlockEntry> getColForBlock(long blockId, BlockMatrixStorage storageC) { + long blockRow = blockId / storageC.blocksInCol; + long blockCol = blockId % storageC.blocksInRow; + + long locBlock = this.blocksInRow * (blockRow) + (blockCol >= this.blocksInRow ? (blocksInRow - 1) : blockCol); + + return getColForBlock(locBlock); + } + + /** + * Build a keyset for this matrix storage. + */ + public Collection<BlockMatrixKey> getAllKeys() { + long maxBlockId = numberOfBlocks(); + Collection<BlockMatrixKey> keys = new LinkedList<>(); + + for (long id = 0; id < maxBlockId; id++) + keys.add(getCacheKey(id)); + + return keys; + } + + /** */ + private List<BlockEntry> getRowForBlock(long blockId) { + List<BlockEntry> res = new LinkedList<>(); + + boolean isFirstRow = blockId < blocksInRow; + + long startBlock = isFirstRow ? 0 : blockId - blockId % blocksInRow; + long endBlock = startBlock + blocksInRow - 1; + + for (long i = startBlock; i <= endBlock; i++) + res.add(getEntryById(i)); + + return res; + } + + /** */ + private List<BlockEntry> getColForBlock(long blockId) { + List<BlockEntry> res = new LinkedList<>(); + + long startBlock = blockId % blocksInRow; + long endBlock = startBlock + blocksInRow * (blocksInCol - 1); + + for (long i = startBlock; i <= endBlock; i += blocksInRow) + res.add(getEntryById(i)); + + return res; + } + + /** + * + */ + private BlockEntry getEntryById(long blockId) { + BlockMatrixKey key = getCacheKey(blockId); + + BlockEntry entry = cache.localPeek(key); + entry = entry != null ? entry : cache.get(key); + + if (entry == null) { + long colId = blockId == 0 ? 0 : blockId + 1; + + boolean isLastRow = (blockId) >= blocksInRow * (blocksInCol - 1); + boolean isLastCol = (colId) % blocksInRow == 0; + + entry = new BlockEntry(isLastRow && rows % maxBlockEdge != 0 ? rows % maxBlockEdge : maxBlockEdge, isLastCol && cols % maxBlockEdge != 0 ? cols % maxBlockEdge : maxBlockEdge); + } + + return entry; + } + + /** + * + */ + private long numberOfBlocks() { + int rows = rowSize(); + int cols = columnSize(); + + return ((rows / maxBlockEdge) + (((rows % maxBlockEdge) > 0) ? 1 : 0)) + * ((cols / maxBlockEdge) + (((cols % maxBlockEdge) > 0) ? 1 : 0)); + } + + /** + * TODO: IGNITE-5646, WIP + * + * Get affinity key for the given id. + */ + private IgniteUuid getAffinityKey(long id) { + return null; + } + + /** + * Distributed matrix set. + * + * @param a Row or column index. + * @param b Row or column index. + * @param v New value to set. + */ + private void matrixSet(int a, int b, double v) { + long id = getBlockId(a, b); + // Remote set on the primary node (where given row or column is stored locally). + ignite().compute(groupForKey(ML_BLOCK_CACHE_NAME, id)).run(() -> { + IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(ML_BLOCK_CACHE_NAME); + + BlockMatrixKey key = getCacheKey(getBlockId(a, b)); + + // Local get. + BlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY); + + if (block == null) + block = cache.get(key); //Remote entry get. + + if (block == null) + block = initBlockFor(a, b); + + block.set(a % block.rowSize(), b % block.columnSize(), v); + + // Local put. + cache.put(key, block); + }); + } + + /** */ + private long getBlockId(int x, int y) { + return (y / maxBlockEdge) * blockShift(cols) + (x / maxBlockEdge); + } + + /** */ + private BlockEntry initBlockFor(int x, int y) { + int blockRows = rows - x >= maxBlockEdge ? maxBlockEdge : rows - x; + int blockCols = cols - y >= maxBlockEdge ? maxBlockEdge : cols - y; + + return new BlockEntry(blockRows, blockCols); + } + + /** */ + private int blockShift(int i) { + return (i) / maxBlockEdge + ((i) % maxBlockEdge > 0 ? 1 : 0); + } + + /** + * Distributed matrix get. + * + * @param a Row or column index. + * @param b Row or column index. + * @return Matrix value at (a, b) index. + */ + private double matrixGet(int a, int b) { + // Remote get from the primary node (where given row or column is stored locally). + return ignite().compute(groupForKey(ML_BLOCK_CACHE_NAME, getBlockId(a, b))).call(() -> { + IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(ML_BLOCK_CACHE_NAME); + + BlockMatrixKey key = getCacheKey(getBlockId(a, b)); + + // Local get. + BlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY); + + if (block == null) + block = cache.get(key); + + return block == null ? 0.0 : block.get(a % block.rowSize(), b % block.columnSize()); + }); + } + + /** + * Create new ML cache if needed. + */ + private IgniteCache<BlockMatrixKey, BlockEntry> newCache() { + CacheConfiguration<BlockMatrixKey, BlockEntry> cfg = new CacheConfiguration<>(); + + // Write to primary. + cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC); + + // Atomic transactions only. + cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC); + + // No eviction. + cfg.setEvictionPolicy(null); + + // No copying of values. + cfg.setCopyOnRead(false); + + // Cache is partitioned. + cfg.setCacheMode(CacheMode.PARTITIONED); + + // Random cache name. + cfg.setName(ML_BLOCK_CACHE_NAME); + + return Ignition.localIgnite().getOrCreateCache(cfg); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java index f2efe74..5145376 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/vector/SparseLocalOnHeapVectorStorage.java @@ -46,9 +46,7 @@ public class SparseLocalOnHeapVectorStorage implements VectorStorage, StorageCon // No-op. } - /** - * @param map - */ + /** */ public SparseLocalOnHeapVectorStorage(Map<Integer, Double> map, boolean copy) { assert map.size() > 0; http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java index e406b5b..525e6e9 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/Variance.java @@ -30,6 +30,7 @@ public class Variance { /** */ private double m2; + /** */ public Variance() { mean = 0; n = 0; http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java new file mode 100644 index 0000000..7b65fce --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/statistics/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * <!-- Package description. --> + * Statistics stuff. + */ +package org.apache.ignite.ml.math.statistics; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java index 6c25f0e..9190901 100644 --- a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/MapUtil.java @@ -25,7 +25,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * + * Some {@link Map} related utils. */ public class MapUtil { /** */ http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java new file mode 100644 index 0000000..2507ee4 --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/util/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * <!-- Package description. --> + * Some math utils. + */ +package org.apache.ignite.ml.math.util; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java new file mode 100644 index 0000000..779581b --- /dev/null +++ b/modules/ml/src/main/java/org/apache/ignite/ml/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * <!-- Package description. --> + * Root ML package. + */ +package org.apache.ignite.ml; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java index 9899d3b..5dc860c 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/MathImplDistributedTestSuite.java @@ -18,6 +18,7 @@ package org.apache.ignite.ml.math; import org.apache.ignite.ml.math.impls.matrix.CacheMatrixTest; +import org.apache.ignite.ml.math.impls.matrix.SparseDistributedBlockMatrixTest; import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrixTest; import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorageTest; import org.apache.ignite.ml.math.impls.vector.CacheVectorTest; @@ -33,6 +34,7 @@ import org.junit.runners.Suite; CacheMatrixTest.class, SparseDistributedMatrixStorageTest.class, SparseDistributedMatrixTest.class, + SparseDistributedBlockMatrixTest.class }) public class MathImplDistributedTestSuite { // No-op. http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java new file mode 100644 index 0000000..1228f05 --- /dev/null +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java @@ -0,0 +1,379 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.ml.math.impls.matrix; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.internal.util.IgniteUtils; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.ml.math.Matrix; +import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException; +import org.apache.ignite.ml.math.impls.MathTestConstants; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixKey; +import org.apache.ignite.ml.math.impls.storage.matrix.BlockMatrixStorage; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.testframework.junits.common.GridCommonTest; + +import static org.apache.ignite.ml.math.impls.MathTestConstants.UNEXPECTED_VAL; + +/** + * Tests for {@link SparseBlockDistributedMatrix}. + */ +@GridCommonTest(group = "Distributed Models") +public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest { + /** Number of nodes in grid */ + private static final int NODE_COUNT = 3; + /** Precision. */ + private static final double PRECISION = 0.0; + /** Grid instance. */ + private Ignite ignite; + /** Matrix rows */ + private final int rows = MathTestConstants.STORAGE_SIZE; + /** Matrix cols */ + private final int cols = MathTestConstants.STORAGE_SIZE; + /** Matrix for tests */ + private SparseBlockDistributedMatrix cacheMatrix; + + /** + * Default constructor. + */ + public SparseDistributedBlockMatrixTest() { + super(false); + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + for (int i = 1; i <= NODE_COUNT; i++) + startGrid(i); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + stopAllGrids(); + } + + /** + * {@inheritDoc} + */ + @Override protected void beforeTest() throws Exception { + ignite = grid(NODE_COUNT); + + ignite.configuration().setPeerClassLoadingEnabled(true); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + if (cacheMatrix != null) { + cacheMatrix.destroy(); + cacheMatrix = null; + } + } + + /** */ + public void testGetSet() throws Exception { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + double v = Math.random(); + cacheMatrix.set(i, j, v); + + assertEquals("Unexpected value for matrix element["+ i +" " + j + "]", v, cacheMatrix.get(i, j), PRECISION); + } + } + } + + /** */ + public void testExternalize() throws IOException, ClassNotFoundException { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + cacheMatrix.set(1, 1, 1.0); + + ByteArrayOutputStream byteArrOutputStream = new ByteArrayOutputStream(); + ObjectOutputStream objOutputStream = new ObjectOutputStream(byteArrOutputStream); + + objOutputStream.writeObject(cacheMatrix); + + ByteArrayInputStream byteArrInputStream = new ByteArrayInputStream(byteArrOutputStream.toByteArray()); + ObjectInputStream objInputStream = new ObjectInputStream(byteArrInputStream); + + SparseBlockDistributedMatrix objRestored = (SparseBlockDistributedMatrix)objInputStream.readObject(); + + assertTrue(MathTestConstants.VAL_NOT_EQUALS, cacheMatrix.equals(objRestored)); + assertEquals(MathTestConstants.VAL_NOT_EQUALS, objRestored.get(1, 1), 1.0, PRECISION); + } + + /** Test simple math. */ + public void testMath() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + initMtx(cacheMatrix); + + cacheMatrix.assign(2.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 2.0, cacheMatrix.get(i, j), PRECISION); + + cacheMatrix.plus(3.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 5.0, cacheMatrix.get(i, j), PRECISION); + + cacheMatrix.times(2.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 10.0, cacheMatrix.get(i, j), PRECISION); + + cacheMatrix.divide(10.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 1.0, cacheMatrix.get(i, j), PRECISION); + + assertEquals(UNEXPECTED_VAL, cacheMatrix.rowSize() * cacheMatrix.columnSize(), cacheMatrix.sum(), PRECISION); + } + + /** */ + public void testMinMax() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + cacheMatrix.set(i, j, i * cols + j + 1); + + assertEquals(UNEXPECTED_VAL, 1.0, cacheMatrix.minValue(), PRECISION); + assertEquals(UNEXPECTED_VAL, rows * cols, cacheMatrix.maxValue(), PRECISION); + + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + cacheMatrix.set(i, j, -1.0 * (i * cols + j + 1)); + + assertEquals(UNEXPECTED_VAL, -rows * cols, cacheMatrix.minValue(), PRECISION); + assertEquals(UNEXPECTED_VAL, -1.0, cacheMatrix.maxValue(), PRECISION); + + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + cacheMatrix.set(i, j, i * cols + j); + + assertEquals(UNEXPECTED_VAL, 0.0, cacheMatrix.minValue(), PRECISION); + assertEquals(UNEXPECTED_VAL, rows * cols - 1.0, cacheMatrix.maxValue(), PRECISION); + } + + /** */ + public void testMap() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + initMtx(cacheMatrix); + + cacheMatrix.map(i -> 100.0); + for (int i = 0; i < cacheMatrix.rowSize(); i++) + for (int j = 0; j < cacheMatrix.columnSize(); j++) + assertEquals(UNEXPECTED_VAL, 100.0, cacheMatrix.get(i, j), PRECISION); + } + + /** */ + public void testCopy() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + try { + cacheMatrix.copy(); + fail("UnsupportedOperationException expected."); + } + catch (UnsupportedOperationException e) { + return; + } + fail("UnsupportedOperationException expected."); + } + + /** */ + public void testCacheBehaviour(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + SparseBlockDistributedMatrix cacheMatrix1 = new SparseBlockDistributedMatrix(rows, cols); + SparseBlockDistributedMatrix cacheMatrix2 = new SparseBlockDistributedMatrix(rows, cols); + + initMtx(cacheMatrix1); + initMtx(cacheMatrix2); + + Collection<String> cacheNames = ignite.cacheNames(); + + assert cacheNames.contains(BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + + IgniteCache<BlockMatrixKey, Object> cache = ignite.getOrCreateCache(BlockMatrixStorage.ML_BLOCK_CACHE_NAME); + + Set<BlockMatrixKey> keySet1 = buildKeySet(cacheMatrix1); + Set<BlockMatrixKey> keySet2 = buildKeySet(cacheMatrix2); + + assert cache.containsKeys(keySet1); + assert cache.containsKeys(keySet2); + + cacheMatrix2.destroy(); + + assert cache.containsKeys(keySet1); + assert !cache.containsKeys(keySet2); + + cacheMatrix1.destroy(); + + assert !cache.containsKeys(keySet1); + } + + /** */ + public void testLike() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + assertNotNull(cacheMatrix.like(1, 1)); + } + + /** */ + public void testLikeVector() { + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + cacheMatrix = new SparseBlockDistributedMatrix(rows, cols); + + try { + cacheMatrix.likeVector(1); + fail("UnsupportedOperationException expected."); + } + catch (UnsupportedOperationException e) { + return; + } + fail("UnsupportedOperationException expected."); + } + + /** + * Simple test for two square matrices. + */ + public void testSquareMatrixTimes(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + int size = 100; + + Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size, size); + Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size, size); + + for (int i = 0; i < size; i++) { + cacheMatrix1.setX(i, i, i); + cacheMatrix2.setX(i, i, i); + } + + Matrix res = cacheMatrix1.times(cacheMatrix2); + + for(int i = 0; i < size; i++) + for(int j = 0; j < size; j++) + if (i == j) + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION); + else + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION); + } + + /** + * + */ + public void testNonSquareMatrixTimes(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + int size = BlockEntry.MAX_BLOCK_SIZE + 1; + int size2 = BlockEntry.MAX_BLOCK_SIZE * 2 + 1; + + Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size2, size); + Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size, size2); + + for (int i = 0; i < size; i++) { + cacheMatrix1.setX(i, i, i); + cacheMatrix2.setX(i, i, i); + } + + Matrix res = cacheMatrix1.times(cacheMatrix2); + + for(int i = 0; i < size; i++) + for(int j = 0; j < size; j++) + if (i == j) + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION); + else + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION); + } + + /** + * + */ + public void testNonSquareMatrixTimes2(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + int size = BlockEntry.MAX_BLOCK_SIZE + 1; + int size2 = BlockEntry.MAX_BLOCK_SIZE * 2 + 1; + + Matrix cacheMatrix1 = new SparseBlockDistributedMatrix(size, size2); + Matrix cacheMatrix2 = new SparseBlockDistributedMatrix(size2, size); + + for (int i = 0; i < size; i++) { + cacheMatrix1.setX(i, i, i); + cacheMatrix2.setX(i, i, i); + } + + Matrix res = cacheMatrix1.times(cacheMatrix2); + + for(int i = 0; i < size; i++) + for(int j = 0; j < size; j++) + if (i == j) + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, i * i, res.get(i, j), PRECISION); + else + assertEquals(UNEXPECTED_VAL + " for "+ i +":"+ j, 0, res.get(i, j), PRECISION); + } + + /** */ + private void initMtx(Matrix m) { + for (int i = 0; i < m.rowSize(); i++) + for (int j = 0; j < m.columnSize(); j++) + m.set(i, j, 1.0); + } + + /** Build key set for SparseBlockDistributedMatrix. */ + private Set<BlockMatrixKey> buildKeySet(SparseBlockDistributedMatrix m){ + Set<BlockMatrixKey> set = new HashSet<>(); + + BlockMatrixStorage storage = (BlockMatrixStorage)m.getStorage(); + + IgniteUuid uuid = storage.getUUID(); + + long maxBlock = (rows / 32 + (rows % 32 > 0 ? 1 : 0)) * (cols / 32 + (cols % 32 > 0 ? 1 : 0)); + + for (long i = 0; i < maxBlock; i++) + set.add(new BlockMatrixKey(i,uuid,null)); + + return set; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/0d2b989d/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java ---------------------------------------------------------------------- diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java index a7cd6b5..3fec83c 100644 --- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java +++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrixTest.java @@ -48,10 +48,10 @@ import static org.apache.ignite.ml.math.impls.MathTestConstants.UNEXPECTED_VAL; public class SparseDistributedMatrixTest extends GridCommonAbstractTest { /** Number of nodes in grid */ private static final int NODE_COUNT = 3; - /** Cache name. */ - private static final String CACHE_NAME = "test-cache"; /** Precision. */ private static final double PRECISION = 0.0; + /** */ + private static final int MATRIX_SIZE = 10; /** Grid instance. */ private Ignite ignite; /** Matrix rows */ @@ -90,8 +90,6 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest { /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { - ignite.destroyCache(CACHE_NAME); - if (cacheMatrix != null) { cacheMatrix.destroy(); cacheMatrix = null; @@ -166,7 +164,9 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest { assertEquals(UNEXPECTED_VAL, cacheMatrix.rowSize() * cacheMatrix.columnSize(), cacheMatrix.sum(), PRECISION); } - /** */ + /** + * TODO: IGNITE-5102, wrong min/max, wait for fold/map fix + */ public void testMinMax() { IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); @@ -286,6 +286,28 @@ public class SparseDistributedMatrixTest extends GridCommonAbstractTest { } /** */ + public void testMatrixTimes(){ + IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName()); + + SparseDistributedMatrix cacheMatrix1 = new SparseDistributedMatrix(MATRIX_SIZE, MATRIX_SIZE, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE); + SparseDistributedMatrix cacheMatrix2 = new SparseDistributedMatrix(MATRIX_SIZE, MATRIX_SIZE, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE); + + for (int i = 0; i < MATRIX_SIZE; i++) { + cacheMatrix1.setX(i, i, i); + cacheMatrix2.setX(i, i, i); + } + + Matrix res = cacheMatrix1.times(cacheMatrix2); + + for(int i = 0; i < MATRIX_SIZE; i++) + for(int j = 0; j < MATRIX_SIZE; j++) + if (i == j) + assertEquals(UNEXPECTED_VAL, i * i, res.get(i, j), PRECISION); + else + assertEquals(UNEXPECTED_VAL, 0, res.get(i, j), PRECISION); + } + + /** */ private void initMtx(Matrix m) { for (int i = 0; i < m.rowSize(); i++) for (int j = 0; j < m.columnSize(); j++)