This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 2ab0eb8fcf [SYSTEMDS-3491] CLA Specialized Column-Indexes
2ab0eb8fcf is described below
commit 2ab0eb8fcf5f5928d904e13518f0b99bbf7dc26c
Author: baunsgaard <[email protected]>
AuthorDate: Fri Jan 27 16:05:52 2023 +0100
[SYSTEMDS-3491] CLA Specialized Column-Indexes
This commit adds new specializations of the column indexes in
CLA. To avoid a major commit, this is the first commit in 3 parts
that will change the allocation of the columns from simple arrays to
these specializations. This part 1 is just the interface and classes
of the new arrays.
Closes #1774
---
.../compress/colgroup/indexes/ArrayIndex.java | 97 +++++++++
.../compress/colgroup/indexes/ColIndexFactory.java | 74 +++++++
.../compress/colgroup/indexes/IColIndex.java | 89 ++++++++
.../compress/colgroup/indexes/IIterate.java | 41 ++++
.../compress/colgroup/indexes/RangeIndex.java | 103 +++++++++
.../compress/colgroup/indexes/SingleIndex.java | 82 ++++++++
.../compress/colgroup/indexes/TwoIndex.java | 90 ++++++++
.../component/compress/indexes/IndexesTest.java | 234 +++++++++++++++++++++
.../compress/indexes/NegativeIndexTest.java | 51 +++++
9 files changed, 861 insertions(+)
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ArrayIndex.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ArrayIndex.java
new file mode 100644
index 0000000000..d26307d352
--- /dev/null
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ArrayIndex.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup.indexes;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.sysds.utils.MemoryEstimates;
+
+public class ArrayIndex implements IColIndex {
+ private final int[] cols;
+
+ public ArrayIndex(int[] cols) {
+ this.cols = cols;
+ }
+
+ @Override
+ public int size() {
+ return cols.length;
+ }
+
+ @Override
+ public int get(int i) {
+ return cols[i];
+ }
+
+ @Override
+ public IColIndex shift(int i) {
+ int[] ret = new int[cols.length];
+ for(int j = 0; j < cols.length; j++)
+ ret[j] = cols[j] + i;
+ return new ArrayIndex(ret);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeByte(ColIndexType.ARRAY.ordinal());
+ out.writeInt(cols.length);
+ for(int i = 0; i < cols.length; i++)
+ out.writeInt(cols[i]);
+ }
+
+ public static ArrayIndex read(DataInput in) throws IOException {
+ int size = in.readInt();
+ int[] cols = new int[size];
+ for(int i = 0; i < size; i++)
+ cols[i] = in.readInt();
+ return new ArrayIndex(cols);
+ }
+
+ @Override
+ public long getExactSizeOnDisk() {
+ return 1 + 4 + 4 * cols.length;
+ }
+
+ @Override
+ public long estimateInMemorySize() {
+ return 16 + (long) MemoryEstimates.intArrayCost(cols.length);
+ }
+
+ @Override
+ public IIterate iterator() {
+ return new ArrayIterator();
+ }
+
+ protected class ArrayIterator implements IIterate {
+ int id = 0;
+
+ @Override
+ public int next() {
+ return cols[id++];
+ }
+
+ @Override
+ public boolean hasNext() {
+ return id < cols.length;
+ }
+ }
+}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ColIndexFactory.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ColIndexFactory.java
new file mode 100644
index 0000000000..3926edc09c
--- /dev/null
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/ColIndexFactory.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup.indexes;
+
+import java.io.DataInput;
+import java.io.IOException;
+
+import org.apache.sysds.runtime.compress.DMLCompressionException;
+import
org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex.ColIndexType;
+
+public interface ColIndexFactory {
+
+ public static IColIndex read(DataInput in) throws IOException {
+ final ColIndexType t = ColIndexType.values()[in.readByte()];
+ switch(t) {
+ case SINGLE:
+ return new SingleIndex(in.readInt());
+ case TWO:
+ return new TwoIndex(in.readInt(), in.readInt());
+ case ARRAY:
+ return ArrayIndex.read(in);
+ case RANGE:
+ return RangeIndex.read(in);
+ default:
+ throw new DMLCompressionException("Failed
reading column index of type: " + t);
+ }
+ }
+
+ public static IColIndex create(int[] indexes) {
+ if(indexes.length == 1)
+ return new SingleIndex(indexes[0]);
+ else if(indexes.length == 2)
+ return new TwoIndex(indexes[0], indexes[1]);
+ else if(RangeIndex.isValidRange(indexes))
+ return new RangeIndex(indexes[0], indexes[0] +
indexes.length);
+ else
+ return new ArrayIndex(indexes);
+ }
+
+ public static IColIndex create(int l, int u) {
+ if(u - 1 == l)
+ return new SingleIndex(l);
+ else if(u - 2 == l)
+ return new TwoIndex(l, l + 1);
+ else
+ return new RangeIndex(l, u);
+ }
+
+ public static IColIndex create(int nCol) {
+ if(nCol == 1)
+ return new SingleIndex(0);
+ else if(nCol == 2)
+ return new TwoIndex(0, 1);
+ else
+ return new RangeIndex(nCol);
+ }
+}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IColIndex.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IColIndex.java
new file mode 100644
index 0000000000..e2d7ff8985
--- /dev/null
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IColIndex.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup.indexes;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Class to contain column indexes for the compression column groups.
+ */
+public interface IColIndex {
+
+ public static enum ColIndexType {
+ SINGLE, TWO, ARRAY, RANGE, UNKNOWN;
+ }
+
+ /**
+ * Get the size of the index aka, how many columns is contained
+ *
+ * @return The size of the array
+ */
+ public int size();
+
+ /**
+ * Get the index at a specific location, Note that many of the
underlying implementations does not throw exceptions
+ * on indexes that are completely wrong, so all implementations that
use this index should always be well behaved.
+ *
+ * @param i The index to get
+ * @return the column index at the index.
+ */
+ public int get(int i);
+
+ /**
+ * Return a new column index where the values are shifted by the
specified amount.
+ *
+ * It is returning a new instance of the index.
+ *
+ * @param i The amount to shift
+ * @return the new instance of an index.
+ */
+ public IColIndex shift(int i);
+
+ /**
+ * Write out the IO representation of this column index
+ *
+ * @param out The Output to write into
+ * @throws IOException IO exceptions in case of for instance not enough
disk space
+ */
+ public void write(DataOutput out) throws IOException;
+
+ /**
+ * Get the exact size on disk to enable preallocation of the disk
output buffer sizes
+ *
+ * @return The exact disk representation size
+ */
+ public long getExactSizeOnDisk();
+
+ /**
+ * Get the in memory size of this object.
+ *
+ * @return The memory size of this object
+ */
+ public long estimateInMemorySize();
+
+ /**
+ * A Iterator of the indexes see the iterator interface for details.
+ *
+ * @return A iterator for the indexes contained.
+ */
+ public IIterate iterator();
+
+}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IIterate.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IIterate.java
new file mode 100644
index 0000000000..fae615c9ec
--- /dev/null
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/IIterate.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup.indexes;
+
+/**
+ * Class to iterate through the columns of a IColIndex.
+ *
+ * When initialized it should be at index -1 and then at the call to next you
get the first value
+ */
+public interface IIterate {
+ /**
+ * Get next index
+ *
+ * @return the index.
+ */
+ public int next();
+
+ /**
+ * Get if the index has a next index.
+ *
+ * @return the next index.
+ */
+ public boolean hasNext();
+}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java
new file mode 100644
index 0000000000..87e74185a1
--- /dev/null
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/RangeIndex.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup.indexes;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class RangeIndex implements IColIndex {
+ private final int l;
+ private final int u; // not inclusive
+
+ public RangeIndex(int nCol) {
+ l = 0;
+ u = nCol;
+ }
+
+ public RangeIndex(int l, int u) {
+ this.l = l;
+ this.u = u;
+ }
+
+ @Override
+ public int size() {
+ return u - l;
+ }
+
+ @Override
+ public int get(int i) {
+ return l + i;
+ }
+
+ @Override
+ public IColIndex shift(int i) {
+ return new RangeIndex(l + i, u + i);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeByte(ColIndexType.RANGE.ordinal());
+ out.writeInt(l);
+ out.writeInt(u);
+ }
+
+ public static RangeIndex read(DataInput in) throws IOException {
+ int l = in.readInt();
+ int u = in.readInt();
+ return new RangeIndex(l, u);
+ }
+
+ @Override
+ public long getExactSizeOnDisk() {
+ return 1 + 4 + 4;
+ }
+
+ @Override
+ public long estimateInMemorySize() {
+ return 16 + 8;
+ }
+
+ @Override
+ public IIterate iterator() {
+ return new RangeIterator();
+ }
+
+ protected static boolean isValidRange(int[] indexes) {
+ int len = indexes.length;
+ int first = indexes[0];
+ int last = indexes[indexes.length - 1];
+ return last - first + 1 == len;
+ }
+
+ protected class RangeIterator implements IIterate {
+ int cl = l;
+
+ @Override
+ public int next() {
+ return cl++;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return cl < u;
+ }
+ }
+}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/SingleIndex.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/SingleIndex.java
new file mode 100644
index 0000000000..939fe3a927
--- /dev/null
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/SingleIndex.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup.indexes;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class SingleIndex implements IColIndex {
+ private final int idx;
+
+ public SingleIndex(int idx) {
+ this.idx = idx;
+ }
+
+ @Override
+ public int size() {
+ return 1;
+ }
+
+ @Override
+ public int get(int i) {
+ return idx;
+ }
+
+ @Override
+ public SingleIndex shift(int i) {
+ return new SingleIndex(i + idx);
+ }
+
+ @Override
+ public IIterate iterator() {
+ return new SingleIterator();
+ }
+
+ public void write(DataOutput out) throws IOException {
+ out.writeByte(ColIndexType.SINGLE.ordinal());
+ out.writeInt(idx);
+ }
+
+ @Override
+ public long getExactSizeOnDisk() {
+ return 1 + 4;
+ }
+
+ @Override
+ public long estimateInMemorySize() {
+ return 16 + 4 + 4; // object, int, and padding
+ }
+
+ protected class SingleIterator implements IIterate {
+ boolean taken = false;
+
+ @Override
+ public int next() {
+ taken = true;
+ return idx;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !taken;
+ }
+ }
+
+}
diff --git
a/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/TwoIndex.java
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/TwoIndex.java
new file mode 100644
index 0000000000..7947d853fa
--- /dev/null
+++
b/src/main/java/org/apache/sysds/runtime/compress/colgroup/indexes/TwoIndex.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.runtime.compress.colgroup.indexes;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class TwoIndex implements IColIndex {
+ private final int id1;
+ private final int id2;
+
+ public TwoIndex(int id1, int id2) {
+ this.id1 = id1;
+ this.id2 = id2;
+ }
+
+ @Override
+ public int size() {
+ return 2;
+ }
+
+ @Override
+ public int get(int i) {
+ if(i == 0)
+ return id1;
+ else
+ return id2;
+ }
+
+ @Override
+ public TwoIndex shift(int i) {
+ return new TwoIndex(id1 + i, id2 + i);
+ }
+
+ @Override
+ public IIterate iterator() {
+ return new TwoIterator();
+ }
+
+ public void write(DataOutput out) throws IOException {
+ out.writeByte(ColIndexType.TWO.ordinal());
+ out.writeInt(id1);
+ out.writeInt(id2);
+ }
+
+ @Override
+ public long getExactSizeOnDisk() {
+ return 1 + 4 + 4;
+ }
+
+ @Override
+ public long estimateInMemorySize() {
+ return 16 + 8; // object, 2x int
+ }
+
+ protected class TwoIterator implements IIterate {
+ int id = 0;
+
+ @Override
+ public int next() {
+ if(id++ == 0)
+ return id1;
+ else
+ return id2;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return id < 2;
+ }
+ }
+
+}
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/indexes/IndexesTest.java
b/src/test/java/org/apache/sysds/test/component/compress/indexes/IndexesTest.java
new file mode 100644
index 0000000000..043b51c5d6
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/component/compress/indexes/IndexesTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.compress.indexes;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
+import org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex;
+import org.apache.sysds.runtime.compress.colgroup.indexes.IIterate;
+import org.apache.sysds.runtime.compress.colgroup.indexes.SingleIndex;
+import org.apache.sysds.runtime.compress.colgroup.indexes.TwoIndex;
+import org.apache.sysds.utils.MemoryEstimates;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(value = Parameterized.class)
+public class IndexesTest {
+
+ private final int[] expected;
+ private final IColIndex actual;
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ List<Object[]> tests = new ArrayList<>();
+
+ try {
+ // single
+ tests.add(new Object[] {new int[] {0}, new
SingleIndex(0)});
+ tests.add(new Object[] {new int[] {334},
ColIndexFactory.create(334, 335)});
+ tests.add(new Object[] {new int[] {0},
ColIndexFactory.create(1)});
+ tests.add(new Object[] {new int[] {0},
ColIndexFactory.create(new int[] {0})});
+ tests.add(new Object[] {new int[] {320},
ColIndexFactory.create(new int[] {320})});
+
+ // two
+ tests.add(new Object[] {new int[] {0, 1}, new
TwoIndex(0, 1)});
+ tests.add(new Object[] {new int[] {3214, 44444}, new
TwoIndex(3214, 44444)});
+ tests.add(new Object[] {new int[] {3214, 44444},
ColIndexFactory.create(new int[] {3214, 44444})});
+ tests.add(new Object[] {new int[] {3214, 3215},
ColIndexFactory.create(3214, 3216)});
+ tests.add(new Object[] {new int[] {0, 1},
ColIndexFactory.create(2)});
+
+ // array
+ tests.add(create(32, 14));
+ tests.add(create(40, 21));
+ tests.add(new Object[] {//
+ new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, //
+ ColIndexFactory.create(0, 10)});
+
+ tests.add(new Object[] {//
+ new int[] {0, 1, 2, 3}, //
+ ColIndexFactory.create(0, 4)});
+
+ tests.add(new Object[] {//
+ new int[] {0, 1, 2, 3}, //
+ ColIndexFactory.create(4)});
+
+ tests.add(new Object[] {//
+ new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, //
+ ColIndexFactory.create(10)});
+
+ tests.add(new Object[] {//
+ new int[] {4, 5, 6, 7, 8, 9}, //
+ ColIndexFactory.create(4, 10)});
+
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ fail("failed constructing tests");
+ }
+
+ return tests;
+ }
+
+ public IndexesTest(int[] expected, IColIndex actual) {
+ this.expected = expected;
+ this.actual = actual;
+ }
+
+ @Test
+ public void testGet() {
+ for(int i = 0; i < expected.length; i++) {
+ assertEquals(expected[i], actual.get(i));
+ }
+ }
+
+ @Test
+ public void testSerialize() {
+ try {
+ // Serialize out
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream fos = new DataOutputStream(bos);
+ actual.write(fos);
+
+ // Serialize in
+ ByteArrayInputStream bis = new
ByteArrayInputStream(bos.toByteArray());
+ DataInputStream fis = new DataInputStream(bis);
+
+ IColIndex n = ColIndexFactory.read(fis);
+
+ compare(actual, n);
+ }
+ catch(IOException e) {
+ throw new RuntimeException("Error in io", e);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testSerializeSize() {
+ try {
+ // Serialize out
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream fos = new DataOutputStream(bos);
+ actual.write(fos);
+
+ long actualSize = bos.size();
+ long expectedSize = actual.getExactSizeOnDisk();
+
+ assertEquals(expectedSize, actualSize);
+ }
+ catch(IOException e) {
+ throw new RuntimeException("Error in io", e);
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @Test
+ public void testSize() {
+ assertEquals(expected.length, actual.size());
+ }
+
+
+ @Test
+ public void iterator() {
+ compare(expected, actual.iterator());
+ }
+
+ @Test
+ public void factoryCreate() {
+ compare(expected, ColIndexFactory.create(expected));
+ }
+
+ @Test
+ public void shift() {
+ shift(5);
+ }
+
+ @Test
+ public void shift2() {
+ shift(1342);
+ }
+
+ @Test
+ public void estimateInMemorySizeIsNotToBig() {
+ assertTrue(MemoryEstimates.intArrayCost(expected.length) >=
actual.estimateInMemorySize() - 16);
+ }
+
+ private void shift(int i) {
+ compare(expected, actual.shift(i), i);
+ }
+
+ private static void compare(int[] expected, IColIndex actual) {
+ assertEquals(expected.length, actual.size());
+ for(int i = 0; i < expected.length; i++)
+ assertEquals(expected[i], actual.get(i));
+ }
+
+ private static void compare(int[] expected, IColIndex actual, int off) {
+ assertEquals(expected.length, actual.size());
+ for(int i = 0; i < expected.length; i++)
+ assertEquals(expected[i] + off, actual.get(i));
+ }
+
+ private static void compare(IColIndex expected, IColIndex actual) {
+ assertEquals(expected.size(), actual.size());
+ for(int i = 0; i < expected.size(); i++)
+ assertEquals(expected.get(i), actual.get(i));
+ }
+
+ private static void compare(int[] expected, IIterate actual) {
+ for(int i = 0; i < expected.length; i++) {
+ assertTrue(actual.hasNext());
+ assertEquals(expected[i], actual.next());
+ }
+ assertFalse(actual.hasNext());
+ }
+
+ private static Object[] create(int size, int seed) {
+ int[] cols = new int[size];
+ Random r = new Random(seed);
+ cols[0] = r.nextInt(1000) + 1;
+ for(int i = 1; i < size; i++) {
+ cols[i] = cols[i - 1] + r.nextInt(1000) + 1;
+ }
+ return new Object[] {cols, ColIndexFactory.create(cols)};
+ }
+}
diff --git
a/src/test/java/org/apache/sysds/test/component/compress/indexes/NegativeIndexTest.java
b/src/test/java/org/apache/sysds/test/component/compress/indexes/NegativeIndexTest.java
new file mode 100644
index 0000000000..adda18c57b
--- /dev/null
+++
b/src/test/java/org/apache/sysds/test/component/compress/indexes/NegativeIndexTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.test.component.compress.indexes;
+
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.sysds.runtime.compress.DMLCompressionException;
+import org.apache.sysds.runtime.compress.colgroup.indexes.ColIndexFactory;
+import
org.apache.sysds.runtime.compress.colgroup.indexes.IColIndex.ColIndexType;
+import org.junit.Test;
+
+public class NegativeIndexTest {
+ @Test(expected = DMLCompressionException.class)
+ public void notValidRead() {
+ try {
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ DataOutputStream fos = new DataOutputStream(bos);
+ fos.writeByte(ColIndexType.UNKNOWN.ordinal());
+ ByteArrayInputStream bis = new
ByteArrayInputStream(bos.toByteArray());
+ DataInputStream fis = new DataInputStream(bis);
+ ColIndexFactory.read(fis);
+ }
+ catch(IOException e) {
+ fail("Wrong type of exception");
+ }
+ }
+}
