This is an automated email from the ASF dual-hosted git repository. baunsgaard pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/systemds.git
commit b244eea45cb4f286072b4e2a012f2f0b724b0e6d Author: baunsgaard <[email protected]> AuthorDate: Mon Jan 23 11:11:54 2023 +0100 [MINOR] 100% Array Tests coverage This commit adds tests and fixes minor bugs in Frame arrays, even when all of the types were implemented together, there was some slight inconsistencies between the different value types. Adding more tests and cases removed these inconsistencies, and fixed a number of significant bugs in the new Optional Array. Closes #1769 --- .../runtime/frame/data/columns/ABooleanArray.java | 41 ++ .../sysds/runtime/frame/data/columns/Array.java | 10 +- .../runtime/frame/data/columns/ArrayFactory.java | 32 +- .../runtime/frame/data/columns/BitSetArray.java | 68 ++- .../runtime/frame/data/columns/BooleanArray.java | 63 +- .../runtime/frame/data/columns/CharArray.java | 69 +-- .../runtime/frame/data/columns/DoubleArray.java | 20 +- .../runtime/frame/data/columns/FloatArray.java | 20 +- .../runtime/frame/data/columns/IntegerArray.java | 18 +- .../runtime/frame/data/columns/LongArray.java | 21 +- .../runtime/frame/data/columns/OptionalArray.java | 77 +-- .../runtime/frame/data/columns/StringArray.java | 73 ++- .../frame/data/lib/FrameLibRemoveEmpty.java | 2 +- .../apache/sysds/runtime/util/UtilFunctions.java | 1 - .../component/frame/array/ColumnMetadataTests.java | 15 + .../component/frame/array/CustomArrayTests.java | 546 ++++++++++++++++- .../component/frame/array/FrameArrayTests.java | 672 ++++++++++++++++++++- .../component/frame/array/NegativeArrayTests.java | 83 +++ 18 files changed, 1623 insertions(+), 208 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ABooleanArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ABooleanArray.java new file mode 100644 index 0000000000..5b7c84605d --- /dev/null +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ABooleanArray.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysds.runtime.frame.data.columns; + +public abstract class ABooleanArray extends Array<Boolean> { + + public ABooleanArray(int size) { + super(size); + } + + public abstract boolean isAllTrue(); + + @Override + public abstract ABooleanArray slice(int rl, int ru); + + @Override + public abstract ABooleanArray clone(); + + @Override + public abstract ABooleanArray select(int[] indices); + + @Override + public abstract ABooleanArray select(boolean[] select, int nTrue); +} diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java index 81bc3f17e7..2be9e10170 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/Array.java @@ -166,7 +166,7 @@ public abstract class Array<T> implements Writable { /** * Set non default values from the value array given * - * @param value array of same type + * @param value array of same type and length */ public final void setNz(Array<T> value) { setNz(0, value.size() - 1, value); @@ -302,12 +302,12 @@ public abstract class Array<T> implements Writable { */ public abstract long getExactSerializedSize(); - public Array<Boolean> getNulls() { + public ABooleanArray getNulls() { return null; } public Array<?> changeTypeWithNulls(ValueType t) { - final Array<Boolean> nulls = getNulls(); + final ABooleanArray nulls = getNulls(); if(nulls == null) return changeType(t); @@ -499,7 +499,9 @@ public abstract class Array<T> implements Writable { * @param select modify this to true in indexes that are empty. */ public void findEmptyInverse(boolean[] select) { - throw new NotImplementedException(); + for(int i = 0; i < select.length; i++) + if(!isNotEmpty(i)) + select[i] = true; } /** diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java index a40c3c1279..88c6ff2040 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/ArrayFactory.java @@ -108,20 +108,20 @@ public interface ArrayFactory { switch(v) { case BOOLEAN: if(nRow > bitSetSwitchPoint) - return new OptionalArray<>(new BitSetArray(nRow)); + return new OptionalArray<>(new BitSetArray(nRow), true); else - return new OptionalArray<>(new BooleanArray(new boolean[nRow])); + return new OptionalArray<>(new BooleanArray(new boolean[nRow]), true); case UINT8: case INT32: - return new OptionalArray<>(new IntegerArray(new int[nRow])); + return new OptionalArray<>(new IntegerArray(new int[nRow]), true); case INT64: - return new OptionalArray<>(new LongArray(new long[nRow])); + return new OptionalArray<>(new LongArray(new long[nRow]), true); case FP32: - return new OptionalArray<>(new FloatArray(new float[nRow])); + return new OptionalArray<>(new FloatArray(new float[nRow]), true); case FP64: - return new OptionalArray<>(new DoubleArray(new double[nRow])); + return new OptionalArray<>(new DoubleArray(new double[nRow]), true); case CHARACTER: - return new OptionalArray<>(new CharArray(new char[nRow])); + return new OptionalArray<>(new CharArray(new char[nRow]), true); case UNKNOWN: case STRING: default: @@ -129,13 +129,17 @@ public interface ArrayFactory { } } + public static ABooleanArray allocateBoolean(int nRow) { + if(nRow > bitSetSwitchPoint) + return new BitSetArray(nRow); + else + return new BooleanArray(new boolean[nRow]); + } + public static Array<?> allocate(ValueType v, int nRow) { switch(v) { case BOOLEAN: - if(nRow > bitSetSwitchPoint) - return new BitSetArray(nRow); - else - return new BooleanArray(new boolean[nRow]); + return allocateBoolean(nRow); case UINT8: case INT32: return new IntegerArray(new int[nRow]); @@ -230,8 +234,12 @@ public interface ArrayFactory { target = allocateOptional(src.getValueType(), rlen); else target = allocate(src.getValueType(), rlen); - } + else if(target.getFrameArrayType() != FrameArrayType.OPTIONAL // + && src.getFrameArrayType() == FrameArrayType.OPTIONAL) { + target = new OptionalArray<>(target, false); + } + final ValueType ta = target.getValueType(); final ValueType tb = src.getValueType(); final ValueType tc = ValueType.getHighestCommonType(ta, tb); diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java index 0222a621b9..70dbde179e 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/BitSetArray.java @@ -34,7 +34,7 @@ import org.apache.sysds.runtime.matrix.data.Pair; import org.apache.sysds.runtime.util.UtilFunctions; import org.apache.sysds.utils.MemoryEstimates; -public class BitSetArray extends Array<Boolean> { +public class BitSetArray extends ABooleanArray { private static final boolean useVectorizedKernel = true; @@ -59,12 +59,12 @@ public class BitSetArray extends Array<Boolean> { if(_size > _data.length * 64) throw new DMLRuntimeException("Invalid allocation long array must be long enough"); if(_data.length > longSize(_size)) - throw new DMLRuntimeException( - "Invalid allocation long array must not be to long: " + _data.length + " " + _size + " " + (longSize(_size))); + throw new DMLRuntimeException("Invalid allocation long array must not be to long: " + _data.length + " " + + _size + " " + (longSize(_size))); } - private static int longSize(int size){ - return Math.max(size >> 6, 0) +1; + private static int longSize(int size) { + return Math.max(size >> 6, 0) + 1; } public BitSetArray(BitSet data, int size) { @@ -247,12 +247,18 @@ public class BitSetArray extends Array<Boolean> { } @Override - public BitSetArray append(Array<Boolean> other) { + public Array<Boolean> append(Array<Boolean> other) { final int endSize = this._size + other.size(); - final BitSetArray retBS = new BitSetArray(endSize); - retBS.set(0, this._size - 1, this, 0); - retBS.set(this._size, endSize - 1, other, 0); - return retBS; + final Array<Boolean> retBS = ArrayFactory.allocateBoolean(endSize); + retBS.set(0, this._size - 1, this); + if(other instanceof OptionalArray) { + retBS.set(this._size, endSize - 1, ((OptionalArray<Boolean>) other)._a); + return OptionalArray.appendOther((OptionalArray<Boolean>) other, retBS); + } + else { + retBS.set(this._size, endSize - 1, other); + return retBS; + } } @Override @@ -461,22 +467,11 @@ public class BitSetArray extends Array<Boolean> { @Override public void fill(Boolean value) { + value = value != null ? value : false; for(int i = 0; i < _size / 64 + 1; i++) _data[i] = value ? -1L : 0L; } - @Override - public String toString() { - StringBuilder sb = new StringBuilder(_size + 10); - sb.append(super.toString() + ":["); - if(_size > 0) { - for(int i = 0; i < _size; i++) - sb.append((get(i) ? 1 : 0)); - } - sb.append("]"); - return sb.toString(); - } - @Override public double getAsDouble(int i) { return get(i) ? 1.0 : 0.0; @@ -489,15 +484,22 @@ public class BitSetArray extends Array<Boolean> { @Override public boolean isEmpty() { - for(int i = 0; i < _data.length; i++) { + for(int i = 0; i < _data.length; i++) if(_data[i] != 0L) return false; - } return true; } @Override - public Array<Boolean> select(int[] indices) { + public boolean isAllTrue() { + for(int i = 0; i < _data.length; i++) + if(_data[i] != -1L) + return false; + return true; + } + + @Override + public ABooleanArray select(int[] indices) { // TODO vectorize final boolean[] ret = new boolean[indices.length]; for(int i = 0; i < indices.length; i++) @@ -506,7 +508,7 @@ public class BitSetArray extends Array<Boolean> { } @Override - public Array<Boolean> select(boolean[] select, int nTrue) { + public ABooleanArray select(boolean[] select, int nTrue) { final boolean[] ret = new boolean[nTrue]; int k = 0; for(int i = 0; i < select.length; i++) @@ -530,10 +532,20 @@ public class BitSetArray extends Array<Boolean> { public static String longToBits(long l) { String bits = Long.toBinaryString(l); StringBuilder sb = new StringBuilder(64); - for(int i = 0; i < 64 - bits.length(); i++) { + for(int i = 0; i < 64 - bits.length(); i++) sb.append('0'); - } sb.append(bits); return sb.toString(); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(_size + 10); + sb.append(super.toString() + ":["); + for(int i = 0; i < _size; i++) + sb.append((get(i) ? 1 : 0)); + + sb.append("]"); + return sb.toString(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java index 77190c58fa..ba91f96d5d 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/BooleanArray.java @@ -26,21 +26,15 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; -import org.apache.commons.lang.NotImplementedException; import org.apache.sysds.common.Types.ValueType; import org.apache.sysds.runtime.frame.data.columns.ArrayFactory.FrameArrayType; import org.apache.sysds.runtime.matrix.data.Pair; import org.apache.sysds.runtime.util.UtilFunctions; import org.apache.sysds.utils.MemoryEstimates; -public class BooleanArray extends Array<Boolean> { +public class BooleanArray extends ABooleanArray { protected boolean[] _data; - private BooleanArray(int size) { - super(size); - _data = new boolean[size]; - } - public BooleanArray(boolean[] data) { super(data.length); _data = data; @@ -85,7 +79,7 @@ public class BooleanArray extends Array<Boolean> { @Override public void set(int rl, int ru, Array<Boolean> value, int rlSrc) { if(value instanceof BooleanArray) - System.arraycopy(((BooleanArray) value)._data, rlSrc, _data, rl, ru - rl + 1); + System.arraycopy((boolean[]) value.get(), rlSrc, _data, rl, ru - rl + 1); else for(int i = rl, off = rlSrc; i <= ru; i++, off++) _data[i] = value.get(off); @@ -133,16 +127,14 @@ public class BooleanArray extends Array<Boolean> { @Override public Array<Boolean> append(Array<Boolean> other) { final int endSize = this._size + other.size(); - if(other instanceof BooleanArray && endSize < ArrayFactory.bitSetSwitchPoint) { - final boolean[] ret = new boolean[endSize]; - System.arraycopy(_data, 0, ret, 0, this._size); - System.arraycopy((boolean[]) other.get(), 0, ret, this._size, other.size()); - return new BooleanArray(ret); + final ABooleanArray retBS = ArrayFactory.allocateBoolean(endSize); + retBS.set(0, this._size - 1, this); + if(other instanceof OptionalArray) { + retBS.set(this._size, endSize - 1, ((OptionalArray<Boolean>) other)._a); + return OptionalArray.appendOther((OptionalArray<Boolean>) other, retBS); } else { - final BooleanArray retBS = new BooleanArray(endSize); - retBS.set(0, this._size - 1, this, 0); - retBS.set(this._size, endSize - 1, other, 0); + retBS.set(this._size, endSize - 1, other); return retBS; } } @@ -162,12 +154,12 @@ public class BooleanArray extends Array<Boolean> { } @Override - public Array<Boolean> clone() { + public ABooleanArray clone() { return new BooleanArray(Arrays.copyOf(_data, _size)); } @Override - public Array<Boolean> slice(int rl, int ru) { + public ABooleanArray slice(int rl, int ru) { return new BooleanArray(Arrays.copyOfRange(_data, rl, ru)); } @@ -223,12 +215,12 @@ public class BooleanArray extends Array<Boolean> { } @Override - protected Array<Boolean> changeTypeBitSet() { + protected ABooleanArray changeTypeBitSet() { return new BitSetArray(_data); } @Override - protected Array<Boolean> changeTypeBoolean() { + protected ABooleanArray changeTypeBoolean() { return this; } @@ -287,6 +279,7 @@ public class BooleanArray extends Array<Boolean> { @Override public void fill(Boolean value) { + value = value != null ? value : false; Arrays.fill(_data, value); } @@ -309,7 +302,15 @@ public class BooleanArray extends Array<Boolean> { } @Override - public Array<Boolean> select(int[] indices) { + public boolean isAllTrue() { + for(int i = 0; i < _data.length; i++) + if(!_data[i]) + return false; + return true; + } + + @Override + public ABooleanArray select(int[] indices) { final boolean[] ret = new boolean[indices.length]; for(int i = 0; i < indices.length; i++) ret[i] = _data[indices[i]]; @@ -317,7 +318,7 @@ public class BooleanArray extends Array<Boolean> { } @Override - public Array<Boolean> select(boolean[] select, int nTrue) { + public ABooleanArray select(boolean[] select, int nTrue) { final boolean[] ret = new boolean[nTrue]; int k = 0; for(int i = 0; i < select.length; i++) @@ -331,25 +332,19 @@ public class BooleanArray extends Array<Boolean> { return _data[i]; } - @Override - public void findEmptyInverse(boolean[] select){ - throw new NotImplementedException(); - - } - public static boolean parseBoolean(String value) { - return value != null && (Boolean.parseBoolean(value) || value.equals("1") || value.equals("1.0")); + return value != null && // + !value.isEmpty() && // + (Boolean.parseBoolean(value) || value.equals("1") || value.equals("1.0") || value.equals("t")); } @Override public String toString() { StringBuilder sb = new StringBuilder(_data.length * 2 + 10); sb.append(super.toString() + ":["); - if(_size > 0) { - for(int i = 0; i < _size - 1; i++) - sb.append((_data[i] ? 1 : 0) + ","); - sb.append(_data[_size - 1] ? 1 : 0); - } + for(int i = 0; i < _size - 1; i++) + sb.append((_data[i] ? 1 : 0) + ","); + sb.append(_data[_size - 1] ? 1 : 0); sb.append("]"); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/CharArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/CharArray.java index f44763e69f..73389beb9c 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/CharArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/CharArray.java @@ -137,7 +137,10 @@ public class CharArray extends Array<Character> { final char[] ret = new char[endSize]; System.arraycopy(_data, 0, ret, 0, this._size); System.arraycopy((char[]) other.get(), 0, ret, this._size, other.size()); - return new CharArray(ret); + if(other instanceof OptionalArray) + return OptionalArray.appendOther((OptionalArray<Character>) other, new CharArray(ret)); + else + return new CharArray(ret); } @Override @@ -187,50 +190,44 @@ public class CharArray extends Array<Character> { @Override protected Array<Boolean> changeTypeBitSet() { - BitSet ret = new BitSet(size()); + final BitSet ret = new BitSet(size()); for(int i = 0; i < size(); i++) { - if(_data[i] != 0 && _data[i] != 1) - throw new DMLRuntimeException("Unable to change to boolean from char array because of value:" + _data[i]); - ret.set(i, _data[i] != 0); + final int di = (int) _data[i]; + if(di != 0 && di != 1) + throw new DMLRuntimeException("Unable to change to boolean from char array because of value:" // + + _data[i] + " (as int: " + di + ")"); + ret.set(i, di != 0); } return new BitSetArray(ret, size()); } @Override protected Array<Boolean> changeTypeBoolean() { - boolean[] ret = new boolean[size()]; + final boolean[] ret = new boolean[size()]; for(int i = 0; i < size(); i++) { - if(_data[i] != 0 && _data[i] != 1) - throw new DMLRuntimeException("Unable to change to boolean from char array because of value:" + _data[i]); - ret[i] = _data[i] != 0; + final int di = (int) _data[i]; + if(di != 0 && di != 1) + throw new DMLRuntimeException("Unable to change to boolean from char array because of value:" // + + _data[i] + " (as int: " + di + ")"); + ret[i] = di != 0; } return new BooleanArray(ret); } @Override protected Array<Double> changeTypeDouble() { - try { - double[] ret = new double[size()]; - for(int i = 0; i < size(); i++) - ret[i] = (int) _data[i]; - return new DoubleArray(ret); - } - catch(NumberFormatException e) { - throw new DMLRuntimeException("Invalid parsing of char to double", e); - } + double[] ret = new double[size()]; + for(int i = 0; i < size(); i++) + ret[i] = (int) _data[i]; + return new DoubleArray(ret); } @Override protected Array<Float> changeTypeFloat() { - try { - float[] ret = new float[size()]; - for(int i = 0; i < size(); i++) - ret[i] = (int) _data[i]; - return new FloatArray(ret); - } - catch(NumberFormatException e) { - throw new DMLRuntimeException("Invalid parsing of char to float", e); - } + float[] ret = new float[size()]; + for(int i = 0; i < size(); i++) + ret[i] = (int) _data[i]; + return new FloatArray(ret); } @Override @@ -238,7 +235,6 @@ public class CharArray extends Array<Character> { int[] ret = new int[size()]; for(int i = 0; i < size(); i++) ret[i] = (int) _data[i]; - return new IntegerArray(ret); } @@ -269,8 +265,9 @@ public class CharArray extends Array<Character> { } @Override - public void fill(Character val) { - Arrays.fill(_data, (char) val); + public void fill(Character value) { + value = value != null ? value : 0; + Arrays.fill(_data, value); } @Override @@ -324,19 +321,17 @@ public class CharArray extends Array<Character> { public final boolean isNotEmpty(int i) { return _data[i] != 0; } - + @Override public String toString() { StringBuilder sb = new StringBuilder(_data.length * 2 + 15); sb.append(super.toString()); sb.append(":["); - if(_size > 0) { - for(int i = 0; i < _size - 1; i++) { - sb.append(_data[i]); - sb.append(','); - } - sb.append(_data[_size - 1]); + for(int i = 0; i < _size - 1; i++) { + sb.append(_data[i]); + sb.append(','); } + sb.append(_data[_size - 1]); sb.append("]"); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java index 0b52744792..b529a4287b 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/DoubleArray.java @@ -115,12 +115,15 @@ public class DoubleArray extends Array<Double> { } @Override - public DoubleArray append(Array<Double> other) { + public Array<Double> append(Array<Double> other) { final int endSize = this._size + other.size(); final double[] ret = new double[endSize]; System.arraycopy(_data, 0, ret, 0, this._size); System.arraycopy((double[]) other.get(), 0, ret, this._size, other.size()); - return new DoubleArray(ret); + if(other instanceof OptionalArray) + return OptionalArray.appendOther((OptionalArray<Double>) other, new DoubleArray(ret)); + else + return new DoubleArray(ret); } @Override @@ -175,7 +178,7 @@ public class DoubleArray extends Array<Double> { public Pair<ValueType, Boolean> analyzeValueType() { ValueType state = FrameUtil.isType(_data[0]); for(int i = 0; i < _size; i++) { - ValueType c = FrameUtil.isType(_data[i]); + ValueType c = FrameUtil.isType(_data[i], state); if(state == ValueType.FP64) return new Pair<ValueType, Boolean>(ValueType.FP64, false); @@ -204,6 +207,7 @@ public class DoubleArray extends Array<Double> { default: } break; + default: case BOOLEAN: switch(c) { case FP64: @@ -214,7 +218,6 @@ public class DoubleArray extends Array<Double> { default: } break; - default: } } return new Pair<ValueType, Boolean>(state, false); @@ -319,6 +322,7 @@ public class DoubleArray extends Array<Double> { @Override public void fill(Double value) { + value = value != null ? value : 0.0d; Arrays.fill(_data, value); } @@ -374,11 +378,9 @@ public class DoubleArray extends Array<Double> { public String toString() { StringBuilder sb = new StringBuilder(_data.length * 5 + 2); sb.append(super.toString() + ":["); - if(_size > 0) { - for(int i = 0; i < _size - 1; i++) - sb.append(_data[i] + ","); - sb.append(_data[_size - 1]); - } + for(int i = 0; i < _size - 1; i++) + sb.append(_data[i] + ","); + sb.append(_data[_size - 1]); sb.append("]"); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java index 7515db84b3..e53486506e 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/FloatArray.java @@ -114,12 +114,15 @@ public class FloatArray extends Array<Float> { } @Override - public FloatArray append(Array<Float> other) { + public Array<Float> append(Array<Float> other) { final int endSize = this._size + other.size(); final float[] ret = new float[endSize]; System.arraycopy(_data, 0, ret, 0, this._size); System.arraycopy((float[]) other.get(), 0, ret, this._size, other.size()); - return new FloatArray(ret); + if(other instanceof OptionalArray) + return OptionalArray.appendOther((OptionalArray<Float>) other, new FloatArray(ret)); + else + return new FloatArray(ret); } @Override @@ -271,6 +274,7 @@ public class FloatArray extends Array<Float> { @Override public void fill(Float value) { + value = value != null ? value : 0.0f; Arrays.fill(_data, value); } @@ -280,7 +284,7 @@ public class FloatArray extends Array<Float> { } public static float parseFloat(String value) { - if(value == null) + if(value == null || value.isEmpty()) return 0.0f; else return Float.parseFloat(value); @@ -294,7 +298,7 @@ public class FloatArray extends Array<Float> { @Override public boolean isEmpty() { for(int i = 0; i < _data.length; i++) - if(_data[i] != 0.0 || Float.isNaN(_data[i])) + if(isNotEmpty(i)) return false; return true; } @@ -326,11 +330,9 @@ public class FloatArray extends Array<Float> { public String toString() { StringBuilder sb = new StringBuilder(_data.length * 5 + 2); sb.append(super.toString() + ":["); - if(_size > 0) { - for(int i = 0; i < _size - 1; i++) - sb.append(_data[i] + ","); - sb.append(_data[_size - 1]); - } + for(int i = 0; i < _size - 1; i++) + sb.append(_data[i] + ","); + sb.append(_data[_size - 1]); sb.append("]"); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java index 5cd2a05006..999025524c 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/IntegerArray.java @@ -115,12 +115,15 @@ public class IntegerArray extends Array<Integer> { } @Override - public IntegerArray append(Array<Integer> other) { + public Array<Integer> append(Array<Integer> other) { final int endSize = this._size + other.size(); final int[] ret = new int[endSize]; System.arraycopy(_data, 0, ret, 0, this._size); System.arraycopy((int[]) other.get(), 0, ret, this._size, other.size()); - return new IntegerArray(ret); + if(other instanceof OptionalArray) + return OptionalArray.appendOther((OptionalArray<Integer>) other, new IntegerArray(ret)); + else + return new IntegerArray(ret); } @Override @@ -268,6 +271,7 @@ public class IntegerArray extends Array<Integer> { @Override public void fill(Integer value) { + value = value != null ? value : 0; Arrays.fill(_data, value); } @@ -277,7 +281,7 @@ public class IntegerArray extends Array<Integer> { } public static int parseInt(String s) { - if(s == null) + if(s == null || s.isEmpty()) return 0; try { return Integer.parseInt(s); @@ -331,11 +335,9 @@ public class IntegerArray extends Array<Integer> { public String toString() { StringBuilder sb = new StringBuilder(_data.length * 5 + 2); sb.append(super.toString() + ":["); - if(_size > 0) { - for(int i = 0; i < _size - 1; i++) - sb.append(_data[i] + ","); - sb.append(_data[_size - 1]); - } + for(int i = 0; i < _size - 1; i++) + sb.append(_data[i] + ","); + sb.append(_data[_size - 1]); sb.append("]"); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java index 6185fa3bb2..aa2e1d5961 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/LongArray.java @@ -80,7 +80,7 @@ public class LongArray extends Array<Long> { @Override public void set(int rl, int ru, Array<Long> value, int rlSrc) { - System.arraycopy(((LongArray) value)._data, rlSrc, _data, rl, ru - rl + 1); + System.arraycopy((long[]) value.get(), rlSrc, _data, rl, ru - rl + 1); } @Override @@ -114,12 +114,15 @@ public class LongArray extends Array<Long> { } @Override - public LongArray append(Array<Long> other) { + public Array<Long> append(Array<Long> other) { final int endSize = this._size + other.size(); final long[] ret = new long[endSize]; System.arraycopy(_data, 0, ret, 0, this._size); System.arraycopy((long[]) other.get(), 0, ret, this._size, other.size()); - return new LongArray(ret); + if(other instanceof OptionalArray) + return OptionalArray.appendOther((OptionalArray<Long>) other, new LongArray(ret)); + else + return new LongArray(ret); } @Override @@ -263,6 +266,7 @@ public class LongArray extends Array<Long> { @Override public void fill(Long value) { + value = value != null ? value : 0L; Arrays.fill(_data, value); } @@ -272,7 +276,7 @@ public class LongArray extends Array<Long> { } public static long parseLong(String s) { - if(s == null) + if(s == null || s.isEmpty()) return 0; try { return Long.parseLong(s); @@ -329,16 +333,13 @@ public class LongArray extends Array<Long> { return _data[i] != 0; } - @Override public String toString() { StringBuilder sb = new StringBuilder(_data.length * 5 + 2); sb.append(super.toString() + ":["); - if(_size > 0) { - for(int i = 0; i < _size - 1; i++) - sb.append(_data[i] + ","); - sb.append(_data[_size - 1]); - } + for(int i = 0; i < _size - 1; i++) + sb.append(_data[i] + ","); + sb.append(_data[_size - 1]); sb.append("]"); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/OptionalArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/OptionalArray.java index baee5be772..32a9c867ed 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/OptionalArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/OptionalArray.java @@ -35,7 +35,7 @@ public class OptionalArray<T> extends Array<T> { /** Underlying values not able to contain null values */ protected Array<T> _a; /** A Bitset specifying where there are null, in it false means null */ - protected Array<Boolean> _n; + protected ABooleanArray _n; @SuppressWarnings("unchecked") public OptionalArray(T[] a) { @@ -51,39 +51,44 @@ public class OptionalArray<T> extends Array<T> { _a = (Array<T>) ArrayFactory.allocate(ValueType.FP32, a.length); else if(a instanceof Long[]) _a = (Array<T>) ArrayFactory.allocate(ValueType.INT64, a.length); - else // Character + else if(a instanceof Character[]) // Character _a = (Array<T>) ArrayFactory.allocate(ValueType.CHARACTER, a.length); + else + throw new DMLRuntimeException("Invalid type for Optional Array: " + a.getClass().getSimpleName()); - _n = (Array<Boolean>) ArrayFactory.allocate(ValueType.BOOLEAN, a.length); + _n = ArrayFactory.allocateBoolean(a.length); for(int i = 0; i < a.length; i++) { _a.set(i, a[i]); _n.set(i, a[i] != null); } - - if(_a instanceof OptionalArray) - throw new DMLRuntimeException("Not allowed optional optional array"); } - public OptionalArray(Array<T> a) { + public OptionalArray(Array<T> a, boolean empty) { super(a.size()); if(a instanceof OptionalArray) throw new DMLRuntimeException("Not allowed optional optional array"); + else if(a instanceof StringArray) + throw new DMLRuntimeException("Not allowed StringArray in OptionalArray"); _a = a; - _n = new BitSetArray(a.size()); + _n = ArrayFactory.allocateBoolean(a.size()); + if(!empty) + _n.fill(true); } - public OptionalArray(Array<T> a, Array<Boolean> n) { + public OptionalArray(Array<T> a, ABooleanArray n) { super(a.size()); if(a instanceof OptionalArray) throw new DMLRuntimeException("Not allowed optional optional array"); + else if(a instanceof StringArray) + throw new DMLRuntimeException("Not allowed StringArray in OptionalArray"); + if(n.size() != a.size()) + throw new DMLRuntimeException("Incompatible sizes of arrays for optional array"); _a = a; _n = n; } @Override public void write(DataOutput out) throws IOException { - if(_a instanceof OptionalArray) - throw new DMLRuntimeException("Not allowed optional optional array"); out.writeByte(FrameArrayType.OPTIONAL.ordinal()); _a.write(out); _n.write(out); @@ -95,24 +100,14 @@ public class OptionalArray<T> extends Array<T> { } @Override - @SuppressWarnings("unchecked") public void readFields(DataInput in) throws IOException { - // throw new DMLRuntimeException("Should not be called"); - // // (FrameArrayType.OPTIONAL.ordinal()); - in.readByte();// ignore ordinal - _a = (Array<T>) ArrayFactory.read(in, _size); - // if(_a instanceof OptionalArray) - // throw new DMLRuntimeException("Invalid nested Optional array read"); - _n = (BitSetArray) ArrayFactory.read(in, _size); - + throw new DMLRuntimeException("Should not be called"); } @SuppressWarnings("unchecked") protected static OptionalArray<?> readOpt(DataInput in, int nRow) throws IOException { - Array<?> a = ArrayFactory.read(in, nRow); - Array<Boolean> n = (Array<Boolean>) ArrayFactory.read(in, nRow); - if(a instanceof OptionalArray) - throw new DMLRuntimeException("Not allowed optional optional array"); + final Array<?> a = ArrayFactory.read(in, nRow); + final ABooleanArray n = (ABooleanArray) ArrayFactory.read(in, nRow); switch(a.getValueType()) { case BOOLEAN: return new OptionalArray<Boolean>((Array<Boolean>) a, n); @@ -205,6 +200,10 @@ public class OptionalArray<T> extends Array<T> { Array<Boolean> nulls = value.getNulls(); if(nulls != null) _n.set(rl, ru, nulls, rlSrc); + else{ + for(int i = rl; i <= ru; i++) + _n.set(i, true); + } } private static <T> Array<T> getBasic(Array<T> value) { @@ -249,14 +248,22 @@ public class OptionalArray<T> extends Array<T> { @Override public Array<T> append(Array<T> other) { - BitSetArray otherB = new BitSetArray(other.size()); - for(int i = 0; i < other.size(); i++) - otherB.set(i, other.get(i) != null); - Array<Boolean> n = _n.append(otherB); - Array<T> a = _a.append(other); + OptionalArray<T> otherOpt = (other instanceof OptionalArray) ? // + (OptionalArray<T>) other : new OptionalArray<T>(other, false); + ABooleanArray n = (ABooleanArray) _n.append(otherOpt._n); + Array<T> a = _a.append(otherOpt._a); return new OptionalArray<T>(a, n); } + public static <T> OptionalArray<T> appendOther(OptionalArray<T> that, Array<T> appended) { + final int endSize = appended.size(); + ABooleanArray nullsThat = that._n; + ABooleanArray optsEnd = ArrayFactory.allocateBoolean(endSize); + optsEnd.fill(true); + optsEnd.set(endSize - that.size(), endSize - 1, nullsThat); + return new OptionalArray<T>(appended, optsEnd); + } + @Override public Array<T> slice(int rl, int ru) { return new OptionalArray<T>(_a.slice(rl, ru), _n.slice(rl, ru)); @@ -360,7 +367,7 @@ public class OptionalArray<T> extends Array<T> { } @Override - public Array<Boolean> getNulls() { + public ABooleanArray getNulls() { return _n; } @@ -371,7 +378,7 @@ public class OptionalArray<T> extends Array<T> { @Override public boolean isEmpty() { - return !_n.isEmpty() && _a.isEmpty(); + return !_n.isAllTrue(); } @Override @@ -422,11 +429,9 @@ public class OptionalArray<T> extends Array<T> { public String toString() { StringBuilder sb = new StringBuilder(_size + 2); sb.append(super.toString() + "<" + _a.getClass().getSimpleName() + ">:["); - if(_size > 0) { - for(int i = 0; i < _size - 1; i++) - sb.append(get(i) + ","); - sb.append(get(_size - 1)); - } + for(int i = 0; i < _size - 1; i++) + sb.append(get(i) + ","); + sb.append(get(_size - 1)); sb.append("]"); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java b/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java index 0be09a0042..250b876152 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/columns/StringArray.java @@ -122,12 +122,11 @@ public class StringArray extends Array<String> { } @Override - public StringArray append(Array<String> other) { + public Array<String> append(Array<String> other) { final int endSize = this._size + other.size(); final String[] ret = new String[endSize]; System.arraycopy(_data, 0, ret, 0, this._size); System.arraycopy((String[]) other.get(), 0, ret, this._size, other.size()); - materializedSize = -1; return new StringArray(ret); } @@ -294,15 +293,25 @@ public class StringArray extends Array<String> { @Override protected Array<Boolean> changeTypeBoolean() { + String firstNN = _data[0]; + int i = 1; + while(firstNN == null && i < size()){ + firstNN = _data[i++]; + } + // detect type of transform. - if(_data[0].toLowerCase().equals("true") || _data[0].toLowerCase().equals("false")) + if(i == size()) // if all null return empty boolean. + return ArrayFactory.allocateBoolean(size()); + else if(firstNN.toLowerCase().equals("true") || firstNN.toLowerCase().equals("false")) return changeTypeBooleanStandard(); - else if(_data[0].equals("0") || _data[0].equals("1")) + else if(firstNN.equals("0") || firstNN.equals("1")) return changeTypeBooleanNumeric(); - else if(_data[0].toLowerCase().equals("t") || _data[0].toLowerCase().equals("f")) + else if(firstNN.equals("0.0") || firstNN.equals("1.0")) + return changeTypeBooleanFloat(); + else if(firstNN.toLowerCase().equals("t") || firstNN.toLowerCase().equals("f")) return changeTypeBooleanCharacter(); else - throw new DMLRuntimeException("Not supported type of Strings to change to Booleans value: " + _data[0]); + throw new DMLRuntimeException("Not supported type of Strings to change to Booleans value: " + firstNN); } protected Array<Boolean> changeTypeBooleanStandard() { @@ -405,6 +414,48 @@ public class StringArray extends Array<String> { return new BooleanArray(ret); } + protected Array<Boolean> changeTypeBooleanFloat() { + if(size() > ArrayFactory.bitSetSwitchPoint) + return changeTypeBooleanFloatBitSet(); + else + return changeTypeBooleanFloatArray(); + } + + + protected Array<Boolean> changeTypeBooleanFloatBitSet() { + BitSet ret = new BitSet(size()); + for(int i = 0; i < size(); i++) { + final String s = _data[i]; + if(s != null) { + + final boolean zero = _data[i].equals("0.0"); + final boolean one = _data[i].equals("1.0"); + if(zero | one) + ret.set(i, one); + else + throw new DMLRuntimeException("Unable to change to Boolean from String array, value:" + _data[i]); + } + } + return new BitSetArray(ret, size()); + } + + protected Array<Boolean> changeTypeBooleanFloatArray() { + boolean[] ret = new boolean[size()]; + for(int i = 0; i < size(); i++) { + final String s = _data[i]; + if(s != null) { + final boolean zero = _data[i].equals("0.0"); + final boolean one = _data[i].equals("1.0"); + if(zero | one) + ret[i] = one; + else + throw new DMLRuntimeException("Unable to change to Boolean from String array, value:" + _data[i]); + } + + } + return new BooleanArray(ret); + } + @Override protected Array<Double> changeTypeDouble() { try { @@ -519,7 +570,7 @@ public class StringArray extends Array<String> { @Override public boolean isShallowSerialize() { long s = getInMemorySize(); - return s / _size < 100; + return _size < 100 || s / _size < 100; } @Override @@ -557,11 +608,9 @@ public class StringArray extends Array<String> { public String toString() { StringBuilder sb = new StringBuilder(_size * 5 + 2); sb.append(super.toString() + ":["); - if(_size > 0) { - for(int i = 0; i < _size - 1; i++) - sb.append(_data[i] + ","); - sb.append(_data[_size - 1]); - } + for(int i = 0; i < _size - 1; i++) + sb.append(_data[i] + ","); + sb.append(_data[_size - 1]); sb.append("]"); return sb.toString(); } diff --git a/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibRemoveEmpty.java b/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibRemoveEmpty.java index 2deeca2320..213459404d 100644 --- a/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibRemoveEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/frame/data/lib/FrameLibRemoveEmpty.java @@ -133,7 +133,7 @@ public class FrameLibRemoveEmpty { return new FrameBlock(schema, colNames); } - private static int getNumberTrue(boolean[] select) { + public static int getNumberTrue(boolean[] select) { int i = 0; for(boolean b : select) i += b ? 1 : 0; diff --git a/src/main/java/org/apache/sysds/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysds/runtime/util/UtilFunctions.java index ec39986322..ea7717388d 100644 --- a/src/main/java/org/apache/sysds/runtime/util/UtilFunctions.java +++ b/src/main/java/org/apache/sysds/runtime/util/UtilFunctions.java @@ -45,7 +45,6 @@ import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.TensorIndexes; import org.apache.sysds.runtime.frame.data.FrameBlock; -import org.apache.sysds.runtime.frame.data.columns.ArrayFactory; import org.apache.sysds.runtime.frame.data.columns.CharArray; import org.apache.sysds.runtime.instructions.spark.data.IndexedMatrixValue; import org.apache.sysds.runtime.matrix.data.MatrixIndexes; diff --git a/src/test/java/org/apache/sysds/test/component/frame/array/ColumnMetadataTests.java b/src/test/java/org/apache/sysds/test/component/frame/array/ColumnMetadataTests.java index ba3045c844..ba2881c8bc 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/array/ColumnMetadataTests.java +++ b/src/test/java/org/apache/sysds/test/component/frame/array/ColumnMetadataTests.java @@ -19,6 +19,7 @@ package org.apache.sysds.test.component.frame.array; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -181,6 +182,20 @@ public class ColumnMetadataTests { assertTrue(d.equals(serializeAndBack(d))); } + @Test + public void serializeSize() { + try { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + DataOutputStream fos = new DataOutputStream(bos); + d.write(fos); + assertEquals(d.getExactSerializedSize(), (long) fos.size()); + } + catch(Exception e) { + e.printStackTrace(); + throw new RuntimeException("Error in io", e); + } + } + public static ColumnMetadata serializeAndBack(ColumnMetadata a) { try { ByteArrayOutputStream bos = new ByteArrayOutputStream(); diff --git a/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java b/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java index e9b0c8bd00..c68332e735 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java +++ b/src/test/java/org/apache/sysds/test/component/frame/array/CustomArrayTests.java @@ -24,8 +24,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.lang.ref.SoftReference; +import java.util.Arrays; import java.util.BitSet; +import java.util.HashMap; +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysds.common.Types.ValueType; @@ -35,12 +39,17 @@ import org.apache.sysds.runtime.frame.data.columns.ArrayFactory; import org.apache.sysds.runtime.frame.data.columns.BitSetArray; import org.apache.sysds.runtime.frame.data.columns.BooleanArray; import org.apache.sysds.runtime.frame.data.columns.CharArray; +import org.apache.sysds.runtime.frame.data.columns.DoubleArray; +import org.apache.sysds.runtime.frame.data.columns.FloatArray; import org.apache.sysds.runtime.frame.data.columns.IntegerArray; import org.apache.sysds.runtime.frame.data.columns.LongArray; +import org.apache.sysds.runtime.frame.data.columns.OptionalArray; import org.apache.sysds.runtime.frame.data.columns.StringArray; import org.apache.sysds.runtime.matrix.data.Pair; import org.junit.Test; +import scala.util.Random; + public class CustomArrayTests { protected static final Log LOG = LogFactory.getLog(CustomArrayTests.class.getName()); @@ -96,7 +105,7 @@ public class CustomArrayTests { } @Test - public void changeTypeBoolean() { + public void changeTypeBoolean1() { StringArray a = ArrayFactory.create(new String[] {"1", "0", "0"}); BooleanArray ai = (BooleanArray) a.changeType(ValueType.BOOLEAN); assertTrue(ai.get(0)); @@ -104,6 +113,69 @@ public class CustomArrayTests { assertTrue(!ai.get(2)); } + @Test + public void changeTypeBoolean2() { + StringArray a = ArrayFactory.create(new String[] {"1.0", "0.0", "0.0"}); + BooleanArray ai = (BooleanArray) a.changeType(ValueType.BOOLEAN); + assertTrue(ai.get(0)); + assertTrue(!ai.get(1)); + assertTrue(!ai.get(2)); + } + + @Test + public void changeTypeBoolean3() { + StringArray a = ArrayFactory.create(new String[] {"1", null, "0"}); + BooleanArray ai = (BooleanArray) a.changeType(ValueType.BOOLEAN); + assertTrue(ai.get(0)); + assertTrue(!ai.get(1)); + assertTrue(!ai.get(2)); + } + + @Test + public void changeTypeBoolean4() { + StringArray a = ArrayFactory.create(new String[] {"1.0", null, "0.0"}); + BooleanArray ai = (BooleanArray) a.changeType(ValueType.BOOLEAN); + assertTrue(ai.get(0)); + assertTrue(!ai.get(1)); + assertTrue(!ai.get(2)); + } + + @Test + public void changeTypeBoolean5() { + StringArray a = ArrayFactory.create(new String[] {"t", null, "f"}); + BooleanArray ai = (BooleanArray) a.changeType(ValueType.BOOLEAN); + assertTrue(ai.get(0)); + assertTrue(!ai.get(1)); + assertTrue(!ai.get(2)); + } + + @Test + public void changeTypeBoolean6() { + StringArray a = ArrayFactory.create(new String[] {"true", null, "false"}); + BooleanArray ai = (BooleanArray) a.changeType(ValueType.BOOLEAN); + assertTrue(ai.get(0)); + assertTrue(!ai.get(1)); + assertTrue(!ai.get(2)); + } + + @Test + public void changeTypeBoolean7() { + StringArray a = ArrayFactory.create(new String[] {"True", null, "False"}); + BooleanArray ai = (BooleanArray) a.changeType(ValueType.BOOLEAN); + assertTrue(ai.get(0)); + assertTrue(!ai.get(1)); + assertTrue(!ai.get(2)); + } + + @Test + public void changeTypeBoolean8() { + StringArray a = ArrayFactory.create(new String[] {"0.0", null, "1.0"}); + BooleanArray ai = (BooleanArray) a.changeType(ValueType.BOOLEAN); + assertTrue(!ai.get(0)); + assertTrue(!ai.get(1)); + assertTrue(ai.get(2)); + } + @Test public void analyzeValueTypeStringBoolean() { StringArray a = ArrayFactory.create(new String[] {"1", "0", "0"}); @@ -209,6 +281,105 @@ public class CustomArrayTests { assertEquals(ValueType.INT32, t); } + @Test + public void analyzeValueTypeFromString() { + StringArray a = ArrayFactory.create(new String[] {"1.1", "1.2", "1.232132512451241", "3"}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP64, t); + } + + @Test + public void analyzeValueTypeFromString2() { + StringArray a = ArrayFactory.create(new String[] {"1", "2", "3", "30000000000"}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.INT64, t); + } + + @Test + public void analyzeValueTypeFromString3() { + StringArray a = ArrayFactory.create(new String[] {"1", "2", "3", "30000000000", "321321324215.213215"}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP64, t); + } + + @Test + public void analyzeValueTypeFromString4() { + StringArray a = ArrayFactory.create(new String[] {"1", "2", "3", "30000000000", "1.5"}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP32, t); + } + + @Test + public void analyzeValueTypeDouble() { + DoubleArray a = ArrayFactory + .create(new double[] {3214161624124214.23214d, 32141521421312.2321d, 32135215213.223d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP64, t); + } + + @Test + public void analyzeValueTypeDouble2() { + DoubleArray a = ArrayFactory.create(new double[] {1.0d, 32141521421312.2321d, 32135215213.223323d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP64, t); + } + + @Test + public void analyzeValueTypeDouble3() { + DoubleArray a = ArrayFactory.create(new double[] {1.0d, 1.1d, 2.2d, 1.0d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP32, t); + } + + @Test + public void analyzeValueTypeDouble4() { + DoubleArray a = ArrayFactory.create(new double[] {1.0d, 2.0d, 3.0d, 1.0d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.INT32, t); + } + + @Test + public void analyzeValueTypeDouble5() { + DoubleArray a = ArrayFactory.create(new double[] {10000000000.0d, 20000000000.0d, 30000000000.0d, 1.0d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.INT64, t); + } + + @Test + public void analyzeValueTypeDouble6() { + DoubleArray a = ArrayFactory.create(new double[] {1.0d, 20000000000.0d, 30000000000.0d, 1.0d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.INT64, t); + } + + @Test + public void analyzeValueTypeDouble7() { + DoubleArray a = ArrayFactory.create(new double[] {1.0d, 0.0d, 1.0d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.BOOLEAN, t); + } + + @Test + public void analyzeValueTypeDouble8() { + DoubleArray a = ArrayFactory.create(new double[] {1.0d, 1.1321321312512312d, 2.2d, 1.0d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP64, t); + } + + @Test + public void analyzeValueTypeDouble9() { + DoubleArray a = ArrayFactory.create(new double[] {1.1d, 1.1321321312512312d, 2.2d, 1.0d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP64, t); + } + + @Test + public void analyzeValueTypeDouble10() { + DoubleArray a = ArrayFactory.create(new double[] {10.d, 1.1d, 2.2d, 1.0d}); + ValueType t = a.analyzeValueType().getKey(); + assertEquals(ValueType.FP32, t); + } + @Test public void getNulls() { StringArray a = ArrayFactory.create(new String[] {"1", "2", null, "3"}); @@ -617,4 +788,377 @@ public class CustomArrayTests { for(int i = 0; i < c.size(); i++) assertEquals(i + 1, Integer.parseInt(c.get(i))); } + + @Test + public void testSetRange_1() { + Array<Integer> a = new IntegerArray(new int[] {1, 2, 3}); + Array<Long> b = new LongArray(new long[] {55L}); + Array<Long> c = ArrayFactory.set(a, b, 2, 2, 3); + assertEquals(c.get(0), Long.valueOf(1L)); + assertEquals(c.get(1), Long.valueOf(2L)); + assertEquals(c.get(2), Long.valueOf(55L)); + assertEquals(c.size(), 3); + } + + @Test + public void testSetRange_2() { + Array<Integer> a = new IntegerArray(new int[] {1, 2, 3, 4}); + Array<Long> b = new LongArray(new long[] {55L}); + Array<Long> c = ArrayFactory.set(a, b, 2, 2, 3); + assertEquals(c.get(0), Long.valueOf(1L)); + assertEquals(c.get(1), Long.valueOf(2L)); + assertEquals(c.get(2), Long.valueOf(55L)); + assertEquals(c.get(3), Long.valueOf(4L)); + assertEquals(c.size(), 4); + } + + @Test + public void testSetRange_nullIn() { + Array<Integer> a = null; + Array<Long> b = new LongArray(new long[] {55L}); + Array<Long> c = ArrayFactory.set(a, b, 2, 2, 4); + assertEquals(c.get(0), Long.valueOf(0L)); + assertEquals(c.get(1), Long.valueOf(0L)); + assertEquals(c.get(2), Long.valueOf(55L)); + assertEquals(c.get(3), Long.valueOf(0L)); + assertEquals(c.size(), 4); + } + + @Test + public void testSetOptional_nulInn() { + Array<Integer> a = null; + Array<Long> b = new OptionalArray<>(new LongArray(new long[] {55L}), false); + Array<Long> c = ArrayFactory.set(a, b, 2, 2, 4); + assertEquals(c.get(0), null); + assertEquals(c.get(1), null); + assertEquals(c.get(2), Long.valueOf(55L)); + assertEquals(c.get(3), null); + assertEquals(c.size(), 4); + } + + @Test + public void testSetBChangeType() { + Array<Long> a = new LongArray(new long[] {1, 2, 3, 4}); + Array<Integer> b = new IntegerArray(new int[] {55}); + Array<Long> c = ArrayFactory.set(a, b, 2, 2, 4); + assertEquals(c.get(0), Long.valueOf(1)); + assertEquals(c.get(1), Long.valueOf(2)); + assertEquals(c.get(2), Long.valueOf(55L)); + assertEquals(c.get(3), Long.valueOf(4)); + assertEquals(c.size(), 4); + } + + @Test + public void testSetOptionalB() { + try { + Array<Long> a = new LongArray(new long[] {1, 2, 3, 4}); + Array<Integer> b = new OptionalArray<>(new IntegerArray(new int[] {132}), + new BooleanArray(new boolean[] {false})); + Array<Long> c = ArrayFactory.set(a, b, 2, 2, 4); + assertEquals(c.get(0), Long.valueOf(1)); + assertEquals(c.get(1), Long.valueOf(2)); + assertEquals(c.get(2), null); + assertEquals(c.get(3), Long.valueOf(4)); + assertEquals(c.size(), 4); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testSetOptionalEmptyB() { + Array<Long> a = new OptionalArray<>(new LongArray(new long[] {1, 2, 3, 4}), true); + Array<Integer> b = new OptionalArray<>(new IntegerArray(new int[] {132}), false); + Array<Long> c = ArrayFactory.set(a, b, 2, 2, 4); + assertEquals(c.get(0), null); + assertEquals(c.get(1), null); + assertEquals(c.get(2), Long.valueOf(132)); + assertEquals(c.get(3), null); + assertEquals(c.size(), 4); + } + + @Test + public void isEmpty() { + for(ValueType t : ValueType.values()) + assertTrue(ArrayFactory.allocate(t, 10).isEmpty()); + } + + @Test + public void isNotEmpty() { + for(ValueType t : ValueType.values()) + assertFalse(ArrayFactory.allocate(t, 10, "1").isEmpty()); + } + + @Test + public void isEmptyOptional() { + assertTrue(new OptionalArray<>(ArrayFactory.allocate(ValueType.INT32, 10, "1"), true).isEmpty()); + } + + @Test + public void isEmptyOptionalFull() { + assertFalse(new OptionalArray<>(ArrayFactory.allocate(ValueType.INT32, 10), false).isEmpty()); + } + + @Test + public void isEmptyOptionalBig() { + assertTrue(new OptionalArray<>(ArrayFactory.allocate(ValueType.INT32, 200, "1"), true).isEmpty()); + } + + @Test + public void isEmptyOptionalFullBig() { + assertFalse(new OptionalArray<>(ArrayFactory.allocate(ValueType.INT32, 200), false).isEmpty()); + } + + @Test + public void allocateOptional() { + for(ValueType t : ValueType.values()) { + Array<?> a = ArrayFactory.allocateOptional(t, 10); + for(int i = 0; i < a.size(); i++) { + assertEquals(null, a.get(i)); + } + } + } + + @Test + public void allocateOptionalLarge() { + for(ValueType t : ValueType.values()) { + Array<?> a = ArrayFactory.allocateOptional(t, 66); + for(int i = 0; i < a.size(); i++) { + assertEquals(null, a.get(i)); + } + } + } + + @Test + public void setNzBooleanDifferentTypesIntoBooleanArray() { + BitSetArray a = new BitSetArray(new boolean[] {false, false, false, true, false}); + BooleanArray b = new BooleanArray(new boolean[] {true, true, false, false, false}); + + b.setNz(a); + assertTrue(b.get(0)); + assertTrue(b.get(1)); + assertFalse(b.get(2)); + assertTrue(b.get(3)); + assertFalse(b.get(4)); + } + + @Test + public void setNzBooleanDifferentTypesIntoBitSetArray() { + BooleanArray a = new BooleanArray(new boolean[] {false, false, false, true, false}); + BitSetArray b = new BitSetArray(new boolean[] {true, true, false, false, false}); + + b.setNz(a); + assertTrue(b.get(0)); + assertTrue(b.get(1)); + assertFalse(b.get(2)); + assertTrue(b.get(3)); + assertFalse(b.get(4)); + } + + @Test + public void parseDoubleEmpty() { + assertEquals(0.0, DoubleArray.parseDouble(""), 0.0); + } + + @Test + public void parseFloatEmpty() { + assertEquals(0.0, FloatArray.parseFloat(""), 0.0); + } + + @Test + public void parseIntegerEmpty() { + assertEquals(0, IntegerArray.parseInt("")); + } + + @Test + public void parseLongEmpty() { + assertEquals(0, LongArray.parseLong("")); + } + + @Test + public void parseBooleanEmpty() { + assertEquals(false, BooleanArray.parseBoolean("")); + } + + @Test + public void parseBooleanT() { + assertEquals(true, BooleanArray.parseBoolean("t")); + } + + @Test + @SuppressWarnings("unchecked") + public void optionalAppendNotNull() { + Array<Double> a = (Array<Double>) ArrayFactory.allocateOptional(ValueType.FP64, 10); + a.append(64.21d); + for(int i = 0; i < 10; i++) + assertEquals(null, a.get(i)); + assertEquals(64.21d, a.get(10), 0.0); + } + + @Test + @SuppressWarnings("unchecked") + public void optionalAppendArrayNotOptional() { + Array<Double> a = (Array<Double>) ArrayFactory.allocateOptional(ValueType.FP64, 10); + Array<Double> b = new DoubleArray(new double[] {0, 1, 2}); + a = a.append(b); + for(int i = 0; i < 10; i++) + assertEquals(null, a.get(i)); + for(int i = 10; i < 13; i++) + assertEquals(i - 10, a.get(i), 0.0); + } + + @Test + @SuppressWarnings("unchecked") + public void optionalSetRange() { + Array<Double> a = (Array<Double>) ArrayFactory.allocateOptional(ValueType.FP64, 10); + Array<Double> b = new DoubleArray(new double[] {0, 1, 2}); + a.set(3, 5, b, 0); + + for(int i = 0; i < 3; i++) + assertEquals(null, a.get(i)); + for(int i = 3; i < 6; i++) + assertEquals(i - 3, a.get(i), 0.0); + } + + @Test + public void optionalChangeToBoolean() { + Array<?> a = new OptionalArray<>(new Double[3]).changeTypeWithNulls(ValueType.BOOLEAN); + for(int i = 0; i < a.size(); i++) + assertEquals(null, a.get(i)); + } + + @Test + public void optionalChangeToBoolean2() { + Array<?> a = new OptionalArray<>(new Double[] {1.0, null, null}).changeTypeWithNulls(ValueType.BOOLEAN); + assertEquals(true, a.get(0)); + for(int i = 1; i < a.size(); i++) + assertEquals(null, a.get(i)); + } + + @Test + public void optionalChangeToBoolean3() { + Array<?> a = new OptionalArray<>(new Double[67]).changeTypeWithNulls(ValueType.BOOLEAN); + a.set(0, "true"); + a.set(a.size() - 1, "true"); + assertEquals(true, a.get(0)); + assertEquals(true, a.get(a.size() - 1)); + for(int i = 1; i < a.size() - 1; i++) + assertEquals(null, a.get(i)); + + } + + @Test + public void isNotShallowSerializeString() { + String[] st = new String[102]; + Arrays.fill(st, StringUtils.repeat("a", 100)); + assertFalse(ArrayFactory.create(st).isShallowSerialize()); + } + + @Test + public void isEmptyBitSet() { + Array<?> a = ArrayFactory.allocateBoolean(132); + assertTrue(a.isEmpty()); + a.set(23, "true"); + assertFalse(a.isEmpty()); + a.set(23, "false"); + assertTrue(a.isEmpty()); + } + + @Test + public void changeTypeBitSet() { + Array<?> a = new OptionalArray<>(new Character[324]).changeType(ValueType.BOOLEAN); + assertTrue(a.isEmpty()); + } + + @Test + public void rand1() { + Random r = new Random(13); + for(int i = 0; i < 10; i++) { + int g = r.nextInt(2); + assertTrue(g == 1 || g == 0); + } + } + + @Test + public void indexAsBytesNull() { + assertEquals(new StringArray(new String[10]).getIndexAsBytes(0), null); + } + + @Test + public void indexAsBytes1() { + byte[] b = new StringArray(new String[] {"a"}).getIndexAsBytes(0); + String exp = "[97]"; + assertEquals(exp, Arrays.toString(b)); + } + + @Test + public void indexAsBytes2() { + byte[] b = new StringArray(new String[] {"b"}).getIndexAsBytes(0); + String exp = "[98]"; + assertEquals(exp, Arrays.toString(b)); + } + + @Test + public void changeTypeNullsFromStringToFloat() { + Array<?> a = new StringArray(new String[] {"0.2", null}); + Array<?> b = a.changeTypeWithNulls(ValueType.FP32); + assertEquals(0.2f, b.get(0)); + assertEquals(null, b.get(1)); + } + + @Test + public void changeTypeNullsFromStringToDouble() { + Array<?> a = new StringArray(new String[] {"0.2", null}); + Array<?> b = a.changeTypeWithNulls(ValueType.FP64); + assertEquals(0.2d, b.get(0)); + assertEquals(null, b.get(1)); + } + + @Test + public void changeTypeNullsFromStringToInt() { + Array<?> a = new StringArray(new String[] {"3241", null}); + Array<?> b = a.changeTypeWithNulls(ValueType.INT32); + assertEquals(3241, b.get(0)); + assertEquals(null, b.get(1)); + } + + @Test + public void changeTypeNullsFromStringToLong() { + Array<?> a = new StringArray(new String[] {"3241", null}); + Array<?> b = a.changeTypeWithNulls(ValueType.INT64); + assertEquals(3241L, b.get(0)); + assertEquals(null, b.get(1)); + } + + @Test + public void changeTypeNullsFromStringToCharacter() { + Array<?> a = new StringArray(new String[] {"a", null}); + Array<?> b = a.changeTypeWithNulls(ValueType.CHARACTER); + assertEquals('a', b.get(0)); + assertEquals(null, b.get(1)); + } + + @Test + public void changeTypeNullsFromStringToBoolean() { + Array<?> a = new StringArray(new String[] {"1", null}); + Array<?> b = a.changeTypeWithNulls(ValueType.BOOLEAN); + assertEquals(true, b.get(0)); + assertEquals(null, b.get(1)); + } + + @Test + public void mappingCache() { + Array<?> a = new StringArray(new String[] {"1", null}); + assertEquals(null, a.getCache()); + a.setCache(new SoftReference<HashMap<String, Long>>(null)); + assertTrue(null != a.getCache()); + a.setCache(new SoftReference<HashMap<String, Long>>(new HashMap<>())); + assertTrue(null != a.getCache()); + HashMap<String, Long> hm = a.getCache().get(); + hm.put("1", 0L); + hm.put(null, 2L); + assertEquals(Long.valueOf(0L), a.getCache().get().get("1")); + } } diff --git a/src/test/java/org/apache/sysds/test/component/frame/array/FrameArrayTests.java b/src/test/java/org/apache/sysds/test/component/frame/array/FrameArrayTests.java index 0653d24699..fae84b39d0 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/array/FrameArrayTests.java +++ b/src/test/java/org/apache/sysds/test/component/frame/array/FrameArrayTests.java @@ -43,7 +43,15 @@ import org.apache.sysds.runtime.frame.data.columns.ArrayFactory; import org.apache.sysds.runtime.frame.data.columns.ArrayFactory.FrameArrayType; import org.apache.sysds.runtime.frame.data.columns.BitSetArray; import org.apache.sysds.runtime.frame.data.columns.BooleanArray; +import org.apache.sysds.runtime.frame.data.columns.CharArray; +import org.apache.sysds.runtime.frame.data.columns.DoubleArray; +import org.apache.sysds.runtime.frame.data.columns.FloatArray; +import org.apache.sysds.runtime.frame.data.columns.IntegerArray; +import org.apache.sysds.runtime.frame.data.columns.LongArray; +import org.apache.sysds.runtime.frame.data.columns.OptionalArray; import org.apache.sysds.runtime.frame.data.columns.StringArray; +import org.apache.sysds.runtime.frame.data.lib.FrameLibRemoveEmpty; +import org.apache.sysds.runtime.matrix.data.Pair; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -71,6 +79,14 @@ public class FrameArrayTests { tests.add(new Object[] {create(t, 124, s), t}); tests.add(new Object[] {create(t, 130, s), t}); tests.add(new Object[] {create(t, 200, s), t}); + if(t != FrameArrayType.STRING) { + tests.add(new Object[] {createOptional(t, 13, s), FrameArrayType.OPTIONAL}); + tests.add(new Object[] {createOptional(t, 321, s), FrameArrayType.OPTIONAL}); + } + else { + tests.add(new Object[] {createOptional(t, 13, s), FrameArrayType.STRING}); + tests.add(new Object[] {createOptional(t, 312, s), FrameArrayType.STRING}); + } } } // Booleans @@ -98,11 +114,20 @@ public class FrameArrayTests { tests.add(new Object[] {ArrayFactory.create(generateRandomTFString(150, 221)), FrameArrayType.STRING}); tests.add(new Object[] {ArrayFactory.create(generateRandomTFString(22, 2)), FrameArrayType.STRING}); tests.add(new Object[] {ArrayFactory.create(generateRandomTFString(142, 4)), FrameArrayType.STRING}); + tests.add(new Object[] {ArrayFactory.create(generateRandomDouble01(90, 32)), FrameArrayType.FP64}); + tests.add(new Object[] {ArrayFactory.create(generateRandomFloat01(90, 14)), FrameArrayType.FP32}); + tests.add(new Object[] {ArrayFactory.create(generateRandomInteger01(90, 55)), FrameArrayType.INT32}); + tests.add(new Object[] {ArrayFactory.create(generateRandomLong01(90, 55)), FrameArrayType.INT64}); + tests.add(new Object[] {ArrayFactory.create(generateRandomNullZeroString(33, 21)), FrameArrayType.STRING}); + tests.add(new Object[] {ArrayFactory.create(generateRandomNullZeroString(67, 21)), FrameArrayType.STRING}); + tests.add(new Object[] {ArrayFactory.create(generateRandomNullFloatString(67, 21)), FrameArrayType.STRING}); + tests.add(new Object[] {ArrayFactory.create(new String[30]), FrameArrayType.STRING}); // all null tests.add(new Object[] {ArrayFactory.create(new char[] {0, 0, 0, 0, 1, 1, 1}), FrameArrayType.CHARACTER}); tests.add(new Object[] {ArrayFactory.create(new char[] {'t', 't', 'f', 'f', 'T'}), FrameArrayType.CHARACTER}); tests.add(new Object[] {ArrayFactory.create(new char[] {'0', '2', '3', '4', '9'}), FrameArrayType.CHARACTER}); tests.add(new Object[] {ArrayFactory.create(generateRandom01chars(150, 221)), FrameArrayType.CHARACTER}); + tests.add(new Object[] {ArrayFactory.create(generateRandom01chars(67, 221)), FrameArrayType.CHARACTER}); // Long to int tests.add(new Object[] {ArrayFactory.create(new long[] {3214, 424, 13, 22, 111, 134}), FrameArrayType.INT64}); @@ -111,7 +136,7 @@ public class FrameArrayTests { tests.add(new Object[] {ArrayFactory.create(new float[] {// Float.NaN, 424, 13, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, 134}), FrameArrayType.FP32}); - } + } catch(Exception e) { e.printStackTrace(); fail("failed constructing tests"); @@ -427,6 +452,7 @@ public class FrameArrayTests { @Test @SuppressWarnings("unchecked") public void set() { + Array<?> a = this.a.clone(); switch(a.getValueType()) { case FP64: Double vd = 1324.42d; @@ -472,6 +498,7 @@ public class FrameArrayTests { @SuppressWarnings("unchecked") public void setDouble() { Double vd = 1.0d; + Array<?> a = this.a.clone(); a.set(0, vd); switch(a.getValueType()) { case FP64: @@ -504,6 +531,7 @@ public class FrameArrayTests { @SuppressWarnings("unchecked") public void setDouble_2() { Double vd = 0.0d; + Array<?> a = this.a.clone(); a.set(0, vd); switch(a.getValueType()) { case FP64: @@ -604,6 +632,7 @@ public class FrameArrayTests { @Test public void setNull() { + Array<?> a = this.a.clone(); // should not crash a.set(0, (String) null); } @@ -960,6 +989,528 @@ public class FrameArrayTests { } } + @Test + public void isShallowSerialize() { + + assertTrue(a.toString(), a.isShallowSerialize()); + } + + @Test + public void emptyInverse() { + boolean[] inverse = new boolean[a.size()]; + a.findEmptyInverse(inverse); + boolean[] normal = new boolean[a.size()]; + a.findEmpty(normal); + for(int i = 0; i < a.size(); i++) { + assertEquals(normal[i], !inverse[i]); + } + } + + @Test + public void getDouble() { + try { + for(int i = 0; i < a.size(); i++) { + double d = a.getAsDouble(i); + if(a.get(i) == null) + assertEquals(0.0, d, 0.0); + } + } + catch(Exception e) { + if(a.getValueType() != ValueType.STRING) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + } + + @Test + public void getDoubleNaN() { + try { + for(int i = 0; i < a.size(); i++) { + double d = a.getAsNaNDouble(i); + if(a.get(i) == null) + assertEquals(Double.NaN, d, 0.0); + } + } + catch(Exception e) { + if(a.getValueType() != ValueType.STRING) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + } + + @Test + @SuppressWarnings("unchecked") + public void setNullType() { + Array<?> aa = a.clone(); + switch(aa.getValueType()) { + case BOOLEAN: + ((Array<Boolean>) aa).set(0, (Boolean) null); + assertTrue(aa.get(0) == null || aa.get(0).equals(Boolean.valueOf(false))); + break; + case CHARACTER: + ((Array<Character>) aa).set(0, (Character) null); + assertTrue(aa.get(0) == null || aa.get(0).equals(Character.valueOf((char) 0))); + break; + + case FP32: + ((Array<Float>) aa).set(0, (Float) null); + assertTrue(aa.get(0) == null || aa.get(0).equals(Float.valueOf(0.0f))); + break; + case FP64: + ((Array<Double>) aa).set(0, (Double) null); + assertTrue(aa.get(0) == null || aa.get(0).equals(Double.valueOf(0.0d))); + break; + case INT32: + ((Array<Integer>) aa).set(0, (Integer) null); + assertTrue(aa.get(0) == null || aa.get(0).equals(Integer.valueOf(0))); + break; + case INT64: + ((Array<Long>) aa).set(0, (Long) null); + assertTrue(aa.get(0) == null || aa.get(0).equals(Long.valueOf(0))); + break; + case UINT8: + ((Array<Integer>) aa).set(0, (Integer) null); + assertTrue(aa.get(0) == null || aa.get(0).equals(Integer.valueOf(0))); + break; + default: + case STRING: + case UNKNOWN: + aa.set(0, (String) null); + assertTrue(aa.get(0) == null); + break; + } + } + + @Test + @SuppressWarnings("unchecked") + public void testAppendArray() { + Array<?> aa = a.clone(); + + switch(a.getValueType()) { + case BOOLEAN: + aa = ((Array<Boolean>) aa).append(new BooleanArray(new boolean[10])); + assertEquals(aa.size(), a.size() + 10); + for(int i = 0; i < 10; i++) + assertEquals(aa.get(i + a.size()), false); + break; + case CHARACTER: + aa = ((Array<Character>) aa).append(new CharArray(new char[10])); + assertEquals(aa.size(), a.size() + 10); + for(int i = 0; i < 10; i++) + assertEquals(aa.get(i + a.size()), (char) 0); + break; + case FP32: + aa = ((Array<Float>) aa).append(new FloatArray(new float[10])); + assertEquals(aa.size(), a.size() + 10); + for(int i = 0; i < 10; i++) + assertEquals(aa.get(i + a.size()), 0.0f); + break; + case FP64: + aa = ((Array<Double>) aa).append(new DoubleArray(new double[10])); + assertEquals(aa.size(), a.size() + 10); + for(int i = 0; i < 10; i++) + assertEquals(aa.get(i + a.size()), 0.0d); + break; + case UINT8: + case INT32: + aa = ((Array<Integer>) aa).append(new IntegerArray(new int[10])); + assertEquals(aa.size(), a.size() + 10); + for(int i = 0; i < 10; i++) + assertEquals(aa.get(i + a.size()), 0); + break; + case INT64: + aa = ((Array<Long>) aa).append(new LongArray(new long[10])); + assertEquals(aa.size(), a.size() + 10); + for(int i = 0; i < 10; i++) + assertEquals(aa.get(i + a.size()), 0L); + break; + case STRING: + aa = ((Array<String>) aa).append(new StringArray(new String[10])); + assertEquals(aa.size(), a.size() + 10); + for(int i = 0; i < 10; i++) + assertEquals(aa.get(i + a.size()), null); + break; + case UNKNOWN: + default: + throw new NotImplementedException("Not supported"); + } + + for(int i = 0; i < a.size(); i++) + assertEquals(a.get(i), aa.get(i)); + + } + + @Test + @SuppressWarnings("unchecked") + public void testAppendValue() { + Array<?> aa = a.clone(); + boolean isOptional = aa instanceof OptionalArray; + switch(a.getValueType()) { + case BOOLEAN: + ((Array<Boolean>) aa).append((Boolean) null); + assertEquals(aa.size(), a.size() + 1); + if(!isOptional) + assertEquals(aa.get(a.size()), false); + break; + case CHARACTER: + ((Array<Character>) aa).append((Character) null); + assertEquals(aa.size(), a.size() + 1); + if(!isOptional) + assertEquals(aa.get(a.size()), (char) 0); + break; + case FP32: + ((Array<Float>) aa).append((Float) null); + assertEquals(aa.size(), a.size() + 1); + if(!isOptional) + assertEquals(aa.get(a.size()), 0.0f); + break; + case FP64: + ((Array<Double>) aa).append((Double) null); + assertEquals(aa.size(), a.size() + 1); + if(!isOptional) + assertEquals(aa.get(a.size()), 0.0d); + break; + case UINT8: + case INT32: + ((Array<Integer>) aa).append((Integer) null); + assertEquals(aa.size(), a.size() + 1); + if(!isOptional) + assertEquals(aa.get(a.size()), 0); + break; + case INT64: + ((Array<Long>) aa).append((Long) null); + assertEquals(aa.size(), a.size() + 1); + if(!isOptional) + assertEquals(aa.get(a.size()), 0L); + break; + case STRING: + aa.append((String) null); + assertEquals(aa.size(), a.size() + 1); + if(!isOptional) + assertEquals(aa.get(a.size()), null); + break; + case UNKNOWN: + default: + throw new NotImplementedException("Not supported"); + } + + for(int i = 0; i < a.size(); i++) + assertEquals(a.get(i), aa.get(i)); + if(isOptional) + assertEquals(aa.get(a.size()), null); + } + + @Test + @SuppressWarnings("unchecked") + public void testAppendArrayOptional() { + Array<?> aa = a.clone(); + + switch(a.getValueType()) { + case BOOLEAN: + try { + aa = ((Array<Boolean>) aa).append(new OptionalArray<>(new Boolean[10])); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + break; + case CHARACTER: + aa = ((Array<Character>) aa).append(new OptionalArray<>(new Character[10])); + break; + case FP32: + aa = ((Array<Float>) aa).append(new OptionalArray<>(new Float[10])); + break; + case FP64: + aa = ((Array<Double>) aa).append(new OptionalArray<>(new Double[10])); + break; + case UINT8: + case INT32: + aa = ((Array<Integer>) aa).append(new OptionalArray<>(new Integer[10])); + break; + case INT64: + aa = ((Array<Long>) aa).append(new OptionalArray<>(new Long[10])); + break; + case STRING: + return; // not relevant + case UNKNOWN: + default: + throw new NotImplementedException("Not supported"); + } + + assertEquals(aa.size(), a.size() + 10); + + for(int i = 0; i < a.size(); i++) + assertEquals(a.get(i), aa.get(i)); + + for(int i = 0; i < 10; i++) + assertEquals(null, aa.get(i + a.size())); + } + + @Test + public void fillNull() { + Array<?> aa = a.clone(); + boolean isOptional = aa instanceof OptionalArray; + aa.fill((String) null); + switch(a.getValueType()) { + case BOOLEAN: + if(!isOptional) + + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), false); + break; + case CHARACTER: + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), (char) 0); + break; + case FP32: + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 0.0f); + break; + case FP64: + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 0.0d); + break; + case UINT8: + case INT32: + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 0); + break; + case INT64: + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 0L); + break; + case STRING: + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), null); + break; + case UNKNOWN: + default: + throw new NotImplementedException("Not supported"); + } + + if(isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), null); + } + + @Test + public void fill1String() { + Array<?> aa = a.clone(); + // boolean isOptional = aa instanceof OptionalArray; + aa.fill("1"); + switch(a.getValueType()) { + case BOOLEAN: + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), true); + break; + case CHARACTER: + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), '1'); + break; + case FP32: + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 1.0f); + break; + case FP64: + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 1.0d); + break; + case UINT8: + case INT32: + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 1); + break; + case INT64: + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 1L); + break; + case STRING: + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), "1"); + break; + case UNKNOWN: + default: + throw new NotImplementedException("Not supported"); + } + } + + @Test + @SuppressWarnings("unchecked") + public void fill1Value() { + Array<?> aa = a.clone(); + switch(a.getValueType()) { + case BOOLEAN: + ((Array<Boolean>) aa).fill(true); + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), true); + break; + case CHARACTER: + ((Array<Character>) aa).fill('1'); + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), '1'); + break; + case FP32: + ((Array<Float>) aa).fill(1.0f); + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 1.0f); + break; + case FP64: + ((Array<Double>) aa).fill(1.0d); + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 1.0d); + break; + case UINT8: + case INT32: + ((Array<Integer>) aa).fill(1); + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 1); + break; + case INT64: + ((Array<Long>) aa).fill(1L); + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 1L); + break; + case STRING: + aa.fill("1"); + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), "1"); + break; + case UNKNOWN: + default: + throw new NotImplementedException("Not supported"); + } + } + + @Test + @SuppressWarnings("unchecked") + public void fill1ValueNull() { + try { + + Array<?> aa = a.clone(); + boolean isOptional = aa instanceof OptionalArray; + switch(a.getValueType()) { + case BOOLEAN: + ((Array<Boolean>) aa).fill((Boolean) null); + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), false); + break; + case CHARACTER: + ((Array<Character>) aa).fill((Character) null); + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), (char)0); + break; + case FP32: + ((Array<Float>) aa).fill((Float) null); + if(!isOptional) + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 0.0f); + break; + case FP64: + ((Array<Double>) aa).fill((Double) null); + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 0.0d); + break; + case UINT8: + case INT32: + ((Array<Integer>) aa).fill((Integer) null); + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 0); + break; + case INT64: + ((Array<Long>) aa).fill((Long) null); + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), 0L); + break; + case STRING: + aa.fill((String) null); + if(!isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), null); + break; + case UNKNOWN: + default: + throw new NotImplementedException("Not supported"); + } + if(isOptional) + for(int i = 0; i < aa.size(); i++) + assertEquals(aa.get(i), null); + } + catch(Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testBooleanSelect() { + boolean[] select = generateRandomBoolean(a.size(), 31); + int nTrue = FrameLibRemoveEmpty.getNumberTrue(select); + if(nTrue > 0) { + Array<?> aa = a.select(select, nTrue); + assertEquals(nTrue, aa.size()); + int k = 0; + for(int i = 0; i < a.size(); i++) { + if(select[i]) { + assertEquals(a.get(i), aa.get(k++)); + } + } + } + } + + @Test(expected = DMLRuntimeException.class) + public void testBooleanSelectEmpty() { + boolean[] select = new boolean[a.size()]; + a.select(select, 0); + } + + @Test + public void testIndexSelect() { + int[] select = generateRandomIntegerMax(Math.min(a.size(), 10), a.size(), 31); + + Array<?> aa = a.select(select); + assertEquals(select.length, aa.size()); + for(int i = 0; i < select.length; i++) { + assertEquals(a.get(select[i]), aa.get(i)); + } + } + + @Test + public void changeTypeWithNulls() { + try { + Pair<ValueType, Boolean> vtb = a.analyzeValueType(); + ValueType vt = vtb.getKey(); + boolean containsNull = vtb.getValue(); + if(vt != a.getValueType() && containsNull) { + Array<?> aa = a.changeTypeWithNulls(vt); + for(int i = 0; i < aa.size(); i++) { + if(a.get(i) == null) { + assertEquals(null, aa.get(i)); + } + } + } + } + catch(Exception e) { + e.printStackTrace(); + LOG.error(a); + fail(e.getMessage()); + } + } + protected static void compare(Array<?> a, Array<?> b) { int size = a.size(); String err = a.getClass().getSimpleName() + " " + a.getValueType() + " " + b.getClass().getSimpleName() + " " @@ -1010,6 +1561,46 @@ public class FrameArrayTests { } } + protected static Array<?> createOptional(FrameArrayType t, int size, int seed) { + switch(t) { + case STRING: + return ArrayFactory.create(generateRandomStringOpt(size, seed)); + case BITSET: + // return ArrayFactory.create(generateRandomBitSet(size, seed), size); + case BOOLEAN: + return ArrayFactory.create(generateRandomBooleanOpt(size, seed)); + case INT32: + return ArrayFactory.create(generateRandomIntegerOpt(size, seed)); + case INT64: + return ArrayFactory.create(generateRandomLongOpt(size, seed)); + case FP32: + return ArrayFactory.create(generateRandomFloatOpt(size, seed)); + case FP64: + return ArrayFactory.create(generateRandomDoubleOpt(size, seed)); + case CHARACTER: + return ArrayFactory.create(generateRandomCharacterOpt(size, seed)); + case OPTIONAL: + Random r = new Random(seed); + switch(r.nextInt(7)) { + case 0: + return ArrayFactory.create(generateRandomIntegerOpt(size, seed)); + case 1: + return ArrayFactory.create(generateRandomLongOpt(size, seed)); + case 2: + return ArrayFactory.create(generateRandomDoubleOpt(size, seed)); + case 3: + return ArrayFactory.create(generateRandomFloatOpt(size, seed)); + case 4: + return ArrayFactory.create(generateRandomCharacterOpt(size, seed)); + default: + return ArrayFactory.create(generateRandomBooleanOpt(size, seed)); + } + default: + throw new DMLRuntimeException("Unsupported value type: " + t); + + } + } + protected static Array<?> create(FrameArrayType t, int size, int seed) { switch(t) { case STRING: @@ -1046,7 +1637,6 @@ public class FrameArrayTests { } default: throw new DMLRuntimeException("Unsupported value type: " + t); - } } @@ -1068,11 +1658,40 @@ public class FrameArrayTests { return ret; } + public static String[] generateRandomStringOpt(int size, int seed) { + Random r = new Random(seed); + String[] ret = new String[size]; + for(int i = 0; i < size; i++) { + if(r.nextBoolean()) + ret[i] = r.nextInt(99) + "ad " + r.nextInt(99); + } + return ret; + } + public static String[] generateRandom01String(int size, int seed) { Random r = new Random(seed); String[] ret = new String[size]; for(int i = 0; i < size; i++) - ret[i] = r.nextInt(1) + ""; + ret[i] = r.nextInt(2) + ""; + return ret; + } + + public static String[] generateRandomNullZeroString(int size, int seed) { + Random r = new Random(seed); + String[] ret = new String[size]; + for(int i = 0; i < size; i++) + ret[i] = r.nextBoolean() ? null : "0"; + + return ret; + } + + // generateRandomNullFloatString + public static String[] generateRandomNullFloatString(int size, int seed) { + Random r = new Random(seed); + String[] ret = new String[size]; + for(int i = 0; i < size; i++) + ret[i] = r.nextBoolean() ? null : r.nextBoolean() ? "1.0" : "0.0"; + return ret; } @@ -1080,7 +1699,7 @@ public class FrameArrayTests { Random r = new Random(seed); char[] ret = new char[size]; for(int i = 0; i < size; i++) - ret[i] = (char) r.nextInt(1); + ret[i] = (char) r.nextInt(2); return ret; } @@ -1088,7 +1707,7 @@ public class FrameArrayTests { Random r = new Random(seed); String[] ret = new String[size]; for(int i = 0; i < size; i++) - ret[i] = r.nextInt(1) == 1 ? "true" : "false"; + ret[i] = r.nextInt(2) == 1 ? "true" : "false"; return ret; } @@ -1096,7 +1715,7 @@ public class FrameArrayTests { Random r = new Random(seed); String[] ret = new String[size]; for(int i = 0; i < size; i++) - ret[i] = r.nextInt(1) == 1 ? "t" : "f"; + ret[i] = r.nextInt(2) == 1 ? "t" : "f"; return ret; } @@ -1188,6 +1807,22 @@ public class FrameArrayTests { return ret; } + protected static int[] generateRandomInteger01(int size, int seed) { + Random r = new Random(seed); + int[] ret = new int[size]; + for(int i = 0; i < size; i++) + ret[i] = r.nextBoolean() ? 1 : 0; + return ret; + } + + protected static int[] generateRandomIntegerMax(int size, int max, int seed) { + Random r = new Random(seed); + int[] ret = new int[size]; + for(int i = 0; i < size; i++) + ret[i] = r.nextInt(max); + return ret; + } + protected static char[] generateRandomChar(int size, int seed) { Random r = new Random(seed); char[] ret = new char[size]; @@ -1212,6 +1847,14 @@ public class FrameArrayTests { return ret; } + protected static long[] generateRandomLong01(int size, int seed) { + Random r = new Random(seed); + long[] ret = new long[size]; + for(int i = 0; i < size; i++) + ret[i] = r.nextBoolean() ? 1L : 0L; + return ret; + } + protected static float[] generateRandomFloat(int size, int seed) { Random r = new Random(seed); float[] ret = new float[size]; @@ -1220,6 +1863,14 @@ public class FrameArrayTests { return ret; } + protected static float[] generateRandomFloat01(int size, int seed) { + Random r = new Random(seed); + float[] ret = new float[size]; + for(int i = 0; i < size; i++) + ret[i] = r.nextBoolean() ? 1.0f : 0.0f; + return ret; + } + protected static double[] generateRandomDouble(int size, int seed) { Random r = new Random(seed); double[] ret = new double[size]; @@ -1228,6 +1879,14 @@ public class FrameArrayTests { return ret; } + protected static double[] generateRandomDouble01(int size, int seed) { + Random r = new Random(seed); + double[] ret = new double[size]; + for(int i = 0; i < size; i++) + ret[i] = r.nextBoolean() ? 1.0 : 0.0; + return ret; + } + protected static BitSet generateRandomBitSet(int size, int seed) { Random r = new Random(seed); int nLongs = size / 64 + 1; @@ -1237,4 +1896,5 @@ public class FrameArrayTests { return BitSet.valueOf(longs); } + } diff --git a/src/test/java/org/apache/sysds/test/component/frame/array/NegativeArrayTests.java b/src/test/java/org/apache/sysds/test/component/frame/array/NegativeArrayTests.java index 3361ae5121..84b43f6e72 100644 --- a/src/test/java/org/apache/sysds/test/component/frame/array/NegativeArrayTests.java +++ b/src/test/java/org/apache/sysds/test/component/frame/array/NegativeArrayTests.java @@ -20,6 +20,9 @@ package org.apache.sysds.test.component.frame.array; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.IOException; import org.apache.commons.lang.NotImplementedException; import org.apache.sysds.common.Types.ValueType; @@ -109,6 +112,21 @@ public class NegativeArrayTests { a.changeType(ValueType.BOOLEAN); } + @Test(expected = DMLRuntimeException.class) + public void changeTypeBoolean_5() { + StringArray a = ArrayFactory.create(new String[] {"0.0", null, "1.1"}); + a.changeType(ValueType.BOOLEAN); + } + + @Test(expected = DMLRuntimeException.class) + public void changeTypeBoolean_6() { + String[] s = new String[100]; + s[0] = "1.0"; + s[1] = "1.2"; + StringArray a = ArrayFactory.create(s); + a.changeType(ValueType.BOOLEAN); + } + @Test(expected = DMLRuntimeException.class) public void invalidConstructionBitArrayToSmall() { new BitSetArray(new long[0], 10); @@ -168,4 +186,69 @@ public class NegativeArrayTests { public void zLenAllocationString() { new StringArray(new String[0]); } + + @Test(expected = DMLRuntimeException.class) + public void createOptionalWithOptionalConstructor1() { + new OptionalArray<>(new OptionalArray<>(new Integer[1]), false); + } + + @Test(expected = DMLRuntimeException.class) + public void createOptionalWithOptionalConstructor2() { + new OptionalArray<>(new OptionalArray<>(new Integer[1]), new BooleanArray(new boolean[1])); + } + + @Test(expected = DMLRuntimeException.class) + public void readFields() { + try { + new OptionalArray<>(new Integer[1]).readFields(null); + } + catch(IOException e) { + fail("not correct exception"); + } + } + + @Test(expected = NullPointerException.class) + public void invalidConstructOptional1() { + new OptionalArray<>(ArrayFactory.allocate(ValueType.CHARACTER, 10), null); + } + + @Test(expected = DMLRuntimeException.class) + public void invalidConstructOptional2() { + new OptionalArray<>(ArrayFactory.allocate(ValueType.CHARACTER, 10), new BooleanArray(new boolean[3])); + } + + @Test(expected = DMLRuntimeException.class) + public void invalidConstrucOptionalString1() { + new OptionalArray<>(new String[2]); + } + + @Test(expected = DMLRuntimeException.class) + public void invalidConstrucOptionalString2() { + new OptionalArray<>(ArrayFactory.allocate(ValueType.STRING, 10), false); + } + + @Test(expected = DMLRuntimeException.class) + public void invalidConstrucOptionalString3() { + new OptionalArray<>(ArrayFactory.allocate(ValueType.STRING, 10), new BooleanArray(new boolean[10])); + } + + @Test(expected = NumberFormatException.class) + public void parseLong() { + LongArray.parseLong("notANumber"); + } + + @Test(expected = NumberFormatException.class) + public void parseInt() { + IntegerArray.parseInt("notANumber"); + } + + @Test(expected = NotImplementedException.class) + public void optionalChangeToUInt8() { + new OptionalArray<>(new Double[3]).changeTypeWithNulls(ValueType.UINT8); + } + + @Test(expected = NotImplementedException.class) + public void byteArrayString(){ + new StringArray(new String[10]).getAsByteArray(); + } }
