>From Preetham Poluparthi <[email protected]>:
Preetham Poluparthi has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20470?usp=email )
Change subject: WIP: Storage debug
......................................................................
WIP: Storage debug
Change-Id: I87e8c54deb257fa28ff7b2730adcb682783a2338
---
M asterixdb/asterix-app/src/test/resources/cc-main.conf
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/AbstractColumnValuesWriter.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/BooleanColumnValuesWriter.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/DoubleColumnValuesWriter.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/FloatColumnValuesWriter.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/LongColumnValuesWriter.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/NullMissingColumnValuesWriter.java
M
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/StringColumnValuesWriter.java
8 files changed, 95 insertions(+), 1 deletion(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/70/20470/1
diff --git a/asterixdb/asterix-app/src/test/resources/cc-main.conf
b/asterixdb/asterix-app/src/test/resources/cc-main.conf
index cdc3e6b..f5a3dfe 100644
--- a/asterixdb/asterix-app/src/test/resources/cc-main.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc-main.conf
@@ -41,6 +41,7 @@
jvm.args=-Xmx4096m
-Dnode.Resolver="org.apache.asterix.external.util.IdentitiyResolverFactory"
storage.buffercache.size=128MB
storage.memorycomponent.globalbudget=512MB
+storage.max.columns.in.zeroth.segment=800
[cc]
address = 127.0.0.1
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/AbstractColumnValuesWriter.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/AbstractColumnValuesWriter.java
index accf4a0..5dd9d78 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/AbstractColumnValuesWriter.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/AbstractColumnValuesWriter.java
@@ -20,8 +20,12 @@
import java.io.DataInput;
import java.io.DataOutput;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.List;
import
org.apache.asterix.column.bytes.encoder.ParquetRunLengthBitPackingHybridEncoder;
import org.apache.asterix.column.util.ColumnValuesUtil;
@@ -40,11 +44,12 @@
import org.apache.parquet.bytes.BytesUtils;
public abstract class AbstractColumnValuesWriter implements
IColumnValuesWriter {
- private static final Logger LOGGER = LogManager.getLogger();
+ private static final Logger LOGGER =
LogManager.getLogger(AbstractColumnValuesWriter.class);
// For 3 integers (count, defSize, and valueSize)
private static final int COUNT_DEF_SIZE_VALUE_SIZE = Integer.BYTES * 3;
protected final AbstractColumnFilterWriter filterWriter;
protected final ParquetRunLengthBitPackingHybridEncoder definitionLevels;
+ protected final List<Integer> definitionLevelsList = new
java.util.ArrayList<>();
protected final int level;
private final int columnIndex;
@@ -111,6 +116,7 @@
try {
for (int i = 0; i < count; i++) {
definitionLevels.writeInt(level);
+ definitionLevelsList.add(level);
}
} catch (IOException e) {
throw HyracksDataException.create(e);
@@ -174,9 +180,17 @@
public final void flush(OutputStream out) throws HyracksDataException {
BytesInput values;
BytesInput defLevelBytes;
+
+ LOGGER.info("@{} Flushing columnIndex {} at level {} with null bit
mask {}",this.hashCode() ,columnIndex, level, nullBitMask);
+
try {
BytesUtils.writeZigZagVarInt(level, out);
defLevelBytes = definitionLevels.toBytes();
+
+ LOGGER.info("@{} LEVEL: {}, COUNT: {}",this.hashCode(), level,
count);
+ LOGGER.info("@{} DEFINITION LEVELS: {}",this.hashCode()
,definitionLevelsList);
+ LOGGER.info("@{} VALUES: {}",this.hashCode(), getValuesString());
+
BytesUtils.writeZigZagVarInt((int) defLevelBytes.size(), out);
BytesUtils.writeZigZagVarInt(count, out);
defLevelBytes.writeAllTo(out);
@@ -230,6 +244,7 @@
try {
writeValues = writeValues || this.level == level;
definitionLevels.writeInt(level);
+ definitionLevelsList.add(level);
count++;
} catch (IOException e) {
throw HyracksDataException.create(e);
@@ -255,4 +270,7 @@
protected abstract void resetValues() throws HyracksDataException;
protected abstract void closeValues();
+
+ protected abstract String getValuesString();
+
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/BooleanColumnValuesWriter.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/BooleanColumnValuesWriter.java
index 7d50cb1..5cba64a 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/BooleanColumnValuesWriter.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/BooleanColumnValuesWriter.java
@@ -19,6 +19,7 @@
package org.apache.asterix.column.values.writer;
import java.io.IOException;
+import java.util.List;
import
org.apache.asterix.column.bytes.encoder.ParquetRunLengthBitPackingHybridEncoder;
import org.apache.asterix.column.values.IColumnValuesReader;
@@ -30,6 +31,10 @@
public final class BooleanColumnValuesWriter extends
AbstractColumnValuesWriter {
private final ParquetRunLengthBitPackingHybridEncoder booleanWriter;
+ private final List<Boolean> booleanList = new java.util.ArrayList<>();
+
+
+
public BooleanColumnValuesWriter(int columnIndex, int level, boolean
collection, boolean filtered) {
super(columnIndex, level, collection, filtered);
@@ -40,6 +45,7 @@
protected void addValue(ATypeTag tag, IValueReference value) throws
IOException {
byte booleanValue = value.getByteArray()[value.getStartOffset()];
booleanWriter.writeInt(booleanValue);
+ booleanList.add(booleanValue!=0);
filterWriter.addLong(booleanValue);
}
@@ -72,6 +78,7 @@
protected void addValue(IColumnValuesReader reader) throws IOException {
int value = reader.getBoolean() ? 1 : 0;
booleanWriter.writeInt(value);
+ booleanList.add(value!=0);
filterWriter.addLong(value);
}
@@ -86,6 +93,11 @@
}
@Override
+ protected String getValuesString() {
+ return booleanList.toString();
+ }
+
+ @Override
protected ATypeTag getTypeTag() {
return ATypeTag.BOOLEAN;
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/DoubleColumnValuesWriter.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/DoubleColumnValuesWriter.java
index 9e6f906..52c7f4a 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/DoubleColumnValuesWriter.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/DoubleColumnValuesWriter.java
@@ -18,7 +18,11 @@
*/
package org.apache.asterix.column.values.writer;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import
org.apache.asterix.column.bytes.encoder.ParquetPlainFixedLengthValuesWriter;
import org.apache.asterix.column.values.IColumnValuesReader;
@@ -38,6 +42,7 @@
public final class DoubleColumnValuesWriter extends AbstractColumnValuesWriter
{
private final ParquetPlainFixedLengthValuesWriter doubleWriter;
+ private final List<Double> doubleList = new ArrayList<>();
public DoubleColumnValuesWriter(Mutable<IColumnWriteMultiPageOp>
multiPageOpRef, int columnIndex, int level,
boolean collection, boolean filtered) {
@@ -49,6 +54,7 @@
protected void addValue(ATypeTag tag, IValueReference value) throws
IOException {
final double normalizedDouble = getValue(tag, value.getByteArray(),
value.getStartOffset());
doubleWriter.writeDouble(normalizedDouble);
+ doubleList.add(normalizedDouble);
filterWriter.addDouble(normalizedDouble);
}
@@ -100,6 +106,7 @@
protected void addValue(IColumnValuesReader reader) throws IOException {
double value = reader.getDouble();
doubleWriter.writeDouble(value);
+ doubleList.add(value);
filterWriter.addDouble(value);
}
@@ -114,6 +121,11 @@
}
@Override
+ protected String getValuesString() {
+ return doubleList.toString();
+ }
+
+ @Override
protected ATypeTag getTypeTag() {
return ATypeTag.DOUBLE;
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/FloatColumnValuesWriter.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/FloatColumnValuesWriter.java
index 39abcad..1f05f4f 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/FloatColumnValuesWriter.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/FloatColumnValuesWriter.java
@@ -18,7 +18,11 @@
*/
package org.apache.asterix.column.values.writer;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import
org.apache.asterix.column.bytes.encoder.ParquetPlainFixedLengthValuesWriter;
import org.apache.asterix.column.values.IColumnValuesReader;
@@ -38,6 +42,7 @@
public final class FloatColumnValuesWriter extends AbstractColumnValuesWriter {
private final ParquetPlainFixedLengthValuesWriter floatWriter;
+ private final List<Float> floatList= new ArrayList<>();
public FloatColumnValuesWriter(Mutable<IColumnWriteMultiPageOp>
multiPageOpRef, int columnIndex, int level,
boolean collection, boolean filtered) {
@@ -49,6 +54,7 @@
protected void addValue(ATypeTag tag, IValueReference value) throws
IOException {
final float normalizedDouble = getValue(tag, value.getByteArray(),
value.getStartOffset());
floatWriter.writeFloat(normalizedDouble);
+ floatList.add(normalizedDouble);
filterWriter.addDouble(normalizedDouble);
}
@@ -100,6 +106,7 @@
protected void addValue(IColumnValuesReader reader) throws IOException {
float value = reader.getFloat();
floatWriter.writeFloat(value);
+ floatList.add(value);
filterWriter.addDouble(value);
}
@@ -114,6 +121,11 @@
}
@Override
+ protected String getValuesString() {
+ return floatList.toString();
+ }
+
+ @Override
protected ATypeTag getTypeTag() {
return ATypeTag.FLOAT;
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/LongColumnValuesWriter.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/LongColumnValuesWriter.java
index 516f56d..b0dd3aa9 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/LongColumnValuesWriter.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/LongColumnValuesWriter.java
@@ -20,7 +20,12 @@
import static
org.apache.asterix.column.util.ColumnValuesUtil.getNormalizedTypeTag;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
import org.apache.asterix.column.bytes.encoder.AbstractParquetValuesWriter;
import
org.apache.asterix.column.bytes.encoder.ParquetDeltaBinaryPackingValuesWriterForLong;
@@ -41,6 +46,7 @@
final class LongColumnValuesWriter extends AbstractColumnValuesWriter {
private final AbstractParquetValuesWriter longWriter;
private final ATypeTag typeTag;
+ private final List<Long> longList = new ArrayList<>();
public LongColumnValuesWriter(Mutable<IColumnWriteMultiPageOp>
multiPageOpRef, int columnIndex, int level,
boolean collection, boolean filtered, ATypeTag typeTag) {
@@ -55,6 +61,7 @@
protected void addValue(ATypeTag tag, IValueReference value) {
final long normalizedInt = getValue(tag, value.getByteArray(),
value.getStartOffset());
longWriter.writeLong(normalizedInt);
+ longList.add(normalizedInt);
filterWriter.addLong(normalizedInt);
}
@@ -102,6 +109,7 @@
protected void addValue(IColumnValuesReader reader) throws IOException {
long value = reader.getLong();
longWriter.writeLong(value);
+ longList.add(value);
filterWriter.addLong(value);
}
@@ -116,6 +124,11 @@
}
@Override
+ protected String getValuesString() {
+ return longList.toString();
+ }
+
+ @Override
protected ATypeTag getTypeTag() {
return typeTag;
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/NullMissingColumnValuesWriter.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/NullMissingColumnValuesWriter.java
index 2d9f5bf..0375473 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/NullMissingColumnValuesWriter.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/NullMissingColumnValuesWriter.java
@@ -18,13 +18,19 @@
*/
package org.apache.asterix.column.values.writer;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.util.List;
+import java.util.UUID;
+import jnr.ffi.annotations.In;
import org.apache.asterix.column.util.RunLengthIntArray;
import org.apache.asterix.column.values.IColumnValuesReader;
import
org.apache.asterix.column.values.writer.filters.AbstractColumnFilterWriter;
import org.apache.asterix.column.values.writer.filters.NoOpColumnFilterWriter;
import org.apache.asterix.om.types.ATypeTag;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.parquet.bytes.BytesInput;
@@ -32,6 +38,7 @@
public class NullMissingColumnValuesWriter extends AbstractColumnValuesWriter {
private static final BytesInput EMPTY = BytesInput.empty();
private final RunLengthIntArray defLevelsIntArray;
+ private final List<Pair<Integer, Integer>> definitionLevelsList = new
java.util.ArrayList<>();
NullMissingColumnValuesWriter(int columnIndex, int level, boolean
collection, boolean filtered) {
super(columnIndex, level, collection, filtered);
@@ -41,12 +48,14 @@
@Override
protected void addLevel(int level) throws HyracksDataException {
defLevelsIntArray.add(level);
+ definitionLevelsList.add(Pair.of(level,1));
super.addLevel(level);
}
@Override
public void writeLevels(int level, int count) throws HyracksDataException {
defLevelsIntArray.add(level, count);
+ definitionLevelsList.add(Pair.of(level,count));
super.writeLevels(level, count);
}
@@ -101,6 +110,11 @@
}
@Override
+ protected String getValuesString() {
+ return definitionLevelsList.toString();
+ }
+
+ @Override
public RunLengthIntArray getDefinitionLevelsIntArray() {
return defLevelsIntArray;
}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/StringColumnValuesWriter.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/StringColumnValuesWriter.java
index 5b1977f..d9c0d47 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/StringColumnValuesWriter.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/writer/StringColumnValuesWriter.java
@@ -18,7 +18,11 @@
*/
package org.apache.asterix.column.values.writer;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.asterix.column.bytes.encoder.AbstractParquetValuesWriter;
import org.apache.asterix.column.bytes.encoder.ParquetDeltaByteArrayWriter;
@@ -35,6 +39,7 @@
public class StringColumnValuesWriter extends AbstractColumnValuesWriter {
private final AbstractParquetValuesWriter stringWriter;
+ private final List<String> stringList = new ArrayList<>();
private final boolean skipLengthBytes;
public StringColumnValuesWriter(Mutable<IColumnWriteMultiPageOp>
multiPageOpRef, int columnIndex, int level,
@@ -53,6 +58,7 @@
@Override
protected final void addValue(ATypeTag tag, IValueReference value) throws
IOException {
stringWriter.writeBytes(value, skipLengthBytes);
+ stringList.add(new String(value.getByteArray(),
value.getStartOffset(), value.getLength()));
filterWriter.addValue(value);
}
@@ -86,6 +92,7 @@
IValueReference value = reader.getBytes();
stringWriter.writeBytes(value, skipLengthBytes);
filterWriter.addValue(value);
+ stringList.add(new String(value.getByteArray(),
value.getStartOffset(), value.getLength()));
}
@Override
@@ -99,6 +106,11 @@
}
@Override
+ protected String getValuesString() {
+ return stringList.toString();
+ }
+
+ @Override
protected ATypeTag getTypeTag() {
return ATypeTag.STRING;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20470?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I87e8c54deb257fa28ff7b2730adcb682783a2338
Gerrit-Change-Number: 20470
Gerrit-PatchSet: 1
Gerrit-Owner: Preetham Poluparthi <[email protected]>
