This is an automated email from the ASF dual-hosted git repository.
ashingau pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 26818de9c8d [feature](jni) support complex types in jni framework
(#24810)
26818de9c8d is described below
commit 26818de9c8d7871bce044618ce89e93744029365
Author: Ashin Gau <[email protected]>
AuthorDate: Wed Sep 27 14:47:41 2023 +0800
[feature](jni) support complex types in jni framework (#24810)
Support complex types in jni framework, and successfully run end-to-end on
hudi.
### How to Use
Other scanners only need to implement three interfaces in `ColumnValue`:
```
// Get array elements and append into values
void unpackArray(List<ColumnValue> values);
// Get map key array&value array, and append into keys&values
void unpackMap(List<ColumnValue> keys, List<ColumnValue> values);
// Get the struct fields specified by `structFieldIndex`, and append into
values
void unpackStruct(List<Integer> structFieldIndex, List<ColumnValue> values);
```
Developers can take `HudiColumnValue` as an example.
---
be/src/vec/exec/jni_connector.cpp | 259 +++++++++++++++------
be/src/vec/exec/jni_connector.h | 12 +
.../org/apache/doris/hudi/HudiColumnValue.java | 114 ++++-----
.../java/org/apache/doris/hudi/HudiJniScanner.java | 2 +-
.../apache/doris/common/jni/MockJniScanner.java | 33 ++-
.../apache/doris/common/jni/vec/VectorColumn.java | 204 ++++++++++++++--
.../apache/doris/common/jni/vec/VectorTable.java | 2 +-
.../apache/doris/common/jni/JniScannerTest.java | 7 +-
.../data/external_table_p2/hive/test_hive_hudi.out | 102 ++++++++
.../external_table_p2/hive/test_hive_hudi.groovy | 2 +
10 files changed, 572 insertions(+), 165 deletions(-)
diff --git a/be/src/vec/exec/jni_connector.cpp
b/be/src/vec/exec/jni_connector.cpp
index bad7f52cc3b..88b860f64b1 100644
--- a/be/src/vec/exec/jni_connector.cpp
+++ b/be/src/vec/exec/jni_connector.cpp
@@ -18,7 +18,6 @@
#include "jni_connector.h"
#include <glog/logging.h>
-#include <stdint.h>
#include <sstream>
#include <variant>
@@ -27,13 +26,17 @@
#include "runtime/decimalv2_value.h"
#include "runtime/runtime_state.h"
#include "util/jni-util.h"
-#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_string.h"
+#include "vec/columns/column_struct.h"
#include "vec/core/block.h"
-#include "vec/core/column_with_type_and_name.h"
#include "vec/core/types.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_map.h"
#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_struct.h"
namespace doris {
class RuntimeProfile;
@@ -301,6 +304,12 @@ Status JniConnector::_fill_column(ColumnPtr& doris_column,
DataTypePtr& data_typ
[[fallthrough]];
case TypeIndex::FixedString:
return _fill_string_column(data_column, num_rows);
+ case TypeIndex::Array:
+ return _fill_array_column(data_column, data_type, num_rows);
+ case TypeIndex::Map:
+ return _fill_map_column(data_column, data_type, num_rows);
+ case TypeIndex::Struct:
+ return _fill_struct_column(data_column, data_type, num_rows);
default:
return Status::InvalidArgument("Unsupported type {} in jni scanner",
getTypeName(logical_type));
@@ -308,17 +317,88 @@ Status JniConnector::_fill_column(ColumnPtr&
doris_column, DataTypePtr& data_typ
return Status::OK();
}
+Status JniConnector::_fill_array_column(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
+ size_t num_rows) {
+ ColumnPtr& element_column =
static_cast<ColumnArray&>(*doris_column).get_data_ptr();
+ DataTypePtr& element_type = const_cast<DataTypePtr&>(
+ (reinterpret_cast<const
DataTypeArray*>(remove_nullable(data_type).get()))
+ ->get_nested_type());
+ ColumnArray::Offsets64& offsets_data =
static_cast<ColumnArray&>(*doris_column).get_offsets();
+
+ int64* offsets = reinterpret_cast<int64*>(_next_meta_as_ptr());
+ size_t origin_size = offsets_data.size();
+ offsets_data.resize(origin_size + num_rows);
+ size_t start_offset = offsets_data[origin_size - 1];
+ for (size_t i = 0; i < num_rows; ++i) {
+ offsets_data[origin_size + i] = offsets[i] + start_offset;
+ }
+
+ // offsets[num_rows - 1] == offsets_data[origin_size + num_rows - 1] -
start_offset
+ // but num_row equals 0 when there are all empty arrays
+ return _fill_column(element_column, element_type,
+ offsets_data[origin_size + num_rows - 1] -
start_offset);
+}
+
+Status JniConnector::_fill_map_column(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
+ size_t num_rows) {
+ auto& map = static_cast<ColumnMap&>(*doris_column);
+ DataTypePtr& key_type = const_cast<DataTypePtr&>(
+ reinterpret_cast<const
DataTypeMap*>(remove_nullable(data_type).get())->get_key_type());
+ DataTypePtr& value_type = const_cast<DataTypePtr&>(
+ reinterpret_cast<const
DataTypeMap*>(remove_nullable(data_type).get())
+ ->get_value_type());
+ ColumnPtr& key_column = map.get_keys_ptr();
+ ColumnPtr& value_column = map.get_values_ptr();
+ ColumnArray::Offsets64& map_offsets = map.get_offsets();
+
+ int64* offsets = reinterpret_cast<int64*>(_next_meta_as_ptr());
+ size_t origin_size = map_offsets.size();
+ map_offsets.resize(origin_size + num_rows);
+ size_t start_offset = map_offsets[origin_size - 1];
+ for (size_t i = 0; i < num_rows; ++i) {
+ map_offsets[origin_size + i] = offsets[i] + start_offset;
+ }
+
+ RETURN_IF_ERROR(_fill_column(key_column, key_type,
+ map_offsets[origin_size + num_rows - 1] -
start_offset));
+ return _fill_column(value_column, value_type,
+ map_offsets[origin_size + num_rows - 1] -
start_offset);
+}
+
+Status JniConnector::_fill_struct_column(MutableColumnPtr& doris_column,
DataTypePtr& data_type,
+ size_t num_rows) {
+ auto& doris_struct = static_cast<ColumnStruct&>(*doris_column);
+ const DataTypeStruct* doris_struct_type =
+ reinterpret_cast<const
DataTypeStruct*>(remove_nullable(data_type).get());
+ for (int i = 0; i < doris_struct.tuple_size(); ++i) {
+ ColumnPtr& struct_field = doris_struct.get_column_ptr(i);
+ DataTypePtr& field_type =
const_cast<DataTypePtr&>(doris_struct_type->get_element(i));
+ RETURN_IF_ERROR(_fill_column(struct_field, field_type, num_rows));
+ }
+ return Status::OK();
+}
+
Status JniConnector::_fill_string_column(MutableColumnPtr& doris_column,
size_t num_rows) {
+ if (num_rows == 0) {
+ return Status::OK();
+ }
+ auto& string_col = static_cast<const ColumnString&>(*doris_column);
+ ColumnString::Chars& string_chars =
const_cast<ColumnString::Chars&>(string_col.get_chars());
+ ColumnString::Offsets& string_offsets =
+ const_cast<ColumnString::Offsets&>(string_col.get_offsets());
int* offsets = reinterpret_cast<int*>(_next_meta_as_ptr());
- char* data = reinterpret_cast<char*>(_next_meta_as_ptr());
- std::vector<StringRef> string_values;
- string_values.reserve(num_rows);
+ char* chars = reinterpret_cast<char*>(_next_meta_as_ptr());
+
+ size_t origin_chars_size = string_chars.size();
+ string_chars.resize(origin_chars_size + offsets[num_rows - 1]);
+ memcpy(string_chars.data() + origin_chars_size, chars, offsets[num_rows -
1]);
+
+ size_t origin_offsets_size = string_offsets.size();
+ size_t start_offset = string_offsets[origin_offsets_size - 1];
+ string_offsets.resize(origin_offsets_size + num_rows);
for (size_t i = 0; i < num_rows; ++i) {
- int start_offset = i == 0 ? 0 : offsets[i - 1];
- int end_offset = offsets[i];
- string_values.emplace_back(data + start_offset, end_offset -
start_offset);
+ string_offsets[origin_offsets_size + i] = offsets[i] + start_offset;
}
- doris_column->insert_many_strings(&string_values[0], num_rows);
return Status::OK();
}
@@ -418,77 +498,108 @@ std::string JniConnector::get_hive_type(const
TypeDescriptor& desc) {
}
}
-Status JniConnector::generate_meta_info(Block* block, std::unique_ptr<long[]>&
meta) {
- std::vector<long> meta_data;
- // insert number of rows
- meta_data.emplace_back(block->rows());
- for (int i = 0; i < block->columns(); ++i) {
- auto& column_with_type_and_name = block->get_by_position(i);
- auto& column_ptr = column_with_type_and_name.column;
- auto& column_type = column_with_type_and_name.type;
- TypeIndex logical_type = remove_nullable(column_type)->get_type_id();
-
- // insert null map address
- MutableColumnPtr data_column;
- if (column_ptr->is_nullable()) {
- auto* nullable_column =
reinterpret_cast<vectorized::ColumnNullable*>(
- column_ptr->assume_mutable().get());
- data_column = nullable_column->get_nested_column_ptr();
- NullMap& null_map = nullable_column->get_null_map_data();
- meta_data.emplace_back((long)null_map.data());
- } else {
- meta_data.emplace_back(0);
- data_column = column_ptr->assume_mutable();
- }
-
- switch (logical_type) {
+void JniConnector::_fill_column_meta(ColumnPtr& doris_column, DataTypePtr&
data_type,
+ std::vector<long>& meta_data) {
+ TypeIndex logical_type = remove_nullable(data_type)->get_type_id();
+ // insert null map address
+ MutableColumnPtr data_column;
+ if (doris_column->is_nullable()) {
+ auto* nullable_column =
+
reinterpret_cast<vectorized::ColumnNullable*>(doris_column->assume_mutable().get());
+ data_column = nullable_column->get_nested_column_ptr();
+ NullMap& null_map = nullable_column->get_null_map_data();
+ meta_data.emplace_back((long)null_map.data());
+ } else {
+ meta_data.emplace_back(0);
+ data_column = doris_column->assume_mutable();
+ }
+ switch (logical_type) {
#define DISPATCH(NUMERIC_TYPE, CPP_NUMERIC_TYPE)
\
case NUMERIC_TYPE: {
\
meta_data.emplace_back(_get_numeric_data_address<CPP_NUMERIC_TYPE>(data_column));
\
break;
\
}
- FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
+ FOR_LOGICAL_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
- case TypeIndex::Decimal128:
- [[fallthrough]];
- case TypeIndex::Decimal128I: {
-
meta_data.emplace_back(_get_decimal_data_address<Int128>(data_column));
- break;
- }
- case TypeIndex::Decimal32: {
-
meta_data.emplace_back(_get_decimal_data_address<Int32>(data_column));
- break;
- }
- case TypeIndex::Decimal64: {
-
meta_data.emplace_back(_get_decimal_data_address<Int64>(data_column));
- break;
- }
- case TypeIndex::DateV2: {
-
meta_data.emplace_back(_get_time_data_address<UInt32>(data_column));
- break;
- }
- case TypeIndex::DateTimeV2: {
-
meta_data.emplace_back(_get_time_data_address<UInt64>(data_column));
- break;
- }
- case TypeIndex::String:
- [[fallthrough]];
- case TypeIndex::FixedString: {
- auto& string_column = static_cast<ColumnString&>(*data_column);
- // inert offsets
- meta_data.emplace_back((long)string_column.get_offsets().data());
- meta_data.emplace_back((long)string_column.get_chars().data());
- break;
- }
- case TypeIndex::Array:
- [[fallthrough]];
- case TypeIndex::Struct:
- [[fallthrough]];
- case TypeIndex::Map:
- return Status::IOError("Unhandled type {}",
getTypeName(logical_type));
- default:
- return Status::IOError("Unsupported type {}",
getTypeName(logical_type));
+ case TypeIndex::Decimal128:
+ [[fallthrough]];
+ case TypeIndex::Decimal128I: {
+ meta_data.emplace_back(_get_decimal_data_address<Int128>(data_column));
+ break;
+ }
+ case TypeIndex::Decimal32: {
+ meta_data.emplace_back(_get_decimal_data_address<Int32>(data_column));
+ break;
+ }
+ case TypeIndex::Decimal64: {
+ meta_data.emplace_back(_get_decimal_data_address<Int64>(data_column));
+ break;
+ }
+ case TypeIndex::DateV2: {
+ meta_data.emplace_back(_get_time_data_address<UInt32>(data_column));
+ break;
+ }
+ case TypeIndex::DateTimeV2: {
+ meta_data.emplace_back(_get_time_data_address<UInt64>(data_column));
+ break;
+ }
+ case TypeIndex::String:
+ [[fallthrough]];
+ case TypeIndex::FixedString: {
+ auto& string_column = static_cast<ColumnString&>(*data_column);
+ // inert offsets
+ meta_data.emplace_back((long)string_column.get_offsets().data());
+ meta_data.emplace_back((long)string_column.get_chars().data());
+ break;
+ }
+ case TypeIndex::Array: {
+ ColumnPtr& element_column =
static_cast<ColumnArray&>(*data_column).get_data_ptr();
+
meta_data.emplace_back((long)static_cast<ColumnArray&>(*data_column).get_offsets().data());
+ DataTypePtr& element_type = const_cast<DataTypePtr&>(
+ (reinterpret_cast<const
DataTypeArray*>(remove_nullable(data_type).get()))
+ ->get_nested_type());
+ _fill_column_meta(element_column, element_type, meta_data);
+ break;
+ }
+ case TypeIndex::Struct: {
+ auto& doris_struct = static_cast<ColumnStruct&>(*data_column);
+ const DataTypeStruct* doris_struct_type =
+ reinterpret_cast<const
DataTypeStruct*>(remove_nullable(data_type).get());
+ for (int i = 0; i < doris_struct.tuple_size(); ++i) {
+ ColumnPtr& struct_field = doris_struct.get_column_ptr(i);
+ DataTypePtr& field_type =
const_cast<DataTypePtr&>(doris_struct_type->get_element(i));
+ _fill_column_meta(struct_field, field_type, meta_data);
}
+ break;
+ }
+ case TypeIndex::Map: {
+ auto& map = static_cast<ColumnMap&>(*data_column);
+ DataTypePtr& key_type = const_cast<DataTypePtr&>(
+ reinterpret_cast<const
DataTypeMap*>(remove_nullable(data_type).get())
+ ->get_key_type());
+ DataTypePtr& value_type = const_cast<DataTypePtr&>(
+ reinterpret_cast<const
DataTypeMap*>(remove_nullable(data_type).get())
+ ->get_value_type());
+ ColumnPtr& key_column = map.get_keys_ptr();
+ ColumnPtr& value_column = map.get_values_ptr();
+ meta_data.emplace_back((long)map.get_offsets().data());
+ _fill_column_meta(key_column, key_type, meta_data);
+ _fill_column_meta(value_column, value_type, meta_data);
+ break;
+ }
+ default:
+ return;
+ }
+}
+
+Status JniConnector::generate_meta_info(Block* block, std::unique_ptr<long[]>&
meta) {
+ std::vector<long> meta_data;
+ // insert number of rows
+ meta_data.emplace_back(block->rows());
+ for (int i = 0; i < block->columns(); ++i) {
+ auto& column_with_type_and_name = block->get_by_position(i);
+ _fill_column_meta(column_with_type_and_name.column,
column_with_type_and_name.type,
+ meta_data);
}
meta.reset(new long[meta_data.size()]);
diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h
index 1cadc37a1b0..969dd14723d 100644
--- a/be/src/vec/exec/jni_connector.h
+++ b/be/src/vec/exec/jni_connector.h
@@ -298,6 +298,18 @@ private:
Status _fill_column(ColumnPtr& doris_column, DataTypePtr& data_type,
size_t num_rows);
+ Status _fill_map_column(MutableColumnPtr& doris_column, DataTypePtr&
data_type,
+ size_t num_rows);
+
+ Status _fill_array_column(MutableColumnPtr& doris_column, DataTypePtr&
data_type,
+ size_t num_rows);
+
+ Status _fill_struct_column(MutableColumnPtr& doris_column, DataTypePtr&
data_type,
+ size_t num_rows);
+
+ static void _fill_column_meta(ColumnPtr& doris_column, DataTypePtr&
data_type,
+ std::vector<long>& meta_data);
+
template <typename CppType>
Status _fill_numeric_column(MutableColumnPtr& doris_column, CppType* ptr,
size_t num_rows) {
auto& column_data =
static_cast<ColumnVector<CppType>&>(*doris_column).get_data();
diff --git
a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java
b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java
index 7d402e84292..1c489affe16 100644
---
a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java
+++
b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java
@@ -17,13 +17,13 @@
package org.apache.doris.hudi;
-
import org.apache.doris.common.jni.vec.ColumnType;
import org.apache.doris.common.jni.vec.ColumnValue;
-import org.apache.doris.common.jni.vec.NativeColumnValue;
import org.apache.spark.sql.catalyst.InternalRow;
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow;
+import org.apache.spark.sql.catalyst.expressions.SpecializedGetters;
+import org.apache.spark.sql.catalyst.util.ArrayData;
+import org.apache.spark.sql.catalyst.util.MapData;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -33,41 +33,33 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.List;
-public class HudiColumnValue implements ColumnValue, NativeColumnValue {
- private boolean isUnsafe;
- private InternalRow internalRow;
+public class HudiColumnValue implements ColumnValue {
+ private SpecializedGetters data;
private int ordinal;
- private int precision;
- private int scale;
+ private ColumnType columnType;
HudiColumnValue() {
}
- HudiColumnValue(InternalRow internalRow, int ordinal, int precision, int
scale) {
- this.isUnsafe = internalRow instanceof UnsafeRow;
- this.internalRow = internalRow;
+ HudiColumnValue(SpecializedGetters data, int ordinal, ColumnType
columnType) {
+ this.data = data;
this.ordinal = ordinal;
- this.precision = precision;
- this.scale = scale;
+ this.columnType = columnType;
}
- public void reset(InternalRow internalRow, int ordinal, int precision, int
scale) {
- this.isUnsafe = internalRow instanceof UnsafeRow;
- this.internalRow = internalRow;
+ public void reset(SpecializedGetters data, int ordinal, ColumnType
columnType) {
+ this.data = data;
this.ordinal = ordinal;
- this.precision = precision;
- this.scale = scale;
+ this.columnType = columnType;
}
- public void reset(int ordinal, int precision, int scale) {
+ public void reset(int ordinal, ColumnType columnType) {
this.ordinal = ordinal;
- this.precision = precision;
- this.scale = scale;
+ this.columnType = columnType;
}
- public void reset(InternalRow internalRow) {
- this.isUnsafe = internalRow instanceof UnsafeRow;
- this.internalRow = internalRow;
+ public void reset(SpecializedGetters data) {
+ this.data = data;
}
@Override
@@ -77,42 +69,42 @@ public class HudiColumnValue implements ColumnValue,
NativeColumnValue {
@Override
public boolean isNull() {
- return internalRow.isNullAt(ordinal);
+ return data.isNullAt(ordinal);
}
@Override
public boolean getBoolean() {
- return internalRow.getBoolean(ordinal);
+ return data.getBoolean(ordinal);
}
@Override
public byte getByte() {
- return internalRow.getByte(ordinal);
+ return data.getByte(ordinal);
}
@Override
public short getShort() {
- return internalRow.getShort(ordinal);
+ return data.getShort(ordinal);
}
@Override
public int getInt() {
- return internalRow.getInt(ordinal);
+ return data.getInt(ordinal);
}
@Override
public float getFloat() {
- return internalRow.getFloat(ordinal);
+ return data.getFloat(ordinal);
}
@Override
public long getLong() {
- return internalRow.getLong(ordinal);
+ return data.getLong(ordinal);
}
@Override
public double getDouble() {
- return internalRow.getDouble(ordinal);
+ return data.getDouble(ordinal);
}
@Override
@@ -122,78 +114,74 @@ public class HudiColumnValue implements ColumnValue,
NativeColumnValue {
@Override
public BigDecimal getDecimal() {
- return internalRow.getDecimal(ordinal, precision,
scale).toJavaBigDecimal();
+ return data.getDecimal(ordinal, columnType.getPrecision(),
columnType.getScale()).toJavaBigDecimal();
}
@Override
public String getString() {
- return internalRow.getUTF8String(ordinal).toString();
+ return data.getUTF8String(ordinal).toString();
}
@Override
public byte[] getStringAsBytes() {
- return internalRow.getUTF8String(ordinal).getBytes();
+ return data.getUTF8String(ordinal).getBytes();
}
@Override
public LocalDate getDate() {
- return LocalDate.ofEpochDay(internalRow.getInt(ordinal));
+ return LocalDate.ofEpochDay(data.getInt(ordinal));
}
@Override
public LocalDateTime getDateTime() {
- long datetime = internalRow.getLong(ordinal);
+ long datetime = data.getLong(ordinal);
long seconds;
long nanoseconds;
- if (precision == 3) {
+ if (columnType.getPrecision() == 3) {
seconds = datetime / 1000;
nanoseconds = (datetime % 1000) * 1000000;
- } else if (precision == 6) {
+ } else if (columnType.getPrecision() == 6) {
seconds = datetime / 1000000;
nanoseconds = (datetime % 1000000) * 1000;
} else {
- throw new RuntimeException("Hoodie timestamp only support
milliseconds and microseconds");
+ throw new RuntimeException("Hoodie timestamp only support
milliseconds and microseconds, wrong precision = "
+ + columnType.getPrecision());
}
return LocalDateTime.ofInstant(Instant.ofEpochSecond(seconds,
nanoseconds), ZoneId.systemDefault());
}
@Override
public byte[] getBytes() {
- return internalRow.getBinary(ordinal);
+ return data.getBinary(ordinal);
}
@Override
public void unpackArray(List<ColumnValue> values) {
-
+ ArrayData array = data.getArray(ordinal);
+ for (int i = 0; i < array.numElements(); ++i) {
+ values.add(new HudiColumnValue(array, i,
columnType.getChildTypes().get(0)));
+ }
}
@Override
public void unpackMap(List<ColumnValue> keys, List<ColumnValue> values) {
-
+ MapData map = data.getMap(ordinal);
+ ArrayData key = map.keyArray();
+ for (int i = 0; i < key.numElements(); ++i) {
+ keys.add(new HudiColumnValue(key, i,
columnType.getChildTypes().get(0)));
+ }
+ ArrayData value = map.valueArray();
+ for (int i = 0; i < value.numElements(); ++i) {
+ values.add(new HudiColumnValue(value, i,
columnType.getChildTypes().get(1)));
+ }
}
@Override
public void unpackStruct(List<Integer> structFieldIndex, List<ColumnValue>
values) {
-
- }
-
- @Override
- public NativeValue getNativeValue(ColumnType.Type type) {
- if (isUnsafe) {
- UnsafeRow unsafeRow = (UnsafeRow) internalRow;
- switch (type) {
- case CHAR:
- case VARCHAR:
- case BINARY:
- case STRING:
- long offsetAndSize = unsafeRow.getLong(ordinal);
- int offset = (int) (offsetAndSize >> 32);
- int size = (int) offsetAndSize;
- return new NativeValue(unsafeRow.getBaseObject(), offset,
size);
- default:
- return null;
- }
+ // todo: support pruned struct fields
+ InternalRow struct = data.getStruct(ordinal, structFieldIndex.size());
+ for (int i : structFieldIndex) {
+ values.add(new HudiColumnValue(struct, i,
columnType.getChildTypes().get(i)));
}
- return null;
}
}
diff --git
a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
index 64c4fd70e7b..daf8d4a21fa 100644
---
a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
+++
b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
@@ -224,7 +224,7 @@ public class HudiJniScanner extends JniScanner {
while (readRowNumbers < fetchSize && recordIterator.hasNext()) {
columnValue.reset(recordIterator.next());
for (int i = 0; i < numFields; i++) {
- columnValue.reset(i, columnTypes[i].getPrecision(),
columnTypes[i].getScale());
+ columnValue.reset(i, columnTypes[i]);
appendData(i, columnValue);
}
readRowNumbers++;
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java
index 3557b3b9032..bc7561e2a23 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java
@@ -42,6 +42,10 @@ public class MockJniScanner extends JniScanner {
private int i;
private int j;
+ public MockColumnValue(int i, int j) {
+ set(i, j);
+ }
+
public MockColumnValue() {
}
@@ -132,17 +136,40 @@ public class MockJniScanner extends JniScanner {
@Override
public void unpackArray(List<ColumnValue> values) {
-
+ for (int m = 1; m < i; ++m) {
+ if (m % 3 == 0) {
+ values.add(null);
+ } else {
+ values.add(new MockColumnValue(i, j + m));
+ }
+ }
}
@Override
public void unpackMap(List<ColumnValue> keys, List<ColumnValue>
values) {
-
+ for (int m = 0; m < i; ++m) {
+ values.add(new MockColumnValue(i + m, j));
+ }
+ for (int m = 0; m < i; ++m) {
+ if (m % 3 == 0) {
+ values.add(null);
+ } else {
+ values.add(new MockColumnValue(i, j + m));
+ }
+ }
}
@Override
public void unpackStruct(List<Integer> structFieldIndex,
List<ColumnValue> values) {
-
+ structFieldIndex.clear();
+ structFieldIndex.add(0);
+ structFieldIndex.add(1);
+ if ((i + j) % 4 == 0) {
+ values.add(null);
+ } else {
+ values.add(new MockColumnValue(i, j));
+ }
+ values.add(new MockColumnValue(i, j + 3));
}
}
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
index 3998a1a3270..6d0ede0a4f3 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
@@ -28,6 +28,8 @@ import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
/**
@@ -46,6 +48,9 @@ public class VectorColumn {
// For String / Array / Map.
private long offsets;
+ // String's offset is int32, while Array&Map's offset is int64
+ // todo: how to solve the overflow of offsets when taking Array&Map's
offset as int32
+ private boolean isComplexType = false;
// Number of elements in vector column
private int capacity;
// Upper limit for the maximum capacity for this column.
@@ -59,6 +64,10 @@ public class VectorColumn {
// For nested column type: String / Array/ Map / Struct
private VectorColumn[] childColumns;
+ // For struct, only support to read all fields in struct now
+ // todo: support pruned struct fields
+ private List<Integer> structFieldIndex;
+
public VectorColumn(ColumnType columnType, int capacity) {
this.columnType = columnType;
this.capacity = 0;
@@ -68,11 +77,18 @@ public class VectorColumn {
this.numNulls = 0;
this.appendIndex = 0;
if (columnType.isComplexType()) {
+ isComplexType = true;
List<ColumnType> children = columnType.getChildTypes();
childColumns = new VectorColumn[children.size()];
for (int i = 0; i < children.size(); ++i) {
childColumns[i] = new VectorColumn(children.get(i), capacity);
}
+ if (columnType.isStruct()) {
+ structFieldIndex = new ArrayList<>();
+ for (int i = 0; i < children.size(); ++i) {
+ structFieldIndex.add(i);
+ }
+ }
} else if (columnType.isStringType()) {
childColumns = new VectorColumn[1];
childColumns[0] = new VectorColumn(new ColumnType("#stringBytes",
Type.BYTE),
@@ -91,6 +107,13 @@ public class VectorColumn {
this.offsets = 0;
this.numNulls = 0;
this.appendIndex = capacity;
+ if (columnType.isStruct()) {
+ List<ColumnType> children = columnType.getChildTypes();
+ structFieldIndex = new ArrayList<>();
+ for (int i = 0; i < children.size(); ++i) {
+ structFieldIndex.add(i);
+ }
+ }
}
// restore block column
@@ -114,8 +137,20 @@ public class VectorColumn {
this.appendIndex = numRows;
if (columnType.isComplexType()) {
- // todo: support complex type
- throw new RuntimeException("Unhandled type: " + columnType);
+ isComplexType = true;
+ int childRows = numRows;
+ if (!columnType.isStruct()) {
+ this.offsets = OffHeap.getLong(null, address);
+ address += 8;
+ childRows = getArrayEndOffset(numRows - 1);
+ }
+ this.data = 0;
+ List<ColumnType> children = columnType.getChildTypes();
+ childColumns = new VectorColumn[children.size()];
+ for (int i = 0; i < children.size(); ++i) {
+ childColumns[i] = new VectorColumn(children.get(i), childRows,
address);
+ address += children.get(i).metaSize() * 8L;
+ }
} else if (columnType.isStringType()) {
this.offsets = OffHeap.getLong(null, address);
address += 8;
@@ -130,6 +165,19 @@ public class VectorColumn {
}
}
+ private int getArrayEndOffset(int rowId) {
+ if (rowId >= 0 && rowId < appendIndex) {
+ if (isComplexType) {
+ // maybe overflowed
+ return (int) OffHeap.getLong(null, offsets + 8L * rowId);
+ } else {
+ return OffHeap.getInt(null, offsets + 4L * rowId);
+ }
+ } else {
+ return 0;
+ }
+ }
+
public long nullMapAddress() {
return nullMap;
}
@@ -200,21 +248,21 @@ public class VectorColumn {
}
private void reserveCapacity(int newCapacity) {
+ long offsetLength = isComplexType ? 8L : 4L;
long oldCapacity = capacity;
- long oldOffsetSize = capacity * 4L;
- long newOffsetSize = newCapacity * 4L;
+ long oldOffsetSize = capacity * offsetLength;
+ long newOffsetSize = newCapacity * offsetLength;
long typeSize = columnType.getTypeSize();
if (columnType.isUnsupported()) {
// do nothing
return;
} else if (typeSize != -1) {
this.data = OffHeap.reallocateMemory(data, oldCapacity * typeSize,
newCapacity * typeSize);
- } else if (columnType.isStringType()) {
+ } else if (columnType.isStringType() || columnType.isArray() ||
columnType.isMap()) {
this.offsets = OffHeap.reallocateMemory(offsets, oldOffsetSize,
newOffsetSize);
- } else {
+ } else if (!columnType.isStruct()) {
throw new RuntimeException("Unhandled type: " + columnType);
}
- // todo: support complex type
if (!"#stringBytes".equals(columnType.getName())) {
this.nullMap = OffHeap.reallocateMemory(nullMap, oldCapacity,
newCapacity);
OffHeap.setMemory(nullMap + oldCapacity, (byte) 0, newCapacity -
oldCapacity);
@@ -292,6 +340,12 @@ public class VectorColumn {
case STRING:
case BINARY:
return appendBytesAndOffset(new byte[0]);
+ case ARRAY:
+ return appendArray(Collections.emptyList());
+ case MAP:
+ return appendMap(Collections.emptyList(),
Collections.emptyList());
+ case STRUCT:
+ return appendStruct(structFieldIndex, null);
default:
throw new RuntimeException("Unknown type value: " + typeValue);
}
@@ -530,6 +584,45 @@ public class VectorColumn {
return new String(bytes, StandardCharsets.UTF_8);
}
+ public int appendArray(List<ColumnValue> values) {
+ int length = values.size();
+ int startOffset = childColumns[0].appendIndex;
+ for (ColumnValue v : values) {
+ childColumns[0].appendValue(v);
+ }
+ reserve(appendIndex + 1);
+ OffHeap.putLong(null, offsets + 8L * appendIndex, startOffset +
length);
+ return appendIndex++;
+ }
+
+ public int appendMap(List<ColumnValue> keys, List<ColumnValue> values) {
+ int length = keys.size();
+ int startOffset = childColumns[0].appendIndex;
+ for (ColumnValue k : keys) {
+ childColumns[0].appendValue(k);
+ }
+ for (ColumnValue v : values) {
+ childColumns[1].appendValue(v);
+ }
+ reserve(appendIndex + 1);
+ OffHeap.putInt(null, offsets + 8L * appendIndex, startOffset + length);
+ return appendIndex++;
+ }
+
+ public int appendStruct(List<Integer> structFieldIndex, List<ColumnValue>
values) {
+ if (values == null) {
+ for (int i : structFieldIndex) {
+ childColumns[i].appendValue(null);
+ }
+ } else {
+ for (int i = 0; i < structFieldIndex.size(); ++i) {
+
childColumns[structFieldIndex.get(i)].appendValue(values.get(i));
+ }
+ }
+ reserve(appendIndex + 1);
+ return appendIndex++;
+ }
+
public void updateMeta(VectorColumn meta) {
if (columnType.isUnsupported()) {
meta.appendLong(0);
@@ -558,19 +651,7 @@ public class VectorColumn {
return;
}
NativeValue nativeValue = o.getNativeValue(typeValue);
- if (nativeValue == null) {
- // can't get native value, fall back to materialized value
- appendValue((ColumnValue) o);
- return;
- }
- if (nativeValue.length == -1) {
- // java origin types
- long typeSize = typeValue.size;
- reserve(appendIndex + 1);
- OffHeap.copyMemory(nativeValue.baseObject, nativeValue.offset,
- null, data + typeSize * appendIndex, typeSize);
- appendIndex++;
- } else {
+ if (nativeValue != null && columnType.isStringType()) {
int byteLength = nativeValue.length;
VectorColumn bytesColumn = childColumns[0];
int startOffset = bytesColumn.appendIndex;
@@ -580,6 +661,9 @@ public class VectorColumn {
bytesColumn.appendIndex += byteLength;
OffHeap.putInt(null, offsets + 4L * appendIndex, startOffset +
byteLength);
appendIndex++;
+ } else {
+ // can't get native value, fall back to materialized value
+ appendValue((ColumnValue) o);
}
}
@@ -639,6 +723,25 @@ public class VectorColumn {
case BINARY:
appendBytesAndOffset(o.getBytes());
break;
+ case ARRAY: {
+ List<ColumnValue> values = new ArrayList<>();
+ o.unpackArray(values);
+ appendArray(values);
+ break;
+ }
+ case MAP: {
+ List<ColumnValue> keys = new ArrayList<>();
+ List<ColumnValue> values = new ArrayList<>();
+ o.unpackMap(keys, values);
+ appendMap(keys, values);
+ break;
+ }
+ case STRUCT: {
+ List<ColumnValue> values = new ArrayList<>();
+ o.unpackStruct(structFieldIndex, values);
+ appendStruct(structFieldIndex, values);
+ break;
+ }
default:
throw new RuntimeException("Unknown type value: " + typeValue);
}
@@ -695,6 +798,67 @@ public class VectorColumn {
case BINARY:
sb.append(getStringWithOffset(i));
break;
+ case ARRAY: {
+ int begin = getArrayEndOffset(i - 1);
+ int end = getArrayEndOffset(i);
+ sb.append("[");
+ for (int rowId = begin; rowId < end; rowId++) {
+ if (rowId != begin) {
+ sb.append(",");
+ }
+ childColumns[0].dump(sb, rowId);
+ }
+ sb.append("]");
+ break;
+ }
+ case MAP: {
+ int begin = getArrayEndOffset(i - 1);
+ int end = getArrayEndOffset(i);
+ sb.append("{");
+ VectorColumn key = childColumns[0];
+ VectorColumn value = childColumns[1];
+ for (int rowId = begin; rowId < end; rowId++) {
+ if (rowId != begin) {
+ sb.append(",");
+ }
+ if (key.columnType.isStringType()) {
+ sb.append("\"");
+ }
+ key.dump(sb, rowId);
+ if (key.columnType.isStringType()) {
+ sb.append("\"");
+ }
+ sb.append(":");
+ if (value.columnType.isStringType()) {
+ sb.append("\"");
+ }
+ value.dump(sb, rowId);
+ if (value.columnType.isStringType()) {
+ sb.append("\"");
+ }
+ }
+ sb.append("}");
+ break;
+ }
+ case STRUCT: {
+ sb.append("{");
+ for (int fieldIndex = 0; fieldIndex < childColumns.length;
++fieldIndex) {
+ VectorColumn child = childColumns[fieldIndex];
+ if (fieldIndex != 0) {
+ sb.append(",");
+ }
+
sb.append("\"").append(child.columnType.getName()).append("\":");
+ if (child.columnType.isStringType()) {
+ sb.append("\"");
+ }
+ child.dump(sb, i);
+ if (child.columnType.isStringType()) {
+ sb.append("\"");
+ }
+ }
+ sb.append("}");
+ break;
+ }
default:
throw new RuntimeException("Unknown type value: " + typeValue);
}
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java
index 63b6f1ac2a9..bb6e455f87a 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java
@@ -58,7 +58,7 @@ public class VectorTable {
this.numRows = (int) OffHeap.getLong(null, address);
address += 8;
- int metaSize = 1; // number of rows
+ int metaSize = 1; // stores the number of rows + other columns meta
data
for (int i = 0; i < types.length; i++) {
columns[i] = new VectorColumn(types[i], numRows, address);
metaSize += types[i].metaSize();
diff --git
a/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java
b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java
index e8a6d49df79..6f47b5bbb71 100644
---
a/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java
+++
b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java
@@ -35,9 +35,10 @@ public class JniScannerTest {
{
put("mock_rows", "128");
put("required_fields",
"boolean,tinyint,smallint,int,bigint,largeint,float,double,"
- +
"date,timestamp,char,varchar,string,decimalv2,decimal64");
+ +
"date,timestamp,char,varchar,string,decimalv2,decimal64,array,map,struct");
put("columns_types",
"boolean#tinyint#smallint#int#bigint#largeint#float#double#"
- +
"date#timestamp#char(10)#varchar(10)#string#decimalv2(12,4)#decimal64(10,3)");
+ +
"date#timestamp#char(10)#varchar(10)#string#decimalv2(12,4)#decimal64(10,3)#"
+ +
"array<array<string>>#map<string,array<int>>#struct<col1:timestamp(6),col2:array<char(10)>>");
}
});
scanner.open();
@@ -50,7 +51,7 @@ public class JniScannerTest {
VectorTable restoreTable = new
VectorTable(scanner.getTable().getColumnTypes(),
scanner.getTable().getFields(), metaAddress);
- System.out.println(restoreTable.dump((int) rows));
+ System.out.println(restoreTable.dump((int) rows).substring(0,
128));
// Restored table is release by the origin table.
}
scanner.resetTable();
diff --git a/regression-test/data/external_table_p2/hive/test_hive_hudi.out
b/regression-test/data/external_table_p2/hive/test_hive_hudi.out
index a695d3cdb7d..3bee0e318da 100644
--- a/regression-test/data/external_table_p2/hive/test_hive_hudi.out
+++ b/regression-test/data/external_table_p2/hive/test_hive_hudi.out
@@ -17,6 +17,108 @@ row_1 2011-11-11 1 v_1
row_2 2021-01-01 0 v_0
row_4 2021-02-01 4 v_4
+-- !complex_types --
+20230922203209630 20230922203209630_0_10000 10002
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
10002 a9999 [[9999], [3333], [9999, 8999, NULL]] {"k9999":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 00:26:48.000000, [10008,
9999, NULL]}
+20230922203209630 20230922203209630_0_9999 10001
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
10001 a9998 [[9998], [3332], [9998, 8998, NULL]] {"k9998":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 23:25:47.000000, [10007,
9998, NULL]}
+20230922203209630 20230922203209630_0_9998 10000
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
10000 a9997 [[9997], [3332], [9997, 8997, NULL]] {"k9997":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 22:24:46.000000, [10006,
9997, NULL]}
+20230922203209630 20230922203209630_0_9997 9999
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9999 a9996 [[9996], [3332], [9996, 8996, NULL]] {"k9996":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 21:23:45.000000, [10005,
9996, NULL]}
+20230922203209630 20230922203209630_0_9996 9998
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9998 a9995 [[9995], [], [9995, 8995, NULL]] {"k9995":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 20:22:44.000000, [10004,
9995, NULL]}
+20230922203209630 20230922203209630_0_9995 9997
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9997 a9994 [[9994], [3331], [9994, 8994, NULL]] {"k9994":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 19:21:43.000000, [10003,
9994, NULL]}
+20230922203209630 20230922203209630_0_9994 9996
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9996 a9993 [[9993], [3331], [9993, 8993, NULL]] {"k9993":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 18:20:42.000000, [10002,
9993, NULL]}
+20230922204306289 20230922204306289_0_542 9995
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9995 a9992-1 [[9992], [3330], [9992,
8992, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9992":[1, NULL, 9], "k2":[]}
{2012-02-02 17:19:41.000000, [10001, 9992, NULL]}
+20230922203209630 20230922203209630_0_9992 9994
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9994 a9991 [[9991], [3330], [9991, 8991, NULL]] {"k9991":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 16:18:40.000000, [10000,
9991, NULL]}
+20230922203209630 20230922203209630_0_9991 9993
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9993 a9990 [NULL, [], [9990, 8990, NULL]] {"k9990":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 15:17:39.000000, [9999, NULL, NULL]}
+20230922203209630 20230922203209630_0_1000 1002
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
1002 a999 [[999], [333], [999, -1, NULL]] {"k999":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 14:32:13.000000, [1008, 999, NULL]}
+20230922203209630 20230922203209630_0_9990 9992
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9992 a9989 [[9989], [3329], [9989, 8989, NULL]] {"k9989":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 14:16:38.000000, [9998,
9989, NULL]}
+20230922203209630 20230922203209630_0_9989 9991
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9991 a9988 [[9988], [3329], [9988, 8988, NULL]] {"k9988":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 13:15:37.000000, [9997,
9988, NULL]}
+20230922203209630 20230922203209630_0_9988 9990
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9990 a9987 [[9987], [3329], [9987, 8987, NULL]] {"k9987":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 12:14:36.000000, [9996,
9987, NULL]}
+20230922203209630 20230922203209630_0_9987 9989
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9989 a9986 [[9986], [3328], [9986, 8986, NULL]] {"k9986":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 11:13:35.000000, [9995,
9986, NULL]}
+20230922203209630 20230922203209630_0_9986 9988
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9988 a9985 [[9985], [], [9985, 8985, NULL]] {"k9985":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 10:12:34.000000, [9994,
9985, NULL]}
+20230922203209630 20230922203209630_0_9985 9987
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9987 a9984 [[9984], [3328], [9984, 8984, NULL]] {"k9984":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 09:11:33.000000, [9993,
9984, NULL]}
+20230922203209630 20230922203209630_0_9984 9986
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9986 a9983 [[9983], [3327], [9983, 8983, NULL]] {"k9983":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 08:10:32.000000, [9992,
9983, NULL]}
+20230922204306289 20230922204306289_0_32 9985
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9985 a9982-1 [[9982], [3327], [9982,
8982, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9982":[1, NULL, 9], "k2":[]}
{2012-02-02 07:09:31.000000, [9991, 9982, NULL]}
+20230922203209630 20230922203209630_0_9982 9984
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9984 a9981 [[9981], [3327], [9981, 8981, NULL]] {"k9981":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 06:08:30.000000, [9990,
9981, NULL]}
+20230922203209630 20230922203209630_0_9981 9983
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9983 a9980 [NULL, [], [9980, 8980, NULL]] {"k9980":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 05:07:29.000000, [9989, NULL, NULL]}
+20230922203209630 20230922203209630_0_999 1001
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
1001 a998 [[998], [332], [998, -2, NULL]] {"k998":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 13:31:12.000000, [1007, 998, NULL]}
+20230922203209630 20230922203209630_0_9980 9982
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9982 a9979 [[9979], [3326], [9979, 8979, NULL]] {"k9979":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 04:06:28.000000, [9988,
9979, NULL]}
+20230922203209630 20230922203209630_0_9979 9981
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9981 a9978 [[9978], [3326], [9978, 8978, NULL]] {"k9978":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 03:05:27.000000, [9987,
9978, NULL]}
+20230922203209630 20230922203209630_0_9978 9980
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9980 a9977 [[9977], [3325], [9977, 8977, NULL]] {"k9977":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 02:04:26.000000, [9986,
9977, NULL]}
+20230922203209630 20230922203209630_0_9977 9979
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9979 a9976 [[9976], [3325], [9976, 8976, NULL]] {"k9976":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 01:03:25.000000, [9985,
9976, NULL]}
+20230922203209630 20230922203209630_0_9976 9978
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9978 a9975 [[9975], [], [9975, 8975, NULL]] {"k9975":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 10:02:24.000000, [9984,
9975, NULL]}
+20230922203209630 20230922203209630_0_9975 9977
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9977 a9974 [[9974], [3324], [9974, 8974, NULL]] {"k9974":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 09:58:23.000000, [9983,
9974, NULL]}
+20230922203209630 20230922203209630_0_9974 9976
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9976 a9973 [[9973], [3324], [9973, 8973, NULL]] {"k9973":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 08:57:22.000000, [9982,
9973, NULL]}
+20230922204306289 20230922204306289_0_939 9975
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9975 a9972-1 [[9972], [3324], [9972,
8972, NULL]] {"k9972":[1, NULL, 9], "k3":NULL, "k4":[NULL, 9], "k2":[]}
{2012-02-04 07:56:21.000000, [9981, 9972, NULL]}
+20230922203209630 20230922203209630_0_9972 9974
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9974 a9971 [[9971], [3323], [9971, 8971, NULL]] {"k9971":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 06:55:20.000000, [9980,
9971, NULL]}
+20230922203209630 20230922203209630_0_9971 9973
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9973 a9970 [NULL, [], [9970, 8970, NULL]] {"k9970":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-04 05:54:19.000000, [9979, NULL, NULL]}
+20230922203209630 20230922203209630_0_998 1000
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
1000 a997 [[997], [332], [997, -3, NULL]] {"k997":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 12:30:11.000000, [1006, 997, NULL]}
+20230922203209630 20230922203209630_0_9970 9972
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9972 a9969 [[9969], [3323], [9969, 8969, NULL]] {"k9969":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 04:53:18.000000, [9978,
9969, NULL]}
+20230922203209630 20230922203209630_0_9969 9971
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9971 a9968 [[9968], [3322], [9968, 8968, NULL]] {"k9968":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 03:52:17.000000, [9977,
9968, NULL]}
+20230922203209630 20230922203209630_0_9968 9970
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9970 a9967 [[9967], [3322], [9967, 8967, NULL]] {"k9967":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 02:51:16.000000, [9976,
9967, NULL]}
+20230922203209630 20230922203209630_0_9967 9969
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9969 a9966 [[9966], [3322], [9966, 8966, NULL]] {"k9966":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 01:50:15.000000, [9975,
9966, NULL]}
+20230922203209630 20230922203209630_0_9966 9968
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9968 a9965 [[9965], [], [9965, 8965, NULL]] {"k9965":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 00:49:14.000000, [9974,
9965, NULL]}
+20230922203209630 20230922203209630_0_9965 9967
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9967 a9964 [[9964], [3321], [9964, 8964, NULL]] {"k9964":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 23:48:13.000000, [9973,
9964, NULL]}
+20230922203209630 20230922203209630_0_9964 9966
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9966 a9963 [[9963], [3321], [9963, 8963, NULL]] {"k9963":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 22:47:12.000000, [9972,
9963, NULL]}
+20230922204306289 20230922204306289_0_173 9965
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9965 a9962-1 [[9962], [3320], [9962,
8962, NULL]] {"k3":NULL, "k9962":[1, NULL, 9], "k4":[NULL, 9], "k2":[]}
{2012-02-03 21:46:11.000000, [9971, 9962, NULL]}
+20230922203209630 20230922203209630_0_9962 9964
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9964 a9961 [[9961], [3320], [9961, 8961, NULL]] {"k9961":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 20:45:10.000000, [9970,
9961, NULL]}
+20230922203209630 20230922203209630_0_9961 9963
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9963 a9960 [NULL, [], [9960, 8960, NULL]] {"k9960":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-03 19:44:09.000000, [9969, NULL, NULL]}
+20230922203209630 20230922203209630_0_997 999
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
999 a996 [[996], [332], [996, -4, NULL]] {"k996":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 11:29:10.000000, [1005, 996, NULL]}
+20230922203209630 20230922203209630_0_9960 9962
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9962 a9959 [[9959], [3319], [9959, 8959, NULL]] {"k9959":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 18:43:08.000000, [9968,
9959, NULL]}
+20230922203209630 20230922203209630_0_9959 9961
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9961 a9958 [[9958], [3319], [9958, 8958, NULL]] {"k9958":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 17:42:07.000000, [9967,
9958, NULL]}
+20230922203209630 20230922203209630_0_9958 9960
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9960 a9957 [[9957], [3319], [9957, 8957, NULL]] {"k9957":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 16:41:06.000000, [9966,
9957, NULL]}
+20230922203209630 20230922203209630_0_9957 9959
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9959 a9956 [[9956], [3318], [9956, 8956, NULL]] {"k9956":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 15:40:05.000000, [9965,
9956, NULL]}
+20230922203209630 20230922203209630_0_9956 9958
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9958 a9955 [[9955], [], [9955, 8955, NULL]] {"k9955":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 14:39:04.000000, [9964,
9955, NULL]}
+20230922203209630 20230922203209630_0_9955 9957
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9957 a9954 [[9954], [3318], [9954, 8954, NULL]] {"k9954":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 13:38:58.000000, [9963,
9954, NULL]}
+20230922203209630 20230922203209630_0_9954 9956
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9956 a9953 [[9953], [3317], [9953, 8953, NULL]] {"k9953":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 12:37:57.000000, [9962,
9953, NULL]}
+20230922204306289 20230922204306289_0_665 9955
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9955 a9952-1 [[9952], [3317], [9952,
8952, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9952":[1, NULL, 9], "k2":[]}
{2012-02-03 11:36:56.000000, [9961, 9952, NULL]}
+20230922203209630 20230922203209630_0_9952 9954
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9954 a9951 [[9951], [3317], [9951, 8951, NULL]] {"k9951":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 10:35:55.000000, [9960,
9951, NULL]}
+20230922203209630 20230922203209630_0_9951 9953
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9953 a9950 [NULL, [], [9950, 8950, NULL]] {"k9950":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-03 09:34:54.000000, [9959, NULL, NULL]}
+20230922203209630 20230922203209630_0_996 998
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
998 a995 [[995], [], [995, -5, NULL]] {"k995":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 10:28:09.000000, [1004, 995, NULL]}
+20230922203209630 20230922203209630_0_9950 9952
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9952 a9949 [[9949], [3316], [9949, 8949, NULL]] {"k9949":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 08:33:53.000000, [9958,
9949, NULL]}
+20230922203209630 20230922203209630_0_9949 9951
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9951 a9948 [[9948], [3316], [9948, 8948, NULL]] {"k9948":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 07:32:52.000000, [9957,
9948, NULL]}
+20230922203209630 20230922203209630_0_9948 9950
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9950 a9947 [[9947], [3315], [9947, 8947, NULL]] {"k9947":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 06:31:51.000000, [9956,
9947, NULL]}
+20230922203209630 20230922203209630_0_9947 9949
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9949 a9946 [[9946], [3315], [9946, 8946, NULL]] {"k9946":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 05:30:50.000000, [9955,
9946, NULL]}
+20230922203209630 20230922203209630_0_9946 9948
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9948 a9945 [[9945], [], [9945, 8945, NULL]] {"k9945":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 04:29:49.000000, [9954,
9945, NULL]}
+20230922203209630 20230922203209630_0_9945 9947
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9947 a9944 [[9944], [3314], [9944, 8944, NULL]] {"k9944":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 03:28:48.000000, [9953,
9944, NULL]}
+20230922203209630 20230922203209630_0_9944 9946
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9946 a9943 [[9943], [3314], [9943, 8943, NULL]] {"k9943":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 02:27:47.000000, [9952,
9943, NULL]}
+20230922204306289 20230922204306289_0_53 9945
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9945 a9942-1 [[9942], [3314], [9942,
8942, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9942":[1, NULL, 9], "k2":[]}
{2012-02-03 01:26:46.000000, [9951, 9942, NULL]}
+20230922203209630 20230922203209630_0_9942 9944
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9944 a9941 [[9941], [3313], [9941, 8941, NULL]] {"k9941":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-03 00:25:45.000000, [9950,
9941, NULL]}
+20230922203209630 20230922203209630_0_9941 9943
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9943 a9940 [NULL, [], [9940, 8940, NULL]] {"k9940":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 23:24:44.000000, [9949, NULL, NULL]}
+20230922203209630 20230922203209630_0_995 997
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
997 a994 [[994], [331], [994, -6, NULL]] {"k994":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 09:27:08.000000, [1003, 994, NULL]}
+20230922203209630 20230922203209630_0_9940 9942
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9942 a9939 [[9939], [3313], [9939, 8939, NULL]] {"k9939":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 22:23:43.000000, [9948,
9939, NULL]}
+20230922203209630 20230922203209630_0_9939 9941
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9941 a9938 [[9938], [3312], [9938, 8938, NULL]] {"k9938":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 21:22:42.000000, [9947,
9938, NULL]}
+20230922203209630 20230922203209630_0_9938 9940
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9940 a9937 [[9937], [3312], [9937, 8937, NULL]] {"k9937":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 20:21:41.000000, [9946,
9937, NULL]}
+20230922203209630 20230922203209630_0_9937 9939
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9939 a9936 [[9936], [3312], [9936, 8936, NULL]] {"k9936":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 19:20:40.000000, [9945,
9936, NULL]}
+20230922203209630 20230922203209630_0_9936 9938
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9938 a9935 [[9935], [], [9935, 8935, NULL]] {"k9935":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 18:19:39.000000, [9944,
9935, NULL]}
+20230922203209630 20230922203209630_0_9935 9937
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9937 a9934 [[9934], [3311], [9934, 8934, NULL]] {"k9934":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 17:18:38.000000, [9943,
9934, NULL]}
+20230922203209630 20230922203209630_0_9934 9936
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9936 a9933 [[9933], [3311], [9933, 8933, NULL]] {"k9933":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 16:17:37.000000, [9942,
9933, NULL]}
+20230922204306289 20230922204306289_0_991 9935
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9935 a9932-1 [[9932], [3310], [9932,
8932, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9932":[1, NULL, 9], "k2":[]}
{2012-02-02 15:16:36.000000, [9941, 9932, NULL]}
+20230922203209630 20230922203209630_0_9932 9934
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9934 a9931 [[9931], [3310], [9931, 8931, NULL]] {"k9931":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 14:15:35.000000, [9940,
9931, NULL]}
+20230922203209630 20230922203209630_0_9931 9933
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9933 a9930 [NULL, [], [9930, 8930, NULL]] {"k9930":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 13:14:34.000000, [9939, NULL, NULL]}
+20230922203209630 20230922203209630_0_994 996
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
996 a993 [[993], [331], [993, -7, NULL]] {"k993":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 08:26:07.000000, [1002, 993, NULL]}
+20230922203209630 20230922203209630_0_9930 9932
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9932 a9929 [[9929], [3309], [9929, 8929, NULL]] {"k9929":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 12:13:33.000000, [9938,
9929, NULL]}
+20230922203209630 20230922203209630_0_9929 9931
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9931 a9928 [[9928], [3309], [9928, 8928, NULL]] {"k9928":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 11:12:32.000000, [9937,
9928, NULL]}
+20230922203209630 20230922203209630_0_9928 9930
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9930 a9927 [[9927], [3309], [9927, 8927, NULL]] {"k9927":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 10:11:31.000000, [9936,
9927, NULL]}
+20230922203209630 20230922203209630_0_9927 9929
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9929 a9926 [[9926], [3308], [9926, 8926, NULL]] {"k9926":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 09:10:30.000000, [9935,
9926, NULL]}
+20230922203209630 20230922203209630_0_9926 9928
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9928 a9925 [[9925], [], [9925, 8925, NULL]] {"k9925":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 08:09:29.000000, [9934,
9925, NULL]}
+20230922203209630 20230922203209630_0_9925 9927
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9927 a9924 [[9924], [3308], [9924, 8924, NULL]] {"k9924":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 07:08:28.000000, [9933,
9924, NULL]}
+20230922203209630 20230922203209630_0_9924 9926
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9926 a9923 [[9923], [3307], [9923, 8923, NULL]] {"k9923":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 06:07:27.000000, [9932,
9923, NULL]}
+20230922204306289 20230922204306289_0_778 9925
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9925 a9922-1 [[9922], [3307], [9922,
8922, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9922":[1, NULL, 9], "k2":[]}
{2012-02-02 05:06:26.000000, [9931, 9922, NULL]}
+20230922203209630 20230922203209630_0_9922 9924
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9924 a9921 [[9921], [3307], [9921, 8921, NULL]] {"k9921":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 04:05:25.000000, [9930,
9921, NULL]}
+20230922203209630 20230922203209630_0_9921 9923
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9923 a9920 [NULL, [], [9920, 8920, NULL]] {"k9920":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 03:04:24.000000, [9929, NULL, NULL]}
+20230922204306289 20230922204306289_0_100 995
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 995 a992-1 [[992], [330], [992,
-8, NULL]] {"k3":NULL, "k4":[NULL, 9], "k992":[1, NULL, 9], "k2":[]}
{2012-02-02 07:25:06.000000, [1001, 992, NULL]}
+20230922203209630 20230922203209630_0_9920 9922
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9922 a9919 [[9919], [3306], [9919, 8919, NULL]] {"k9919":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 02:03:23.000000, [9928,
9919, NULL]}
+20230922203209630 20230922203209630_0_9919 9921
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9921 a9918 [[9918], [3306], [9918, 8918, NULL]] {"k9918":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-02 01:02:22.000000, [9927,
9918, NULL]}
+20230922203209630 20230922203209630_0_9918 9920
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9920 a9917 [[9917], [3305], [9917, 8917, NULL]] {"k9917":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 10:58:21.000000, [9926,
9917, NULL]}
+20230922203209630 20230922203209630_0_9917 9919
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9919 a9916 [[9916], [3305], [9916, 8916, NULL]] {"k9916":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 09:57:20.000000, [9925,
9916, NULL]}
+20230922203209630 20230922203209630_0_9916 9918
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9918 a9915 [[9915], [], [9915, 8915, NULL]] {"k9915":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 08:56:19.000000, [9924,
9915, NULL]}
+20230922203209630 20230922203209630_0_9915 9917
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9917 a9914 [[9914], [3304], [9914, 8914, NULL]] {"k9914":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 07:55:18.000000, [9923,
9914, NULL]}
+20230922203209630 20230922203209630_0_9914 9916
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9916 a9913 [[9913], [3304], [9913, 8913, NULL]] {"k9913":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 06:54:17.000000, [9922,
9913, NULL]}
+20230922204306289 20230922204306289_0_750 9915
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0 9915 a9912-1 [[9912], [3304], [9912,
8912, NULL]] {"k3":NULL, "k4":[NULL, 9], "k9912":[1, NULL, 9], "k2":[]}
{2012-02-04 05:53:16.000000, [9921, 9912, NULL]}
+20230922203209630 20230922203209630_0_9912 9914
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9914 a9911 [[9911], [3303], [9911, 8911, NULL]] {"k9911":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 04:52:15.000000, [9920,
9911, NULL]}
+20230922203209630 20230922203209630_0_9911 9913
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9913 a9910 [NULL, [], [9910, 8910, NULL]] {"k9910":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-04 03:51:14.000000, [9919, NULL, NULL]}
+20230922203209630 20230922203209630_0_992 994
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
994 a991 [[991], [330], [991, -9, NULL]] {"k991":[1, NULL, 9], "k2":[],
"k3":NULL, "k4":[NULL, 9]} {2012-02-02 06:24:05.000000, [1000, 991, NULL]}
+20230922203209630 20230922203209630_0_9910 9912
8b83d7ed-c150-4177-9dbb-d41169e8b9c7-0_0-163-0_20230922203209630.parquet
9912 a9909 [[9909], [3303], [9909, 8909, NULL]] {"k9909":[1, NULL, 9],
"k2":[], "k3":NULL, "k4":[NULL, 9]} {2012-02-04 02:50:13.000000, [9918,
9909, NULL]}
+
-- !skip_merge --
20230605145009209 20230605145009209_0_0 rowId:row_1
partitionId=2021-01-01/versionId=v_0
65ffc5d9-397a-456e-a735-30f3ad37466f-0_0-33-96_20230605145009209.parquet
row_1 2021-01-01 0 bob v_0 toBeDel0 0 1000000
20230605145403388 20230605145403388_2_0 rowId:row_1
partitionId=2011-11-11/versionId=v_1
dbff8acb-42bc-400c-be33-47d9e0bae9b7-0_2-83-222_20230605145403388.parquet
row_1 2011-11-11 1 bob v_1 toBeDel1 0 1000001
diff --git
a/regression-test/suites/external_table_p2/hive/test_hive_hudi.groovy
b/regression-test/suites/external_table_p2/hive/test_hive_hudi.groovy
index abdd5b34dcb..8648aa4bc98 100644
--- a/regression-test/suites/external_table_p2/hive/test_hive_hudi.groovy
+++ b/regression-test/suites/external_table_p2/hive/test_hive_hudi.groovy
@@ -39,6 +39,8 @@ suite("test_hive_hudi", "p2,external,hive,hudi") {
// match colum name in lower case
qt_lowercase_column """select RoWiD, PaRtiTionID, PrEComB, VerSIonID
from partitioned_mor_rt order by rowid, versionid"""
+ // test complex types
+ qt_complex_types """select * from complex_type_rt order by name desc
limit 100"""
// skip logs
sql """drop catalog if exists ${catalog_name};"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
