This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 23832b9513 [feature](array-type) Unique table support array value
(#17024) (#19371)
23832b9513 is described below
commit 23832b951383b2d3548fb79ba69442e600fff281
Author: hqx871 <[email protected]>
AuthorDate: Mon Aug 14 23:06:26 2023 +0800
[feature](array-type) Unique table support array value (#17024) (#19371)
- #17024
- #19751
---
.../aggregate_function_reader_first_last.h | 25 +-
be/src/vec/columns/column_complex.h | 16 +-
be/src/vec/functions/function_case.h | 1 +
be/src/vec/utils/template_helpers.hpp | 2 +
be/test/CMakeLists.txt | 1 +
.../vec/aggregate_functions/agg_replace_test.cpp | 462 +++++++++++++++++++++
be/test/vec/core/column_complex_test.cpp | 7 +
.../org/apache/doris/analysis/CreateTableStmt.java | 7 +-
.../data/query_p0/show/complex_unique_1.csv | 2 +
.../data/query_p0/show/complex_unique_2.csv | 2 +
.../query_p0/show/test_complex_type_unique_key.out | 15 +
.../show/test_complex_type_unique_key.groovy | 99 +++++
12 files changed, 615 insertions(+), 24 deletions(-)
diff --git
a/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
b/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
index 4b7c1e0c98..8529dd0cfb 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_reader_first_last.h
@@ -44,12 +44,12 @@ public:
return false;
}
- StringRef get_value() const {
+ void insert_into(IColumn& to) const {
if constexpr (arg_is_nullable) {
auto* col = assert_cast<const ColumnNullable*>(_ptr);
- return assert_cast<const
ColVecType&>(col->get_nested_column()).get_data_at(_offset);
+ assert_cast<ColVecType&>(to).insert_from(col->get_nested_column(),
_offset);
} else {
- return assert_cast<const ColVecType*>(_ptr)->get_data_at(_offset);
+ assert_cast<ColVecType&>(to).insert_from(*_ptr, _offset);
}
}
@@ -71,7 +71,7 @@ protected:
template <typename ColVecType, bool arg_is_nullable>
struct CopiedValue : public Value<ColVecType, arg_is_nullable> {
public:
- StringRef get_value() const { return _copied_value; }
+ void insert_into(IColumn& to) const {
assert_cast<ColVecType&>(to).insert(_copied_value); }
bool is_null() const { return this->_ptr == nullptr; }
@@ -86,17 +86,16 @@ public:
this->reset();
return;
} else {
- _copied_value = assert_cast<const
ColVecType&>(col->get_nested_column())
- .get_data_at(row)
- .to_string();
+ auto& nested_col = assert_cast<const
ColVecType&>(col->get_nested_column());
+ nested_col.get(row, _copied_value);
}
} else {
- _copied_value = assert_cast<const
ColVecType*>(column)->get_data_at(row).to_string();
+ column->get(row, _copied_value);
}
}
private:
- std::string _copied_value;
+ Field _copied_value;
};
template <typename ColVecType, bool result_is_nullable, bool arg_is_nullable,
bool is_copy>
@@ -118,15 +117,11 @@ public:
col.insert_default();
} else {
auto& col = assert_cast<ColumnNullable&>(to);
- //get_value will never get null value
- const StringRef& value = _data_value.get_value();
col.get_null_map_data().push_back(0);
- assert_cast<ColVecType&>(col.get_nested_column())
- .insert_data(value.data, value.size);
+ _data_value.insert_into(col.get_nested_column());
}
} else {
- const StringRef& value = _data_value.get_value();
- assert_cast<ColVecType&>(to).insert_data(value.data, value.size);
+ _data_value.insert_into(to);
}
}
diff --git a/be/src/vec/columns/column_complex.h
b/be/src/vec/columns/column_complex.h
index 4b6562d534..005153853d 100644
--- a/be/src/vec/columns/column_complex.h
+++ b/be/src/vec/columns/column_complex.h
@@ -130,15 +130,19 @@ public:
MutableColumnPtr clone_resized(size_t size) const override;
- [[noreturn]] void insert(const Field& x) override {
- LOG(FATAL) << "insert field not implemented";
+ void insert(const Field& x) override {
+ const String& s = doris::vectorized::get<const String&>(x);
+ data.push_back(*reinterpret_cast<const T*>(s.c_str()));
}
- [[noreturn]] Field operator[](size_t n) const override {
- LOG(FATAL) << "operator[] not implemented";
+ Field operator[](size_t n) const override {
+ assert(n < size());
+ return Field(reinterpret_cast<const char*>(&data[n]), sizeof(data[n]));
}
- [[noreturn]] void get(size_t n, Field& res) const override {
- LOG(FATAL) << "get field not implemented";
+
+ void get(size_t n, Field& res) const override {
+ assert(n < size());
+ res.assign_string(reinterpret_cast<const char*>(&data[n]),
sizeof(data[n]));
}
[[noreturn]] UInt64 get64(size_t n) const override {
diff --git a/be/src/vec/functions/function_case.h
b/be/src/vec/functions/function_case.h
index a4049f7840..6b628339d6 100644
--- a/be/src/vec/functions/function_case.h
+++ b/be/src/vec/functions/function_case.h
@@ -235,6 +235,7 @@ public:
if constexpr (std::is_same_v<ColumnType, ColumnString> ||
std::is_same_v<ColumnType, ColumnBitmap> ||
+ std::is_same_v<ColumnType, ColumnArray> ||
std::is_same_v<ColumnType, ColumnHLL>) {
// result_column and all then_column is not nullable.
// can't simd when type is string.
diff --git a/be/src/vec/utils/template_helpers.hpp
b/be/src/vec/utils/template_helpers.hpp
index 265f74d53d..9cb93833e8 100644
--- a/be/src/vec/utils/template_helpers.hpp
+++ b/be/src/vec/utils/template_helpers.hpp
@@ -22,6 +22,7 @@
#include "http/http_status.h"
#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column_array.h"
#include "vec/columns/column_complex.h"
#include "vec/columns/columns_number.h"
#include "vec/data_types/data_type.h"
@@ -54,6 +55,7 @@
M(DateTimeV2, ColumnUInt64)
#define COMPLEX_TYPE_TO_COLUMN_TYPE(M) \
+ M(Array, ColumnArray) \
M(BitMap, ColumnBitmap) \
M(HLL, ColumnHLL)
diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt
index 40933d2634..4ff2a9998e 100644
--- a/be/test/CMakeLists.txt
+++ b/be/test/CMakeLists.txt
@@ -333,6 +333,7 @@ set(VEC_TEST_FILES
vec/aggregate_functions/agg_collect_test.cpp
vec/aggregate_functions/agg_test.cpp
vec/aggregate_functions/agg_min_max_test.cpp
+ vec/aggregate_functions/agg_replace_test.cpp
vec/aggregate_functions/vec_window_funnel_test.cpp
vec/aggregate_functions/vec_retention_test.cpp
vec/aggregate_functions/vec_sequence_match_test.cpp
diff --git a/be/test/vec/aggregate_functions/agg_replace_test.cpp
b/be/test/vec/aggregate_functions/agg_replace_test.cpp
new file mode 100644
index 0000000000..cbc673340a
--- /dev/null
+++ b/be/test/vec/aggregate_functions/agg_replace_test.cpp
@@ -0,0 +1,462 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include "common/logging.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/aggregate_functions/aggregate_function_reader.h"
+#include "vec/aggregate_functions/aggregate_function_simple_factory.h"
+#include "vec/columns/column_complex.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_hll.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+namespace doris::vectorized {
+
+class VAggReplaceTest : public testing::Test {
+public:
+ void SetUp() {
+ AggregateFunctionSimpleFactory factory =
AggregateFunctionSimpleFactory::instance();
+ register_aggregate_function_replace_reader_load(factory);
+ }
+
+ void TearDown() {}
+
+ template <typename DataType, bool nullable>
+ void check_column_basic(const IColumn* column, int64_t expect_num, size_t
pos = 0) {
+ //expect basic column[pos]=expect_num
+ EXPECT_FALSE(column->is_null_at(pos));
+ auto* unwrap_col = column;
+ if constexpr (nullable) {
+ auto* nullable_col = assert_cast<const
ColumnNullable*>(unwrap_col);
+ EXPECT_FALSE(nullable_col->is_null_at(pos));
+ unwrap_col = nullable_col->get_nested_column_ptr().get();
+ }
+ if constexpr (std::is_same_v<DataType, DataTypeString>) {
+ auto str = unwrap_col->get_data_at(pos).to_string();
+ EXPECT_EQ("item" + std::to_string(expect_num), str);
+ } else if constexpr (std::is_same_v<DataType, DataTypeBitMap>) {
+ auto& container = assert_cast<const
ColumnBitmap*>(unwrap_col)->get_data();
+ auto& bitmap = container[pos];
+ EXPECT_TRUE(bitmap.contains(static_cast<uint64_t>(expect_num)));
+ } else if constexpr (std::is_same_v<DataType, DataTypeHLL>) {
+ auto& container = assert_cast<const
ColumnHLL*>(unwrap_col)->get_data();
+ auto& hll = container[pos];
+ auto expect = hll.estimate_cardinality();
+
const_cast<HyperLogLog*>(&hll)->update(static_cast<uint64_t>(expect_num));
+ auto actual = hll.estimate_cardinality();
+ EXPECT_EQ(expect, actual);
+ } else {
+ EXPECT_EQ(expect_num, unwrap_col->get_int(pos));
+ }
+ }
+
+ template <typename DataType, bool nullable>
+ void check_column_array(const IColumn* column, int32_t expect_num) {
+ //expect array column:[[0..expect_num-1]]
+ EXPECT_EQ(column->size(), 1);
+ auto* unwrap_col = column;
+ if constexpr (nullable) {
+ auto* nullable_col = assert_cast<const
ColumnNullable*>(unwrap_col);
+ EXPECT_FALSE(nullable_col->is_null_at(0));
+ unwrap_col = nullable_col->get_nested_column_ptr().get();
+ }
+ auto* array_col = assert_cast<const ColumnArray*>(unwrap_col);
+ EXPECT_EQ(array_col->get_offsets()[0], expect_num);
+ auto* data_col = array_col->get_data_ptr().get();
+ EXPECT_EQ(data_col->size(), expect_num);
+ for (size_t i = 0; i < expect_num; ++i) {
+ check_column_basic<DataType, nullable>(data_col, i, i);
+ }
+ }
+
+ template <typename DataType, bool nullable>
+ void add_elements(MutableColumnPtr& input_col, size_t input_nums) {
+ //fill column: [0..input_nums-1]
+ using FieldType = typename DataType::FieldType;
+ Field field;
+ for (size_t i = 0; i < input_nums; ++i) {
+ if constexpr (std::is_same_v<DataType, DataTypeString>) {
+ auto item = std::string("item") + std::to_string(i);
+ input_col->insert_data(item.c_str(), item.size());
+ EXPECT_EQ(item, input_col->get_data_at(i).to_string());
+ } else if constexpr (std::is_same_v<DataType, DataTypeBitMap>) {
+ BitmapValue bitmap;
+ bitmap.add(i);
+ input_col->insert_data(reinterpret_cast<const char*>(&bitmap),
sizeof(bitmap));
+ } else if constexpr (std::is_same_v<DataType, DataTypeHLL>) {
+ HyperLogLog hll;
+ hll.update(i);
+ input_col->insert_data(reinterpret_cast<const char*>(&hll),
sizeof(hll));
+ } else {
+ auto item = FieldType(static_cast<uint64_t>(i));
+ input_col->insert_data(reinterpret_cast<const char*>(&item),
0);
+ }
+ }
+ EXPECT_EQ(input_col->size(), input_nums);
+ }
+
+ template <typename DataType, bool nullable>
+ void agg_replace_add_elements(MutableColumnPtr& input_col,
AggregateFunctionPtr agg_function,
+ AggregateDataPtr place, size_t input_nums) {
+ //fill column: [0..input_nums-1]
+ add_elements<DataType, nullable>(input_col, input_nums);
+ const IColumn* column[1] = {input_col.get()};
+ for (int i = 0; i < input_col->size(); i++) {
+ agg_function->add(place, column, i, &_agg_arena_pool);
+ }
+ }
+
+ template <typename DataType, bool nullable>
+ void array_add_elements(MutableColumnPtr& input_col, size_t input_nums) {
+ //fill array column: [[],[0],[0,1]..[0..input_nums-1]]
+ using FieldType = typename DataType::FieldType;
+ Field field;
+ for (int32_t i = 0; i <= input_nums; ++i) {
+ doris::vectorized::Array array(i);
+ for (int32_t j = 0; j < i; ++j) {
+ if constexpr (std::is_same_v<DataType, DataTypeString>) {
+ auto item = std::string("item") + std::to_string(j);
+ array[j] = std::move(item);
+ } else if constexpr (IsDecimalNumber<FieldType>) {
+ auto item = FieldType(static_cast<uint64_t>(j));
+ array[j] = std::move(DecimalField<FieldType>(item, 20));
+ } else {
+ array[j] = std::move(FieldType(static_cast<uint64_t>(j)));
+ }
+ }
+ input_col->insert(array);
+ }
+
+ EXPECT_EQ(input_col->size(), input_nums + 1);
+ }
+
+ template <typename DataType, bool nullable>
+ void agg_replace_array_add_elements(MutableColumnPtr& input_col,
+ AggregateFunctionPtr agg_function,
AggregateDataPtr place,
+ size_t input_nums) {
+ //fill array column: [[],[0],[0,1]..[0..input_nums-1]]
+ array_add_elements<DataType, nullable>(input_col, input_nums);
+ const IColumn* column[1] = {input_col.get()};
+ for (size_t i = 0; i < input_col->size(); ++i) {
+ agg_function->add(place, column, i, &_agg_arena_pool);
+ }
+ }
+
+ template <typename DataType, bool nullable, bool array>
+ vectorized::DataTypePtr get_data_type() {
+ vectorized::DataTypePtr data_type = get_basic_type<DataType>();
+ if constexpr (nullable) {
+ data_type =
std::make_shared<vectorized::DataTypeNullable>(data_type);
+ }
+ if constexpr (array) {
+ data_type = std::make_shared<vectorized::DataTypeArray>(data_type);
+ if constexpr (nullable) {
+ data_type =
std::make_shared<vectorized::DataTypeNullable>(data_type);
+ }
+ }
+ return data_type;
+ }
+
+ template <typename DataType>
+ vectorized::DataTypePtr get_basic_type() {
+ using FieldType = typename DataType::FieldType;
+ if constexpr (IsDecimalNumber<FieldType>) {
+ //decimal column get_int will return (data * scale), so let scale
be 1.
+ return std::make_shared<DataType>(27, 1);
+ }
+ return std::make_shared<DataType>();
+ }
+
+ template <typename DataType, bool nullable>
+ void test_agg_replace(const std::string& fn_name, size_t input_nums,
size_t expect_num) {
+ vectorized::DataTypePtr data_type = get_data_type<DataType, nullable,
false>();
+ DataTypes data_types = {data_type};
+ LOG(INFO) << "test_agg_replace for " << fn_name << "(" <<
data_types[0]->get_name() << ")";
+ Array array;
+ AggregateFunctionSimpleFactory factory =
AggregateFunctionSimpleFactory::instance();
+ auto agg_function = factory.get(fn_name, data_types, array, nullable);
+ EXPECT_NE(agg_function, nullptr);
+
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+ //EXPECT_EQ(3, 0);
+
+ auto data_column = data_type->create_column();
+ agg_replace_add_elements<DataType, nullable>(data_column,
agg_function, place, input_nums);
+
+ //EXPECT_EQ(4, 0);
+ auto column_result = data_type->create_column();
+ agg_function->insert_result_into(place, *column_result);
+ check_column_basic<DataType, nullable>(column_result.get(),
expect_num);
+ agg_function->destroy(place);
+ }
+
+ template <typename DataType, bool nullable>
+ void test_agg_array_replace(const std::string& fn_name, size_t input_nums,
size_t expect_num) {
+ vectorized::DataTypePtr data_type = get_data_type<DataType, nullable,
true>();
+ DataTypes data_types = {data_type};
+ LOG(INFO) << "test_agg_replace for " << fn_name << "(" <<
data_types[0]->get_name() << ")";
+ Array array;
+ AggregateFunctionSimpleFactory factory =
AggregateFunctionSimpleFactory::instance();
+ auto agg_function = factory.get(fn_name, data_types, array, nullable);
+ EXPECT_NE(agg_function, nullptr);
+
+ std::unique_ptr<char[]> memory(new char[agg_function->size_of_data()]);
+ AggregateDataPtr place = memory.get();
+ agg_function->create(place);
+
+ auto input_column = data_type->create_column();
+ agg_replace_array_add_elements<DataType, nullable>(input_column,
agg_function, place,
+ input_nums);
+
+ auto column_result = data_type->create_column();
+ agg_function->insert_result_into(place, *column_result);
+ check_column_array<DataType, nullable>(column_result.get(),
expect_num);
+
+ agg_function->destroy(place);
+ }
+
+ template <typename DataType, typename ColumnType, bool nullable>
+ void test_basic_data(int8_t input_nums) {
+ vectorized::DataTypePtr data_type = get_data_type<DataType, nullable,
false>();
+
+ auto data_column = data_type->create_column();
+ add_elements<DataType, nullable>(data_column, input_nums);
+
+ EXPECT_EQ(input_nums, data_column->size());
+ //test Value
+ {
+ Value<ColumnType, nullable> value;
+ EXPECT_TRUE(value.is_null());
+ for (int64_t i = 0; i < input_nums; ++i) {
+ value.set_value(data_column.get(), i);
+ EXPECT_FALSE(value.is_null());
+ auto to_column = data_type->create_column();
+ if constexpr (nullable) {
+ auto& nullable_col =
assert_cast<ColumnNullable&>(*to_column);
+ value.insert_into(nullable_col.get_nested_column());
+ } else {
+ value.insert_into(*to_column);
+ }
+
+ EXPECT_EQ(1, to_column->size());
+ check_column_basic<DataType, nullable>(to_column.get(), i);
+ }
+ }
+ //test CopiedValue
+ {
+ CopiedValue<ColumnType, nullable> value;
+ EXPECT_TRUE(value.is_null());
+ for (int64_t i = 0; i < input_nums; ++i) {
+ value.set_value(data_column.get(), i);
+ EXPECT_FALSE(value.is_null());
+ auto to_column = data_type->create_column();
+ if constexpr (nullable) {
+ auto& nullable_col =
assert_cast<ColumnNullable&>(*to_column);
+ value.insert_into(nullable_col.get_nested_column());
+ } else {
+ value.insert_into(*to_column);
+ }
+ EXPECT_EQ(1, to_column->size());
+ check_column_basic<DataType, nullable>(to_column.get(), i);
+ }
+ }
+ }
+
+ template <typename DataType, typename ColumnType, bool nullable>
+ void test_array_data(int8_t input_nums) {
+ vectorized::DataTypePtr data_type = get_data_type<DataType, nullable,
true>();
+
+ auto data_column = data_type->create_column();
+ array_add_elements<DataType, nullable>(data_column, input_nums);
+
+ EXPECT_EQ(input_nums + 1, data_column->size());
+ //test Value
+ {
+ Value<ColumnArray, nullable> value;
+ EXPECT_TRUE(value.is_null());
+ for (int64_t i = 0; i <= input_nums; ++i) {
+ value.set_value(data_column.get(), i);
+ EXPECT_FALSE(value.is_null());
+ auto to_column = data_type->create_column();
+ if constexpr (nullable) {
+ auto& nullable_col =
assert_cast<ColumnNullable&>(*to_column);
+ value.insert_into(nullable_col.get_nested_column());
+ } else {
+ value.insert_into(*to_column);
+ }
+ EXPECT_EQ(1, to_column->size());
+ check_column_array<DataType, nullable>(to_column.get(), i);
+ }
+ }
+ //test CopiedValue
+ {
+ CopiedValue<ColumnArray, nullable> value;
+ EXPECT_TRUE(value.is_null());
+ for (int64_t i = 0; i <= input_nums; ++i) {
+ value.set_value(data_column.get(), i);
+ EXPECT_FALSE(value.is_null());
+ auto to_column = data_type->create_column();
+ if constexpr (nullable) {
+ auto& nullable_col =
assert_cast<ColumnNullable&>(*to_column);
+ value.insert_into(nullable_col.get_nested_column());
+ } else {
+ value.insert_into(*to_column);
+ }
+ EXPECT_EQ(1, to_column->size());
+ check_column_array<DataType, nullable>(to_column.get(), i);
+ }
+ }
+ }
+
+private:
+ Arena _agg_arena_pool;
+};
+
+TEST_F(VAggReplaceTest, test_basic_data) {
+ test_basic_data<DataTypeInt8, ColumnInt8, false>(11);
+ test_basic_data<DataTypeInt16, ColumnInt16, false>(11);
+ test_basic_data<DataTypeInt32, ColumnInt32, false>(11);
+ test_basic_data<DataTypeInt64, ColumnInt64, false>(11);
+ test_basic_data<DataTypeInt128, ColumnInt128, false>(11);
+ test_basic_data<DataTypeDecimal<Decimal128>, ColumnDecimal128, false>(11);
+ test_basic_data<DataTypeString, ColumnString, false>(11);
+ test_basic_data<DataTypeInt128, ColumnInt128, false>(11);
+ test_basic_data<DataTypeDate, ColumnDate, false>(11);
+ test_basic_data<DataTypeDateTime, ColumnDateTime, false>(11);
+}
+
+TEST_F(VAggReplaceTest, test_array_data) {
+ test_array_data<DataTypeInt8, ColumnArray, false>(11);
+ test_array_data<DataTypeInt16, ColumnArray, false>(11);
+ test_array_data<DataTypeInt32, ColumnArray, false>(11);
+ test_array_data<DataTypeInt64, ColumnArray, false>(11);
+ test_array_data<DataTypeInt128, ColumnArray, false>(11);
+ test_array_data<DataTypeDecimal<Decimal128>, ColumnArray, false>(11);
+ test_array_data<DataTypeString, ColumnArray, false>(11);
+ test_array_data<DataTypeInt128, ColumnArray, false>(11);
+ test_array_data<DataTypeDate, ColumnArray, false>(11);
+ test_array_data<DataTypeDateTime, ColumnArray, false>(11);
+}
+
+TEST_F(VAggReplaceTest, test_basic_replace_reader) {
+ test_agg_replace<DataTypeInt8, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeInt16, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeInt32, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeInt64, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeInt128, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeDecimal<Decimal128>, false>("replace_reader", 10,
0);
+ test_agg_replace<DataTypeString, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeDate, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeDateTime, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeDateTime, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeBitMap, false>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeHLL, false>("replace_reader", 10, 0);
+
+ test_agg_replace<DataTypeInt8, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeInt16, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeInt32, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeInt64, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeInt128, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeDecimal<Decimal128>, true>("replace_reader", 10,
0);
+ test_agg_replace<DataTypeString, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeDate, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeDateTime, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeBitMap, true>("replace_reader", 10, 0);
+ test_agg_replace<DataTypeHLL, true>("replace_reader", 10, 0);
+}
+
+TEST_F(VAggReplaceTest, test_basic_replace_load) {
+ test_agg_replace<DataTypeInt8, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeInt16, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeInt32, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeInt64, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeInt128, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeDecimal<Decimal128>, false>("replace_load", 10,
9);
+ test_agg_replace<DataTypeString, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeDate, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeDateTime, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeBitMap, false>("replace_load", 10, 9);
+ test_agg_replace<DataTypeHLL, false>("replace_load", 10, 9);
+
+ test_agg_replace<DataTypeInt8, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeInt16, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeInt32, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeInt64, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeInt128, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeDecimal<Decimal128>, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeString, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeDate, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeDateTime, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeBitMap, true>("replace_load", 10, 9);
+ test_agg_replace<DataTypeHLL, true>("replace_load", 10, 9);
+}
+
+TEST_F(VAggReplaceTest, test_array_replace_reader) {
+ test_agg_array_replace<DataTypeInt8, false>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeInt16, false>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeInt32, false>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeInt64, false>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeInt128, false>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeDecimal<Decimal128>,
false>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeString, false>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeDate, false>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeDateTime, false>("replace_reader", 10, 0);
+
+ test_agg_array_replace<DataTypeInt8, true>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeInt16, true>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeInt32, true>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeInt64, true>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeInt128, true>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeDecimal<Decimal128>,
true>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeString, true>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeDate, true>("replace_reader", 10, 0);
+ test_agg_array_replace<DataTypeDateTime, true>("replace_reader", 10, 0);
+}
+
+TEST_F(VAggReplaceTest, test_array_replace_load) {
+ test_agg_array_replace<DataTypeInt8, false>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeInt16, false>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeInt32, false>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeInt64, false>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeInt128, false>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeDecimal<Decimal128>, false>("replace_load",
10, 10);
+ test_agg_array_replace<DataTypeString, false>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeDate, false>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeDateTime, false>("replace_load", 10, 10);
+
+ test_agg_array_replace<DataTypeInt8, true>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeInt16, true>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeInt32, true>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeInt64, true>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeInt128, true>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeDecimal<Decimal128>, true>("replace_load",
10, 10);
+ test_agg_array_replace<DataTypeString, true>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeDate, true>("replace_load", 10, 10);
+ test_agg_array_replace<DataTypeDateTime, true>("replace_load", 10, 10);
+}
+
+} // namespace doris::vectorized
diff --git a/be/test/vec/core/column_complex_test.cpp
b/be/test/vec/core/column_complex_test.cpp
index 87e5998fa3..3c597ca1b9 100644
--- a/be/test/vec/core/column_complex_test.cpp
+++ b/be/test/vec/core/column_complex_test.cpp
@@ -104,6 +104,13 @@ TEST_F(ColumnBitmapTest, SerializeAndDeserialize) {
data[row_size - 1].add(33333);
data[row_size - 1].add(0);
check_serialize_and_deserialize(column);
+
+ Field field;
+ column->get(0, field);
+ auto str = field.get<String>();
+ auto* bitmap = reinterpret_cast<const BitmapValue*>(str.c_str());
+ EXPECT_TRUE(bitmap->contains(10));
+ EXPECT_TRUE(bitmap->contains(1000000));
}
} // namespace doris::vectorized
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index d68a80f3da..347f6c555b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -438,10 +438,11 @@ public class CreateTableStmt extends DdlStmt {
columnDef.analyze(engineName.equals("olap"));
if (columnDef.getType().isArrayType() &&
engineName.equals("olap")) {
- if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
- throw new AnalysisException("Array column can't be used in
unique table");
+ if (keysDesc.getKeysType() == KeysType.AGG_KEYS) {
+ throw new AnalysisException("Array column can't be used in
aggregate table");
}
- if (columnDef.getAggregateType() != null &&
columnDef.getAggregateType() != AggregateType.NONE) {
+ if (columnDef.getAggregateType() != null &&
columnDef.getAggregateType() != AggregateType.NONE
+ && columnDef.getAggregateType() !=
AggregateType.REPLACE) {
throw new AnalysisException("Array column can't support
aggregation "
+ columnDef.getAggregateType());
}
diff --git a/regression-test/data/query_p0/show/complex_unique_1.csv
b/regression-test/data/query_p0/show/complex_unique_1.csv
new file mode 100644
index 0000000000..285573913f
--- /dev/null
+++ b/regression-test/data/query_p0/show/complex_unique_1.csv
@@ -0,0 +1,2 @@
+1 [1, 2, 3] [2022-07-13]
+2 [] [2023-12-24]
diff --git a/regression-test/data/query_p0/show/complex_unique_2.csv
b/regression-test/data/query_p0/show/complex_unique_2.csv
new file mode 100644
index 0000000000..221dcbd8eb
--- /dev/null
+++ b/regression-test/data/query_p0/show/complex_unique_2.csv
@@ -0,0 +1,2 @@
+3 [1, 2, 3] [2022-07-13]
+4 [] [2023-12-24]
diff --git
a/regression-test/data/query_p0/show/test_complex_type_unique_key.out
b/regression-test/data/query_p0/show/test_complex_type_unique_key.out
new file mode 100644
index 0000000000..5df6bd3ee4
--- /dev/null
+++ b/regression-test/data/query_p0/show/test_complex_type_unique_key.out
@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+1 [1, 2, 3] [2022-07-13]
+2 [] [2023-12-24]
+
+-- !select --
+1 [1, 2, 3] [2022-07-13]
+2 [] [2023-12-24]
+
+-- !select --
+1 [1, 2, 3] [2022-07-13]
+2 [] [2023-12-24]
+3 [1, 2, 3] [2022-07-13]
+4 [] [2023-12-24]
+
diff --git
a/regression-test/suites/query_p0/show/test_complex_type_unique_key.groovy
b/regression-test/suites/query_p0/show/test_complex_type_unique_key.groovy
new file mode 100644
index 0000000000..ab72d4205a
--- /dev/null
+++ b/regression-test/suites/query_p0/show/test_complex_type_unique_key.groovy
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_complex_type_unique_key", "p0") {
+ // define a sql table
+ def testTable = "tbl_test_complex_unique"
+ def dataFile = "complex_unique_1.csv"
+ def dataFile1 = "complex_unique_2.csv"
+
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ sql """
+ CREATE TABLE IF NOT EXISTS tbl_test_complex_unique (
+ id INT,
+ a ARRAY<SMALLINT> NOT NULL COMMENT "",
+ an ARRAY<DATE>
+ )
+ UNIQUE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+
+ // insert into valid json rows
+ sql """INSERT INTO ${testTable} VALUES (2, [], ['2023-12-24']);"""
+ sql """INSERT INTO ${testTable} VALUES (1, [1, 2, 3], ['2022-07-13'])"""
+
+ // check result
+ qt_select "SELECT * FROM ${testTable} ORDER BY id"
+
+ // stream_load for same key
+ streamLoad {
+ table testTable
+
+ file dataFile // import csv file
+ time 10000 // limit inflight 10s
+ set 'strict_mode', 'true'
+
+ // if declared a check callback, the default check condition will
ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(2, json.NumberTotalRows)
+ assertEquals(2, json.NumberLoadedRows)
+ assertEquals(0, json.NumberFilteredRows)
+ assertTrue(json.LoadBytes > 0)
+ }
+ }
+
+ sql """sync"""
+ // check result : now only 2 rows
+ qt_select "SELECT * FROM ${testTable} ORDER BY id"
+
+ // stream load different key
+ streamLoad {
+ table testTable
+
+ file dataFile1 // import csv file
+ time 10000 // limit inflight 10s
+ set 'strict_mode', 'true'
+
+ // if declared a check callback, the default check condition will
ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(2, json.NumberTotalRows)
+ assertEquals(2, json.NumberLoadedRows)
+ assertEquals(0, json.NumberFilteredRows)
+ assertTrue(json.LoadBytes > 0)
+ }
+ }
+
+ sql """sync"""
+ // check result : now 4 rows
+ qt_select "SELECT * FROM ${testTable} ORDER BY id"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
