This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from babec88aa90 fix cloud mode from PR #32748
new 36a1bf1d73f [feature][insert]Adapt the create table statement to the
nereids sql (#32458)
new c68b3530175 [feature][insert]add FE UT and support CTAS for external
table (#32525)
new 70489fe749c [fix](insert)fix hive table sink write path (#32587)
new 3343322965d [fix](insert)fix conversion of doris type to hive type
(#32735)
new b98d225183f [fix](insert)fix hive table sink type coercion and unify
coercion (#32762)
new bd364897d42 [feature](hive/iceberg)add doris's version in table
properties (#32774)
new 9ada38327b8 [feature](txn insert) txn insert support insert into
select (#31666)
new f3a6132214a [chore] Format regression-conf.groovy (#32713)
new 716c1467505 [fix](insert)fix hive external return msgs and exception
and pass all columns to BE (#32824)
new 07f296734a2 [regression](insert)add hive DDL and CTAS regression case
(#32924)
new 3c9c6c18a80 [Enhancement](hive-writer) Write only regular fields to
file in the hive-writer. (#33000)
new 01b21da82d8 [feature](insert)add hive insert plan ut and remove
redundant fields (#33051)
new 7a05396cd11 [feature](multi-catalog)support catalog name when
create/drop db (#33116)
new f0ac21e231e [feature](external) process tbl/db exist when create/drop
db/tbl (#33119)
new e11db3f050c [feature](hive)support ExternalTransaction for writing
exteral table (#32726)
new 31a7060dbdb [testcase](hive)add exception test for hive txn (#33278)
new 18fb8407ae1 [feature](insert)use optional location and add hive
regression test (#33153)
new f0463a90343 [Feature][Enhancement](hive-writer) Add hive-writer
runtime profiles, change output file names (#33245)
new d31bca199f0 [feature](iceberg)The new DDL syntax is added to create
iceberg partitioned tables (#33338)
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/config.h | 2 +-
be/src/vec/runtime/vorc_transformer.cpp | 104 +-
be/src/vec/runtime/vorc_transformer.h | 10 +-
be/src/vec/runtime/vparquet_transformer.cpp | 32 +-
be/src/vec/runtime/vparquet_transformer.h | 10 +-
be/src/vec/sink/writer/vhive_partition_writer.cpp | 169 +--
be/src/vec/sink/writer/vhive_partition_writer.h | 17 +-
be/src/vec/sink/writer/vhive_table_writer.cpp | 265 ++--
be/src/vec/sink/writer/vhive_table_writer.h | 30 +-
.../main/java/org/apache/doris/common/Config.java | 6 +-
fe/fe-core/src/main/cup/sql_parser.cup | 20 +-
.../org/apache/doris/analysis/CreateDbStmt.java | 16 +-
.../java/org/apache/doris/analysis/DbName.java | 83 ++
.../java/org/apache/doris/analysis/DropDbStmt.java | 10 +-
.../apache/doris/analysis/ShowCreateDbStmt.java | 13 +-
.../main/java/org/apache/doris/catalog/Env.java | 16 +-
.../doris/catalog/InternalSchemaInitializer.java | 5 +-
.../org/apache/doris/common/util/LocationPath.java | 7 +
.../apache/doris/datasource/ExternalCatalog.java | 18 +-
.../apache/doris/datasource/InternalCatalog.java | 4 +-
.../apache/doris/datasource/hive/HMSCommitter.java | 754 -----------
.../doris/datasource/hive/HMSExternalCatalog.java | 6 +-
.../doris/datasource/hive/HMSTransaction.java | 1315 ++++++++++++++++++++
.../datasource/hive/HiveDatabaseMetadata.java | 9 +
.../datasource/hive/HiveMetaStoreClientHelper.java | 103 +-
.../doris/datasource/hive/HiveMetadataOps.java | 193 ++-
.../datasource/hive/HivePartitionStatistics.java | 2 +-
.../hive/HivePartitionWithStatistics.java | 6 +-
.../doris/datasource/hive/HiveTableMetadata.java | 86 +-
.../org/apache/doris/datasource/hive/HiveUtil.java | 213 +++-
.../datasource/hive/ThriftHMSCachedClient.java | 166 +--
.../datasource/iceberg/DorisTypeToIcebergType.java | 14 +-
.../datasource/iceberg/IcebergMetadataOps.java | 42 +-
.../doris/datasource/iceberg/IcebergUtils.java | 94 +-
.../datasource/operations/ExternalMetadataOps.java | 2 +
.../main/java/org/apache/doris/fs/FileSystem.java | 9 +
.../org/apache/doris/fs/LocalDfsFileSystem.java | 245 ++++
.../java/org/apache/doris/fs/LocalFileSystem.java | 76 --
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 45 +
.../doris/insertoverwrite/InsertOverwriteUtil.java | 45 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 9 +-
.../doris/nereids/parser/PartitionTableInfo.java | 7 +-
.../DistributionSpecTableSinkHashPartitioned.java | 3 -
.../doris/nereids/rules/analysis/BindSink.java | 131 +-
...ogicalHiveTableSinkToPhysicalHiveTableSink.java | 3 +-
.../trees/plans/commands/CreateTableCommand.java | 9 +-
.../trees/plans/commands/info/CreateTableInfo.java | 97 +-
.../commands/insert/AbstractInsertExecutor.java | 2 +-
.../commands/insert/HiveInsertCommandContext.java | 2 +-
.../plans/commands/insert/HiveInsertExecutor.java | 54 +-
.../commands/insert/InsertIntoTableCommand.java | 11 +-
.../insert/InsertOverwriteTableCommand.java | 92 +-
.../trees/plans/commands/insert/InsertUtils.java | 5 +
.../plans/commands/insert/OlapInsertExecutor.java | 30 +-
.../trees/plans/logical/LogicalHiveTableSink.java | 27 +-
.../trees/plans/logical/LogicalOlapTableSink.java | 8 +-
.../trees/plans/logical/LogicalTableSink.java | 17 +-
.../plans/physical/PhysicalHiveTableSink.java | 25 +-
.../trees/plans/physical/PhysicalTableSink.java | 3 +
.../org/apache/doris/planner/HiveTableSink.java | 13 +-
.../java/org/apache/doris/qe/ConnectContext.java | 13 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 30 +-
.../apache/doris/qe/InsertStreamTxnExecutor.java | 3 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 6 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 96 +-
.../doris/transaction/HiveTransactionManager.java | 79 ++
.../Transaction.java} | 15 +-
.../apache/doris/transaction/TransactionEntry.java | 163 ++-
.../TransactionManager.java} | 22 +-
.../TransactionManagerFactory.java} | 17 +-
.../apache/doris/analysis/CreateDbStmtTest.java | 9 +-
.../org/apache/doris/analysis/DropDbStmtTest.java | 7 +-
.../doris/analysis/ShowCreateDbStmtTest.java | 5 +-
.../doris/datasource/HMSCachedClientTest.java | 328 +++++
.../datasource/hive/HiveDDLAndDMLPlanTest.java | 704 +++++++++++
.../doris/datasource/hive/HiveMetadataOpsTest.java | 173 +++
.../doris/datasource/hive/HmsCommitTest.java | 375 ++++--
.../datasource/iceberg/CreateIcebergTableTest.java | 196 +++
.../java/org/apache/doris/qe/ShowExecutorTest.java | 5 +-
.../apache/doris/utframe/TestWithFeService.java | 18 +
gensrc/thrift/DataSinks.thrift | 3 +-
regression-test/conf/regression-conf.groovy | 5 +-
.../external_table_p0/hive/ddl/test_hive_ctas.out | 92 ++
.../external_table_p0/hive/ddl/test_hive_ddl.out | 60 +
.../hive/ddl/test_hive_write_type.out | 18 +
.../hive/write/test_hive_write_insert.out | 66 +
.../hive/write/test_hive_write_partitions.out | 50 +
regression-test/data/insert_p0/txn_insert.out | 259 ++++
.../pipeline/external/conf/regression-conf.groovy | 10 +-
.../pipeline/p0/conf/regression-conf.groovy | 17 +-
.../pipeline/p1/conf/regression-conf.groovy | 17 +-
.../hive/ddl/test_hive_ctas.groovy | 515 ++++++++
.../hive/ddl/test_hive_ddl.groovy | 676 ++++++++++
.../hive/ddl/test_hive_write_type.groovy | 285 +++++
.../hive/write/test_hive_write_insert.groovy | 786 ++++++------
.../hive/write/test_hive_write_partitions.groovy | 3 +-
regression-test/suites/insert_p0/txn_insert.groovy | 263 ++--
97 files changed, 7845 insertions(+), 2354 deletions(-)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/analysis/DbName.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/fs/LocalDfsFileSystem.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/fs/LocalFileSystem.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/transaction/HiveTransactionManager.java
copy
fe/fe-core/src/main/java/org/apache/doris/{datasource/hive/HiveDatabaseMetadata.java
=> transaction/Transaction.java} (70%)
copy
fe/fe-core/src/main/java/org/apache/doris/{datasource/hive/HiveDatabaseMetadata.java
=> transaction/TransactionManager.java} (70%)
copy
fe/fe-core/src/main/java/org/apache/doris/{datasource/hive/HiveDatabaseMetadata.java
=> transaction/TransactionManagerFactory.java} (70%)
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/HMSCachedClientTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
create mode 100644
regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out
create mode 100644
regression-test/data/external_table_p0/hive/ddl/test_hive_ddl.out
create mode 100644
regression-test/data/external_table_p0/hive/ddl/test_hive_write_type.out
create mode 100644
regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy
create mode 100644
regression-test/suites/external_table_p0/hive/ddl/test_hive_ddl.groovy
create mode 100644
regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
