This is an automated email from the ASF dual-hosted git repository. xuba pushed a commit to branch v0.7.x-test-front in repository https://gitbox.apache.org/repos/asf/amoro.git
commit ef68e0e51a2f94b2189725a56357ac0553a78719 Merge: 5a4e133ca 13501f406 Author: Xavier Bai <[email protected]> AuthorDate: Wed Apr 24 17:47:36 2024 +0800 Merge branch 'refs/heads/before-move-package' into v0.7.x # Conflicts: # ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java # ams/api/src/main/java/com/netease/arctic/api/config/DataExpirationConfig.java # ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/LoginController.java # ams/server/src/main/java/com/netease/arctic/server/dashboard/controller/OptimizerController.java # ams/server/src/main/java/com/netease/arctic/server/optimizing/OptimizingQueue.java # ams/server/src/main/java/com/netease/arctic/server/optimizing/TaskRuntime.java # ams/server/src/main/java/com/netease/arctic/server/table/DataExpirationConfig.java .github/ISSUE_TEMPLATE/config.yml | 2 +- .github/ISSUE_TEMPLATE/improvement.yml | 2 +- .github/ISSUE_TEMPLATE/subtask.yml | 2 +- .github/PULL_REQUEST_TEMPLATE | 4 +- .github/workflows/core-hadoop2-ci.yml | 1 + .github/workflows/core-hadoop3-ci.yml | 1 + .github/workflows/docker-images.yml | 73 ++++ .github/workflows/trino-ci.yml | 1 + .gitignore | 4 + .idea/vcs.xml | 4 +- .rat-excludes | 1 - CONTRIBUTING.md | 2 +- NOTICE | 5 + README.md | 24 +- ams/api/pom.xml | 6 +- .../netease/arctic/api/config/ConfigHelpers.java} | 0 .../netease/arctic/api/config}/ConfigOption.java | 9 +- .../netease/arctic/api/config}/ConfigOptions.java | 4 +- .../netease/arctic/api/config}/Configurations.java | 12 +- .../arctic/api/config/DataExpirationConfig.java | 117 +++--- .../arctic/api/config/OptimizingConfig.java | 171 ++++++-- .../api/config}/StructuredOptionsSplitter.java | 6 +- .../arctic/api/config/TableConfiguration.java | 112 ++--- .../arctic/api/config/TagConfiguration.java | 128 +++++- .../netease/arctic/api/metrics/MetricDefine.java | 2 +- .../com/netease/arctic/api/metrics/MetricKey.java | 4 +- ams/dashboard/README.md | 2 - ams/dashboard/package.json | 2 +- ams/dashboard/pnpm-lock.yaml | 165 +++----- ams/dashboard/pom.xml | 161 +++---- ams/dashboard/src/components/Topbar.vue | 6 +- .../src/components/tables-sub-menu/TablesMenu.vue | 54 ++- ams/dashboard/src/main.ts | 17 +- .../main/resources/static/assets/404-6Zzsn426.css | 20 - .../main/resources/static/assets/404-Cfg0Wzoe.js | 20 - .../main/resources/static/assets/Home-BVx4TYHz.css | 20 - .../main/resources/static/assets/Home-C66Dh-wI.js | 2 - .../assets/QuestionCircleOutlined-BjpJO9pY.css | 20 - .../assets/QuestionCircleOutlined-BsAdzrND.js | 20 - .../main/resources/static/assets/apex-BL_CmIl-.js | 20 - .../resources/static/assets/arctic-ICNTAog3.svg | 20 - .../main/resources/static/assets/azcli-1IWB1ccx.js | 20 - .../main/resources/static/assets/bat-DPkNLes8.js | 20 - .../resources/static/assets/clojure-C7lEYwxn.js | 20 - .../resources/static/assets/coffee-B7EJu28W.js | 20 - .../static/assets/common.type-BewOZShK.js | 20 - .../main/resources/static/assets/cpp-BEB0WcoC.js | 20 - .../resources/static/assets/create-CMcV8tL0.js | 20 - .../resources/static/assets/create-CmP2qTs8.css | 20 - .../resources/static/assets/csharp-DWIp4pSg.js | 20 - .../main/resources/static/assets/csp-C46ZqvIl.js | 20 - .../main/resources/static/assets/css-DUIZ76Ch.js | 22 - .../resources/static/assets/cssMode-BWxpU_eN.js | 23 - .../resources/static/assets/dockerfile-C1cGoIR-.js | 20 - .../resources/static/assets/fsharp-fd1GTHhf.js | 20 - .../main/resources/static/assets/go-O9LJTZXk.js | 20 - .../resources/static/assets/graphql-DV_i635h.js | 20 - .../resources/static/assets/handlebars-BvkMZJ20.js | 20 - .../main/resources/static/assets/hive-Dn5cA63E.svg | 20 - .../static/assets/home-content-iEWjcHtP.png | Bin 700160 -> 0 bytes .../main/resources/static/assets/html-3yF9IKqR.js | 20 - .../resources/static/assets/htmlMode-FDWWjQuC.js | 23 - .../resources/static/assets/iceberg-Cuu4gQ9t.svg | 20 - .../resources/static/assets/index--RLxQK15.css | 20 - .../resources/static/assets/index--Wdj70yL.css | 20 - .../resources/static/assets/index-BbPIP2jN.css | 20 - .../resources/static/assets/index-BgUGsUZ9.css | 20 - .../main/resources/static/assets/index-BjI98gEp.js | 20 - .../main/resources/static/assets/index-Bjo7HzMG.js | 23 - .../resources/static/assets/index-Btpcwc4_.css | 20 - .../resources/static/assets/index-CBlaEa6X.css | 20 - .../main/resources/static/assets/index-CIrzahzi.js | 20 - .../resources/static/assets/index-Cavyp6vq.css | 20 - .../resources/static/assets/index-CmZEm_Zd.css | 20 - .../resources/static/assets/index-DCSSQW8n.css | 20 - .../main/resources/static/assets/index-DL_oEwV3.js | 20 - .../resources/static/assets/index-Df_5A3gy.css | 20 - .../main/resources/static/assets/index-DlWIyNCI.js | 373 ----------------- .../resources/static/assets/index-Dq0y8BTZ.css | 20 - .../main/resources/static/assets/index-Dq3nmUzZ.js | 20 - .../main/resources/static/assets/index-Dt04o-8L.js | 64 --- .../main/resources/static/assets/index-QO7atDR1.js | 20 - .../resources/static/assets/index-RrNRkkHx.css | 20 - .../resources/static/assets/index-SXqAZhkD.css | 20 - .../resources/static/assets/index-XWqLvTvc.css | 20 - .../main/resources/static/assets/index-_IAmf6sY.js | 20 - .../resources/static/assets/index-hyzR3Ge3.css | 20 - .../resources/static/assets/index-mIXv5212.css | 20 - .../resources/static/assets/index-pblPB3fl.css | 20 - .../main/resources/static/assets/ini-BvajGCUy.js | 20 - .../main/resources/static/assets/java-DPgNP7lh.js | 20 - .../resources/static/assets/javascript-BhEWbzVX.js | 20 - .../resources/static/assets/jsonMode-CqSy5WHl.js | 25 -- .../resources/static/assets/kotlin-DNEyQ-_b.js | 20 - .../main/resources/static/assets/less-B86-qBTk.js | 21 - .../resources/static/assets/logo-all1-D7rKIuV6.svg | 20 - .../main/resources/static/assets/lua-D28Ae8-K.js | 20 - .../resources/static/assets/markdown-B78DZXM-.js | 20 - .../main/resources/static/assets/msdax-CYqgjx_P.js | 20 - .../main/resources/static/assets/mysql-BAlWi9Ou.js | 20 - .../static/assets/objective-c-B1aVtJYH.js | 20 - .../resources/static/assets/optimize-DFb8yrlB.css | 20 - .../static/assets/optimize.service-gp8YzoRE.js | 20 - .../resources/static/assets/overview-Cxckk5vc.svg | 29 -- .../resources/static/assets/pascal-DTluBHOt.js | 20 - .../main/resources/static/assets/perl-DlYyT36c.js | 20 - .../main/resources/static/assets/pgsql-M-WMQIqP.js | 20 - .../main/resources/static/assets/php-120yhfDK.js | 20 - .../resources/static/assets/postiats-CQpG440k.js | 20 - .../resources/static/assets/powerquery-DdJtto1Z.js | 20 - .../resources/static/assets/powershell-Bu_VLpJB.js | 20 - .../main/resources/static/assets/pug-kFxLfcjb.js | 20 - .../resources/static/assets/python-BkTgHGXl.js | 20 - .../src/main/resources/static/assets/r-DTzboqR4.js | 20 - .../main/resources/static/assets/razor-erwnNA65.js | 20 - .../main/resources/static/assets/redis-CHOsPHWR.js | 20 - .../resources/static/assets/redshift-C7PkzcsH.js | 20 - .../main/resources/static/assets/ruby-Dg4z_gXy.js | 20 - .../main/resources/static/assets/rust-BCqVQ9nX.js | 20 - .../main/resources/static/assets/sb-BYAiYHFx.js | 20 - .../resources/static/assets/scheme-Dhb-2j9p.js | 20 - .../main/resources/static/assets/scss-CdQcnAnq.js | 22 - .../static/assets/setting.services-D6jcsF9r.js | 20 - .../resources/static/assets/settings-6RYF0iyM.svg | 30 -- .../main/resources/static/assets/shell-CmmbcNmB.js | 20 - .../resources/static/assets/solidity-CME5AdoB.js | 20 - .../main/resources/static/assets/sql-CbGH2FIt.js | 20 - .../main/resources/static/assets/st-h_7JEduv.js | 20 - .../main/resources/static/assets/swift-CDHAPRNA.js | 22 - .../static/assets/table.service-Ca4OvdR2.js | 20 - .../main/resources/static/assets/tcl-CVK_QlCp.js | 20 - .../resources/static/assets/tsMode-D4NVmffY.js | 28 -- .../resources/static/assets/typescript-TCW5qV2A.js | 20 - .../resources/static/assets/upgrade-B5IwYnbA.css | 20 - .../resources/static/assets/upgrade-DQoaXczL.js | 20 - .../static/assets/usePagination-flyrWnsD.js | 20 - .../static/assets/usePlaceholder-DDgQeLdU.js | 20 - .../main/resources/static/assets/vb-DgkWD2cI.js | 20 - .../main/resources/static/assets/xml-DfgkohBE.js | 20 - .../main/resources/static/assets/yaml-BR93gJIW.js | 20 - .../src/main/resources/static/favicon-bak.ico | Bin 4286 -> 0 bytes .../src/main/resources/static/favicon.ico | Bin 1272 -> 0 bytes ams/dashboard/src/main/resources/static/index.html | 32 -- ams/dashboard/src/views/catalogs/Detail.vue | 31 +- ams/dashboard/vite.config.ts | 10 +- ams/dist/pom.xml | 14 +- ams/dist/src/main/arctic-bin/conf/config.yaml | 21 + ams/dist/src/main/assemblies/bin.xml | 12 +- ams/metric-reporter/pom.xml | 4 +- ams/metric-reporter/prometheus-reporter/pom.xml | 6 +- ams/optimizer/common/pom.xml | 12 +- .../optimizer/common/TestOptimizerExecutor.java | 2 +- .../optimizer/common/TestOptimizerToucher.java | 2 +- ams/optimizer/flink-optimizer/pom.xml | 10 +- .../arctic/optimizer/flink/FlinkExecutor.java | 9 +- .../arctic/optimizer/flink/FlinkOptimizer.java | 4 +- .../optimizer/flink/FlinkOptimizerExecutor.java | 2 +- ams/optimizer/pom.xml | 4 +- ams/optimizer/spark-optimizer/pom.xml | 12 +- .../arctic/optimizer/spark/SparkOptimizer.java | 9 +- ams/optimizer/standalone-optimizer/pom.xml | 10 +- ams/pom.xml | 4 +- ams/server/pom.xml | 31 +- .../arctic/server/ArcticManagementConf.java | 23 +- .../server/ArcticManagementConfValidator.java | 94 +++++ .../arctic/server/ArcticServiceContainer.java | 90 +--- .../arctic/server/DefaultOptimizingService.java | 6 +- .../arctic/server/HighAvailabilityContainer.java | 2 +- .../netease/arctic/server/RestCatalogService.java | 6 +- .../arctic/server/TableManagementService.java | 2 +- .../arctic/server/catalog/CatalogBuilder.java | 20 +- .../arctic/server/catalog/ExternalCatalog.java | 2 +- .../arctic/server/catalog/InternalCatalog.java | 2 +- .../server/catalog/InternalIcebergCatalogImpl.java | 2 +- .../server/catalog/InternalMixedCatalogImpl.java | 2 +- .../server/catalog/MixedHiveCatalogImpl.java | 5 +- .../arctic/server/dashboard/DashboardServer.java | 3 +- .../dashboard/MixedAndIcebergTableDescriptor.java | 5 +- .../server/dashboard/ServerTableDescriptor.java | 2 +- .../dashboard/component/reverser/DDLReverser.java | 2 +- .../component/reverser/PaimonTableMetaExtract.java | 2 +- .../dashboard/controller/CatalogController.java | 26 +- .../dashboard/controller/LoginController.java | 2 +- .../dashboard/controller/OptimizerController.java | 4 +- .../controller/PlatformFileInfoController.java | 17 +- .../dashboard/controller/SettingController.java | 2 +- .../dashboard/controller/TableController.java | 68 +-- .../dashboard/model/AmoroSnapshotsOfTable.java | 18 +- .../arctic/server/dashboard/model/TableMeta.java | 8 +- .../dashboard/model/TableOptimizingInfo.java | 2 +- .../arctic/server/dashboard/utils/AmsUtil.java | 27 +- .../server/exception/ArcticRuntimeException.java | 2 +- .../server/exception/ObjectNotExistsException.java | 2 +- .../server/manager/FlinkOptimizerContainer.java | 297 ++++++++++--- .../server/manager/SparkOptimizerContainer.java | 434 +++++++++++++++++++ .../arctic/server/metrics/MetricRegistry.java | 8 +- .../arctic/server/optimizing/OptimizingConfig.java | 447 -------------------- .../arctic/server/optimizing/OptimizingQueue.java | 32 +- .../arctic/server/optimizing/SchedulingPolicy.java | 4 +- .../arctic/server/optimizing/TaskRuntime.java | 5 +- .../server/optimizing/UnKeyedTableCommit.java | 2 +- .../maintainer/AutoCreateIcebergTagAction.java | 2 +- .../maintainer/IcebergTableMaintainer.java | 194 +++++---- .../maintainer/MixedTableMaintainer.java | 6 +- .../optimizing/maintainer/TableMaintainer.java | 1 + .../optimizing/plan/AbstractPartitionPlan.java | 2 +- .../optimizing/plan/CommonPartitionEvaluator.java | 2 +- .../optimizing/plan/MixedHivePartitionPlan.java | 2 +- .../server/persistence/NestedSqlSession.java | 2 +- .../arctic/server/persistence/PersistentBase.java | 2 +- .../persistence/SqlSessionFactoryProvider.java | 2 +- .../server/persistence/StatedPersistentBase.java | 8 +- .../persistence/mapper/CatalogMetaMapper.java | 2 +- .../persistence/mapper/OptimizingMapper.java | 2 +- .../persistence/mapper/TableBlockerMapper.java | 2 +- .../server/persistence/mapper/TableMetaMapper.java | 2 +- .../arctic/server/resource/ContainerMetadata.java | 4 +- .../arctic/server/resource/OptimizerThread.java | 8 +- .../arctic/server/resource/ResourceContainers.java | 4 +- .../arctic/server/table/DataExpirationConfig.java | 349 ---------------- .../arctic/server/table/DefaultTableService.java | 22 +- .../arctic/server/table/RuntimeHandlerChain.java | 1 + .../arctic/server/table/ServerTableIdentifier.java | 145 ------- .../arctic/server/table/TableConfiguration.java | 200 --------- .../netease/arctic/server/table/TableManager.java | 1 + .../netease/arctic/server/table/TableMetadata.java | 3 +- .../netease/arctic/server/table/TableMetrics.java | 1 + .../netease/arctic/server/table/TableRuntime.java | 5 +- .../arctic/server/table/TableRuntimeHandler.java | 1 + .../arctic/server/table/TableRuntimeMeta.java | 1 + .../netease/arctic/server/table/TableService.java | 1 + .../arctic/server/table/TagConfiguration.java | 240 ----------- .../arctic/server/table/blocker/TableBlocker.java | 2 +- .../server/table/executor/AsyncTableExecutors.java | 2 +- .../server/table/executor/BaseTableExecutor.java | 4 +- .../DanglingDeleteFilesCleaningExecutor.java | 2 +- .../table/executor/DataExpiringExecutor.java | 2 +- .../table/executor/HiveCommitSyncExecutor.java | 2 +- .../executor/OrphanFilesCleaningExecutor.java | 2 +- .../table/executor/SnapshotsExpiringExecutor.java | 2 +- .../executor/TableRuntimeRefreshExecutor.java | 2 +- .../table/executor/TagsAutoCreatingExecutor.java | 2 +- .../internal/IcebergInternalTableOperations.java | 2 +- .../table/internal/InternalIcebergCreator.java | 2 +- .../internal/InternalMixedIcebergCreator.java | 2 +- .../MixedIcebergInternalTableStoreOperations.java | 2 +- .../arctic/server/terminal/ExecutionResult.java | 6 +- .../arctic/server/terminal/SparkContextUtil.java | 18 +- .../arctic/server/terminal/TerminalManager.java | 23 +- .../server/terminal/TerminalSessionContext.java | 2 +- .../server/terminal/TerminalSessionFactory.java | 8 +- .../kyuubi/KyuubiTerminalSessionFactory.java | 25 +- .../server/terminal/local/LocalSessionFactory.java | 6 +- .../arctic/server/utils/FlinkClientUtil.java | 48 +++ .../server/utils/StructuredOptionsSplitter.java | 2 +- ...m.netease.arctic.api.resource.ResourceContainer | 1 + .../com/netease/arctic/server/AmsEnvironment.java | 2 +- .../arctic/server/RestCatalogServiceTestBase.java | 2 +- .../server/TestArcticManagementConfValidator.java | 154 +++++++ .../server/TestDefaultOptimizingService.java | 2 +- .../server/TestInternalMixedCatalogService.java | 165 ++++---- .../dashboard/TestServerTableDescriptor.java | 2 +- .../manager/TestSparkOptimizerContainer.java | 61 +++ .../optimizing/TestIcebergHadoopOptimizing.java | 2 +- .../server/optimizing/TestOptimizingQueue.java | 10 +- .../optimizing/flow/CompleteOptimizingFlow.java | 6 +- .../flow/view/AbstractTableDataView.java | 2 +- .../optimizing/flow/view/KeyedTableDataView.java | 2 +- .../maintainer/TestAutoCreateIcebergTagAction.java | 2 +- .../optimizing/maintainer/TestDataExpire.java | 4 +- .../optimizing/maintainer/TestOrphanFileClean.java | 4 +- .../maintainer/TestOrphanFileCleanIceberg.java | 2 +- .../optimizing/maintainer/TestSnapshotExpire.java | 4 +- .../optimizing/plan/MixedTablePlanTestBase.java | 8 +- .../plan/TestHiveKeyedPartitionPlan.java | 4 +- .../plan/TestHiveUnkeyedPartitionPlan.java | 4 +- .../optimizing/plan/TestIcebergPartitionPlan.java | 2 +- .../scan/TableFileScanHelperTestBase.java | 2 +- .../persistence/TestStatedPersistentBase.java | 9 +- .../arctic/server/table/AMSTableTestBase.java | 41 +- .../arctic/server/table/DerbyPersistence.java | 2 +- .../arctic/server/table/TableServiceTestBase.java | 2 +- .../arctic/server/table/TestDatabaseService.java | 2 +- .../server/table/TestTableRuntimeHandler.java | 11 +- .../server/table/TestTableRuntimeManager.java | 1 + .../arctic/server/table/TestTableService.java | 29 +- .../executor/TestBlockerExpiringExecutor.java | 2 +- .../arctic/server/util/TestConfigurationUtil.java | 4 +- charts/amoro/Chart.yaml | 6 +- charts/amoro/README.md | 8 +- charts/amoro/templates/NOTES.txt | 8 +- charts/amoro/templates/_helpers.tpl | 1 - charts/amoro/templates/_optimizer.tpl | 28 ++ charts/amoro/templates/_pod.tpl | 30 +- charts/amoro/templates/amoro-configmap.yaml | 4 + charts/amoro/values.yaml | 29 +- core/pom.xml | 4 +- .../netease/arctic/catalog/BasicArcticCatalog.java | 38 +- .../com/netease/arctic/catalog/MixedTables.java | 4 + .../com/netease/arctic/client/ArcticThriftUrl.java | 4 +- .../java/com/netease/arctic/data/ChangedLsn.java | 8 +- .../java/com/netease/arctic/data/DataTreeNode.java | 10 +- .../com/netease/arctic/data/FileNameRules.java | 2 +- .../arctic/formats/iceberg/IcebergCatalog.java | 80 ++-- .../java/com/netease/arctic/io/ArcticFileIO.java | 15 + .../com/netease/arctic/io/ArcticFileIOAdapter.java | 12 + .../com/netease/arctic/io/ArcticHadoopFileIO.java | 15 + .../arctic/io/reader/AbstractKeyedDataReader.java | 2 +- .../io/reader/AbstractUnkeyedDataReader.java | 2 +- .../io/writer/ArcticTreeNodePosDeleteWriter.java | 4 +- .../netease/arctic/io/writer/TaskWriterKey.java | 8 +- .../java/com/netease/arctic/log/Converter.java | 2 +- .../arctic/mixed/BasicMixedIcebergCatalog.java | 30 +- .../arctic/mixed/InternalMixedIcebergCatalog.java | 2 +- .../arctic/properties}/HiveTableProperties.java | 2 +- .../netease/arctic/utils/ArcticCatalogUtil.java | 41 ++ .../com/netease/arctic/utils/TableFileUtil.java | 82 +++- .../netease/arctic/MockArcticMetastoreServer.java | 33 +- .../formats/IcebergHadoopCatalogTestHelper.java | 16 +- .../MixedIcebergHadoopCatalogTestHelper.java | 14 +- .../formats/PaimonHadoopCatalogTestHelper.java | 4 +- .../netease/arctic/io/IcebergDataTestHelpers.java | 2 +- .../io/TestIcebergCombinedReaderVariousTypes.java | 3 +- .../arctic/utils/map/TestSimpleSpillableMap.java | 4 +- .../map/TestStructLikeWrapperSizeEstimator.java | 2 +- .../org/apache/iceberg/TestIcebergFindFiles.java | 18 +- docker/amoro/Dockerfile | 1 + docker/build.sh | 69 ++- docker/optimizer-flink/.gitignore | 2 - docker/optimizer-flink/Dockerfile | 15 +- .../Dockerfile | 21 +- docs/admin-guides/deployment-on-kubernetes.md | 6 +- docs/admin-guides/deployment.md | 20 +- docs/admin-guides/managing-optimizers.md | 154 ++++++- docs/engines/flink/flink-dml.md | 58 ++- docs/engines/flink/using-logstore.md | 2 +- docs/engines/spark/spark-get-started.md | 2 +- docs/images/flink/double-write.png | Bin 0 -> 54092 bytes mixed-format/flink/flink-common-format/pom.xml | 6 +- .../netease/arctic/flink/shuffle/LogRecordV1.java | 2 +- .../flink/flink-common-iceberg-bridge/pom.xml | 12 +- mixed-format/flink/flink-common/pom.xml | 18 +- .../com/netease/arctic/flink/FlinkSchemaUtil.java | 1 + .../arctic/flink/catalog/FlinkUnifiedCatalog.java | 28 +- .../netease/arctic/flink/catalog/MixedCatalog.java | 2 +- .../flink/lookup/filter/RowDataPredicate.java | 2 +- .../enumerator/ContinuousSplitPlannerImpl.java | 6 +- .../enumerator/MergeOnReadIncrementalPlanner.java | 2 +- .../hybrid/enumerator/MergeOnReadPlannerImpl.java | 4 +- .../read/hybrid/split/TemporalJoinSplits.java | 8 +- .../com/netease/arctic/flink/util/ArcticUtils.java | 4 +- .../arctic/flink/write/FlinkTaskWriterBuilder.java | 2 +- .../flink/catalog/TestFlinkUnifiedCatalogs.java | 2 +- .../netease/arctic/flink/lookup/TestKVTable.java | 5 +- .../arctic/flink/read/TestArcticSource.java | 68 +-- .../netease/arctic/flink/read/TestFlinkSource.java | 4 +- .../flink/read/hidden/kafka/TestKafkaConsumer.java | 6 +- .../read/hidden/kafka/TestKafkaSourceReader.java | 7 +- .../kafka/TestLogKafkaPartitionSplitReader.java | 6 +- .../enumerator/TestArcticSourceEnumerator.java | 18 +- .../enumerator/TestContinuousSplitPlannerImpl.java | 34 +- .../hybrid/reader/MixedIncrementalLoaderTest.java | 2 +- .../hybrid/reader/TestRowDataReaderFunction.java | 60 +-- .../arctic/flink/shuffle/TestLogRecordV1.java | 4 +- .../arctic/flink/write/TestAutomaticLogWriter.java | 6 +- .../flink/write/hidden/kafka/TestBaseLog.java | 10 +- .../hidden/kafka/TestHiddenKafkaProducer.java | 9 +- .../write/hidden/kafka/TestHiddenLogOperators.java | 18 +- mixed-format/flink/pom.xml | 4 +- mixed-format/flink/v1.15/flink-runtime/pom.xml | 10 +- mixed-format/flink/v1.15/flink/pom.xml | 8 +- mixed-format/flink/v1.16/flink-runtime/pom.xml | 10 +- mixed-format/flink/v1.16/flink/pom.xml | 8 +- mixed-format/flink/v1.17/flink-runtime/pom.xml | 10 +- mixed-format/flink/v1.17/flink/pom.xml | 8 +- mixed-format/hive/pom.xml | 10 +- .../java/com/netease/arctic/hive/HMSClient.java | 2 + .../com/netease/arctic/hive/HMSClientImpl.java | 6 + .../arctic/hive/catalog/ArcticHiveCatalog.java | 463 +++++++++++++++++++-- .../arctic/hive/catalog/MixedHiveTables.java | 144 ++++++- .../writer/AdaptHiveGenericTaskWriterBuilder.java | 2 +- .../arctic/hive/op/ReplaceHivePartitions.java | 2 +- .../netease/arctic/hive/op/UpdateHiveFiles.java | 2 +- .../netease/arctic/hive/table/KeyedHiveTable.java | 2 +- .../arctic/hive/table/UnkeyedHiveTable.java | 4 +- .../hive/utils/CompatibleHivePropertyUtil.java | 2 +- .../arctic/hive/utils/HiveMetaSynchronizer.java | 8 +- .../netease/arctic/hive/utils/HiveTableUtil.java | 2 +- .../arctic/hive/utils/UpgradeHiveTableUtil.java | 2 +- .../arctic/hive/catalog/HiveCatalogTestHelper.java | 5 +- .../arctic/hive/catalog/TestMixedHiveCatalog.java | 4 +- .../hive/formats/IcebergHiveCatalogTestHelper.java | 2 +- .../arctic/hive/io/HiveDataTestHelpers.java | 2 +- .../netease/arctic/hive/op/TestOverwriteFiles.java | 2 +- .../netease/arctic/hive/op/TestRewriteFiles.java | 2 +- .../arctic/hive/op/TestRewritePartitions.java | 2 +- .../netease/arctic/hive/op/TestSyncHiveMeta.java | 2 +- .../arctic/hive/op/UpdateHiveFilesTestHelpers.java | 2 +- .../hive/utils/TestCompatibleHivePropertyUtil.java | 2 +- .../hive/utils/TestHiveMetaSynchronizer.java | 2 +- .../hive/utils/TestUpgradeHiveTableUtil.java | 2 +- mixed-format/pom.xml | 4 +- mixed-format/spark/common/pom.xml | 14 +- .../arctic/spark/mixed/MixedFormatSparkUtil.java | 2 +- .../spark/mixed/MixedSessionCatalogBase.java | 4 +- .../arctic/spark/test/MixedTableTestBase.java | 8 +- .../netease/arctic/spark/test/SparkTestBase.java | 22 +- .../test/unified/UnifiedCatalogTestSuites.java | 6 +- .../arctic/spark/test/utils/RecordGenerator.java | 2 +- .../arctic/spark/test/utils/TestTables.java | 38 +- mixed-format/spark/pom.xml | 4 +- mixed-format/spark/v3.2/spark-runtime/pom.xml | 21 +- mixed-format/spark/v3.2/spark/pom.xml | 18 +- .../com/netease/arctic/spark/io/TaskWriters.java | 2 +- .../arctic/spark/reader/SparkScanBuilder.java | 2 +- .../arctic/spark/reader/UnkeyedSparkBatchScan.java | 2 +- .../arctic/spark/ArcticSparkExtensions.scala | 7 + .../catalyst/analysis/RewriteArcticCommand.scala | 2 +- .../arctic/spark/test/TestImpalaParquet.java | 2 +- .../test/suites/catalog/TestSessionCatalog.java | 14 +- .../test/suites/sql/TestArcticSessionCatalog.java | 28 +- .../test/suites/sql/TestCreateTableAsSelect.java | 34 +- .../test/suites/sql/TestCreateTableLikeSQL.java | 17 +- .../spark/test/suites/sql/TestInsertIntoSQL.java | 90 ++-- .../test/suites/sql/TestInsertOverwriteSQL.java | 196 ++++----- .../spark/test/suites/sql/TestMergeIntoSQL.java | 4 +- .../spark/test/suites/sql/TestSelectSQL.java | 4 +- .../arctic/spark/writer/TestMixedWriter.java | 108 ++--- mixed-format/spark/v3.3/spark-runtime/pom.xml | 21 +- mixed-format/spark/v3.3/spark/pom.xml | 18 +- .../com/netease/arctic/spark/io/TaskWriters.java | 2 +- .../arctic/spark/reader/SparkScanBuilder.java | 2 +- .../arctic/spark/reader/UnkeyedSparkBatchScan.java | 2 +- .../arctic/spark/ArcticSparkExtensions.scala | 7 + .../catalyst/analysis/RewriteArcticCommand.scala | 2 +- .../arctic/spark/test/TestImpalaParquet.java | 2 +- .../test/suites/catalog/TestSessionCatalog.java | 14 +- .../test/suites/sql/TestArcticSessionCatalog.java | 28 +- .../test/suites/sql/TestCreateTableAsSelect.java | 34 +- .../test/suites/sql/TestCreateTableLikeSQL.java | 17 +- .../spark/test/suites/sql/TestInsertIntoSQL.java | 90 ++-- .../test/suites/sql/TestInsertOverwriteSQL.java | 196 ++++----- .../spark/test/suites/sql/TestMergeIntoSQL.java | 4 +- .../spark/test/suites/sql/TestSelectSQL.java | 4 +- .../arctic/spark/writer/TestMixedWriter.java | 108 ++--- mixed-format/trino/pom.xml | 14 +- .../com/netease/arctic/trino/ArcticConnector.java | 8 +- .../arctic/trino/ArcticConnectorMetadata.java | 4 +- .../com/netease/arctic/trino/ArcticPlugin.java | 2 +- .../arctic/trino/ArcticSessionProperties.java | 2 +- .../arctic/trino/ArcticTransactionManager.java | 4 +- .../arctic/trino/delete/DummyFileScanTask.java | 2 +- .../arctic/trino/delete/TrinoDeleteFile.java | 6 +- .../com/netease/arctic/trino/delete/TrinoRow.java | 4 +- .../arctic/trino/keyed/KeyedConnectorMetadata.java | 16 +- .../trino/keyed/KeyedConnectorPageSource.java | 6 +- .../arctic/trino/keyed/KeyedDeleteFilter.java | 2 +- .../trino/keyed/KeyedPageSourceProvider.java | 2 +- .../trino/unkeyed/AdaptHiveIcebergTableHandle.java | 6 +- .../arctic/trino/unkeyed/ArcticTrinoCatalog.java | 2 +- .../arctic/trino/unkeyed/IcebergMetadata.java | 32 +- .../arctic/trino/unkeyed/IcebergPageSource.java | 4 +- .../trino/unkeyed/IcebergPageSourceProvider.java | 18 +- .../netease/arctic/trino/unkeyed/IcebergSplit.java | 6 +- .../arctic/trino/unkeyed/IcebergSplitManager.java | 2 +- .../arctic/trino/unkeyed/IcebergSplitSource.java | 24 +- .../arctic/trino/unkeyed/TrinoDeleteFilter.java | 4 +- .../hadoop/hive/metastore/HiveMetaStoreClient.java | 185 ++++---- .../arctic/trino/arctic/ArcticQueryRunner.java | 2 +- .../arctic/trino/arctic/TableTestBaseForTrino.java | 4 +- .../trino/arctic/TestBaseArcticPrimaryTable.java | 2 +- .../netease/arctic/trino/arctic/TestHiveTable.java | 4 +- .../trino/arctic/TestHiveTableBaseForTrino.java | 6 +- .../arctic/trino/iceberg/ArcticQueryRunner.java | 9 +- .../trino/iceberg/ArcticQueryRunnerForClient.java | 2 +- .../arctic/trino/iceberg/BaseConnectorTest.java | 16 +- .../arctic/trino/iceberg/SchemaInitializer.java | 4 +- .../arctic/trino/iceberg/TestArcticPlugin.java | 2 +- .../trino/iceberg/TestBaseArcticConnectorTest.java | 16 +- pom.xml | 57 ++- tools/maven/checkstyle.xml | 307 ++++++++++++++ tools/maven/suppressions.xml | 115 +++++ 482 files changed, 5348 insertions(+), 6284 deletions(-) diff --cc .rat-excludes index 9605e1ddc,26655c8ac..2200a8195 --- a/.rat-excludes +++ b/.rat-excludes @@@ -16,10 -16,5 +16,9 @@@ docs/* **/node_modules/** **/pnpm-lock.yaml - **/src/main/resources/static/** **/ams-dashboard/dist/** -**/components.d.ts +**/components.d.ts + +# WAP +wbx3-policies/** +frontend/** diff --cc ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java index def707f4d,693c55084..def707f4d mode 100755,100644..100755 --- a/ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java +++ b/ams/api/src/main/java/com/netease/arctic/api/config/ConfigHelpers.java
