This is an automated email from the ASF dual-hosted git repository. lta pushed a commit to branch cluster_scalability in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2c7f2a1ec1048cc57353492735b62a2b87217299 Merge: 74cfc17 db71701 Author: lta <[email protected]> AuthorDate: Fri May 7 15:23:11 2021 +0800 merge master and fix all conflicts .github/workflows/client-go.yml | 3 + .github/workflows/client.yml | 5 +- .github/workflows/e2e.yml | 11 +- .github/workflows/main-unix.yml | 5 + .github/workflows/main-win.yml | 3 + .github/workflows/sonar-coveralls.yml | 3 + Code Summary.md | 59 ++ README.md | 2 +- antlr/pom.xml | 2 +- .../antlr4/org/apache/iotdb/db/qp/sql/SqlBase.g4 | 30 +- cli/pom.xml | 5 +- .../org/apache/iotdb/tool/AbstractCsvTool.java | 1 + client-cpp/pom.xml | 2 +- client-cpp/src/main/Session.cpp | 11 +- client-cpp/src/main/Session.h | 16 +- client-py/README.md | 2 +- client-py/pom.xml | 2 +- cluster/pom.xml | 15 +- .../resources/conf/iotdb-cluster.properties | 8 +- .../java/org/apache/iotdb/cluster/ClientMain.java | 5 +- .../java/org/apache/iotdb/cluster/ClusterMain.java | 5 +- .../cluster/client/sync/SyncClientAdaptor.java | 6 +- .../apache/iotdb/cluster/config/ClusterConfig.java | 37 +- .../iotdb/cluster/coordinator/Coordinator.java | 79 ++- .../iotdb/cluster/log/applier/BaseApplier.java | 4 +- .../iotdb/cluster/log/catchup/CatchUpTask.java | 48 +- .../apache/iotdb/cluster/metadata/CMManager.java | 198 +++++- .../apache/iotdb/cluster/metadata/MetaPuller.java | 21 +- .../cluster/partition/slot/SlotPartitionTable.java | 5 +- .../cluster/query/ClusterPhysicalGenerator.java | 7 + .../iotdb/cluster/query/ClusterPlanRouter.java | 11 +- .../iotdb/cluster/query/LocalQueryExecutor.java | 22 +- .../cluster/query/reader/ClusterReaderFactory.java | 16 +- .../apache/iotdb/cluster/server/ClientServer.java | 1 + .../caller/PullMeasurementSchemaHandler.java | 13 +- .../cluster/server/member/DataGroupMember.java | 41 +- .../cluster/server/member/MetaGroupMember.java | 17 +- .../iotdb/cluster/server/member/RaftMember.java | 31 +- .../apache/iotdb/cluster/utils/PartitionUtils.java | 8 +- .../apache/iotdb/cluster/utils/StatusUtils.java | 3 +- .../cluster/client/async/AsyncClientPoolTest.java | 10 +- .../cluster/client/sync/SyncClientAdaptorTest.java | 7 +- .../org/apache/iotdb/cluster/common/IoTDBTest.java | 6 +- .../org/apache/iotdb/cluster/common/TestUtils.java | 9 +- .../query/ClusterAggregateExecutorTest.java | 4 +- .../query/ClusterDataQueryExecutorTest.java | 10 +- .../cluster/query/ClusterFillExecutorTest.java | 4 +- .../cluster/query/ClusterPlanExecutorTest.java | 2 +- .../cluster/query/ClusterQueryRouterTest.java | 12 +- .../ClusterGroupByNoVFilterDataSetTest.java | 2 +- .../groupby/ClusterGroupByVFilterDataSetTest.java | 2 +- .../caller/PullMeasurementSchemaHandlerTest.java | 12 +- .../cluster/server/member/DataGroupMemberTest.java | 4 +- .../cluster/server/member/MetaGroupMemberTest.java | 14 +- cluster/src/test/resources/logback.xml | 2 +- .../resources/node1conf/iotdb-cluster.properties | 2 +- .../resources/node2conf/iotdb-cluster.properties | 2 +- .../resources/node3conf/iotdb-cluster.properties | 2 +- code-coverage/pom.xml | 2 +- compile-tools/pom.xml | 10 +- compile-tools/thrift/pom.xml | 2 +- cross-tests/pom.xml | 3 +- distribution/pom.xml | 2 +- docker/src/main/{Dockerfile => Dockerfile-cluster} | 4 +- docker/src/main/{Dockerfile => Dockerfile-single} | 2 +- docs/Development/VoteRelease.md | 8 +- docs/SystemDesign/TsFile/Format.md | 12 +- docs/UserGuide/API/Programming-Java-Native-API.md | 61 +- .../UserGuide/API/Programming-Python-Native-API.md | 2 +- docs/UserGuide/API/Programming-TsFile-API.md | 4 +- docs/UserGuide/Advanced-Features/Alerting.md | 385 ++++++++++++ docs/UserGuide/Advanced-Features/Triggers.md | 390 +++++++++++- .../Advanced-Features/UDF-User-Defined-Function.md | 8 +- docs/UserGuide/Appendix/SQL-Reference.md | 30 +- docs/UserGuide/CLI/Command-Line-Interface.md | 69 +- docs/UserGuide/Cluster/Cluster-Setup-Example.md | 8 +- .../Data-Concept/Data-Model-and-Terminology.md | 24 +- .../UserGuide/Ecosystem Integration/Hive TsFile.md | 8 +- .../Ecosystem Integration/MapReduce TsFile.md | 2 +- .../UserGuide/Ecosystem Integration/Spark IoTDB.md | 8 +- .../Ecosystem Integration/Spark TsFile.md | 2 +- .../Ecosystem Integration/Zeppelin-IoTDB.md | 2 +- .../DDL-Data-Definition-Language.md | 42 +- .../DML-Data-Manipulation-Language.md | 19 +- docs/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +- docs/zh/Development/VoteRelease.md | 8 +- docs/zh/SystemDesign/TsFile/Format.md | 12 +- docs/zh/UserGuide/API/Programming-JDBC.md | 2 +- .../UserGuide/API/Programming-Java-Native-API.md | 41 +- .../UserGuide/API/Programming-Python-Native-API.md | 2 +- docs/zh/UserGuide/API/Programming-TsFile-API.md | 4 +- docs/zh/UserGuide/Advanced-Features/Alerting.md | 385 ++++++++++++ docs/zh/UserGuide/Advanced-Features/Triggers.md | 389 +++++++++++- .../Advanced-Features/UDF-User-Defined-Function.md | 8 +- docs/zh/UserGuide/Appendix/SQL-Reference.md | 30 +- docs/zh/UserGuide/CLI/Command-Line-Interface.md | 75 ++- docs/zh/UserGuide/Cluster/Cluster-Setup-Example.md | 8 +- .../Data-Concept/Data-Model-and-Terminology.md | 24 +- .../UserGuide/Ecosystem Integration/Hive TsFile.md | 8 +- .../Ecosystem Integration/MapReduce TsFile.md | 2 +- .../UserGuide/Ecosystem Integration/Spark IoTDB.md | 8 +- .../Ecosystem Integration/Spark TsFile.md | 2 +- .../Ecosystem Integration/Zeppelin-IoTDB.md | 2 +- .../DDL-Data-Definition-Language.md | 39 ++ .../DML-Data-Manipulation-Language.md | 21 +- docs/zh/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +- example/client-cpp-example/pom.xml | 2 +- example/flink/pom.xml | 2 +- example/hadoop/pom.xml | 5 +- .../apache/iotdb/hadoop/tsfile/TsFileHelper.java | 5 +- example/jdbc/pom.xml | 2 +- example/kafka/pom.xml | 4 +- example/kafka/readme.md | 4 +- example/mqtt/pom.xml | 2 +- .../java/org/apache/iotdb/mqtt/MQTTClient.java | 1 + example/pom.xml | 3 +- example/pulsar/pom.xml | 2 +- example/rocketmq/pom.xml | 2 +- example/rocketmq/readme.md | 4 +- example/session/pom.xml | 2 +- .../org/apache/iotdb/DataMigrationExample.java | 3 +- .../main/java/org/apache/iotdb/SessionExample.java | 6 +- .../org/apache/iotdb/VectorSessionExample.java | 337 ++++++++++ example/{udf => trigger}/pom.xml | 6 +- .../org/apache/iotdb/trigger/AlertingExample.java | 107 ++++ .../org/apache/iotdb/trigger/TriggerExample.java | 127 ++++ example/tsfile/pom.xml | 2 +- example/tsfile/readme.md | 2 +- .../apache/iotdb/tsfile/TsFileWriteWithTablet.java | 5 +- example/udf/pom.xml | 4 +- flink-iotdb-connector/pom.xml | 2 +- flink-tsfile-connector/pom.xml | 2 +- grafana/pom.xml | 3 +- hadoop/README.md | 2 +- hadoop/pom.xml | 5 +- .../iotdb/hadoop/tsfile/TsFileTestHelper.java | 3 +- hive-connector/pom.xml | 7 +- .../org/apache/iotdb/hive/TsFileTestHelper.java | 3 +- jdbc/README.md | 4 +- jdbc/pom.xml | 3 +- .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 14 +- pom.xml | 61 +- server/pom.xml | 13 +- .../resources/conf/iotdb-engine.properties | 153 ++++- server/src/assembly/resources/conf/logback.xml | 24 +- .../iotdb/db/auth/authorizer/OpenIdAuthorizer.java | 3 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 58 +- .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 4 + .../org/apache/iotdb/db/conf/IoTDBConstant.java | 1 + .../org/apache/iotdb/db/engine/StorageEngine.java | 48 +- .../db/engine/cache/CacheHitRatioMonitor.java | 28 + .../engine/cache/CacheHitRatioMonitorMXBean.java | 10 + .../db/engine/cache/TimeSeriesMetadataCache.java | 7 +- .../compaction/CompactionMergeTaskPoolManager.java | 38 +- .../db/engine/compaction/TsFileManagement.java | 19 +- .../level/LevelCompactionTsFileManagement.java | 274 ++++---- .../no/NoCompactionTsFileManagement.java | 132 ++-- .../iotdb/db/engine/flush/MemTableFlushTask.java | 119 +++- .../iotdb/db/engine/flush/NotifyFlushMemTable.java | 4 +- .../iotdb/db/engine/memtable/AbstractMemTable.java | 193 ++++-- .../apache/iotdb/db/engine/memtable/IMemTable.java | 19 +- .../db/engine/memtable/IWritableMemChunk.java | 43 +- .../db/engine/memtable/PrimitiveMemTable.java | 9 +- .../iotdb/db/engine/memtable/WritableMemChunk.java | 87 ++- .../db/engine/merge/manage/MergeResource.java | 14 +- .../iotdb/db/engine/merge/task/MergeFileTask.java | 17 +- .../db/engine/merge/task/MergeMultiChunkTask.java | 6 +- .../iotdb/db/engine/merge/task/MergeTask.java | 5 +- .../db/engine/querycontext/QueryDataSource.java | 12 +- .../db/engine/querycontext/ReadOnlyMemChunk.java | 145 ++++- .../db/engine/storagegroup/StorageGroupInfo.java | 6 +- .../engine/storagegroup/StorageGroupProcessor.java | 183 +++--- .../db/engine/storagegroup/TsFileProcessor.java | 214 +++++-- .../engine/storagegroup/TsFileProcessorInfo.java | 6 +- .../db/engine/storagegroup/TsFileResource.java | 152 +++-- .../storagegroup/timeindex/DeviceTimeIndex.java | 18 - .../storagegroup/timeindex/FileTimeIndex.java | 5 - .../engine/storagegroup/timeindex/ITimeIndex.java | 8 - ...eption.java => AlignedTimeseriesException.java} | 19 +- .../metadata/DuplicatedTemplateException.java} | 17 +- .../metadata/StorageGroupNotSetException.java | 4 + .../metadata/UndefinedTemplateException.java} | 17 +- .../apache/iotdb/db/metadata/MLogTxtWriter.java | 36 +- .../org/apache/iotdb/db/metadata/MManager.java | 640 ++++++++++++++++--- .../java/org/apache/iotdb/db/metadata/MTree.java | 389 ++++++++++-- .../apache/iotdb/db/metadata/MeasurementMeta.java | 14 +- .../org/apache/iotdb/db/metadata/MetaUtils.java | 24 + .../iotdb/db/metadata/MetadataOperationType.java | 2 + .../org/apache/iotdb/db/metadata/PartialPath.java | 6 +- .../iotdb/db/metadata/VectorPartialPath.java | 68 ++ .../iotdb/db/metadata/logfile/MLogWriter.java | 38 +- .../org/apache/iotdb/db/metadata/mnode/MNode.java | 98 +++ .../iotdb/db/metadata/mnode/MeasurementMNode.java | 20 +- .../iotdb/db/metadata/mnode/StorageGroupMNode.java | 26 +- .../iotdb/db/metadata/template/Template.java | 170 +++++ .../apache/iotdb/db/qp/executor/PlanExecutor.java | 142 ++++- .../org/apache/iotdb/db/qp/logical/Operator.java | 7 +- .../iotdb/db/qp/logical/crud/InsertOperator.java | 10 +- .../apache/iotdb/db/qp/physical/PhysicalPlan.java | 25 + .../iotdb/db/qp/physical/crud/AggregationPlan.java | 5 + .../db/qp/physical/crud/CreateTemplatePlan.java | 240 +++++++ .../iotdb/db/qp/physical/crud/FillQueryPlan.java | 5 + .../iotdb/db/qp/physical/crud/GroupByTimePlan.java | 2 + .../iotdb/db/qp/physical/crud/InsertRowPlan.java | 110 ++-- .../db/qp/physical/crud/InsertTabletPlan.java | 70 ++- .../iotdb/db/qp/physical/crud/LastQueryPlan.java | 5 + .../iotdb/db/qp/physical/crud/QueryIndexPlan.java | 5 + .../iotdb/db/qp/physical/crud/QueryPlan.java | 14 + .../db/qp/physical/crud/RawDataQueryPlan.java | 63 +- .../db/qp/physical/crud/SetDeviceTemplatePlan.java | 94 +++ .../apache/iotdb/db/qp/physical/crud/UDTFPlan.java | 5 + .../qp/physical/sys/AutoCreateDeviceMNodePlan.java | 87 +++ .../physical/sys/CreateAlignedTimeSeriesPlan.java | 259 ++++++++ .../qp/physical/sys/CreateMultiTimeSeriesPlan.java | 7 +- .../db/qp/physical/sys/MeasurementMNodePlan.java | 9 +- .../physical/sys/SetUsingDeviceTemplatePlan.java | 83 +++ .../db/qp/physical/sys/StorageGroupMNodePlan.java | 36 +- .../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 48 +- .../iotdb/db/qp/strategy/PhysicalGenerator.java | 115 +++- .../qp/strategy/optimizer/ConcatPathOptimizer.java | 9 + .../iotdb/db/query/context/QueryContext.java | 4 +- .../db/query/dataset/AlignByDeviceDataSet.java | 25 +- .../dataset/RawQueryDataSetWithValueFilter.java | 8 +- .../dataset/RawQueryDataSetWithoutValueFilter.java | 143 ++++- .../iotdb/db/query/executor/LastQueryExecutor.java | 41 +- .../iotdb/db/query/executor/QueryRouter.java | 16 +- .../db/query/executor/fill/LastPointReader.java | 34 +- .../iotdb/db/query/reader/chunk/MemPageReader.java | 19 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 17 +- .../chunk/metadata/MemChunkMetadataLoader.java | 15 +- .../query/reader/series/SeriesAggregateReader.java | 2 +- .../reader/series/SeriesRawDataBatchReader.java | 4 +- .../iotdb/db/query/reader/series/SeriesReader.java | 129 ++-- .../reader/series/SeriesReaderByTimestamp.java | 4 +- .../query/reader/series/SeriesReaderFactory.java | 94 +++ .../db/query/reader/series/VectorSeriesReader.java | 144 +++++ .../query/timegenerator/ServerTimeGenerator.java | 7 + .../iotdb/db/rescon/PrimitiveArrayManager.java | 7 +- .../org/apache/iotdb/db/rescon/SystemInfo.java | 244 +++---- .../apache/iotdb/db/rescon/TVListAllocator.java | 15 +- .../org/apache/iotdb/db/service/ServiceType.java | 3 +- .../org/apache/iotdb/db/service/TSServiceImpl.java | 163 ++++- .../alertmanager/AlertManagerConfiguration.java} | 17 +- .../db/sink/alertmanager/AlertManagerEvent.java | 123 ++++ .../db/sink/alertmanager/AlertManagerHandler.java | 83 +++ .../Evaluator.java => sink/api/Configuration.java} | 14 +- .../api/Evaluator.java => sink/api/Event.java} | 14 +- .../api/Evaluator.java => sink/api/Handler.java} | 16 +- .../exception/SinkException.java} | 15 +- .../db/sink/local/LocalIoTDBConfiguration.java | 51 ++ .../local/LocalIoTDBEvent.java} | 23 +- .../iotdb/db/sink/local/LocalIoTDBHandler.java | 104 +++ .../iotdb/db/sink/mqtt/MQTTConfiguration.java | 74 +++ .../org/apache/iotdb/db/sink/mqtt/MQTTEvent.java | 51 +- .../org/apache/iotdb/db/sink/mqtt/MQTTHandler.java | 85 +++ .../apache/iotdb/db/tools/TsFileRewriteTool.java | 54 +- .../org/apache/iotdb/db/tools/mlog/MLogParser.java | 4 + .../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 31 +- .../java/org/apache/iotdb/db/utils/AuthUtils.java | 9 + .../org/apache/iotdb/db/utils/CommonUtils.java | 3 + .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 36 +- .../java/org/apache/iotdb/db/utils/MemUtils.java | 119 ++-- .../java/org/apache/iotdb/db/utils/MergeUtils.java | 31 +- .../apache/iotdb/db/utils/QueryDataSetUtils.java | 19 + .../java/org/apache/iotdb/db/utils/QueryUtils.java | 8 +- .../org/apache/iotdb/db/utils/SchemaUtils.java | 6 +- .../org/apache/iotdb/db/utils/SerializeUtils.java | 74 +++ .../apache/iotdb/db/utils/TypeInferenceUtils.java | 2 + .../iotdb/db/utils/datastructure/TVList.java | 88 ++- .../iotdb/db/utils/datastructure/VectorTVList.java | 698 +++++++++++++++++++++ .../iotdb/db/utils/windowing/api/Evaluator.java | 3 +- .../windowing/runtime/WindowEvaluationTask.java | 2 +- .../iotdb/db/writelog/recover/LogReplayer.java | 27 +- .../db/auth/role/LocalFileRoleManagerTest.java | 6 + .../db/auth/user/LocalFileUserManagerTest.java | 16 + .../compaction/LevelCompactionCacheTest.java | 2 +- .../engine/compaction/LevelCompactionLogTest.java | 2 +- .../compaction/LevelCompactionMergeTest.java | 6 +- .../compaction/LevelCompactionMoreDataTest.java | 2 +- .../LevelCompactionTsFileManagementTest.java | 69 ++ .../NoCompactionTsFileManagementTest.java | 71 ++- .../db/engine/memtable/MemTableFlushTaskTest.java | 57 ++ .../db/engine/memtable/MemTableTestUtils.java | 81 ++- .../db/engine/memtable/MemtableBenchmark.java | 1 - .../db/engine/memtable/PrimitiveMemTableTest.java | 130 +++- .../iotdb/db/engine/merge/MergeTaskTest.java | 75 +++ .../db/engine/modification/DeletionQueryTest.java | 10 +- .../storagegroup/StorageGroupProcessorTest.java | 28 +- .../iotdb/db/engine/storagegroup/TTLTest.java | 6 +- .../engine/storagegroup/TsFileProcessorTest.java | 84 ++- .../iotdb/db/integration/IOTDBGroupByIT.java | 19 + .../db/integration/IOTDBInsertAlignedValuesIT.java | 215 +++++++ .../apache/iotdb/db/integration/IOTDBInsertIT.java | 2 +- .../iotdb/db/integration/IoTDBCompressTypeIT.java | 5 - .../iotdb/db/integration/IoTDBDeletionIT.java | 12 +- .../db/integration/IoTDBEngineTimeGeneratorIT.java | 6 +- .../db/integration/IoTDBFlushQueryMergeIT.java | 4 +- .../iotdb/db/integration/IoTDBGroupByFillIT.java | 97 ++- .../iotdb/db/integration/IoTDBGroupByMonthIT.java | 70 ++- ...TDBInsertIT.java => IoTDBInsertMultiRowIT.java} | 31 +- ...oTDBLoadExternalTsFileWithTimePartitionIT.java} | 44 +- .../db/integration/IoTDBSequenceDataQueryIT.java | 6 +- .../iotdb/db/integration/IoTDBSeriesReaderIT.java | 8 +- .../iotdb/db/integration/IoTDBSimpleQueryIT.java | 4 + .../apache/iotdb/db/integration/IoTDBTagIT.java | 40 +- .../apache/iotdb/db/integration/IoTDBTtlIT.java | 2 +- .../integration/IoTDBUDTFAlignByTimeQueryIT.java | 16 + .../aggregation/IoTDBAggregationByLevelIT.java | 21 + .../db/integration/auth/IoTDBAuthorizationIT.java | 108 +++- .../iotdb/db/metadata/MManagerBasicTest.java | 589 ++++++++++++++++- .../iotdb/db/metadata/MManagerImproveTest.java | 17 +- .../iotdb/db/qp/physical/InsertRowPlanTest.java | 246 ++++++++ .../iotdb/db/qp/physical/InsertTabletPlanTest.java | 264 ++++++++ .../db/qp/physical/PhysicalPlanSerializeTest.java | 44 ++ .../iotdb/db/qp/physical/PhysicalPlanTest.java | 15 + .../reader/series/SeriesAggregateReaderTest.java | 6 +- .../reader/series/SeriesReaderByTimestampTest.java | 6 +- .../query/reader/series/SeriesReaderTestUtil.java | 3 +- .../org/apache/iotdb/db/sink/AlertManagerTest.java | 328 ++++++++++ .../apache/iotdb/db/sink/LocalIoTDBSinkTest.java | 217 +++++++ .../org/apache/iotdb/db/sink/MQTTSinkTest.java | 204 ++++++ .../org/apache/iotdb/db/utils/MemUtilsTest.java | 2 +- .../iotdb/db/utils/TsFileRewriteToolTest.java | 57 ++ .../db/utils/datastructure/VectorTVListTest.java | 143 +++++ .../iotdb/db/writelog/recover/LogReplayerTest.java | 22 +- .../db/writelog/recover/SeqTsFileRecoverTest.java | 3 +- .../writelog/recover/UnseqTsFileRecoverTest.java | 5 +- .../src/test/resources/testcontainers.properties | 17 + service-rpc/pom.xml | 3 +- .../org/apache/iotdb/rpc/RpcTransportFactory.java | 10 +- .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 4 +- .../java/org/apache/iotdb/rpc/TSStatusCode.java | 4 + .../rpc/TimeoutChangeableTFastFramedTransport.java | 8 +- .../TimeoutChangeableTSnappyFramedTransport.java | 20 +- session/pom.xml | 3 +- .../java/org/apache/iotdb/session/Session.java | 174 ++++- .../apache/iotdb/session/SessionConnection.java | 60 ++ .../org/apache/iotdb/session/SessionUtils.java | 133 ++-- .../iotdb/session/IoTDBSessionComplexIT.java | 7 +- .../apache/iotdb/session/IoTDBSessionSimpleIT.java | 57 +- .../apache/iotdb/session/SessionCacheLeaderUT.java | 5 +- .../java/org/apache/iotdb/session/SessionUT.java | 52 +- site/pom.xml | 2 +- site/src/main/.vuepress/config.js | 12 +- spark-iotdb-connector/Readme.md | 8 +- spark-iotdb-connector/pom.xml | 2 +- spark-tsfile/README.md | 6 +- spark-tsfile/pom.xml | 4 +- .../iotdb/spark/tsfile/NarrowConverter.scala | 4 +- .../apache/iotdb/spark/tsfile/WideConverter.scala | 5 +- test/e2e/base/docker-compose.yaml | 4 +- test/e2e/cases/cli/cleanup.sh | 0 test/e2e/cases/cli/run.sh | 0 testcontainer/Readme.md | 66 ++ testcontainer/pom.xml | 159 +++++ .../java/org/apache/iotdb/db/sql/ClusterIT.java | 122 ++++ .../java/org/apache/iotdb/db/sql/SingleNodeIT.java | 122 ++++ .../NoProjectNameDockerComposeContainer.java | 24 +- .../src/test/resources/1node}/docker-compose.yaml | 31 +- .../test/resources/1node}/iotdb-cluster.properties | 7 +- .../src/test/resources/3nodes}/docker-compose.yaml | 31 +- .../resources/3nodes}/iotdb-cluster.properties | 5 +- .../src/test/resources/5nodes}/docker-compose.yaml | 32 +- .../resources/5nodes}/iotdb-cluster.properties | 5 +- .../src/test/resources/iotdb-engine.properties | 24 + .../src/test/resources/logback-container.xml | 10 +- .../src/test/resources/logback.xml | 23 +- thrift-cluster/pom.xml | 2 +- thrift-sync/pom.xml | 2 +- thrift/pom.xml | 2 +- thrift/src/main/thrift/rpc.thrift | 31 + tsfile/README.md | 2 +- tsfile/pom.xml | 3 +- .../iotdb/tsfile/encoding/decoder/Decoder.java | 4 + .../iotdb/tsfile/file/header/ChunkHeader.java | 15 +- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 22 +- .../iotdb/tsfile/file/metadata/IChunkMetadata.java | 74 +++ .../metadata/ITimeSeriesMetadata.java} | 23 +- .../tsfile/file/metadata/TimeseriesMetadata.java | 23 +- .../tsfile/file/metadata/VectorChunkMetadata.java | 178 ++++++ .../file/metadata/VectorTimeSeriesMetadata.java | 97 +++ .../tsfile/file/metadata/enums/TSDataType.java | 8 +- .../file/metadata/statistics/Statistics.java | 96 +-- .../file/metadata/statistics/TimeStatistics.java | 161 +++++ .../iotdb/tsfile/read/TsFileSequenceReader.java | 59 +- .../apache/iotdb/tsfile/read/common/BatchData.java | 60 ++ .../org/apache/iotdb/tsfile/read/common/Chunk.java | 5 +- .../read/controller/IChunkMetadataLoader.java | 8 +- .../tsfile/read/controller/IMetadataQuerier.java | 6 +- .../read/controller/MetadataQuerierByFileImpl.java | 13 +- .../expression/impl/SingleSeriesExpression.java | 4 + .../tsfile/read/filter/GroupByMonthFilter.java | 17 +- .../iotdb/tsfile/read/filter/ValueFilter.java | 106 ++++ .../iotdb/tsfile/read/filter/operator/In.java | 6 +- .../query/executor/ExecutorWithTimeGenerator.java | 4 +- .../tsfile/read/query/executor/TsFileExecutor.java | 4 +- .../query/timegenerator/TsFileTimeGenerator.java | 4 +- .../tsfile/read/reader/chunk/ChunkReader.java | 8 +- .../read/reader/chunk/VectorChunkReader.java | 262 ++++++++ .../tsfile/read/reader/page/TimePageReader.java | 92 +++ .../tsfile/read/reader/page/ValuePageReader.java | 218 +++++++ .../tsfile/read/reader/page/VectorChunkReader.java | 15 +- .../tsfile/read/reader/page/VectorPageReader.java | 122 ++++ .../reader/series/AbstractFileSeriesReader.java | 14 +- .../read/reader/series/EmptyFileSeriesReader.java | 6 +- .../read/reader/series/FileSeriesReader.java | 9 +- .../reader/series/FileSeriesReaderByTimestamp.java | 13 +- .../java/org/apache/iotdb/tsfile/utils/Binary.java | 2 + .../java/org/apache/iotdb/tsfile/utils/BitMap.java | 121 ++++ .../org/apache/iotdb/tsfile/utils/PublicBAOS.java | 4 + .../apache/iotdb/tsfile/utils/TsPrimitiveType.java | 88 +++ .../tsfile/v2/read/TsFileSequenceReaderForV2.java | 6 +- .../apache/iotdb/tsfile/write/TsFileWriter.java | 13 +- .../tsfile/write/chunk/ChunkGroupWriterImpl.java | 6 +- .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 32 +- .../tsfile/write/chunk/IChunkGroupWriter.java | 4 +- .../iotdb/tsfile/write/chunk/IChunkWriter.java | 15 +- .../iotdb/tsfile/write/chunk/TimeChunkWriter.java | 255 ++++++++ .../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 252 ++++++++ .../tsfile/write/chunk/VectorChunkWriterImpl.java | 202 ++++++ .../apache/iotdb/tsfile/write/page/PageWriter.java | 6 +- .../iotdb/tsfile/write/page/TimePageWriter.java | 177 ++++++ .../page/{PageWriter.java => ValuePageWriter.java} | 160 ++--- .../apache/iotdb/tsfile/write/record/Tablet.java | 251 +++++--- .../write/record/datapoint/BooleanDataPoint.java | 2 +- .../write/record/datapoint/DoubleDataPoint.java | 2 +- .../write/record/datapoint/FloatDataPoint.java | 2 +- .../write/record/datapoint/IntDataPoint.java | 2 +- .../write/record/datapoint/LongDataPoint.java | 2 +- .../write/record/datapoint/StringDataPoint.java | 2 +- .../tsfile/write/schema/IMeasurementSchema.java | 77 +++ .../tsfile/write/schema/MeasurementSchema.java | 89 ++- .../apache/iotdb/tsfile/write/schema/Schema.java | 25 +- .../write/schema/VectorMeasurementSchema.java | 370 +++++++++++ .../write/writer/RestorableTsFileIOWriter.java | 17 +- .../iotdb/tsfile/write/writer/TsFileIOWriter.java | 39 +- ...easurementChunkMetadataListMapIteratorTest.java | 13 +- .../tsfile/read/controller/ChunkLoaderTest.java | 7 +- .../query/timegenerator/ReaderByTimestampTest.java | 4 +- .../iotdb/tsfile/read/reader/ReaderTest.java | 6 +- .../org/apache/iotdb/tsfile/utils/BitMapTest.java | 67 ++ .../org/apache/iotdb/tsfile/utils/RecordUtils.java | 4 +- .../tsfile/write/DefaultDeviceTemplateTest.java | 5 +- .../iotdb/tsfile/write/TsFileIOWriterTest.java | 3 +- .../write/schema/converter/SchemaBuilderTest.java | 17 +- .../write/writer/RestorableTsFileIOWriterTest.java | 5 +- .../tsfile/write/writer/TestTsFileOutput.java} | 49 +- .../tsfile/write/writer/TimeChunkWriterTest.java | 111 ++++ .../tsfile/write/writer/TimePageWriterTest.java | 171 +++++ .../tsfile/write/writer/ValueChunkWriterTest.java | 109 ++++ .../tsfile/write/writer/ValuePageWriterTest.java | 291 +++++++++ .../write/writer/VectorChunkWriterImplTest.java | 178 ++++++ .../write/writer/VectorMeasurementSchemaStub.java | 128 ++++ zeppelin-interpreter/pom.xml | 2 +- 454 files changed, 19263 insertions(+), 2662 deletions(-) diff --cc cluster/src/assembly/resources/conf/iotdb-cluster.properties index 23cf744,dd0a257..5044017 --- a/cluster/src/assembly/resources/conf/iotdb-cluster.properties +++ b/cluster/src/assembly/resources/conf/iotdb-cluster.properties @@@ -65,11 -67,8 +67,11 @@@ seed_nodes=127.0.0.1:900 # max_concurrent_client_num=10000 # number of replications for one partition - default_replica_num=3 + default_replica_num=1 +# sub raft num for multi-raft +multi_raft_factor=1 + # cluster name to identify different clusters # all node's cluster_name in one cluster are the same # cluster_name=default diff --cc cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java index b928208,dc3e767..7b2ae4c --- a/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/ClusterMain.java @@@ -118,15 -116,10 +118,15 @@@ public class ClusterMain } } else if (MODE_ADD.equals(mode)) { try { + long startTime = System.currentTimeMillis(); metaServer = new MetaClusterServer(); - // preStartCustomize(); + preStartCustomize(); metaServer.start(); metaServer.joinCluster(); + logger.info( + "Adding this node {} to cluster costs {} ms", + metaServer.getMember().getThisNode(), + (System.currentTimeMillis() - startTime)); } catch (TTransportException | StartupException | QueryProcessException diff --cc cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java index 95c52b8,a2bb2de..fc90b5c --- a/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/config/ClusterConfig.java @@@ -41,10 -45,8 +45,10 @@@ public class ClusterConfig @ClusterConsistent private boolean isRpcThriftCompressionEnabled = false; private int maxConcurrentClientNum = 10000; - @ClusterConsistent private int replicationNum = 3; + @ClusterConsistent private int replicationNum = 1; + @ClusterConsistent private int multiRaftFactor = 1; + @ClusterConsistent private String clusterName = "default"; @ClusterConsistent private boolean useAsyncServer = false; diff --cc cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java index 4785718,f37570a2..6ae886d --- a/cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/coordinator/Coordinator.java @@@ -246,19 -264,29 +266,29 @@@ public class Coordinator for (PartitionGroup partitionGroup : partitionGroups) { if (partitionGroup.contains(thisNode)) { // the query should be handled by a group the local node is in, handle it with in the group - logger.debug("Execute {} in a local group of {}", plan, partitionGroup.getHeader()); status = metaGroupMember - .getLocalDataMember(partitionGroup.getHeader()) + .getLocalDataMember(partitionGroup.getHeader(), partitionGroup.getId()) .executeNonQueryPlan(plan); + logger.debug( + "Execute {} in a local group of {} with status {}", + plan, + partitionGroup.getHeader(), + status); } else { // forward the query to the group that should handle it - logger.debug("Forward {} to a remote group of {}", plan, partitionGroup.getHeader()); status = forwardPlan(plan, partitionGroup); + logger.debug( + "Forward {} to a remote group of {} with status {}", + plan, + partitionGroup.getHeader(), + status); } if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode() - && (!(plan instanceof DeleteTimeSeriesPlan) - || status.getCode() != TSStatusCode.TIMESERIES_NOT_EXIST.getStatusCode())) { + && !(plan instanceof SetDeviceTemplatePlan + && status.getCode() == TSStatusCode.DUPLICATED_TEMPLATE.getStatusCode()) + && !(plan instanceof DeleteTimeSeriesPlan + && status.getCode() == TSStatusCode.TIMESERIES_NOT_EXIST.getStatusCode())) { // execution failed, record the error message errorCodePartitionGroups.add( String.format( diff --cc cluster/src/main/java/org/apache/iotdb/cluster/metadata/MetaPuller.java index 42c83cc,35a74b8..6a2132d --- a/cluster/src/main/java/org/apache/iotdb/cluster/metadata/MetaPuller.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/metadata/MetaPuller.java @@@ -124,10 -126,10 +126,10 @@@ public class MetaPuller * @param prefixPaths * @param results */ - private void pullMeasurementSchemas( + public void pullMeasurementSchemas( PartitionGroup partitionGroup, List<PartialPath> prefixPaths, - List<MeasurementSchema> results) { + List<IMeasurementSchema> results) { if (partitionGroup.contains(metaGroupMember.getThisNode())) { // the node is in the target group, synchronize with leader should be enough try { diff --cc cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java index 64c1c63,bdc1578..8a12294 --- a/cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/partition/slot/SlotPartitionTable.java @@@ -411,30 -355,22 +412,30 @@@ public class SlotPartitionTable impleme int prevNodeMapSize = buffer.getInt(); previousNodeMap = new HashMap<>(); for (int i = 0; i < prevNodeMapSize; i++) { - int nodeId = buffer.getInt(); - Node node = idNodeMap.get(nodeId); + node = new Node(); + NodeSerializeUtils.deserialize(node, buffer); + RaftNode raftNode = new RaftNode(node, buffer.getInt()); - Map<Integer, Node> prevHolders = new HashMap<>(); + Map<Integer, PartitionGroup> prevHolders = new HashMap<>(); int holderNum = buffer.getInt(); for (int i1 = 0; i1 < holderNum; i1++) { - int slot = buffer.getInt(); - Node holder = idNodeMap.get(buffer.getInt()); - prevHolders.put(slot, holder); + PartitionGroup group = new PartitionGroup(); + group.deserialize(buffer); + prevHolders.put(buffer.getInt(), group); } - previousNodeMap.put(node, prevHolders); + previousNodeMap.put(raftNode, prevHolders); } - lastLogIndex = buffer.getLong(); - nodeRing.addAll(nodeSlotMap.keySet()); + nodeRemovalResult = new SlotNodeRemovalResult(); + nodeRemovalResult.deserialize(buffer); + + nodeRing.clear(); + for (RaftNode raftNode : nodeSlotMap.keySet()) { + if (!nodeRing.contains(raftNode.getNode())) { + nodeRing.add(raftNode.getNode()); + } + } - nodeRing.sort(Comparator.comparingInt(Node::getNodeIdentifier)); + Collections.sort(nodeRing); logger.info("All known nodes: {}", nodeRing); localGroups = getPartitionGroups(thisNode); diff --cc cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java index f5264b6,5069f60..ffe89ee --- a/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/ClusterPlanRouter.java @@@ -41,9 -36,9 +41,10 @@@ import org.apache.iotdb.db.qp.physical. import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; import org.apache.iotdb.db.qp.physical.sys.AlterTimeSeriesPlan; import org.apache.iotdb.db.qp.physical.sys.CountPlan; + import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; import org.apache.iotdb.db.qp.physical.sys.CreateMultiTimeSeriesPlan; import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.LogPlan; import org.apache.iotdb.db.qp.physical.sys.ShowChildPathsPlan; import org.apache.iotdb.db.qp.physical.sys.ShowPlan.ShowContentType; import org.apache.iotdb.db.service.IoTDB; diff --cc cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java index 5e0e38c,0731ce1..9c2563f --- a/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/LocalQueryExecutor.java @@@ -447,9 -450,10 +457,9 @@@ public class LocalQueryExecutor // collect local timeseries schemas and send to the requester // the measurements in them are the full paths. List<String> prefixPaths = request.getPrefixPaths(); - List<MeasurementSchema> measurementSchemas = new ArrayList<>(); + List<IMeasurementSchema> measurementSchemas = new ArrayList<>(); - for (String prefixPath : prefixPaths) { - getCMManager().collectSeries(new PartialPath(prefixPath), measurementSchemas); - } + + collectSeries(prefixPaths, measurementSchemas); if (logger.isDebugEnabled()) { logger.debug( "{}: Collected {} schemas for {} and other {} paths", @@@ -475,44 -479,6 +485,44 @@@ return resp; } - private void collectSeries(List<String> prefixPaths, List<MeasurementSchema> measurementSchemas) ++ private void collectSeries(List<String> prefixPaths, List<IMeasurementSchema> measurementSchemas) + throws MetadataException { + // Due to add/remove node, some slots may in the state of PULLING, which will not contains the + // corresponding schemas. + // In this case, we need to pull series from previous holder. + Map<PartitionGroup, List<PartialPath>> prePartitionGroupPathMap = new HashMap<>(); + + RaftNode raftNode = new RaftNode(dataGroupMember.getHeader(), dataGroupMember.getRaftGroupId()); + Map<Integer, PartitionGroup> slotPreviousHolderMap = + ((SlotPartitionTable) dataGroupMember.getMetaGroupMember().getPartitionTable()) + .getPreviousNodeMap() + .get(raftNode); + + for (String prefixPath : prefixPaths) { + int slot = + ClusterUtils.getSlotByPathTimeWithSync( + new PartialPath(prefixPath), dataGroupMember.getMetaGroupMember()); + if (dataGroupMember.getSlotManager().checkSlotInMetaMigrationStatus(slot) + && slotPreviousHolderMap.containsKey(slot)) { + prePartitionGroupPathMap + .computeIfAbsent(slotPreviousHolderMap.get(slot), s -> new ArrayList<>()) + .add(new PartialPath(prefixPath)); + } else { + getCMManager().collectSeries(new PartialPath(prefixPath), measurementSchemas); + } + } + + if (prePartitionGroupPathMap.isEmpty()) { + return; + } + for (Map.Entry<PartitionGroup, List<PartialPath>> partitionGroupListEntry : + prePartitionGroupPathMap.entrySet()) { + PartitionGroup partitionGroup = partitionGroupListEntry.getKey(); + List<PartialPath> paths = partitionGroupListEntry.getValue(); + MetaPuller.getInstance().pullMeasurementSchemas(partitionGroup, paths, measurementSchemas); + } + } + /** * Create an IReaderByTime of a path, register it in the query manager to get a reader id for it * and send the id back to the requester. If the reader does not have any data, an id of -1 will diff --cc cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java index f7018d7,bd45755..f8b73e9 --- a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java @@@ -540,19 -508,14 +542,19 @@@ public class ClusterReaderFactory Filter valueFilter, QueryContext context, Node header, - boolean ascending) + int raftId, + boolean ascending, + Set<Integer> requiredSlots) throws StorageEngineException, QueryProcessException { ClusterQueryUtils.checkPathExistence(path); - List<Integer> nodeSlots = - ((SlotPartitionTable) metaGroupMember.getPartitionTable()).getNodeSlots(header); + if (requiredSlots == null) { + List<Integer> nodeSlots = + ((SlotPartitionTable) metaGroupMember.getPartitionTable()).getNodeSlots(header, raftId); + requiredSlots = new HashSet<>(nodeSlots); + } QueryDataSource queryDataSource = QueryResourceManager.getInstance().getQueryDataSource(path, context, timeFilter); - return new SeriesReader( + return SeriesReaderFactory.createSeriesReader( path, allSensors, dataType, diff --cc cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java index 43eaefb,49acf81..c666014 --- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/DataGroupMember.java @@@ -28,9 -28,9 +28,10 @@@ import org.apache.iotdb.cluster.client. import org.apache.iotdb.cluster.client.sync.SyncDataHeartbeatClient; import org.apache.iotdb.cluster.config.ClusterConstant; import org.apache.iotdb.cluster.config.ClusterDescriptor; + import org.apache.iotdb.cluster.exception.CheckConsistencyException; import org.apache.iotdb.cluster.exception.LogExecutionException; import org.apache.iotdb.cluster.exception.SnapshotInstallationException; +import org.apache.iotdb.cluster.log.Log; import org.apache.iotdb.cluster.log.LogApplier; import org.apache.iotdb.cluster.log.Snapshot; import org.apache.iotdb.cluster.log.applier.AsyncDataLogApplier; diff --cc cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java index 01ecfe8,1d75920..7a9e3b2 --- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/MetaGroupMember.java @@@ -93,21 -85,12 +93,20 @@@ import org.apache.iotdb.db.conf.IoTDBDe import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.exception.StartupException; import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.exception.metadata.IllegalPathException; import org.apache.iotdb.db.exception.metadata.MetadataException; +import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.PartialPath; - import org.apache.iotdb.db.qp.executor.PlanExecutor; import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateMultiTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.DeleteTimeSeriesPlan; import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.utils.TestOnly; +import org.apache.iotdb.rpc.RpcUtils; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.EndPoint; import org.apache.iotdb.service.rpc.thrift.TSStatus; import org.apache.iotdb.tsfile.read.filter.basic.Filter; diff --cc cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java index e464747,f2187f9..659f9bb --- a/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/server/member/RaftMember.java @@@ -68,8 -68,9 +69,10 @@@ import org.apache.iotdb.db.exception.me import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.metadata.StorageGroupAlreadySetException; import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; + import org.apache.iotdb.db.exception.query.QueryProcessException; + import org.apache.iotdb.db.qp.executor.PlanExecutor; import org.apache.iotdb.db.qp.physical.PhysicalPlan; +import org.apache.iotdb.db.qp.physical.sys.LogPlan; import org.apache.iotdb.db.utils.TestOnly; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@@ -232,9 -227,11 +235,14 @@@ public abstract class RaftMember */ private LogDispatcher logDispatcher; + /** If this node can not be the leader, this parameter will be set true. */ + private volatile boolean skipElection = false; + + /** + * localExecutor is used to directly execute plans like load configuration in the underlying IoTDB + */ + protected PlanExecutor localExecutor; + protected RaftMember() {} protected RaftMember( @@@ -1022,7 -987,7 +1037,7 @@@ return StatusUtils.OK; } } catch (LogExecutionException e) { - return handleLogExecutionException(log, e); - return handleLogExecutionException(log.getPlan(), IOUtils.getRootCause(e)); ++ return handleLogExecutionException(log, IOUtils.getRootCause(e)); } return StatusUtils.TIME_OUT; } @@@ -1096,7 -1051,7 +1111,7 @@@ break; } } catch (LogExecutionException e) { - return handleLogExecutionException(log, e); - return handleLogExecutionException(log.getPlan(), IOUtils.getRootCause(e)); ++ return handleLogExecutionException(log, IOUtils.getRootCause(e)); } return StatusUtils.TIME_OUT; } @@@ -1536,8 -1483,7 +1551,7 @@@ } } - private TSStatus handleLogExecutionException(Log log, LogExecutionException e) { - Throwable cause = IOUtils.getRootCause(e); - protected TSStatus handleLogExecutionException(PhysicalPlan log, Throwable cause) { ++ protected TSStatus handleLogExecutionException(Object log, Throwable cause) { if (cause instanceof BatchProcessException) { return RpcUtils.getStatus(Arrays.asList(((BatchProcessException) cause).getFailingStatus())); } diff --cc cluster/src/main/java/org/apache/iotdb/cluster/utils/PartitionUtils.java index 7d79308,9bcd9ec..7aae564 --- a/cluster/src/main/java/org/apache/iotdb/cluster/utils/PartitionUtils.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/utils/PartitionUtils.java @@@ -20,11 -20,13 +20,13 @@@ package org.apache.iotdb.cluster.utils; import org.apache.iotdb.cluster.partition.PartitionTable; -import org.apache.iotdb.cluster.rpc.thrift.Node; +import org.apache.iotdb.cluster.rpc.thrift.RaftNode; import org.apache.iotdb.db.engine.StorageEngine; import org.apache.iotdb.db.qp.physical.PhysicalPlan; + import org.apache.iotdb.db.qp.physical.crud.CreateTemplatePlan; import org.apache.iotdb.db.qp.physical.crud.DeletePlan; import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; + import org.apache.iotdb.db.qp.physical.crud.SetDeviceTemplatePlan; import org.apache.iotdb.db.qp.physical.sys.AuthorPlan; import org.apache.iotdb.db.qp.physical.sys.DataAuthPlan; import org.apache.iotdb.db.qp.physical.sys.DeleteStorageGroupPlan; diff --cc cluster/src/main/java/org/apache/iotdb/cluster/utils/StatusUtils.java index b20faed,719e52a..1c3b224 --- a/cluster/src/main/java/org/apache/iotdb/cluster/utils/StatusUtils.java +++ b/cluster/src/main/java/org/apache/iotdb/cluster/utils/StatusUtils.java @@@ -45,9 -45,9 +45,10 @@@ public class StatusUtils public static final TSStatus TIMESERIES_NOT_EXIST_ERROR = getStatus(TSStatusCode.TIMESERIES_NOT_EXIST); public static final TSStatus NO_CONNECTION = getStatus(TSStatusCode.NO_CONNECTION); + public static final TSStatus PARSE_LOG_ERROR = getStatus(TSStatusCode.PARSE_LOG_ERROR); + public static final TSStatus DUPLICATED_TEMPLATE = getStatus(TSStatusCode.DUPLICATED_TEMPLATE); - private static TSStatus getStatus(TSStatusCode statusCode) { + public static TSStatus getStatus(TSStatusCode statusCode) { TSStatus status = new TSStatus(); status.setCode(statusCode.getStatusCode()); switch (statusCode) { diff --cc server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java index 8139dbd,ff9a7eb..15112e8 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java @@@ -2857,14 -2823,14 +2845,14 @@@ public class StorageGroupProcessor /** remove all partitions that satisfy a filter. */ public void removePartitions(TimePartitionFilter filter) { // this requires blocking all other activities - tsFileManagement.writeLock(); - insertLock.writeLock().lock(); + writeLock(); try { - // abort ongoing merges + // abort ongoing comapctions and merges + CompactionMergeTaskPoolManager.getInstance().abortCompaction(logicalStorageGroupName); MergeManager.getINSTANCE().abortMerge(logicalStorageGroupName); // close all working files that should be removed - removePartitions(filter, workSequenceTsFileProcessors.entrySet()); - removePartitions(filter, workUnsequenceTsFileProcessors.entrySet()); + removePartitions(filter, workSequenceTsFileProcessors.entrySet(), true); + removePartitions(filter, workUnsequenceTsFileProcessors.entrySet(), false); // remove data files removePartitions(filter, tsFileManagement.getIterator(true), true); diff --cc server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java index 58a40b0,4c5b1a1..af6a4a0 --- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java +++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java @@@ -349,9 -356,18 +357,21 @@@ public abstract class PhysicalPlan case STOP_TRIGGER: plan = new StopTriggerPlan(); break; + case CLUSTER_LOG: + plan = new LogPlan(); + break; + case CREATE_TEMPLATE: + plan = new CreateTemplatePlan(); + break; + case SET_DEVICE_TEMPLATE: + plan = new SetDeviceTemplatePlan(); + break; + case SET_USING_DEVICE_TEMPLATE: + plan = new SetUsingDeviceTemplatePlan(); + break; + case AUTO_CREATE_DEVICE_MNODE: + plan = new AutoCreateDeviceMNodePlan(); + break; default: throw new IOException("unrecognized log type " + type); }
