This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch native_raft in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 73dfcb9f32ae6a7d37300e5ba825a605c213450f Merge: 0c2fef70c03 a5053662da4 Author: Tian Jiang <[email protected]> AuthorDate: Mon May 29 11:43:42 2023 +0800 Merge branch 'auto_compressor' into native_raft # Conflicts: # client-go # iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java # pom.xml # server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java .github/workflows/client-cpp.yml | 8 +- .github/workflows/client-go.yml | 2 +- .github/workflows/client-python.yml | 6 +- .github/workflows/grafana-plugin.yml | 4 +- .github/workflows/influxdb-protocol.yml | 2 +- .gitignore | 9 + .gitmodules | 6 +- .../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 18 +- client-go | 2 +- .../confignode/client/DataNodeRequestType.java | 5 + .../client/async/AsyncDataNodeClientPool.java | 21 + .../client/async/handlers/AsyncClientHandler.java | 3 + .../heartbeat/DataNodeHeartbeatHandler.java | 10 +- .../consensus/request/ConfigPhysicalPlan.java | 6 +- .../consensus/request/ConfigPhysicalPlanType.java | 1 + .../PipeHandleLeaderChangePlan.java | 2 +- .../pipe/runtime/PipeHandleMetaChangePlan.java | 67 + .../statemachine/ConfigRegionStateMachine.java | 6 +- .../iotdb/confignode/manager/ConfigManager.java | 22 +- .../apache/iotdb/confignode/manager/IManager.java | 3 + .../iotdb/confignode/manager/ProcedureManager.java | 69 + .../manager/load/service/HeartbeatService.java | 16 +- .../iotdb/confignode/manager/node/NodeManager.java | 4 +- .../manager/partition/PartitionMetrics.java | 35 +- .../manager/pipe/runtime/PipeMetaSyncer.java | 21 +- .../pipe/runtime/PipeRuntimeCoordinator.java | 26 +- .../manager/schema/ClusterSchemaManager.java | 3 +- .../persistence/executor/ConfigPlanExecutor.java | 7 +- .../confignode/persistence/pipe/PipeTaskInfo.java | 22 +- .../persistence/pipe/PipeTaskOperation.java | 2 +- .../procedure/env/DataNodeRemoveHandler.java | 2 + .../runtime/PipeHandleLeaderChangeProcedure.java | 2 +- .../runtime/PipeHandleMetaChangeProcedure.java | 287 ++++ .../impl/pipe/task/CreatePipeProcedureV2.java | 7 +- .../impl/schema/DeleteDatabaseProcedure.java | 4 +- .../impl/schema/DeleteLogicalViewProcedure.java | 353 +++++ .../state/schema/DeleteLogicalViewState.java | 9 +- .../procedure/store/ProcedureFactory.java | 12 + .../confignode/procedure/store/ProcedureType.java | 6 +- .../thrift/ConfigNodeRPCServiceHandlerMetrics.java | 8 +- .../thrift/ConfigNodeRPCServiceMetrics.java | 8 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 6 + .../request/ConfigPhysicalPlanSerDeTest.java | 49 +- .../iotdb/confignode/persistence/PipeInfoTest.java | 4 +- .../runtime/PipeHandleMetaChangeProcedureTest.java | 56 +- .../consensus/iot/IoTConsensusServerImpl.java | 94 +- .../consensus/iot/IoTConsensusServerMetrics.java | 327 ++-- .../consensus/iot/client/DispatchLogHandler.java | 25 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 32 +- .../logdispatcher/LogDispatcherThreadMetrics.java | 194 ++- .../ratis/ApplicationStateMachineProxy.java | 2 +- .../iotdb/consensus/simple/SimpleConsensus.java | 2 +- distribution/src/assembly/all.xml | 4 +- distribution/src/assembly/cli.xml | 4 +- distribution/src/assembly/client-cpp.xml | 2 +- distribution/src/assembly/grafana-connector.xml | 2 +- distribution/src/assembly/grafana-plugin.xml | 2 +- distribution/src/assembly/spark-connector.xml | 4 +- docker/src/main/DockerCompose/start-1c1d.sh | 2 +- .../Ecosystem-Integration/Grafana-Plugin.md | 8 +- docs/UserGuide/Monitor-Alert/Metric-Tool.md | 51 +- .../Ecosystem-Integration/Grafana-Plugin.md | 6 +- docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 66 +- example/session/pom.xml | 6 + .../org/apache/iotdb/DataMigrationExample.java | 142 +- grafana-plugin/go.sum | 660 --------- influxdb-protocol/pom.xml | 140 -- .../org/apache/iotdb/influxdb/IoTDBInfluxDB.java | 411 ------ .../iotdb/influxdb/IoTDBInfluxDBFactory.java | 70 - .../iotdb/influxdb/protocol/dto/SessionPoint.java | 85 -- .../protocol/impl/IoTDBInfluxDBService.java | 99 -- .../influxdb/protocol/util/ParameterUtils.java | 36 - .../iotdb/influxdb/session/InfluxDBSession.java | 279 ---- .../influxdb/integration/IoTDBInfluxDBIT.java | 207 --- integration-test/pom.xml | 6 + integration-test/src/assembly/mpp-test.xml | 4 +- .../confignode/it/utils/ConfigNodeTestUtils.java | 1 + integration/README.md | 120 -- integration/checkstyle.xml | 217 --- integration/import-control.xml | 29 - integration/pom.xml | 295 ---- integration/src/assembly/cluster.xml | 47 - .../db/engine/trigger/example/Accumulator.java | 96 -- .../iotdb/db/engine/trigger/example/Counter.java | 97 -- .../iotdb/db/query/udf/example/Accumulator.java | 102 -- .../apache/iotdb/db/query/udf/example/Adder.java | 92 -- .../apache/iotdb/db/query/udf/example/Counter.java | 88 -- .../db/query/udf/example/ExampleUDFConstant.java | 31 - .../org/apache/iotdb/db/query/udf/example/Max.java | 74 - .../iotdb/db/query/udf/example/Multiplier.java | 63 - .../SlidingSizeWindowConstructorTester0.java | 59 - .../SlidingSizeWindowConstructorTester1.java | 64 - .../SlidingTimeWindowConstructionTester.java | 72 - .../db/query/udf/example/TerminateTester.java | 67 - .../iotdb/db/query/udf/example/ValidateTester.java | 44 - .../iotdb/db/query/udf/example/WindowStartEnd.java | 66 - .../iotdb/integration/env/ClusterEnvBase.java | 279 ---- .../iotdb/integration/env/ClusterEnvConfig.java | 154 -- .../apache/iotdb/integration/env/ClusterNode.java | 179 --- .../iotdb/integration/env/ConfigFactory.java | 52 - .../apache/iotdb/integration/env/EnvFactory.java | 57 - .../iotdb/integration/env/FiveNodeCluster1Env.java | 64 - .../iotdb/integration/env/RemoteEnvConfig.java | 23 - .../iotdb/integration/env/RemoteServerEnv.java | 107 -- .../iotdb/itbase/category/LocalStandaloneTest.java | 21 - .../apache/iotdb/itbase/category/RemoteTest.java | 21 - .../org/apache/iotdb/itbase/env/BaseConfig.java | 112 -- .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 39 - .../apache/iotdb/db/integration/IOTDBInsertIT.java | 200 --- .../IoTDBAlignedTimeSeriesCompactionIT.java | 564 ------- .../iotdb/db/integration/IoTDBCheckConfigIT.java | 142 -- .../iotdb/db/integration/IoTDBClearCacheIT.java | 171 --- .../apache/iotdb/db/integration/IoTDBCloseIT.java | 191 --- .../iotdb/db/integration/IoTDBCompactionIT.java | 338 ----- .../integration/IoTDBCompactionWithIDTableIT.java | 352 ----- .../iotdb/db/integration/IoTDBCompressTypeIT.java | 101 -- .../db/integration/IoTDBContinuousQueryIT.java | 544 ------- .../apache/iotdb/db/integration/IoTDBDaemonIT.java | 456 ------ .../iotdb/db/integration/IoTDBDisableAlignIT.java | 392 ----- .../apache/iotdb/db/integration/IoTDBFillIT.java | 1227 --------------- .../db/integration/IoTDBFlushQueryMergeIT.java | 197 --- .../iotdb/db/integration/IoTDBKillQueryIT.java | 80 - .../iotdb/db/integration/IoTDBLargeDataIT.java | 373 ----- .../apache/iotdb/db/integration/IoTDBLastIT.java | 629 -------- .../iotdb/db/integration/IoTDBMaxTimeQueryIT.java | 139 -- .../IoTDBMultiOverlappedChunkInUnseqIT.java | 110 -- .../db/integration/IoTDBMultiOverlappedPageIT.java | 166 --- .../iotdb/db/integration/IoTDBMultiSeriesIT.java | 471 ------ .../db/integration/IoTDBMultiStatementsIT.java | 187 --- .../db/integration/IoTDBNewTsFileCompactionIT.java | 1078 -------------- .../db/integration/IoTDBOverlappedPageIT.java | 194 --- .../iotdb/db/integration/IoTDBQueryTimeoutIT.java | 156 -- .../db/integration/IoTDBQueryWithIDTableIT.java | 290 ---- .../db/integration/IoTDBRemovePartitionIT.java | 331 ----- .../db/integration/IoTDBRepeatPatternNameIT.java | 67 - .../db/integration/IoTDBRewriteTsFileToolIT.java | 618 -------- .../db/integration/IoTDBRpcCompressionIT.java | 140 -- .../iotdb/db/integration/IoTDBSensorUpdateIT.java | 87 -- .../db/integration/IoTDBSessionTimeoutIT.java | 84 -- .../IoTDBSetSystemReadOnlyWritableIT.java | 258 ---- .../integration/IoTDBSizeTieredCompactionIT.java | 1370 ----------------- .../integration/IoTDBSortedShowTimeseriesIT.java | 339 ----- .../iotdb/db/integration/IoTDBTimePartitionIT.java | 90 -- .../iotdb/db/integration/IoTDBTracingIT.java | 84 -- .../apache/iotdb/db/integration/IoTDBTtlIT.java | 354 ----- .../db/integration/IoTDBUDFWindowQueryIT.java | 808 ---------- .../db/integration/IoTDBUDTFHybridQueryIT.java | 160 -- .../db/integration/IoTDBUDTFNonAlignQueryIT.java | 341 ----- .../db/integration/IoTDBUnseqOverlappedPageIT.java | 127 -- .../iotdb/db/integration/IoTDBVersionIT.java | 68 - .../iotdb/db/integration/env/StandaloneEnv.java | 93 -- .../db/integration/env/StandaloneEnvConfig.java | 133 -- .../versionadaption/IoTDBDDLVersionAdaptionIT.java | 362 ----- .../IoTDBDeletionVersionAdaptionIT.java | 303 ---- .../IoTDBQueryVersionAdaptionIT.java | 320 ---- .../session/IoTDBSessionAlignedABDeviceIT.java | 235 --- .../session/IoTDBSessionAlignedAggregationIT.java | 263 ---- .../IoTDBSessionAlignedAggregationWithUnSeqIT.java | 196 --- .../iotdb/session/IoTDBSessionIteratorIT.java | 354 ----- .../src/test/resources/iotdb-datanode.properties | 24 - integration/src/test/resources/logback-test.xml | 58 - {external-api => iotdb-api/external-api}/pom.xml | 2 +- .../iotdb/external/api/IPropertiesLoader.java | 0 .../iotdb/external/api/ISeriesNumerMonitor.java | 0 .../external/api/thrift/JudgableServerContext.java | 0 .../external/api/thrift/ServerContextFactory.java | 0 .../external-pipe-api}/pom.xml | 2 +- .../apache/iotdb/pipe/external/api/DataType.java | 0 .../external/api/ExternalPipeSinkWriterStatus.java | 0 .../pipe/external/api/IExternalPipeSinkWriter.java | 0 .../api/IExternalPipeSinkWriterFactory.java | 0 {pipe-api => iotdb-api/pipe-api}/pom.xml | 2 +- .../org/apache/iotdb/pipe/api/PipeCollector.java | 0 .../org/apache/iotdb/pipe/api/PipeConnector.java | 10 +- .../java/org/apache/iotdb/pipe/api/PipePlugin.java | 0 .../org/apache/iotdb/pipe/api/PipeProcessor.java | 10 +- .../java/org/apache/iotdb/pipe/api/access/Row.java | 0 .../apache/iotdb/pipe/api/access/RowIterator.java | 0 .../iotdb/pipe/api/collector/EventCollector.java | 32 +- .../iotdb/pipe/api/collector/RowCollector.java | 0 .../api/customizer/PipeParameterValidator.java | 0 .../iotdb/pipe/api/customizer/PipeParameters.java | 0 .../api/customizer/PipeRuntimeConfiguration.java | 0 .../iotdb/pipe/api/customizer/PipeStrategy.java | 0 .../PipeCollectorRuntimeConfiguration.java | 0 .../PipeConnectorRuntimeConfiguration.java | 0 .../connector/parallel/ParallelStrategy.java | 0 .../retry/EqualRetryIntervalStrategy.java | 0 .../retry/ExponentialRetryIntervalStrategy.java | 0 .../customizer/connector/retry/RetryStrategy.java | 0 .../customizer/connector/reuse/ReuseStrategy.java | 0 .../PipeProcessorRuntimeConfiguration.java | 0 .../org/apache/iotdb/pipe/api/event/Event.java | 6 +- .../org/apache/iotdb/pipe/api/event/EventType.java | 0 .../event/dml/insertion/TabletInsertionEvent.java | 6 - .../event/dml/insertion/TsFileInsertionEvent.java | 6 - .../PipeAttributeNotProvidedException.java | 0 .../api/exception/PipeConnectionException.java | 0 .../iotdb/pipe/api/exception/PipeException.java | 0 .../api/exception/PipeManagementException.java | 0 .../exception/PipeParameterNotValidException.java | 0 .../exception/PipeRuntimeCriticalException.java | 0 .../pipe/api/exception/PipeRuntimeException.java | 0 .../exception/PipeRuntimeNonCriticalException.java | 0 .../exception/PipeStrategyNotValidException.java | 0 .../org/apache/iotdb/pipe/api/type/Binary.java | 0 .../java/org/apache/iotdb/pipe/api/type/Type.java | 0 .../subscription-api}/pom.xml | 2 +- .../api/SubscriptionConfiguration.java | 0 .../subscription/api/SubscriptionFactory.java | 0 .../iotdb/subscription/api/consumer/Consumer.java | 0 .../api/consumer/pull/PullConsumer.java | 0 .../api/consumer/push/DataArrivalListener.java | 0 .../api/consumer/push/ExceptionListener.java | 0 .../api/consumer/push/PushConsumer.java | 0 .../api/dataset/SubscriptionDataSet.java | 0 .../api/exception/SubscriptionException.java | 0 .../SubscriptionStrategyNotValidException.java | 0 .../api/strategy/SubscriptionStrategy.java | 0 .../disorder/DisorderHandlingStrategy.java | 0 .../api/strategy/disorder/IntolerableStrategy.java | 0 .../api/strategy/disorder/WatermarkStrategy.java | 0 .../strategy/topic/MultipleConnectionStrategy.java | 0 .../api/strategy/topic/SingleTopicStrategy.java | 0 .../api/strategy/topic/TopicsStrategy.java | 0 {trigger-api => iotdb-api/trigger-api}/pom.xml | 2 +- .../java/org/apache/iotdb/trigger/api/Trigger.java | 0 .../iotdb/trigger/api/TriggerAttributes.java | 0 .../iotdb/trigger/api/enums/FailureStrategy.java | 0 .../iotdb/trigger/api/enums/TriggerEvent.java | 0 .../iotdb/trigger/api/enums/TriggerType.java | 0 {udf-api => iotdb-api/udf-api}/pom.xml | 2 +- .../main/java/org/apache/iotdb/udf/api/UDF.java | 0 .../main/java/org/apache/iotdb/udf/api/UDTF.java | 0 .../java/org/apache/iotdb/udf/api/access/Row.java | 0 .../apache/iotdb/udf/api/access/RowIterator.java | 0 .../org/apache/iotdb/udf/api/access/RowWindow.java | 0 .../iotdb/udf/api/collector/PointCollector.java | 0 .../api/customizer/config/UDFConfigurations.java | 0 .../api/customizer/config/UDTFConfigurations.java | 0 .../parameter/UDFParameterValidator.java | 0 .../api/customizer/parameter/UDFParameters.java | 0 .../api/customizer/strategy/AccessStrategy.java | 0 .../strategy/MappableRowByRowAccessStrategy.java | 0 .../strategy/RowByRowAccessStrategy.java | 0 .../strategy/SessionTimeWindowAccessStrategy.java | 0 .../strategy/SlidingSizeWindowAccessStrategy.java | 0 .../strategy/SlidingTimeWindowAccessStrategy.java | 0 .../strategy/StateWindowAccessStrategy.java | 0 .../UDFAttributeNotProvidedException.java | 0 .../iotdb/udf/api/exception/UDFException.java | 0 .../UDFInputSeriesDataTypeNotValidException.java | 0 .../UDFInputSeriesIndexNotValidException.java | 0 .../UDFInputSeriesNumberNotValidException.java | 0 .../udf/api/exception/UDFManagementException.java | 0 .../UDFOutputSeriesDataTypeNotValidException.java | 0 .../exception/UDFParameterNotValidException.java | 0 .../java/org/apache/iotdb/udf/api/type/Binary.java | 0 .../java/org/apache/iotdb/udf/api/type/Type.java | 0 {cli => iotdb-client/cli}/pom.xml | 2 +- {cli => iotdb-client/cli}/src/assembly/cli.xml | 0 .../src/assembly/resources/conf/logback-cli.xml | 0 .../cli}/src/assembly/resources/sbin/start-cli.bat | 0 .../cli}/src/assembly/resources/sbin/start-cli.sh | 0 .../src/assembly/resources/tools/export-csv.bat | 0 .../src/assembly/resources/tools/export-csv.sh | 0 .../src/assembly/resources/tools/export-tsfile.bat | 0 .../src/assembly/resources/tools/export-tsfile.sh | 0 .../src/assembly/resources/tools/import-csv.bat | 0 .../src/assembly/resources/tools/import-csv.sh | 0 .../src/assembly/resources/tools/load-tsfile.bat | 0 .../src/assembly/resources/tools/load-tsfile.sh | 0 .../java/org/apache/iotdb/cli/AbstractCli.java | 0 .../src/main/java/org/apache/iotdb/cli/Cli.java | 0 .../apache/iotdb/cli/IoTDBSyntaxHighlighter.java | 0 .../org/apache/iotdb/cli/utils/IoTPrinter.java | 0 .../org/apache/iotdb/cli/utils/JlineUtils.java | 0 .../apache/iotdb/exception/ArgsErrorException.java | 0 .../org/apache/iotdb/tool/AbstractCsvTool.java | 0 .../org/apache/iotdb/tool/AbstractTsFileTool.java | 0 .../main/java/org/apache/iotdb/tool/ExportCsv.java | 0 .../java/org/apache/iotdb/tool/ExportTsFile.java | 0 .../main/java/org/apache/iotdb/tool/ImportCsv.java | 0 .../java/org/apache/iotdb/cli/AbstractCliIT.java | 0 .../java/org/apache/iotdb/cli/AbstractScript.java | 0 .../iotdb/tool/integration/ExportCsvTestIT.java | 0 .../iotdb/tool/integration/ImportCsvTestIT.java | 0 .../apache/iotdb/tool/unit/WriteCsvFileTestUT.java | 0 .../cli}/src/test/resources/logback.xml | 0 {client-cpp => iotdb-client/client-cpp}/README.md | 0 {client-cpp => iotdb-client/client-cpp}/pom.xml | 24 +- .../client-cpp}/src/assembly/client-cpp.xml | 0 .../client-cpp}/src/main/CMakeLists.txt | 0 .../client-cpp}/src/main/Session.cpp | 0 .../client-cpp}/src/main/Session.h | 0 .../client-cpp}/src/test/CMakeLists.txt | 0 .../client-cpp}/src/test/cpp/sessionIT.cpp | 0 .../client-cpp}/src/test/main.cpp | 0 iotdb-client/client-go | 1 + {client-py => iotdb-client/client-py}/.flake8 | 0 {client-py => iotdb-client/client-py}/.gitignore | 0 {client-py => iotdb-client/client-py}/README.md | 0 .../client-py}/SessionAlignedTimeseriesExample.py | 0 .../client-py}/SessionExample.py | 0 .../client-py}/iotdb/IoTDBContainer.py | 0 .../client-py}/iotdb/Session.py | 0 .../client-py}/iotdb/__init__.py | 0 .../client-py}/iotdb/dbapi/Connection.py | 0 .../client-py}/iotdb/dbapi/Cursor.py | 0 .../client-py}/iotdb/dbapi/Exceptions.py | 0 .../client-py}/iotdb/dbapi/__init__.py | 0 .../client-py}/iotdb/dbapi/tests/__init__.py | 0 .../iotdb/dbapi/tests/test_connection.py | 0 .../client-py}/iotdb/dbapi/tests/test_cursor.py | 0 .../client-py}/iotdb/sqlalchemy/IoTDBDialect.py | 0 .../iotdb/sqlalchemy/IoTDBIdentifierPreparer.py | 0 .../iotdb/sqlalchemy/IoTDBSQLCompiler.py | 0 .../iotdb/sqlalchemy/IoTDBTypeCompiler.py | 0 .../client-py}/iotdb/sqlalchemy/__init__.py | 0 .../client-py}/iotdb/sqlalchemy/tests/__init__.py | 0 .../iotdb/sqlalchemy/tests/test_dialect.py | 0 .../client-py}/iotdb/template/InternalNode.py | 0 .../client-py}/iotdb/template/MeasurementNode.py | 0 .../client-py}/iotdb/template/Template.py | 0 .../client-py}/iotdb/template/TemplateNode.py | 0 .../client-py}/iotdb/template/TemplateQueryType.py | 0 .../client-py}/iotdb/template/__init__.py | 0 .../client-py}/iotdb/tsfile/__init__.py | 0 .../client-py}/iotdb/tsfile/common/__init__.py | 0 .../iotdb/tsfile/common/constant/TsFileConstant.py | 0 .../iotdb/tsfile/common/constant/__init__.py | 0 .../client-py}/iotdb/tsfile/utils/Pair.py | 0 .../iotdb/tsfile/utils/ReadWriteIOUtils.py | 0 .../client-py}/iotdb/tsfile/utils/__init__.py | 0 .../client-py}/iotdb/utils/BitMap.py | 0 .../client-py}/iotdb/utils/Field.py | 0 .../iotdb/utils/IoTDBConnectionException.py | 0 .../client-py}/iotdb/utils/IoTDBConstants.py | 0 .../client-py}/iotdb/utils/IoTDBRpcDataSet.py | 0 .../client-py}/iotdb/utils/NumpyTablet.py | 0 .../client-py}/iotdb/utils/RowRecord.py | 0 .../client-py}/iotdb/utils/SessionDataSet.py | 0 .../client-py}/iotdb/utils/Tablet.py | 0 .../client-py}/iotdb/utils/__init__.py | 0 {client-py => iotdb-client/client-py}/pom.xml | 6 +- .../client-py}/pyproject.toml | 0 {client-py => iotdb-client/client-py}/release.sh | 0 .../client-py}/requirements.txt | 0 .../client-py}/requirements_dev.txt | 0 {client-py => iotdb-client/client-py}/setup.py | 0 .../client-py}/tests/__init__.py | 0 .../tests/tablet_performance_comparison.py | 0 .../client-py}/tests/test_aligned_timeseries.py | 0 .../client-py}/tests/test_dataframe.py | 0 .../client-py}/tests/test_delete_data.py | 0 .../client-py}/tests/test_numpy_tablet.py | 0 .../client-py}/tests/test_one_device.py | 0 .../client-py}/tests/test_session.py | 0 .../client-py}/tests/test_tablet.py | 0 .../client-py}/tests/test_template.py | 0 .../client-py}/tests/test_todf.py | 0 .../compile-tools}/README.md | 0 .../compile-tools}/pom.xml | 2 +- .../compile-tools}/thrift/pom.xml | 0 {isession => iotdb-client/isession}/pom.xml | 1 + .../java/org/apache/iotdb/isession/ISession.java | 0 .../org/apache/iotdb/isession/SessionConfig.java | 0 .../org/apache/iotdb/isession/SessionDataSet.java | 8 + .../apache/iotdb/isession/pool/ISessionPool.java | 0 .../iotdb/isession/pool/SessionDataSetWrapper.java | 0 .../apache/iotdb/isession/template/Template.java | 0 .../iotdb/isession/template/TemplateNode.java | 0 .../apache/iotdb/isession/util/SystemStatus.java | 0 .../org/apache/iotdb/isession/util/Version.java | 0 {jdbc => iotdb-client/jdbc}/README.md | 0 {jdbc => iotdb-client/jdbc}/osgi.bnd | 0 {jdbc => iotdb-client/jdbc}/pom.xml | 2 +- .../jdbc}/src/main/feature/feature.xml | 0 .../iotdb/jdbc/AbstractIoTDBJDBCResultSet.java | 0 .../main/java/org/apache/iotdb/jdbc/Activator.java | 0 .../main/java/org/apache/iotdb/jdbc/Config.java | 0 .../main/java/org/apache/iotdb/jdbc/Constant.java | 0 .../src/main/java/org/apache/iotdb/jdbc/Field.java | 0 .../iotdb/jdbc/GroupedLSBWatermarkEncoder.java | 0 .../org/apache/iotdb/jdbc/IoTDBConnection.java | 0 .../apache/iotdb/jdbc/IoTDBConnectionParams.java | 0 .../org/apache/iotdb/jdbc/IoTDBDataSource.java | 0 .../apache/iotdb/jdbc/IoTDBDataSourceFactory.java | 0 .../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 0 .../java/org/apache/iotdb/jdbc/IoTDBDriver.java | 0 .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 0 .../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 0 .../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 0 .../org/apache/iotdb/jdbc/IoTDBResultMetadata.java | 0 .../org/apache/iotdb/jdbc/IoTDBSQLException.java | 0 .../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 0 .../org/apache/iotdb/jdbc/IoTDBTracingInfo.java | 0 .../org/apache/iotdb/jdbc/IoTDBURLException.java | 0 .../java/org/apache/iotdb/jdbc/StringUtils.java | 0 .../src/main/java/org/apache/iotdb/jdbc/Utils.java | 0 .../org/apache/iotdb/jdbc/WatermarkEncoder.java | 0 .../resources/services/META-INF/java.sql.Driver | 0 .../test/java/org/apache/iotdb/jdbc/BatchTest.java | 0 .../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 0 .../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 0 .../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 0 .../iotdb/jdbc/IoTDBPreparedStatementTest.java | 0 .../apache/iotdb/jdbc/IoTDBResultMetadataTest.java | 0 .../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 0 .../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 0 {service-rpc => iotdb-client/service-rpc}/pom.xml | 2 +- .../org/apache/iotdb/rpc/AutoResizingBuffer.java | 3 - .../iotdb/rpc/AutoScalingBufferReadTransport.java | 0 .../iotdb/rpc/AutoScalingBufferWriteTransport.java | 0 .../apache/iotdb/rpc/BatchExecutionException.java | 0 .../iotdb/rpc/ConfigNodeConnectionException.java | 0 .../apache/iotdb/rpc/ConfigurableTByteBuffer.java | 0 .../iotdb/rpc/InfluxDBSynchronizedHandler.java | 0 .../apache/iotdb/rpc/IoTDBConnectionException.java | 0 .../org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 0 .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 0 .../apache/iotdb/rpc/NoValidValueException.java | 0 .../org/apache/iotdb/rpc/NonOpenTransport.java | 0 .../org/apache/iotdb/rpc/RedirectException.java | 0 .../main/java/org/apache/iotdb/rpc/RpcStat.java | 0 .../org/apache/iotdb/rpc/RpcTransportFactory.java | 0 .../main/java/org/apache/iotdb/rpc/RpcUtils.java | 0 .../iotdb/rpc/StatementExecutionException.java | 0 .../org/apache/iotdb/rpc/SynchronizedHandler.java | 0 .../rpc/TCompressedElasticFramedTransport.java | 0 .../org/apache/iotdb/rpc/TConfigurationConst.java | 0 .../apache/iotdb/rpc/TElasticFramedTransport.java | 0 .../iotdb/rpc/TNonblockingSocketWrapper.java | 0 .../java/org/apache/iotdb/rpc/TSStatusCode.java | 0 .../iotdb/rpc/TSnappyElasticFramedTransport.java | 0 .../java/org/apache/iotdb/rpc/TSocketWrapper.java | 0 .../rpc/TimeoutChangeableTFastFramedTransport.java | 0 .../TimeoutChangeableTSnappyFramedTransport.java | 0 .../iotdb/rpc/TimeoutChangeableTransport.java | 0 .../org/apache/iotdb/rpc/TSStatusCodeTest.java | 0 {session => iotdb-client/session}/pom.xml | 2 +- .../org/apache/iotdb/session/InsertConsumer.java | 0 .../java/org/apache/iotdb/session/Session.java | 7 - .../apache/iotdb/session/SessionConnection.java | 0 .../org/apache/iotdb/session/pool/SessionPool.java | 16 - .../iotdb/session/template/InternalNode.java | 0 .../iotdb/session/template/MeasurementNode.java | 0 .../iotdb/session/template/TemplateQueryType.java | 0 .../apache/iotdb/session/util/SessionUtils.java | 0 .../org/apache/iotdb/session/util/ThreadUtils.java | 0 .../apache/iotdb/session/SessionCacheLeaderUT.java | 0 .../java/org/apache/iotdb/session/TabletTest.java | 0 .../apache/iotdb/session/pool/SessionPoolTest.java | 0 .../apache/iotdb/session/util/ThreadUtilsTest.java | 0 .../src/test/resources/iotdb-common.properties | 0 .../src/test/resources/iotdb-datanode.properties | 0 .../session}/src/test/resources/logback.xml | 0 .../flink-iotdb-connector}/README.md | 0 .../flink-iotdb-connector}/pom.xml | 2 +- .../iotdb/flink/DefaultIoTSerializationSchema.java | 0 .../main/java/org/apache/iotdb/flink/Event.java | 0 .../java/org/apache/iotdb/flink/IoTDBSink.java | 0 .../java/org/apache/iotdb/flink/IoTDBSource.java | 0 .../apache/iotdb/flink/IoTSerializationSchema.java | 0 .../apache/iotdb/flink/options/IoTDBOptions.java | 0 .../iotdb/flink/options/IoTDBSinkOptions.java | 0 .../iotdb/flink/options/IoTDBSourceOptions.java | 0 .../flink/DefaultIoTSerializationSchemaTest.java | 0 .../iotdb/flink/IoTDBSinkBatchInsertTest.java | 0 .../iotdb/flink/IoTDBSinkBatchTimerTest.java | 0 .../apache/iotdb/flink/IoTDBSinkInsertTest.java | 0 .../flink-tsfile-connector}/README.md | 0 .../flink-tsfile-connector}/pom.xml | 2 +- .../apache/iotdb/flink/tsfile/RowRecordParser.java | 0 .../iotdb/flink/tsfile/RowRowRecordParser.java | 0 .../iotdb/flink/tsfile/RowTSRecordConverter.java | 0 .../iotdb/flink/tsfile/TSRecordConverter.java | 0 .../iotdb/flink/tsfile/TSRecordOutputFormat.java | 0 .../iotdb/flink/tsfile/TsFileInputFormat.java | 0 .../iotdb/flink/tsfile/TsFileOutputFormat.java | 0 .../iotdb/flink/tsfile/util/TSFileConfigUtil.java | 0 .../RowTSRecordOutputFormatIntegrationTest.java | 0 .../flink/tsfile/RowTSRecordOutputFormatTest.java | 0 .../flink/tsfile/RowTsFileConnectorTestBase.java | 0 .../RowTsFileInputFormatIntegrationTest.java | 0 .../flink/tsfile/RowTsFileInputFormatTest.java | 0 .../flink/tsfile/RowTsFileInputFormatTestBase.java | 0 .../tsfile/RowTsFileOutputFormatTestBase.java | 0 .../util/TSFileConfigUtilCompletenessTest.java | 0 .../apache/iotdb/flink/util/TsFileWriteUtil.java | 0 .../grafana-connector}/img/add_data_source.png | Bin .../grafana-connector}/img/add_graph.png | Bin .../grafana-connector}/img/edit_data_source.png | Bin .../grafana-connector}/pom.xml | 2 +- .../grafana-connector}/readme.md | 0 .../grafana-connector}/readme_zh.md | 0 .../web/grafana/TsfileWebDemoApplication.java | 0 .../apache/iotdb/web/grafana/bean/TimeValues.java | 0 .../iotdb/web/grafana/conf/MyConfiguration.java | 0 .../controller/DatabaseConnectController.java | 0 .../org/apache/iotdb/web/grafana/dao/BasicDao.java | 0 .../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 0 .../web/grafana/interceptor/LoginInterceptor.java | 0 .../grafana/service/DatabaseConnectService.java | 0 .../service/impl/DatabaseConnectServiceImpl.java | 0 .../src/main/resources/application.properties | 0 .../web/grafana/dao/impl/BasicDaoImplTest.java | 0 .../grafana-plugin}/.gitignore | 0 .../grafana-plugin}/.prettierrc.js | 0 .../grafana-plugin}/CHANGELOG.md | 0 .../grafana-plugin}/LICENSE | 0 .../grafana-plugin}/Magefile.go | 0 .../grafana-plugin}/README.md | 0 .../grafana-plugin}/backend-compile.bat | 0 .../grafana-plugin}/backend-compile.sh | 0 .../grafana-plugin}/go.mod | 0 .../grafana-plugin}/jest.config.js | 0 .../grafana-plugin}/package.json | 0 .../grafana-plugin}/pkg/main.go | 0 .../pkg/plugin/iotdb_resource_handler.go | 0 .../grafana-plugin}/pkg/plugin/plugin.go | 0 .../grafana-plugin}/pom.xml | 2 +- .../grafana-plugin}/src/ConfigEditor.tsx | 0 .../grafana-plugin}/src/QueryEditor.tsx | 0 .../src/componments/AggregateFun.tsx | 0 .../src/componments/ControlValue.tsx | 0 .../grafana-plugin}/src/componments/FillValue.tsx | 0 .../grafana-plugin}/src/componments/Form.tsx | 0 .../grafana-plugin}/src/componments/FromValue.tsx | 0 .../grafana-plugin}/src/componments/GroupBy.tsx | 0 .../src/componments/SelectValue.tsx | 0 .../grafana-plugin}/src/componments/TimeSeries.tsx | 0 .../grafana-plugin}/src/componments/WhereValue.tsx | 0 .../grafana-plugin}/src/datasource.ts | 0 .../grafana-plugin}/src/functions.ts | 0 .../grafana-plugin}/src/img/addIoTDBDataSource.png | Bin .../grafana-plugin}/src/img/logo.svg | 0 .../grafana-plugin}/src/img/showData.png | Bin .../grafana-plugin}/src/module.ts | 0 .../grafana-plugin}/src/plugin.json | 0 .../grafana-plugin}/src/types.ts | 0 .../grafana-plugin}/tsconfig.json | 0 .../grafana-plugin}/yarn.lock | 0 {hadoop => iotdb-connector/hadoop}/README.md | 0 {hadoop => iotdb-connector/hadoop}/pom.xml | 2 +- .../iotdb/hadoop/fileSystem/HDFSConfUtil.java | 0 .../apache/iotdb/hadoop/fileSystem/HDFSFile.java | 0 .../apache/iotdb/hadoop/fileSystem/HDFSInput.java | 0 .../apache/iotdb/hadoop/fileSystem/HDFSOutput.java | 0 .../org/apache/iotdb/hadoop/tsfile/IReaderSet.java | 0 .../iotdb/hadoop/tsfile/TSFHadoopException.java | 0 .../apache/iotdb/hadoop/tsfile/TSFInputFormat.java | 0 .../apache/iotdb/hadoop/tsfile/TSFInputSplit.java | 0 .../iotdb/hadoop/tsfile/TSFOutputFormat.java | 0 .../iotdb/hadoop/tsfile/TSFRecordReader.java | 0 .../iotdb/hadoop/tsfile/TSFRecordWriter.java | 0 .../iotdb/hadoop/tsfile/record/HDFSTSRecord.java | 0 .../apache/iotdb/hadoop/tsfile/TSFHadoopTest.java | 0 .../iotdb/hadoop/tsfile/TSFInputSplitTest.java | 0 .../iotdb/hadoop/tsfile/TsFileTestHelper.java | 0 .../iotdb/hadoop/tsfile/constant/TestConstant.java | 0 .../hadoop}/src/test/resources/logback.xml | 0 .../hive-connector}/pom.xml | 2 +- .../org/apache/iotdb/hive/TSFHiveInputFormat.java | 0 .../org/apache/iotdb/hive/TSFHiveOutputFormat.java | 0 .../org/apache/iotdb/hive/TSFHiveRecordReader.java | 0 .../org/apache/iotdb/hive/TSFHiveRecordWriter.java | 0 .../org/apache/iotdb/hive/TsFileDeserializer.java | 0 .../java/org/apache/iotdb/hive/TsFileSerDe.java | 0 .../apache/iotdb/hive/TsFileSerDeException.java | 0 .../apache/iotdb/hive/TSFHiveInputFormatTest.java | 0 .../apache/iotdb/hive/TSFHiveRecordReaderTest.java | 0 .../apache/iotdb/hive/TsFileDeserializerTest.java | 0 .../org/apache/iotdb/hive/TsFileSerDeTest.java | 0 .../org/apache/iotdb/hive/TsFileTestHelper.java | 0 .../apache/iotdb/hive/constant/TestConstant.java | 0 .../hive-connector}/src/test/resources/logback.xml | 0 .../spark-iotdb-connector}/pom.xml | 2 +- .../spark-iotdb-connector}/scala_2.11/pom.xml | 2 +- .../spark-iotdb-connector}/scala_2.12/pom.xml | 2 +- .../org/apache/iotdb/spark/db/Converter.scala | 0 .../org/apache/iotdb/spark/db/DataFrameTools.scala | 0 .../org/apache/iotdb/spark/db/DefaultSource.scala | 0 .../org/apache/iotdb/spark/db/IoTDBOptions.scala | 0 .../scala/org/apache/iotdb/spark/db/IoTDBRDD.scala | 0 .../org/apache/iotdb/spark/db/IoTDBRelation.scala | 0 .../org/apache/iotdb/spark/db/SQLConstant.scala | 0 .../org/apache/iotdb/spark/db/Transformer.scala | 0 .../scala/org/apache/iotdb/spark/db/package.scala | 0 .../spark-tsfile}/pom.xml | 2 +- .../iotdb/spark/tsfile/io/TsFileOutputFormat.java | 0 .../iotdb/spark/tsfile/io/TsFileRecordWriter.java | 0 .../org/apache/iotdb/spark/tsfile/qp/Executor.java | 0 .../iotdb/spark/tsfile/qp/QueryProcessor.java | 0 .../spark/tsfile/qp/common/BasicOperator.java | 0 .../spark/tsfile/qp/common/FilterOperator.java | 0 .../iotdb/spark/tsfile/qp/common/Operator.java | 0 .../iotdb/spark/tsfile/qp/common/SQLConstant.java | 0 .../iotdb/spark/tsfile/qp/common/SingleQuery.java | 0 .../iotdb/spark/tsfile/qp/common/TSQueryPlan.java | 0 .../qp/exception/BasicOperatorException.java | 0 .../tsfile/qp/exception/DNFOptimizeException.java | 0 .../qp/exception/LogicalOptimizeException.java | 0 .../tsfile/qp/exception/MergeFilterException.java | 0 .../qp/exception/QueryOperatorException.java | 0 .../qp/exception/QueryProcessorException.java | 0 .../tsfile/qp/exception/RemoveNotException.java | 0 .../tsfile/qp/optimizer/DNFFilterOptimizer.java | 0 .../tsfile/qp/optimizer/IFilterOptimizer.java | 0 .../qp/optimizer/MergeSingleFilterOptimizer.java | 0 .../tsfile/qp/optimizer/PhysicalOptimizer.java | 0 .../tsfile/qp/optimizer/RemoveNotOptimizer.java | 0 .../org/apache/iotdb/spark/tsfile/Converter.scala | 0 .../apache/iotdb/spark/tsfile/DefaultSource.scala | 0 .../iotdb/spark/tsfile/NarrowConverter.scala | 0 .../spark/tsfile/NarrowTsFileOutputWriter.scala | 0 .../apache/iotdb/spark/tsfile/Transformer.scala | 0 .../iotdb/spark/tsfile/TsFileWriterFactory.scala | 0 .../apache/iotdb/spark/tsfile/WideConverter.scala | 0 .../spark/tsfile/WideTsFileOutputWriter.scala | 0 .../org/apache/iotdb/spark/tsfile/package.scala | 0 .../apache/iotdb/spark/constant/TestConstant.java | 0 .../org/apache/iotdb/spark/tool/TsFileExample.java | 0 .../apache/iotdb/spark/tool/TsFileWriteTool.java | 0 .../apache/iotdb/spark/tsfile/ConverterTest.scala | 0 .../apache/iotdb/spark/tsfile/HDFSInputTest.java | 0 .../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 0 .../zeppelin-interpreter}/IoTDB-Zeppelin-Demo.zpln | 0 .../zeppelin-interpreter}/pom.xml | 2 +- .../apache/zeppelin/iotdb/IoTDBInterpreter.java | 0 .../src/main/resources/interpreter-setting.json | 0 {openapi => iotdb-protocol/openapi}/pom.xml | 2 +- .../src/main/openapi3/iotdb_rest_common.yaml | 0 .../openapi}/src/main/openapi3/iotdb_rest_v1.yaml | 0 .../openapi}/src/main/openapi3/iotdb_rest_v2.yaml | 0 .../thrift-commons}/pom.xml | 2 +- .../thrift-commons}/src/main/thrift/common.thrift | 0 .../thrift-confignode}/pom.xml | 2 +- .../src/main/thrift/confignode.thrift | 8 + .../thrift-influxdb}/README.md | 0 .../thrift-influxdb}/pom.xml | 2 +- .../src/main/thrift/influxdb.thrift | 0 .../thrift-iot-consensus}/pom.xml | 2 +- .../src/main/thrift/iotconsensus.thrift | 0 .../thrift-mlnode}/pom.xml | 2 +- .../thrift-mlnode}/src/main/thrift/mlnode.thrift | 0 .../thrift-raft}/pom.xml | 2 +- .../thrift-raft}/src/main/thrift/raft.thrift | 0 {thrift => iotdb-protocol/thrift}/README.md | 0 {thrift => iotdb-protocol/thrift}/pom.xml | 2 +- .../thrift}/rpc-changelist.md | 0 .../thrift}/src/main/thrift/client.thrift | 0 .../thrift}/src/main/thrift/datanode.thrift | 23 + .../java/org/apache/iotdb/metrics/type/Gauge.java | 1 - .../resources/conf/iotdb-common.properties | 5 +- .../iotdb/commons/concurrent/ThreadName.java | 1 + .../apache/iotdb/commons/conf/CommonConfig.java | 10 + .../iotdb/commons/conf/CommonDescriptor.java | 3 + .../index/ComparableConsensusRequest.java | 7 +- .../commons/consensus/index/ProgressIndex.java | 79 + .../commons/consensus/index/ProgressIndexType.java | 79 + .../consensus/index/impl/IoTProgressIndex.java | 167 +++ .../consensus/index/impl/MinimumProgressIndex.java | 87 ++ .../builtin/connector/DoNothingConnector.java | 4 +- .../builtin/connector/IoTDBThriftConnector.java | 4 +- .../builtin/processor/DoNothingProcessor.java | 11 +- .../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 76 +- .../commons/schema/view/LogicalViewSchema.java | 30 + .../schema/view/viewExpression/ViewExpression.java | 2 + .../{enums => }/PerformanceOverviewMetrics.java | 4 +- .../iotdb/commons/service/metric/enums/Metric.java | 187 +-- .../iotdb/commons/service/metric/enums/Tag.java | 26 +- .../commons/pipe/task/meta/PipeMetaDeSerTest.java | 7 +- .../iotdb/pipe/api/collector/EventCollector.java | 62 - .../pipe/api/event/dml/deletion/DeletionEvent.java | 48 - pom.xml | 70 +- rewrite-tsfile-tool/pom.xml | 83 -- .../src/assembly/resources/sbin/rewrite-tsfile.bat | 60 - .../src/assembly/resources/sbin/rewrite-tsfile.sh | 48 - rewrite-tsfile-tool/src/assembly/rewriteTsFile.xml | 40 - .../java/org/apache/iotdb/RewriteTsFileTool.java | 914 ------------ schema-engine-rocksdb/README.md | 38 - schema-engine-rocksdb/pom.xml | 79 - .../resources/conf/schema-rocksdb.properties | 94 -- .../src/assembly/schema-engine-rocksdb.xml | 44 - .../schemaregion/rocksdb/CheckKeyResult.java | 50 - .../schemaregion/rocksdb/RSchemaConfLoader.java | 185 --- .../schemaregion/rocksdb/RSchemaConstants.java | 76 - .../schemaregion/rocksdb/RSchemaLogger.java | 61 - .../rocksdb/RSchemaReadWriteHandler.java | 515 ------- .../schemaregion/rocksdb/RSchemaRegion.java | 1558 -------------------- .../schemaregion/rocksdb/RSchemaUtils.java | 592 -------- .../schemaregion/rocksdb/mnode/RDatabaseMNode.java | 117 -- .../schemaregion/rocksdb/mnode/RDeviceMNode.java | 171 --- .../schemaregion/rocksdb/mnode/RInternalMNode.java | 171 --- .../schemaregion/rocksdb/mnode/RMNode.java | 230 --- .../schemaregion/rocksdb/mnode/RMNodeType.java | 49 - .../rocksdb/mnode/RMNodeValueType.java | 47 - .../rocksdb/mnode/RMeasurementMNode.java | 249 ---- .../schemaregion/rocksdb/MRocksDBBenchmark.java | 98 -- .../schemaregion/rocksdb/MRocksDBUnitTest.java | 265 ---- .../rocksdb/RSchemaReadWriteHandlerTest.java | 77 - .../rocksdb/RocksDBBenchmarkEngine.java | 144 -- .../schemaregion/rocksdb/RocksDBBenchmarkTask.java | 109 -- .../schemaregion/rocksdb/RocksDBTestUtils.java | 72 - schema-engine-tag/README.md | 190 --- schema-engine-tag/pom.xml | 81 - .../assembly/resources/conf/schema-tag.properties | 33 - .../src/assembly/schema-engine-tag.xml | 44 - .../metadata/tagSchemaRegion/TagSchemaRegion.java | 611 -------- .../tagSchemaRegion/config/TagSchemaConfig.java | 55 - .../config/TagSchemaDescriptor.java | 93 -- .../idtable/IDTableWithDeviceIDListImpl.java | 138 -- .../tagIndex/ITagInvertedIndex.java | 61 - .../tagIndex/Request/InsertionRequest.java | 59 - .../tagSchemaRegion/tagIndex/TagInvertedIndex.java | 183 --- .../tagIndex/deletion/MemChunkDeletion.java | 57 - .../tagIndex/deletion/MemChunkGroupDeletion.java | 68 - .../tagIndex/deletion/MemTableDeletion.java | 77 - .../tagIndex/deletion/MemTableGroupDeletion.java | 66 - .../tagIndex/insertion/MemChunkGroupInsertion.java | 65 - .../tagIndex/insertion/MemChunkInsertion.java | 58 - .../tagIndex/insertion/MemTableGroupInsertion.java | 74 - .../tagIndex/insertion/MemTableInsertion.java | 67 - .../tagIndex/memtable/MemChunk.java | 54 - .../tagIndex/memtable/MemChunkGroup.java | 56 - .../tagIndex/memtable/MemTable.java | 87 -- .../tagIndex/memtable/MemTableGroup.java | 102 -- .../tagIndex/query/MemChunkGroupQuery.java | 61 - .../tagIndex/query/MemChunkQuery.java | 67 - .../tagIndex/query/MemTableGroupQuery.java | 59 - .../tagIndex/query/MemTableQuery.java | 72 - .../tagIndex/response/QueryResponse.java | 71 - .../tagSchemaRegion/tagIndex/wal/WALEntry.java | 134 -- .../tagSchemaRegion/tagIndex/wal/WALManager.java | 141 -- .../utils/MeasurementPathUtils.java | 80 - .../utils/PathTagConverterUtils.java | 70 - .../utils/ShowTimeSeriesResultUtils.java | 82 -- .../iotdb/lsm/annotation/DeletionProcessor.java | 36 - .../iotdb/lsm/annotation/InsertionProcessor.java | 36 - .../iotdb/lsm/annotation/QueryProcessor.java | 36 - .../applicationcontext/ApplicationContext.java | 61 - .../ApplicationContextGenerator.java | 119 -- .../requestcontext/DeleteRequestContext.java | 34 - .../requestcontext/FlushRequestContext.java | 33 - .../requestcontext/InsertRequestContext.java | 34 - .../requestcontext/QueryRequestContext.java | 34 - .../lsm/context/requestcontext/RequestContext.java | 108 -- .../org/apache/iotdb/lsm/engine/ILSMEngine.java | 73 - .../org/apache/iotdb/lsm/engine/IRecoverable.java | 34 - .../org/apache/iotdb/lsm/engine/LSMEngine.java | 176 --- .../apache/iotdb/lsm/engine/LSMEngineBuilder.java | 290 ---- .../lsm/levelProcess/BasicLevelProcessor.java | 79 - .../lsm/levelProcess/DeleteLevelProcessor.java | 39 - .../lsm/levelProcess/FlushLevelProcessor.java | 38 - .../iotdb/lsm/levelProcess/ILevelProcessor.java | 41 - .../lsm/levelProcess/InsertLevelProcessor.java | 39 - .../lsm/levelProcess/LevelProcessorChain.java | 45 - .../lsm/levelProcess/QueryLevelProcessor.java | 39 - .../apache/iotdb/lsm/manager/BasicLSMManager.java | 55 - .../apache/iotdb/lsm/manager/DeletionManager.java | 48 - .../org/apache/iotdb/lsm/manager/ILSMManager.java | 58 - .../apache/iotdb/lsm/manager/InsertionManager.java | 48 - .../org/apache/iotdb/lsm/manager/QueryManager.java | 32 - .../apache/iotdb/lsm/manager/RecoverManager.java | 49 - .../org/apache/iotdb/lsm/manager/WALManager.java | 126 -- .../apache/iotdb/lsm/request/IDeletionRequest.java | 30 - .../iotdb/lsm/request/IInsertionRequest.java | 30 - .../apache/iotdb/lsm/request/IQueryRequest.java | 35 - .../org/apache/iotdb/lsm/request/IRequest.java | 61 - .../org/apache/iotdb/lsm/request/RequestType.java | 27 - .../org/apache/iotdb/lsm/response/IResponse.java | 46 - .../iotdb/lsm/strategy/BFSAccessStrategy.java | 66 - .../apache/iotdb/lsm/strategy/IAccessStrategy.java | 36 - .../lsm/strategy/PostOrderAccessStrategy.java | 56 - .../iotdb/lsm/strategy/PreOrderAccessStrategy.java | 54 - .../iotdb/lsm/strategy/RBFSAccessStrategy.java | 85 -- .../java/org/apache/iotdb/lsm/wal/IWALReader.java | 48 - .../java/org/apache/iotdb/lsm/wal/IWALRecord.java | 50 - .../java/org/apache/iotdb/lsm/wal/WALReader.java | 99 -- .../java/org/apache/iotdb/lsm/wal/WALWriter.java | 111 -- .../tagSchemaRegion/TagSchemaRegionTest.java | 233 --- .../idtable/IDTableWithDeviceIDListImplTest.java | 213 --- .../tagIndex/TagTagInvertedIndexTest.java | 184 --- .../tools/tsfile/print-iotdb-data-dir.bat | 2 +- .../resources/tools/tsfile/print-iotdb-data-dir.sh | 2 +- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 2 +- .../apache/iotdb/db/client/ConfigNodeClient.java | 22 + .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 5 +- .../IoTConsensusDataRegionStateMachine.java | 11 +- .../org/apache/iotdb/db/engine/StorageEngine.java | 4 +- .../apache/iotdb/db/engine/cache/ChunkCache.java | 11 +- .../db/engine/cache/TimeSeriesMetadataCache.java | 7 +- .../execute/task/AbstractCompactionTask.java | 6 +- .../execute/task/CrossSpaceCompactionTask.java | 4 +- .../execute/task/InnerSpaceCompactionTask.java | 4 +- .../execute/utils/MultiTsFileDeviceIterator.java | 13 +- .../readchunk/AlignedSeriesCompactionExecutor.java | 8 +- .../readchunk/SingleSeriesCompactionExecutor.java | 10 +- .../utils/writer/AbstractCompactionWriter.java | 4 +- .../compaction/schedule/CompactionScheduler.java | 49 +- .../compaction/schedule/CompactionTaskManager.java | 1 + .../schedule/constant/CompactionType.java | 17 +- .../schedule/constant/ProcessChunkType.java | 17 +- .../iotdb/db/engine/flush/FlushManagerMetrics.java | 8 +- .../iotdb/db/engine/flush/MemTableFlushTask.java | 3 +- .../iotdb/db/engine/flush/tasks/FlushContext.java | 6 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 23 +- .../db/engine/storagegroup/DataRegionMetrics.java | 8 +- .../db/engine/storagegroup/TsFileManager.java | 1 + .../db/engine/storagegroup/TsFileProcessor.java | 60 +- .../db/engine/storagegroup/TsFileResource.java | 81 +- .../db/engine/storagegroup/TsFileResourceList.java | 62 +- .../db/metadata/cache/DataNodeSchemaCache.java | 22 +- .../metadata/cache/DataNodeSchemaCacheMetrics.java | 12 +- .../cache/DeviceUsingTemplateSchemaCache.java | 7 + .../iotdb/db/metadata/cache/SchemaCacheEntry.java | 9 + .../db/metadata/cache/TimeSeriesSchemaCache.java | 70 +- .../db/metadata/metric/SchemaEngineMemMetric.java | 5 +- .../db/metadata/mnode/mem/info/BasicMNodeInfo.java | 2 +- .../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 60 +- .../plan/schemaregion/SchemaRegionPlanType.java | 3 + .../plan/schemaregion/SchemaRegionPlanVisitor.java | 17 + .../impl/SchemaRegionPlanDeserializer.java | 33 +- .../impl/SchemaRegionPlanSerializer.java | 49 +- .../impl/SchemaRegionPlanTxtSerializer.java | 25 + .../impl/write/CreateLogicalViewPlanImpl.java | 6 +- .../impl/write/DeleteLogicalViewPlanImpl.java | 25 +- .../impl/write/PreDeleteLogicalViewPlanImpl.java | 25 +- .../RollbackPreDeleteLogicalViewPlanImpl.java | 28 +- .../impl/write/SchemaRegionWritePlanFactory.java | 31 + .../write/view/IDeleteLogicalViewPlan.java} | 22 +- .../write/view/IPreDeleteLogicalViewPlan.java | 39 +- .../view/IRollbackPreDeleteLogicalViewPlan.java | 34 +- .../reader/SchemaReaderLimitOffsetWrapper.java | 86 ++ .../db/metadata/schemaregion/ISchemaRegion.java | 7 + .../schemaregion/SchemaRegionMemoryImpl.java | 103 +- .../schemaregion/SchemaRegionSchemaFileImpl.java | 15 + .../iotdb/db/metadata/view/ViewPathType.java | 9 +- .../apache/iotdb/db/metadata/view/ViewPaths.java | 97 ++ .../metadata/visitor/SchemaExecutionVisitor.java | 44 +- .../iotdb/db/mpp/aggregation/Aggregator.java | 15 +- .../db/mpp/common/header/ColumnHeaderConstant.java | 13 + .../db/mpp/common/header/DatasetHeaderFactory.java | 4 + .../mpp/common/schematree/ClusterSchemaTree.java | 70 +- .../common/schematree/IMeasurementSchemaInfo.java | 3 + .../db/mpp/common/schematree/ISchemaTree.java | 7 + .../common/schematree/MeasurementSchemaInfo.java | 9 + .../schematree/node/SchemaMeasurementNode.java | 8 + .../iotdb/db/mpp/execution/driver/DataDriver.java | 3 +- .../iotdb/db/mpp/execution/driver/Driver.java | 5 +- .../execution/exchange/MPPDataExchangeManager.java | 45 +- .../exchange/MPPDataExchangeServiceMetrics.java | 8 - ...MppDataExchangeServiceThriftHandlerMetrics.java | 35 +- .../mpp/execution/exchange/SharedTsBlockQueue.java | 16 +- .../execution/exchange/sink/LocalSinkChannel.java | 7 +- .../execution/exchange/sink/ShuffleSinkHandle.java | 8 +- .../mpp/execution/exchange/sink/SinkChannel.java | 15 +- .../exchange/source/LocalSourceHandle.java | 11 +- .../execution/exchange/source/SourceHandle.java | 21 +- .../execution/executor/RegionWriteExecutor.java | 4 +- .../fragment/FragmentInstanceManager.java | 16 +- .../iotdb/db/mpp/execution/memory/MemoryPool.java | 28 +- .../schema/source/LogicalViewSchemaSource.java | 175 +++ .../schema/source/SchemaSourceFactory.java | 5 + .../execution/operator/source/SeriesScanUtil.java | 9 +- .../db/mpp/execution/schedule/DriverScheduler.java | 17 +- .../iotdb/db/mpp/metric/ChunkCacheMetrics.java | 14 +- .../db/mpp/metric/DataExchangeCostMetricSet.java | 345 +++-- .../db/mpp/metric/DataExchangeCountMetricSet.java | 211 ++- .../db/mpp/metric/DriverSchedulerMetricSet.java | 98 +- .../db/mpp/metric/QueryExecutionMetricSet.java | 206 ++- .../iotdb/db/mpp/metric/QueryMetricsManager.java | 122 +- .../db/mpp/metric/QueryPlanCostMetricSet.java | 86 +- .../mpp/metric/QueryRelatedResourceMetricSet.java | 162 ++ .../db/mpp/metric/QueryResourceMetricSet.java | 77 +- .../db/mpp/metric/SeriesScanCostMetricSet.java | 1117 +++++++++----- .../mpp/metric/TimeSeriesMetadataCacheMetrics.java | 22 +- .../org/apache/iotdb/db/mpp/plan/Coordinator.java | 4 + .../apache/iotdb/db/mpp/plan/analyze/Analysis.java | 27 + .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 408 +++-- .../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 4 +- .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 37 + .../db/mpp/plan/analyze/cache/PartitionCache.java | 52 +- .../analyze/schema/ClusterSchemaFetchExecutor.java | 19 + .../plan/analyze/schema/ISchemaComputation.java | 39 + .../mpp/plan/analyze/schema/ISchemaValidation.java | 9 + .../plan/analyze/schema/NormalSchemaFetcher.java | 206 ++- .../db/mpp/plan/execution/QueryExecution.java | 17 +- .../plan/execution/config/ConfigTaskVisitor.java | 24 + .../config/executor/ClusterConfigTaskExecutor.java | 206 +++ .../config/executor/IConfigTaskExecutor.java | 12 + .../config/metadata/ShowVariablesTask.java | 4 + .../config/metadata/view/AlterLogicalViewTask.java | 44 + .../metadata/view/DeleteLogicalViewTask.java | 46 + .../metadata/view/RenameLogicalViewTask.java | 45 + .../iotdb/db/mpp/plan/expression/Expression.java | 20 + .../plan/expression/binary/BinaryExpression.java | 22 + .../plan/expression/binary/WhenThenExpression.java | 7 + .../db/mpp/plan/expression/leaf/LeafOperand.java | 5 + .../plan/expression/multi/FunctionExpression.java | 21 + .../expression/other/CaseWhenThenExpression.java | 17 + .../plan/expression/ternary/BetweenExpression.java | 9 + .../plan/expression/ternary/TernaryExpression.java | 18 +- .../db/mpp/plan/expression/unary/InExpression.java | 26 +- .../plan/expression/unary/IsNullExpression.java | 5 + .../mpp/plan/expression/unary/LikeExpression.java | 5 + .../plan/expression/unary/LogicNotExpression.java | 9 + .../plan/expression/unary/NegationExpression.java | 11 + .../plan/expression/unary/RegularExpression.java | 5 + .../mpp/plan/expression/unary/UnaryExpression.java | 6 +- .../visitor/CompleteMeasurementSchemaVisitor.java | 112 ++ .../RemoveWildcardAndViewInExpressionVisitor.java | 95 ++ .../RemoveWildcardAndViewInFilterVisitor.java | 109 ++ .../visitor/ReplaceLogicalViewVisitor.java | 188 +++ .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 99 +- .../db/mpp/plan/parser/StatementGenerator.java | 2 +- .../db/mpp/plan/planner/LocalExecutionPlanner.java | 4 + .../db/mpp/plan/planner/LogicalPlanBuilder.java | 9 + .../db/mpp/plan/planner/LogicalPlanVisitor.java | 52 +- .../iotdb/db/mpp/plan/planner/LogicalPlanner.java | 4 +- .../db/mpp/plan/planner/OperatorTreeGenerator.java | 20 + .../mpp/plan/planner/plan/node/PlanNodeType.java | 20 +- .../db/mpp/plan/planner/plan/node/PlanVisitor.java | 25 +- .../metedata/read/LogicalViewSchemaScanNode.java | 126 ++ .../view/ConstructLogicalViewBlackListNode.java | 91 ++ .../write/{ => view}/CreateLogicalViewNode.java | 2 +- .../metedata/write/view/DeleteLogicalViewNode.java | 91 ++ .../view/RollbackLogicalViewBlackListNode.java | 91 ++ .../plan/node/write/InsertMultiTabletsNode.java | 7 + .../plan/planner/plan/node/write/InsertNode.java | 20 +- .../planner/plan/node/write/InsertRowsNode.java | 7 + .../plan/node/write/InsertRowsOfOneDeviceNode.java | 7 + .../plan/scheduler/AsyncSendPlanNodeHandler.java | 2 +- .../db/mpp/plan/scheduler/ClusterScheduler.java | 7 +- .../scheduler/FixedRateFragInsStateTracker.java | 14 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 10 +- .../plan/scheduler/IFragInstanceStateTracker.java | 4 +- .../mpp/plan/scheduler/SimpleQueryTerminator.java | 52 +- .../iotdb/db/mpp/plan/statement/StatementType.java | 3 + .../db/mpp/plan/statement/StatementVisitor.java | 24 +- .../plan/statement/crud/InsertBaseStatement.java | 93 ++ .../crud/InsertMultiTabletsStatement.java | 17 + .../plan/statement/crud/InsertRowStatement.java | 128 +- .../crud/InsertRowsOfOneDeviceStatement.java | 21 + .../plan/statement/crud/InsertRowsStatement.java | 17 + .../plan/statement/crud/InsertTabletStatement.java | 135 +- .../metadata/view/AlterLogicalViewStatement.java | 115 ++ .../{ => view}/CreateLogicalViewStatement.java | 148 +- .../metadata/view/DeleteLogicalViewStatement.java | 66 + .../metadata/view/RenameLogicalViewStatement.java | 73 + .../metadata/view/ShowLogicalViewStatement.java | 62 + .../iotdb/db/pipe/agent/runtime/PipeLauncher.java | 15 +- .../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 72 +- .../core/collector/IoTDBDataRegionCollector.java | 9 +- .../PipeHistoricalDataRegionTsFileCollector.java | 22 +- .../realtime/PipeRealtimeDataRegionCollector.java | 11 + .../PipeRealtimeDataRegionHybridCollector.java | 53 +- .../realtime/assigner/PipeDataRegionAssigner.java | 7 +- .../listener/PipeInsertionDataNodeListener.java | 10 +- .../impl/iotdb/v1/IoTDBThriftConnectorV1.java | 9 +- .../iotdb/db/pipe/core/event/EnrichedEvent.java | 76 +- .../core/event/impl/PipeTabletInsertionEvent.java | 100 +- .../core/event/impl/PipeTsFileInsertionEvent.java | 71 +- .../event/realtime/PipeRealtimeCollectEvent.java | 49 +- .../realtime/PipeRealtimeCollectEventFactory.java | 7 +- .../event/view/collector/PipeEventCollector.java | 19 +- .../execution/executor/PipeSubtaskExecutor.java | 24 +- .../execution/scheduler/PipeSubtaskScheduler.java | 89 ++ .../execution/scheduler/PipeTaskScheduler.java | 74 - .../db/pipe/resource/PipeResourceManager.java | 9 + .../{ => file}/PipeFileResourceManager.java | 2 +- .../db/pipe/resource/wal/PipeWALResource.java | 162 ++ .../pipe/resource/wal/PipeWALResourceManager.java | 111 ++ .../org/apache/iotdb/db/pipe/task/PipeBuilder.java | 26 +- .../org/apache/iotdb/db/pipe/task/PipeTask.java | 7 +- .../apache/iotdb/db/pipe/task/PipeTaskBuilder.java | 15 +- .../db/pipe/task/stage/PipeTaskCollectorStage.java | 11 +- .../db/pipe/task/stage/PipeTaskProcessorStage.java | 3 +- .../db/pipe/task/subtask/PipeConnectorSubtask.java | 36 +- .../db/pipe/task/subtask/PipeProcessorSubtask.java | 26 +- .../iotdb/db/pipe/task/subtask/PipeSubtask.java | 34 +- .../query/reader/chunk/DiskAlignedChunkLoader.java | 10 +- .../db/query/reader/chunk/DiskChunkLoader.java | 10 +- .../query/reader/chunk/MemAlignedChunkLoader.java | 10 +- .../db/query/reader/chunk/MemChunkLoader.java | 10 +- .../metadata/DiskAlignedChunkMetadataLoader.java | 11 +- .../chunk/metadata/DiskChunkMetadataLoader.java | 11 +- .../metadata/MemAlignedChunkMetadataLoader.java | 10 +- .../chunk/metadata/MemChunkMetadataLoader.java | 9 +- .../service/DataNodeInternalRPCServiceMetrics.java | 8 +- .../apache/iotdb/db/service/RPCServiceMetrics.java | 8 +- .../iotdb/db/service/metrics/CacheMetrics.java | 144 ++ .../db/service/metrics/CompactionMetrics.java | 396 ++++- .../db/service/metrics/DataNodeMetricsHelper.java | 22 +- .../iotdb/db/service/metrics/SystemMetrics.java | 43 +- .../iotdb/db/service/metrics/WritingMetrics.java | 574 +++++++- .../metrics/recorder/CacheMetricsRecorder.java | 57 - .../metrics/recorder/CompactionMetricsManager.java | 240 --- .../metrics/recorder/WritingMetricsManager.java | 351 ----- .../InternalServiceThriftHandlerMetrics.java | 8 +- .../handler/RPCServiceThriftHandlerMetrics.java | 8 +- .../impl/DataNodeInternalRPCServiceImpl.java | 95 ++ .../apache/iotdb/db/tools/IoTDBDataDirViewer.java | 41 +- .../org/apache/iotdb/db/utils/FileLoaderUtils.java | 17 +- .../java/org/apache/iotdb/db/wal/WALManager.java | 6 +- .../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 12 +- .../iotdb/db/wal/checkpoint/CheckpointManager.java | 4 +- .../iotdb/db/wal/checkpoint/CheckpointType.java | 16 +- .../java/org/apache/iotdb/db/wal/node/WALNode.java | 8 +- .../{WALPipeHandler.java => WALEntryHandler.java} | 22 +- .../db/wal/utils/listener/WALFlushListener.java | 10 +- .../engine/compaction/AbstractCompactionTest.java | 53 +- .../ReadPointCompactionPerformerTest.java | 16 +- .../cross/CrossSpaceCompactionExceptionTest.java | 5 +- .../RewriteCrossSpaceCompactionRecoverTest.java | 10 +- ...eCrossSpaceCompactionWithFastPerformerTest.java | 21 +- ...sSpaceCompactionWithReadPointPerformerTest.java | 21 +- .../SizeTieredCompactionRecoverTest.java | 52 +- .../SizeTieredCompactionSelectorTest.java | 38 + .../utils/MultiTsFileDeviceIteratorTest.java | 14 + .../storagegroup/TsFileResourceListTest.java | 110 ++ .../TsFileResourceProgressIndexTest.java | 173 +++ .../execution/exchange/LocalSinkChannelTest.java | 14 +- .../execution/exchange/LocalSourceHandleTest.java | 14 +- .../execution/exchange/SharedTsBlockQueueTest.java | 7 +- .../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 2 +- .../mpp/plan/analyze/ExpressionAnalyzerTest.java | 4 +- .../collector/CachedSchemaPatternMatcherTest.java | 14 +- .../core/collector/PipeRealtimeCollectTest.java | 38 +- .../pipe/resource/PipeFileResourceManagerTest.java | 1 + ...peHandlerTest.java => WALEntryHandlerTest.java} | 18 +- .../iotdb/db/wal/utils/WALInsertNodeCacheTest.java | 8 +- .../apache/iotdb/tsfile/compress/ICompressor.java | 3 + .../iotdb/tsfile/compress/auto/AutoCompressor.java | 121 ++ .../tsfile/compress/auto/AutoUncompressor.java | 75 + .../tsfile/compress/auto/CompressionSampler.java | 251 ++++ .../file/metadata/enums/CompressionType.java | 5 +- 1038 files changed, 12143 insertions(+), 37582 deletions(-) diff --cc client-go index 84b8d45829d,00000000000..dc64b1a7648 mode 160000,000000..160000 --- a/client-go +++ b/client-go @@@ -1,1 -1,0 +1,1 @@@ - Subproject commit 84b8d45829d846440a3246400e7bc5e39587dcb5 ++Subproject commit dc64b1a7648d3c505c10eed5419f422bb49f1def diff --cc iotdb-protocol/thrift-raft/pom.xml index 10548f21cd3,00000000000..b1fa8193192 mode 100644,000000..100644 --- a/iotdb-protocol/thrift-raft/pom.xml +++ b/iotdb-protocol/thrift-raft/pom.xml @@@ -1,67 -1,0 +1,67 @@@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-parent</artifactId> + <version>1.2.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> ++ <relativePath>../../pom.xml</relativePath> + </parent> + <artifactId>iotdb-thrift-raft-consensus</artifactId> + <name>rpc-thrift-raft-consensus</name> + <description>Rpc modules for raft consensus algorithm</description> + <dependencies> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>libthrift</artifactId> + </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-thrift-commons</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>3.2.0</version> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>${project.build.directory}/generated-sources/thrift</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --cc iotdb-protocol/thrift-raft/src/main/thrift/raft.thrift index c3500cf6b4a,00000000000..c3500cf6b4a mode 100644,000000..100644 --- a/iotdb-protocol/thrift-raft/src/main/thrift/raft.thrift +++ b/iotdb-protocol/thrift-raft/src/main/thrift/raft.thrift diff --cc pom.xml index 1319282d50d,17c23f639fe..fa4405baade --- a/pom.xml +++ b/pom.xml @@@ -81,54 -81,47 +81,48 @@@ <modules> <module>tsfile</module> <module>antlr</module> - <module>thrift</module> - <module>thrift-commons</module> - <module>thrift-confignode</module> - <module>thrift-iot-consensus</module> - <module>thrift-influxdb</module> - <module>thrift-raft</module> - <module>thrift-mlnode</module> - <module>service-rpc</module> - <module>jdbc</module> - <module>influxdb-protocol</module> - <module>session</module> - <module>cli</module> - <module>openapi</module> + <module>iotdb-protocol/thrift</module> + <module>iotdb-protocol/thrift-commons</module> + <module>iotdb-protocol/thrift-confignode</module> + <module>iotdb-protocol/thrift-iot-consensus</module> + <module>iotdb-protocol/thrift-influxdb</module> ++ <module>iotdb-protocol/thrift-raft</module> + <module>iotdb-protocol/thrift-mlnode</module> + <module>iotdb-client/service-rpc</module> + <module>iotdb-client/jdbc</module> + <module>iotdb-client/session</module> + <module>iotdb-client/cli</module> + <module>iotdb-protocol/openapi</module> <module>server</module> <module>example</module> - <module>grafana-plugin</module> - <module>grafana-connector</module> - <module>spark-tsfile</module> - <module>hadoop</module> - <module>spark-iotdb-connector/scala_2.11</module> - <module>spark-iotdb-connector/scala_2.12</module> - <module>flink-tsfile-connector</module> - <module>flink-iotdb-connector</module> + <module>iotdb-connector/grafana-plugin</module> + <module>iotdb-connector/grafana-connector</module> + <module>iotdb-connector/spark-tsfile</module> + <module>iotdb-connector/hadoop</module> + <module>iotdb-connector/spark-iotdb-connector/scala_2.11</module> + <module>iotdb-connector/spark-iotdb-connector/scala_2.12</module> + <module>iotdb-connector/flink-tsfile-connector</module> + <module>iotdb-connector/flink-iotdb-connector</module> <module>distribution</module> - <module>hive-connector</module> + <module>iotdb-connector/hive-connector</module> <module>node-commons</module> <module>confignode</module> - <module>zeppelin-interpreter</module> - <module>client-py</module> - <module>compile-tools</module> - <module>client-cpp</module> + <module>iotdb-connector/zeppelin-interpreter</module> + <module>iotdb-client/client-py</module> + <module>iotdb-client/compile-tools</module> + <module>iotdb-client/client-cpp</module> <module>metrics</module> - <!-- Will remove integration at 2023.1.23--> - <module>integration</module> <module>integration-test</module> <module>consensus</module> - <module>external-pipe-api</module> + <module>iotdb-api/external-pipe-api</module> <module>library-udf</module> - <module>schema-engine-rocksdb</module> - <module>udf-api</module> - <module>trigger-api</module> - <module>rewrite-tsfile-tool</module> - <module>external-api</module> - <module>schema-engine-tag</module> - <module>isession</module> + <module>iotdb-api/udf-api</module> + <module>iotdb-api/trigger-api</module> + <module>iotdb-api/external-api</module> + <module>iotdb-client/isession</module> <module>mlnode</module> - <module>pipe-api</module> - <module>subscription-api</module> + <module>iotdb-api/pipe-api</module> + <module>iotdb-api/subscription-api</module> </modules> <!-- Properties Management --> <properties> diff --cc server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java index d16ae562d46,2a81dbe26b3..b51fa9adf32 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java @@@ -65,15 -58,17 +64,15 @@@ public class MemTableFlushTask private static final Logger LOGGER = LoggerFactory.getLogger(MemTableFlushTask.class); private static final FlushSubTaskPoolManager SUB_TASK_POOL_MANAGER = FlushSubTaskPoolManager.getInstance(); - private static final WritingMetricsManager WRITING_METRICS = WritingMetricsManager.getInstance(); + private static final WritingMetrics WRITING_METRICS = WritingMetrics.getInstance(); private static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private final Future<?> encodingTaskFuture; - private final Future<?> ioTaskFuture; - private RestorableTsFileIOWriter writer; + private final DynamicThreadGroup sortTasks; + private final DynamicThreadGroup encodingTasks; + private final DynamicThreadGroup ioTask; - private final LinkedBlockingQueue<Object> encodingTaskQueue = new LinkedBlockingQueue<>(); - private final LinkedBlockingQueue<Object> ioTaskQueue = - (config.isEnableMemControl() && SystemInfo.getInstance().isEncodingFasterThanIo()) - ? new LinkedBlockingQueue<>(config.getIoTaskQueueSizeForFlushing()) - : new LinkedBlockingQueue<>(); + private final LinkedBlockingQueue<Task> sortTaskQueue = new LinkedBlockingQueue<>(); + private final LinkedBlockingQueue<Task> encodingTaskQueue = new LinkedBlockingQueue<>(); + private final LinkedBlockingQueue<Task> ioTaskQueue = new LinkedBlockingQueue<>(); private String storageGroup; private String dataRegionId; diff --cc server/src/main/java/org/apache/iotdb/db/engine/flush/tasks/FlushContext.java index 87523c8bc33,00000000000..5a7c973e18b mode 100644,000000..100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/flush/tasks/FlushContext.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/tasks/FlushContext.java @@@ -1,90 -1,0 +1,90 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.engine.flush.tasks; + +import org.apache.iotdb.db.engine.memtable.IMemTable; - import org.apache.iotdb.db.service.metrics.recorder.WritingMetricsManager; ++import org.apache.iotdb.db.service.metrics.WritingMetrics; +import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class FlushContext { - private final WritingMetricsManager WRITING_METRICS = WritingMetricsManager.getInstance(); ++ private final WritingMetrics WRITING_METRICS = WritingMetrics.getInstance(); + private List<FlushDeviceContext> deviceContexts; + private AtomicInteger cursor = new AtomicInteger(); + private RestorableTsFileIOWriter writer; + private AtomicLong sortTime = new AtomicLong(); + private AtomicLong encodingTime = new AtomicLong(); + private AtomicLong ioTime = new AtomicLong(); + private IMemTable memTable; + + public List<FlushDeviceContext> getDeviceContexts() { + return deviceContexts; + } + + public void setDeviceContexts(List<FlushDeviceContext> deviceContexts) { + this.deviceContexts = deviceContexts; + } + + public int getCursor() { + return cursor.get(); + } + + public void setCursor(int cursor) { + this.cursor.set(cursor); + } + - public WritingMetricsManager getWritingMetrics() { ++ public WritingMetrics getWritingMetrics() { + return WRITING_METRICS; + } + + public AtomicLong getSortTime() { + return sortTime; + } + + public AtomicLong getEncodingTime() { + return encodingTime; + } + + public AtomicLong getIoTime() { + return ioTime; + } + + public RestorableTsFileIOWriter getWriter() { + return writer; + } + + public void setWriter(RestorableTsFileIOWriter writer) { + this.writer = writer; + } + + public IMemTable getMemTable() { + return memTable; + } + + public void setMemTable(IMemTable memTable) { + this.memTable = memTable; + } + + public boolean allFlushed() { + return cursor.get() == deviceContexts.size(); + } +} diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java index 1a143afa51f,168a288ca6a..7bc192cf02b --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java @@@ -28,8 -26,7 +28,8 @@@ import org.apache.iotdb.commons.client. import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.consensus.ConsensusGroupId; - import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; + import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; +import org.apache.iotdb.commons.utils.ThriftUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; import org.apache.iotdb.db.mpp.common.MPPQueryContext; diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/AutoCompressor.java index 00000000000,a349aaa098a..ec6e099632c mode 000000,100644..100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/AutoCompressor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/AutoCompressor.java @@@ -1,0 -1,120 +1,121 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.iotdb.tsfile.compress.auto; + ++import org.apache.iotdb.tsfile.compress.ICompressor; ++import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; ++ + import java.io.IOException; + import java.nio.ByteBuffer; + import java.util.ArrayList; + import java.util.List; -import org.apache.iotdb.tsfile.compress.ICompressor; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; + + public class AutoCompressor implements ICompressor { + + private CompressionSampler sampler; + + public AutoCompressor() { + List<CompressionType> compressionTypes = collectCompressionTypes(); + double alpha = 1.0; + long minSampleIntervalMS = 1000; + sampler = new CompressionSampler(compressionTypes, alpha, minSampleIntervalMS); + } + + public AutoCompressor(double alpha, long minSampleIntervalMS) { + List<CompressionType> compressionTypes = collectCompressionTypes(); + sampler = new CompressionSampler(compressionTypes, alpha, minSampleIntervalMS); + } + + private static List<CompressionType> collectCompressionTypes() { - List<CompressionType> compressionTypeList = new ArrayList<>( - CompressionType.values().length - 1); ++ List<CompressionType> compressionTypeList = ++ new ArrayList<>(CompressionType.values().length - 1); + for (CompressionType type : CompressionType.values()) { + if (!type.equals(CompressionType.AUTO) && !type.equals(CompressionType.UNCOMPRESSED)) { + compressionTypeList.add(type); + } + } + return compressionTypeList; + } + + @Override + public byte[] compress(byte[] data) throws IOException { + if (sampler.shouldSample()) { + return sampler.sample(data); + } + ICompressor preferredSampler = sampler.getPreferredSampler(); + byte[] compress = preferredSampler.compress(data); + byte[] result = new byte[compress.length + 1]; + System.arraycopy(compress, 0, result, 0, compress.length); + // the last byte is for the real compression type + result[compress.length] = preferredSampler.getType().serialize(); + return result; + } + + @Override + public byte[] compress(byte[] data, int offset, int length) throws IOException { + if (sampler.shouldSample()) { + return sampler.sample(data, offset, length); + } + ICompressor preferredSampler = sampler.getPreferredSampler(); + byte[] compress = preferredSampler.compress(data, offset, length); + byte[] result = new byte[compress.length + 1]; + System.arraycopy(compress, 0, result, 0, compress.length); + // the last byte is for the real compression type + result[compress.length] = preferredSampler.getType().serialize(); + return result; + } + + @Override + public int compress(byte[] data, int offset, int length, byte[] compressed) throws IOException { + if (sampler.shouldSample()) { + return sampler.sample(data, offset, length, compressed); + } + ICompressor preferredSampler = sampler.getPreferredSampler(); + int compressedLength = preferredSampler.compress(data, offset, length, compressed); + // the last byte is for the real compression type + compressed[compressedLength] = preferredSampler.getType().serialize(); + return compressedLength + 1; + } + + @Override + public int compress(ByteBuffer data, ByteBuffer compressed) throws IOException { + if (sampler.shouldSample()) { + return sampler.sample(data, compressed); + } + ICompressor preferredSampler = sampler.getPreferredSampler(); + int compressedLength = preferredSampler.compress(data, compressed); + // the last byte is for the real compression type + compressed.mark(); + compressed.position(compressed.position() + compressedLength); + compressed.put(preferredSampler.getType().serialize()); + compressed.reset(); + return compressedLength + 1; + } + + @Override + public int getMaxBytesForCompression(int uncompressedDataSize) { + return sampler.getMaxBytesForCompression(uncompressedDataSize); + } + + @Override + public CompressionType getType() { + return CompressionType.AUTO; + } + } diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/AutoUncompressor.java index 00000000000,037ffedabd7..ef48bc34971 mode 000000,100644..100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/AutoUncompressor.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/AutoUncompressor.java @@@ -1,0 -1,74 +1,75 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.iotdb.tsfile.compress.auto; + -import java.io.IOException; -import java.nio.ByteBuffer; + import org.apache.iotdb.tsfile.compress.IUnCompressor; + import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; + ++import java.io.IOException; ++import java.nio.ByteBuffer; ++ + public class AutoUncompressor implements IUnCompressor { + + @Override + public int getUncompressedLength(byte[] array, int offset, int length) throws IOException { + byte realType = array[offset + length - 1]; - IUnCompressor unCompressor = IUnCompressor.getUnCompressor( - CompressionType.deserialize(realType)); ++ IUnCompressor unCompressor = ++ IUnCompressor.getUnCompressor(CompressionType.deserialize(realType)); + return unCompressor.getUncompressedLength(array, offset, length); + } + + @Override + public int getUncompressedLength(ByteBuffer buffer) throws IOException { + byte realType = buffer.array()[buffer.position() + buffer.remaining() - 1]; - IUnCompressor unCompressor = IUnCompressor.getUnCompressor( - CompressionType.deserialize(realType)); ++ IUnCompressor unCompressor = ++ IUnCompressor.getUnCompressor(CompressionType.deserialize(realType)); + return unCompressor.getUncompressedLength(buffer); + } + + @Override + public byte[] uncompress(byte[] byteArray) throws IOException { + byte realType = byteArray[byteArray.length - 1]; - IUnCompressor unCompressor = IUnCompressor.getUnCompressor( - CompressionType.deserialize(realType)); ++ IUnCompressor unCompressor = ++ IUnCompressor.getUnCompressor(CompressionType.deserialize(realType)); + return unCompressor.uncompress(byteArray); + } + + @Override + public int uncompress(byte[] byteArray, int offset, int length, byte[] output, int outOffset) + throws IOException { + byte realType = byteArray[offset + length - 1]; - IUnCompressor unCompressor = IUnCompressor.getUnCompressor( - CompressionType.deserialize(realType)); ++ IUnCompressor unCompressor = ++ IUnCompressor.getUnCompressor(CompressionType.deserialize(realType)); + return unCompressor.uncompress(byteArray, offset, length, output, outOffset); + } + + @Override + public int uncompress(ByteBuffer compressed, ByteBuffer uncompressed) throws IOException { + byte realType = compressed.array()[compressed.position() + compressed.remaining() - 1]; - IUnCompressor unCompressor = IUnCompressor.getUnCompressor( - CompressionType.deserialize(realType)); ++ IUnCompressor unCompressor = ++ IUnCompressor.getUnCompressor(CompressionType.deserialize(realType)); + return unCompressor.uncompress(compressed, uncompressed); + } + + @Override + public CompressionType getCodecName() { + return null; + } + } diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/CompressionSampler.java index 00000000000,1bdd6505306..a05cd51641d mode 000000,100644..100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/CompressionSampler.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/compress/auto/CompressionSampler.java @@@ -1,0 -1,250 +1,251 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + package org.apache.iotdb.tsfile.compress.auto; + ++import org.apache.iotdb.tsfile.compress.ICompressor; ++import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; ++ ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; ++ + import java.io.IOException; + import java.nio.ByteBuffer; + import java.util.ArrayDeque; + import java.util.ArrayList; + import java.util.List; + import java.util.Queue; -import org.apache.iotdb.tsfile.compress.ICompressor; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; + + public class CompressionSampler { + + private static final Logger logger = LoggerFactory.getLogger(CompressionSampler.class); + + private List<CompressionType> compressionTypes; + private long minSampleInterval; + private long lastSampleTimeMS; + private List<ICompressor> compressors; + private List<CompressionMonitor> monitors; + private int preferredCompressorIndex; + - public CompressionSampler(List<CompressionType> compressionTypes, double alpha, - long minSampleInterval) { ++ public CompressionSampler( ++ List<CompressionType> compressionTypes, double alpha, long minSampleInterval) { + this.compressionTypes = compressionTypes; + this.minSampleInterval = minSampleInterval; + this.monitors = new ArrayList<>(compressionTypes.size()); + this.compressors = new ArrayList<>(compressionTypes.size()); + + int maxSampleNum = 10; + + for (CompressionType compressionType : compressionTypes) { + monitors.add(new CompressionMonitor(maxSampleNum, alpha)); + compressors.add(ICompressor.getCompressor(compressionType)); + } + } + + public boolean shouldSample() { + return System.currentTimeMillis() - lastSampleTimeMS >= minSampleInterval; + } + + public ICompressor getPreferredSampler() { + return compressors.get(preferredCompressorIndex); + } + + public byte[] sample(byte[] data) throws IOException { + return sample(data, 0, data.length); + } + + public byte[] sample(byte[] data, int offset, int length) throws IOException { + CompressionType bestType = CompressionType.UNCOMPRESSED; + int smallestLength = length; + byte[] bestResult = data; + + for (int i = 0; i < compressionTypes.size(); i++) { + ICompressor compressor = compressors.get(i); + CompressionMonitor monitor = monitors.get(i); + long startTime = System.currentTimeMillis(); + byte[] compressed = compressor.compress(data, offset, length); + int bytesBeforeCompression = data.length; + int bytesAfterCompression = compressed.length; + long timeConsumption = System.currentTimeMillis() - startTime; + monitor.addSample(bytesBeforeCompression, bytesAfterCompression, timeConsumption); + + if (bytesAfterCompression < smallestLength) { + smallestLength = bytesAfterCompression; + bestType = compressionTypes.get(i); + bestResult = compressed; + } + } + + lastSampleTimeMS = System.currentTimeMillis(); + updatePreferredIndex(); + + // the last byte is for the real compression type + byte[] result = new byte[bestResult.length + 1]; + System.arraycopy(bestResult, 0, result, 0, bestResult.length); + result[bestResult.length] = bestType.serialize(); + return result; + } + + public int sample(byte[] data, int offset, int length, byte[] compressed) throws IOException { + CompressionType bestType = CompressionType.UNCOMPRESSED; + int smallestLength = length; + + for (int i = 0; i < compressionTypes.size(); i++) { + ICompressor compressor = compressors.get(i); + CompressionMonitor monitor = monitors.get(i); + long startTime = System.currentTimeMillis(); + int bytesAfterCompression = compressor.compress(data, offset, length, compressed); + int bytesBeforeCompression = data.length; + long timeConsumption = System.currentTimeMillis() - startTime; + monitor.addSample(bytesBeforeCompression, bytesAfterCompression, timeConsumption); + + if (bytesAfterCompression < smallestLength) { + smallestLength = bytesAfterCompression; + bestType = compressionTypes.get(i); + } + } + + lastSampleTimeMS = System.currentTimeMillis(); + updatePreferredIndex(); + + // the last byte is for the real compression type + compressed[smallestLength] = bestType.serialize(); + return smallestLength + 1; + } + + public int sample(ByteBuffer data, ByteBuffer compressed) throws IOException { + CompressionType bestType = CompressionType.UNCOMPRESSED; + int smallestLength = data.remaining(); + + for (int i = 0; i < compressionTypes.size(); i++) { + ICompressor compressor = compressors.get(i); + CompressionMonitor monitor = monitors.get(i); + long startTime = System.currentTimeMillis(); + int bytesAfterCompression = compressor.compress(data, compressed); + int bytesBeforeCompression = data.remaining(); + long timeConsumption = System.currentTimeMillis() - startTime; + monitor.addSample(bytesBeforeCompression, bytesAfterCompression, timeConsumption); + + if (bytesAfterCompression < smallestLength) { + smallestLength = bytesAfterCompression; + bestType = compressionTypes.get(i); + } + } + + lastSampleTimeMS = System.currentTimeMillis(); + updatePreferredIndex(); + + // the last byte is for the real compression type + compressed.mark(); + compressed.position(compressed.position() + smallestLength); + compressed.put(bestType.serialize()); + compressed.reset(); + return smallestLength + 1; + } + + public int getMaxBytesForCompression(int uncompressedDataSize) { + int maxBytes = 0; + for (ICompressor compressor : compressors) { + maxBytes = Math.max(maxBytes, compressor.getMaxBytesForCompression(uncompressedDataSize)); + } + // the last byte is for the real compression type + return maxBytes + 1; + } + - + private void updatePreferredIndex() { + double bestScore = 0; + int prevIndex = preferredCompressorIndex; + for (int i = 0; i < monitors.size(); i++) { + double score = monitors.get(i).score(); + if (score > bestScore) { + preferredCompressorIndex = i; + } + } + if (prevIndex != preferredCompressorIndex) { + logger.info("Preferred compressor changed to {}", compressors.get(preferredCompressorIndex)); + } + } + + private static class CompressionSample { + + private long bytesBeforeCompression; + private long bytesAfterCompression; + private long timeConsumptionNS; + - public CompressionSample(long bytesBeforeCompression, long bytesAfterCompression, - long timeConsumptionNS) { ++ public CompressionSample( ++ long bytesBeforeCompression, long bytesAfterCompression, long timeConsumptionNS) { + this.bytesBeforeCompression = bytesBeforeCompression; + this.bytesAfterCompression = bytesAfterCompression; + this.timeConsumptionNS = timeConsumptionNS; + } + } + + private static class CompressionMonitor { + + private Queue<CompressionSample> samples; + private int maxSampleNum; + private double alpha; + private long bytesBeforeCompressionSum; + private long bytesAfterCompressionSum; + private long timeConsumptionSumNS; + + private CompressionMonitor(int maxSampleNum, double alpha) { + this.maxSampleNum = maxSampleNum; + this.samples = new ArrayDeque<>(maxSampleNum); + this.alpha = alpha; + } + + private double compressionRatio() { + return bytesAfterCompressionSum * 1.0 / bytesBeforeCompressionSum; + } + + private double throughput() { + return bytesBeforeCompressionSum * 1.0 / timeConsumptionSumNS; + } + + private double score() { + return Math.pow(throughput(), alpha) / compressionRatio(); + } + - private void addSample(long bytesBeforeCompression, long bytesAfterCompression, - long timeConsumptionNS) { - CompressionSample sample = new CompressionSample(bytesBeforeCompression, - bytesAfterCompression, timeConsumptionNS); ++ private void addSample( ++ long bytesBeforeCompression, long bytesAfterCompression, long timeConsumptionNS) { ++ CompressionSample sample = ++ new CompressionSample(bytesBeforeCompression, bytesAfterCompression, timeConsumptionNS); + if (samples.size() < maxSampleNum) { + addSample(sample); + } else { + removeSample(); + } + } + + private void addSample(CompressionSample sample) { + bytesAfterCompressionSum += sample.bytesAfterCompression; + bytesBeforeCompressionSum += sample.bytesBeforeCompression; + timeConsumptionSumNS += sample.timeConsumptionNS; + samples.add(sample); + } + + private void removeSample() { + CompressionSample sample = samples.remove(); + bytesBeforeCompressionSum -= sample.bytesBeforeCompression; + bytesAfterCompressionSum -= sample.bytesAfterCompression; + timeConsumptionSumNS -= sample.timeConsumptionNS; + } + } + }
