This is an automated email from the ASF dual-hosted git repository. zyk pushed a commit to branch rc/1.1.0 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8877512e66b09d62a9ec5dbe3a0d97eefe1ff7f5 Author: ZhangHongYin <[email protected]> AuthorDate: Wed Mar 22 12:47:10 2023 +0800 [IOTDB-5701] Optimize the management of performance overview related metrics (#9381) (#9396) --- .../consensus/iot/IoTConsensusServerImpl.java | 11 +- .../ratis/ApplicationStateMachineProxy.java | 18 +- .../iotdb/consensus/simple/SimpleConsensus.java | 16 +- .../metric/enums/PerformanceOverviewMetrics.java | 256 +++++++++++++++++++-- .../org/apache/iotdb/db/auth/AuthorityChecker.java | 7 +- .../iotdb/db/engine/storagegroup/DataRegion.java | 16 +- .../db/engine/storagegroup/TsFileProcessor.java | 25 +- .../execution/executor/RegionWriteExecutor.java | 12 +- .../metric/PerformanceOverviewMetricsManager.java | 214 ----------------- .../db/mpp/plan/execution/QueryExecution.java | 11 +- .../db/mpp/plan/parser/StatementGenerator.java | 54 ++--- .../plan/scheduler/AsyncSendPlanNodeHandler.java | 8 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 6 +- .../db/service/metrics/DataNodeMetricsHelper.java | 4 + 14 files changed, 326 insertions(+), 332 deletions(-) diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index 3aadc12f57..201322fa48 100644 --- a/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -94,6 +94,8 @@ public class IoTConsensusServerImpl { private static final String CONFIGURATION_TMP_FILE_NAME = "configuration.dat.tmp"; public static final String SNAPSHOT_DIR_NAME = "snapshot"; private static final Pattern SNAPSHOT_INDEX_PATTEN = Pattern.compile(".*[^\\d](?=(\\d+))"); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); private final Logger logger = LoggerFactory.getLogger(IoTConsensusServerImpl.class); private final Peer thisNode; private final IStateMachine stateMachine; @@ -224,14 +226,7 @@ public class IoTConsensusServerImpl { IConsensusRequest planNode = stateMachine.deserializeRequest(indexedConsensusRequest); long startWriteTime = System.nanoTime(); TSStatus result = stateMachine.write(planNode); - MetricService.getInstance() - .timer( - System.nanoTime() - startWriteTime, - TimeUnit.NANOSECONDS, - Metric.PERFORMANCE_OVERVIEW_STORAGE_DETAIL.toString(), - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.ENGINE); + PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(System.nanoTime() - startWriteTime); long writeToStateMachineEndTime = System.nanoTime(); // statistic the time of writing request into stateMachine diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java index aacb0743be..80f11ac05e 100644 --- a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java +++ b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java @@ -20,16 +20,12 @@ package org.apache.iotdb.consensus.ratis; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.service.metric.MetricService; -import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; -import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.consensus.IStateMachine; import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest; import org.apache.iotdb.consensus.common.request.IConsensusRequest; import org.apache.iotdb.consensus.ratis.metrics.RatisMetricsManager; -import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.ratis.proto.RaftProtos; import org.apache.ratis.proto.RaftProtos.RaftConfigurationProto; @@ -58,7 +54,9 @@ import java.util.concurrent.TimeUnit; public class ApplicationStateMachineProxy extends BaseStateMachine { - private final Logger logger = LoggerFactory.getLogger(ApplicationStateMachineProxy.class); + private static final Logger logger = LoggerFactory.getLogger(ApplicationStateMachineProxy.class); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); private final IStateMachine applicationStateMachine; private final IStateMachine.RetryPolicy retryPolicy; private final SnapshotStorage snapshotStorage; @@ -177,14 +175,8 @@ public class ApplicationStateMachineProxy extends BaseStateMachine { if (isLeader) { // only record time cost for data region in Performance Overview Dashboard if (consensusGroupType == TConsensusGroupType.DataRegion) { - MetricService.getInstance() - .timer( - System.nanoTime() - writeToStateMachineStartTime, - TimeUnit.NANOSECONDS, - Metric.PERFORMANCE_OVERVIEW_STORAGE_DETAIL.toString(), - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.ENGINE); + PERFORMANCE_OVERVIEW_METRICS.recordEngineCost( + System.nanoTime() - writeToStateMachineStartTime); } // statistic the time of write stateMachine RatisMetricsManager.getInstance() diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/simple/SimpleConsensus.java b/consensus/src/main/java/org/apache/iotdb/consensus/simple/SimpleConsensus.java index 27a1687e90..514b75c372 100644 --- a/consensus/src/main/java/org/apache/iotdb/consensus/simple/SimpleConsensus.java +++ b/consensus/src/main/java/org/apache/iotdb/consensus/simple/SimpleConsensus.java @@ -23,10 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; -import org.apache.iotdb.commons.service.metric.MetricService; -import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; -import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.consensus.IConsensus; import org.apache.iotdb.consensus.IStateMachine; @@ -41,7 +38,6 @@ import org.apache.iotdb.consensus.exception.ConsensusGroupAlreadyExistException; import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException; import org.apache.iotdb.consensus.exception.IllegalPeerEndpointException; import org.apache.iotdb.consensus.exception.IllegalPeerNumException; -import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -56,7 +52,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -73,6 +68,8 @@ class SimpleConsensus implements IConsensus { private final File storageDir; private final IStateMachine.Registry registry; private final Map<ConsensusGroupId, SimpleServerImpl> stateMachineMap = new ConcurrentHashMap<>(); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); public SimpleConsensus(ConsensusConfig config, Registry registry) { this.thisNode = config.getThisNodeEndPoint(); @@ -132,14 +129,7 @@ class SimpleConsensus implements IConsensus { long startWriteTime = System.nanoTime(); status = impl.write(request); // only record time cost for data region in Performance Overview Dashboard - MetricService.getInstance() - .timer( - System.nanoTime() - startWriteTime, - TimeUnit.NANOSECONDS, - Metric.PERFORMANCE_OVERVIEW_STORAGE_DETAIL.toString(), - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.ENGINE); + PERFORMANCE_OVERVIEW_METRICS.recordEngineCost(System.nanoTime() - startWriteTime); } else { status = impl.write(request); } diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/PerformanceOverviewMetrics.java b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/PerformanceOverviewMetrics.java index 9b59cefc7d..e82e5d7106 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/PerformanceOverviewMetrics.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/PerformanceOverviewMetrics.java @@ -20,7 +20,9 @@ package org.apache.iotdb.commons.service.metric.enums; import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.impl.DoNothingMetricManager; import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.type.Timer; import org.apache.iotdb.metrics.utils.MetricInfo; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.MetricType; @@ -30,13 +32,20 @@ import java.util.Map; public class PerformanceOverviewMetrics implements IMetricSet { private static final Map<String, MetricInfo> metricInfoMap = new HashMap<>(); + + private PerformanceOverviewMetrics() { + // empty constructor + } + + // region overview + private static final String PERFORMANCE_OVERVIEW_DETAIL = Metric.PERFORMANCE_OVERVIEW_DETAIL.toString(); - public static final String AUTHORITY = "authority"; - public static final String PARSER = "parser"; - public static final String ANALYZER = "analyzer"; - public static final String PLANNER = "planner"; - public static final String SCHEDULER = "scheduler"; + private static final String AUTHORITY = "authority"; + private static final String PARSER = "parser"; + private static final String ANALYZER = "analyzer"; + private static final String PLANNER = "planner"; + private static final String SCHEDULER = "scheduler"; static { metricInfoMap.put( @@ -61,10 +70,45 @@ public class PerformanceOverviewMetrics implements IMetricSet { MetricType.TIMER, PERFORMANCE_OVERVIEW_DETAIL, Tag.STAGE.toString(), SCHEDULER)); } + private Timer authTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer parseTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer analyzeTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer planTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer scheduleTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + /** Record the time cost in authority stage. */ + public void recordAuthCost(long costTimeInNanos) { + authTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost in parse stage. */ + public void recordParseCost(long costTimeInNanos) { + parseTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost in analyze stage. */ + public void recordAnalyzeCost(long costTimeInNanos) { + analyzeTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost in plan stage. */ + public void recordPlanCost(long costTimeInNanos) { + planTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost in schedule stage. */ + public void recordScheduleCost(long costTimeInNanos) { + scheduleTimer.updateNanos(costTimeInNanos); + } + + // endregion + + // region schedule + private static final String PERFORMANCE_OVERVIEW_SCHEDULE_DETAIL = Metric.PERFORMANCE_OVERVIEW_SCHEDULE_DETAIL.toString(); - public static final String LOCAL_SCHEDULE = "local_scheduler"; - public static final String REMOTE_SCHEDULE = "remote_scheduler"; + private static final String LOCAL_SCHEDULE = "local_scheduler"; + private static final String REMOTE_SCHEDULE = "remote_scheduler"; static { metricInfoMap.put( @@ -83,11 +127,27 @@ public class PerformanceOverviewMetrics implements IMetricSet { REMOTE_SCHEDULE)); } + private Timer localScheduleTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer remoteScheduleTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + /** Record the time cost of local schedule. */ + public void recordScheduleLocalCost(long costTimeInNanos) { + localScheduleTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost of remote schedule. */ + public void recordScheduleRemoteCost(long costTimeInNanos) { + remoteScheduleTimer.updateNanos(costTimeInNanos); + } + + // endregion + + // region local schedule private static final String PERFORMANCE_OVERVIEW_LOCAL_DETAIL = Metric.PERFORMANCE_OVERVIEW_LOCAL_DETAIL.toString(); - public static final String SCHEMA_VALIDATE = "schema_validate"; - public static final String TRIGGER = "trigger"; - public static final String STORAGE = "storage"; + private static final String SCHEMA_VALIDATE = "schema_validate"; + private static final String TRIGGER = "trigger"; + private static final String STORAGE = "storage"; static { metricInfoMap.put( @@ -107,9 +167,31 @@ public class PerformanceOverviewMetrics implements IMetricSet { MetricType.TIMER, PERFORMANCE_OVERVIEW_LOCAL_DETAIL, Tag.STAGE.toString(), STORAGE)); } + private Timer schemaValidateTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer triggerTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer storageTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + /** Record the time cost of schema validate stage in local schedule. */ + public void recordScheduleSchemaValidateCost(long costTimeInNanos) { + schemaValidateTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost of trigger stage in local schedule. */ + public void recordScheduleTriggerCost(long costTimeInNanos) { + triggerTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost of storage stage in local schedule. */ + public void recordScheduleStorageCost(long costTimeInNanos) { + storageTimer.updateNanos(costTimeInNanos); + } + + // endregion + + // region storage private static final String PERFORMANCE_OVERVIEW_STORAGE_DETAIL = Metric.PERFORMANCE_OVERVIEW_STORAGE_DETAIL.toString(); - public static final String ENGINE = "engine"; + private static final String ENGINE = "engine"; static { metricInfoMap.put( @@ -118,14 +200,24 @@ public class PerformanceOverviewMetrics implements IMetricSet { MetricType.TIMER, PERFORMANCE_OVERVIEW_STORAGE_DETAIL, Tag.STAGE.toString(), ENGINE)); } + private Timer engineTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + public void recordEngineCost(long costTimeInNanos) { + engineTimer.updateNanos(costTimeInNanos); + } + + // endregion + + // region engine + private static final String PERFORMANCE_OVERVIEW_ENGINE_DETAIL = Metric.PERFORMANCE_OVERVIEW_ENGINE_DETAIL.toString(); - public static final String LOCK = "lock"; - public static final String MEMORY_BLOCK = "memory_block"; - public static final String CREATE_MEMTABLE_BLOCK = "create_memtable_block"; - public static final String WAL = "wal"; - public static final String MEMTABLE = "memtable"; - public static final String LAST_CACHE = "last_cache"; + private static final String LOCK = "lock"; + private static final String MEMORY_BLOCK = "memory_block"; + private static final String CREATE_MEMTABLE_BLOCK = "create_memtable_block"; + private static final String WAL = "wal"; + private static final String MEMTABLE = "memtable"; + private static final String LAST_CACHE = "last_cache"; static { metricInfoMap.put( @@ -163,12 +255,121 @@ public class PerformanceOverviewMetrics implements IMetricSet { LAST_CACHE)); } + private Timer lockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer createMemtableBlockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer memoryBlockTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer walTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer memtableTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + private Timer lastCacheTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + + /** Record the time cost of lock in engine. */ + public void recordScheduleLockCost(long costTimeInNanos) { + lockTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost of create memtable block in engine. */ + public void recordCreateMemtableBlockCost(long costTimeInNanos) { + createMemtableBlockTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost of memory block in engine. */ + public void recordScheduleMemoryBlockCost(long costTimeInNanos) { + memoryBlockTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost of wal in engine. */ + public void recordScheduleWalCost(long costTimeInNanos) { + walTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost of memtable in engine. */ + public void recordScheduleMemTableCost(long costTimeInNanos) { + memtableTimer.updateNanos(costTimeInNanos); + } + + /** Record the time cost of update last cache in engine. */ + public void recordScheduleUpdateLastCacheCost(long costTimeInNanos) { + lastCacheTimer.updateNanos(costTimeInNanos); + } + + // endregion + @Override public void bindTo(AbstractMetricService metricService) { - for (MetricInfo metricInfo : metricInfoMap.values()) { - metricService.getOrCreateTimer( - metricInfo.getName(), MetricLevel.CORE, metricInfo.getTagsInArray()); - } + // bind overview metrics + authTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), AUTHORITY); + parseTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), PARSER); + analyzeTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), ANALYZER); + planTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), PLANNER); + scheduleTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), SCHEDULER); + // bind schedule metrics + localScheduleTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_SCHEDULE_DETAIL, + MetricLevel.CORE, + Tag.STAGE.toString(), + LOCAL_SCHEDULE); + remoteScheduleTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_SCHEDULE_DETAIL, + MetricLevel.CORE, + Tag.STAGE.toString(), + REMOTE_SCHEDULE); + // bind local schedule metrics + schemaValidateTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_LOCAL_DETAIL, + MetricLevel.CORE, + Tag.STAGE.toString(), + SCHEMA_VALIDATE); + triggerTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_LOCAL_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), TRIGGER); + storageTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_LOCAL_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), STORAGE); + // bind storage metrics + engineTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_STORAGE_DETAIL, + MetricLevel.CORE, + Tag.STAGE.toString(), + PerformanceOverviewMetrics.ENGINE); + // bind engine metrics + localScheduleTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_ENGINE_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), LOCK); + createMemtableBlockTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_ENGINE_DETAIL, + MetricLevel.CORE, + Tag.STAGE.toString(), + CREATE_MEMTABLE_BLOCK); + memoryBlockTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_ENGINE_DETAIL, + MetricLevel.CORE, + Tag.STAGE.toString(), + MEMORY_BLOCK); + walTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_ENGINE_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), WAL); + memtableTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_ENGINE_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), MEMTABLE); + lastCacheTimer = + metricService.getOrCreateTimer( + PERFORMANCE_OVERVIEW_ENGINE_DETAIL, MetricLevel.CORE, Tag.STAGE.toString(), LAST_CACHE); } @Override @@ -177,4 +378,17 @@ public class PerformanceOverviewMetrics implements IMetricSet { metricService.remove(MetricType.TIMER, metricInfo.getName(), metricInfo.getTagsInArray()); } } + + private static class PerformanceOverviewMetricsHolder { + + private static final PerformanceOverviewMetrics INSTANCE = new PerformanceOverviewMetrics(); + + private PerformanceOverviewMetricsHolder() { + // empty constructor + } + } + + public static PerformanceOverviewMetrics getInstance() { + return PerformanceOverviewMetricsHolder.INSTANCE; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java b/server/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java index 62738656df..e802701b26 100644 --- a/server/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java +++ b/server/src/main/java/org/apache/iotdb/db/auth/AuthorityChecker.java @@ -23,9 +23,9 @@ import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.commons.utils.AuthUtils; import org.apache.iotdb.db.conf.OperationType; -import org.apache.iotdb.db.mpp.metric.PerformanceOverviewMetricsManager; import org.apache.iotdb.db.mpp.plan.statement.Statement; import org.apache.iotdb.db.mpp.plan.statement.StatementType; import org.apache.iotdb.db.mpp.plan.statement.sys.AuthorStatement; @@ -49,6 +49,9 @@ public class AuthorityChecker { private static final AuthorizerManager authorizerManager = AuthorizerManager.getInstance(); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); + private AuthorityChecker() { // empty constructor } @@ -122,7 +125,7 @@ public class AuthorityChecker { return onQueryException( e, OperationType.CHECK_AUTHORITY.getName(), TSStatusCode.EXECUTE_STATEMENT_ERROR); } finally { - PerformanceOverviewMetricsManager.recordAuthCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordAuthCost(System.nanoTime() - startTime); } return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java index 9fe0e3346f..7aa38a3a68 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java @@ -32,6 +32,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -70,7 +71,6 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.cache.DataNodeSchemaCache; import org.apache.iotdb.db.metadata.idtable.IDTable; import org.apache.iotdb.db.metadata.idtable.IDTableManager; -import org.apache.iotdb.db.mpp.metric.PerformanceOverviewMetricsManager; import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.DeleteDataNode; @@ -277,6 +277,8 @@ public class DataRegion implements IDataRegionForQuery { private final QueryMetricsManager queryMetricsManager = QueryMetricsManager.getInstance(); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); /** * construct a database processor. * @@ -924,7 +926,7 @@ public class DataRegion implements IDataRegionForQuery { } long startTime = System.nanoTime(); writeLock("InsertRow"); - PerformanceOverviewMetricsManager.recordScheduleLockCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime); try { if (deleted) { return; @@ -975,7 +977,7 @@ public class DataRegion implements IDataRegionForQuery { } long startTime = System.nanoTime(); writeLock("insertTablet"); - PerformanceOverviewMetricsManager.recordScheduleLockCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime); try { if (deleted) { return; @@ -1068,8 +1070,7 @@ public class DataRegion implements IDataRegionForQuery { lastFlushTimeMap.getGlobalFlushedTime(insertTabletNode.getDevicePath().getFullPath()); startTime = System.nanoTime(); tryToUpdateBatchInsertLastCache(insertTabletNode, globalLatestFlushedTime); - PerformanceOverviewMetricsManager.recordScheduleUpdateLastCacheCost( - System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleUpdateLastCacheCost(System.nanoTime() - startTime); if (!noFailure) { throw new BatchProcessException(results); @@ -1174,8 +1175,7 @@ public class DataRegion implements IDataRegionForQuery { long startTime = System.nanoTime(); tryToUpdateInsertLastCache(insertRowNode, globalLatestFlushTime); - PerformanceOverviewMetricsManager.recordScheduleUpdateLastCacheCost( - System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleUpdateLastCacheCost(System.nanoTime() - startTime); // check memtable size and may asyncTryToFlush the work memtable if (tsFileProcessor.shouldFlush()) { @@ -3097,7 +3097,7 @@ public class DataRegion implements IDataRegionForQuery { } long startTime = System.nanoTime(); writeLock("InsertRowsOfOneDevice"); - PerformanceOverviewMetricsManager.recordScheduleLockCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleLockCost(System.nanoTime() - startTime); try { if (deleted) { return; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 2d078e4109..3d399ecdff 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -47,7 +48,6 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils; -import org.apache.iotdb.db.mpp.metric.PerformanceOverviewMetricsManager; import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertRowNode; @@ -170,6 +170,9 @@ public class TsFileProcessor { private final QueryMetricsManager queryMetricsManager = QueryMetricsManager.getInstance(); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); + @SuppressWarnings("squid:S107") TsFileProcessor( String storageGroupName, @@ -228,8 +231,7 @@ public class TsFileProcessor { if (workMemTable == null) { long startTime = System.nanoTime(); createNewWorkingMemTable(); - PerformanceOverviewMetricsManager.recordCreateMemtableBlockCost( - System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost(System.nanoTime() - startTime); } long[] memIncrements = null; @@ -246,8 +248,7 @@ public class TsFileProcessor { insertRowNode.getDevicePath().getFullPath(), insertRowNode.getMeasurements(), insertRowNode.getDataTypes(), insertRowNode.getValues()); } - PerformanceOverviewMetricsManager.recordScheduleMemoryBlockCost( - System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(System.nanoTime() - startTime); } long startTime = System.nanoTime(); @@ -266,7 +267,7 @@ public class TsFileProcessor { storageGroupName, tsFileResource.getTsFile().getAbsolutePath()), e); } finally { - PerformanceOverviewMetricsManager.recordScheduleWalCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(System.nanoTime() - startTime); } startTime = System.nanoTime(); @@ -285,7 +286,7 @@ public class TsFileProcessor { tsFileResource.updateEndTime( insertRowNode.getDeviceID().toStringID(), insertRowNode.getTime()); } - PerformanceOverviewMetricsManager.recordScheduleMemTableCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost(System.nanoTime() - startTime); } private void createNewWorkingMemTable() throws WriteProcessException { @@ -310,8 +311,7 @@ public class TsFileProcessor { if (workMemTable == null) { long startTime = System.nanoTime(); createNewWorkingMemTable(); - PerformanceOverviewMetricsManager.recordCreateMemtableBlockCost( - System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost(System.nanoTime() - startTime); } long[] memIncrements = null; @@ -337,8 +337,7 @@ public class TsFileProcessor { start, end); } - PerformanceOverviewMetricsManager.recordScheduleMemoryBlockCost( - System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(System.nanoTime() - startTime); } } catch (WriteProcessException e) { for (int i = start; i < end; i++) { @@ -363,7 +362,7 @@ public class TsFileProcessor { } throw new WriteProcessException(e); } finally { - PerformanceOverviewMetricsManager.recordScheduleWalCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(System.nanoTime() - startTime); } startTime = System.nanoTime(); @@ -392,7 +391,7 @@ public class TsFileProcessor { tsFileResource.updateEndTime( insertTabletNode.getDeviceID().toStringID(), insertTabletNode.getTimes()[end - 1]); } - PerformanceOverviewMetricsManager.recordScheduleMemTableCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost(System.nanoTime() - startTime); } @SuppressWarnings("squid:S3776") // high Cognitive Complexity diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/executor/RegionWriteExecutor.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/executor/RegionWriteExecutor.java index 1d4d1a7c1c..9c5917664b 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/executor/RegionWriteExecutor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/executor/RegionWriteExecutor.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.exception.IoTDBException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.common.response.ConsensusWriteResponse; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -40,7 +41,6 @@ import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; import org.apache.iotdb.db.metadata.schemaregion.SchemaEngine; import org.apache.iotdb.db.metadata.template.ClusterTemplateManager; import org.apache.iotdb.db.metadata.template.Template; -import org.apache.iotdb.db.mpp.metric.PerformanceOverviewMetricsManager; import org.apache.iotdb.db.mpp.plan.analyze.schema.SchemaValidator; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanVisitor; @@ -83,6 +83,9 @@ public class RegionWriteExecutor { private static final DataNodeRegionManager REGION_MANAGER = DataNodeRegionManager.getInstance(); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); + public RegionExecutionResult execute(ConsensusGroupId groupId, PlanNode planNode) { try { WritePlanNodeExecutionContext context = @@ -119,8 +122,7 @@ public class RegionWriteExecutor { long startWriteTime = System.nanoTime(); writeResponse = DataRegionConsensusImpl.getInstance().write(groupId, planNode); - PerformanceOverviewMetricsManager.recordScheduleStorageCost( - System.nanoTime() - startWriteTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleStorageCost(System.nanoTime() - startWriteTime); // fire Trigger after the insertion if (writeResponse.isSuccessful()) { @@ -135,7 +137,7 @@ public class RegionWriteExecutor { triggerCostTime += (System.nanoTime() - startTime); } } - PerformanceOverviewMetricsManager.recordScheduleTriggerCost(triggerCostTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleTriggerCost(triggerCostTime); return writeResponse; } @@ -239,7 +241,7 @@ public class RegionWriteExecutor { } return response; } finally { - PerformanceOverviewMetricsManager.recordScheduleSchemaValidateCost( + PERFORMANCE_OVERVIEW_METRICS.recordScheduleSchemaValidateCost( System.nanoTime() - startTime); } boolean hasFailedMeasurement = insertNode.hasFailedMeasurements(); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/PerformanceOverviewMetricsManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/PerformanceOverviewMetricsManager.java deleted file mode 100644 index 1dbab30601..0000000000 --- a/server/src/main/java/org/apache/iotdb/db/mpp/metric/PerformanceOverviewMetricsManager.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.mpp.metric; - -import org.apache.iotdb.commons.service.metric.MetricService; -import org.apache.iotdb.commons.service.metric.enums.Metric; -import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; -import org.apache.iotdb.commons.service.metric.enums.Tag; -import org.apache.iotdb.metrics.type.Timer; -import org.apache.iotdb.metrics.utils.MetricLevel; - -public class PerformanceOverviewMetricsManager { - private static final MetricService metricService = MetricService.getInstance(); - - // region overview - private static final String PERFORMANCE_OVERVIEW_DETAIL = - Metric.PERFORMANCE_OVERVIEW_DETAIL.toString(); - private static final Timer AUTH_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.AUTHORITY); - private static final Timer PARSER_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.PARSER); - private static final Timer ANALYZE_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.ANALYZER); - private static final Timer PLAN_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.PLANNER); - private static final Timer SCHEDULE_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.SCHEDULER); - - /** Record the time cost in authority stage. */ - public static void recordAuthCost(long costTimeInNanos) { - AUTH_TIMER.updateNanos(costTimeInNanos); - } - - /** Record the time cost in parse stage. */ - public static void recordParseCost(long costTimeInNanos) { - PARSER_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordAnalyzeCost(long costTimeInNanos) { - ANALYZE_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordPlanCost(long costTimeInNanos) { - PLAN_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordScheduleCost(long costTimeInNanos) { - SCHEDULE_TIMER.updateNanos(costTimeInNanos); - } - // endregion - - // region schedule - private static final String PERFORMANCE_OVERVIEW_SCHEDULE_DETAIL = - Metric.PERFORMANCE_OVERVIEW_SCHEDULE_DETAIL.toString(); - private static final Timer LOCAL_SCHEDULE_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_SCHEDULE_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.LOCAL_SCHEDULE); - private static final Timer REMOTE_SCHEDULE_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_SCHEDULE_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.REMOTE_SCHEDULE); - - public static void recordScheduleLocalCost(long costTimeInNanos) { - LOCAL_SCHEDULE_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordScheduleRemoteCost(long costTimeInNanos) { - REMOTE_SCHEDULE_TIMER.updateNanos(costTimeInNanos); - } - - // endregion - - // region local schedule - private static final String PERFORMANCE_OVERVIEW_LOCAL_DETAIL = - Metric.PERFORMANCE_OVERVIEW_LOCAL_DETAIL.toString(); - private static final Timer SCHEMA_VALIDATE_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_LOCAL_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.SCHEMA_VALIDATE); - private static final Timer TRIGGER_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_LOCAL_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.TRIGGER); - private static final Timer STORAGE_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_LOCAL_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.STORAGE); - - public static void recordScheduleSchemaValidateCost(long costTimeInNanos) { - SCHEMA_VALIDATE_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordScheduleTriggerCost(long costTimeInNanos) { - TRIGGER_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordScheduleStorageCost(long costTimeInNanos) { - STORAGE_TIMER.updateNanos(costTimeInNanos); - } - - // endregion - - // region engine - private static final String PERFORMANCE_OVERVIEW_ENGINE_DETAIL = - Metric.PERFORMANCE_OVERVIEW_ENGINE_DETAIL.toString(); - private static final Timer LOCK_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_ENGINE_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.LOCK); - private static final Timer CREATE_MEMTABLE_BLOCK_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_ENGINE_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.CREATE_MEMTABLE_BLOCK); - private static final Timer MEMORY_BLOCK_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_ENGINE_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.MEMORY_BLOCK); - private static final Timer WAL_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_ENGINE_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.WAL); - private static final Timer MEMTABLE_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_ENGINE_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.MEMTABLE); - private static final Timer LAST_CACHE_TIMER = - metricService.getOrCreateTimer( - PERFORMANCE_OVERVIEW_ENGINE_DETAIL, - MetricLevel.IMPORTANT, - Tag.STAGE.toString(), - PerformanceOverviewMetrics.LAST_CACHE); - - public static void recordScheduleLockCost(long costTimeInNanos) { - LOCK_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordCreateMemtableBlockCost(long costTimeInNanos) { - CREATE_MEMTABLE_BLOCK_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordScheduleMemoryBlockCost(long costTimeInNanos) { - MEMORY_BLOCK_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordScheduleWalCost(long costTimeInNanos) { - WAL_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordScheduleMemTableCost(long costTimeInNanos) { - MEMTABLE_TIMER.updateNanos(costTimeInNanos); - } - - public static void recordScheduleUpdateLastCacheCost(long costTimeInNanos) { - LAST_CACHE_TIMER.updateNanos(costTimeInNanos); - } -} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java index de474a5db2..4c2d0c23bf 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.query.KilledByOthersException; @@ -34,7 +35,6 @@ import org.apache.iotdb.db.mpp.execution.QueryState; import org.apache.iotdb.db.mpp.execution.QueryStateMachine; import org.apache.iotdb.db.mpp.execution.exchange.MPPDataExchangeService; import org.apache.iotdb.db.mpp.execution.exchange.source.ISourceHandle; -import org.apache.iotdb.db.mpp.metric.PerformanceOverviewMetricsManager; import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.analyze.Analyzer; @@ -134,6 +134,9 @@ public class QueryExecution implements IQueryExecution { private static final QueryMetricsManager QUERY_METRICS = QueryMetricsManager.getInstance(); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); + public QueryExecution( Statement statement, MPPQueryContext context, @@ -211,7 +214,7 @@ public class QueryExecution implements IQueryExecution { if (context.getQueryType() == QueryType.READ) { initResultHandle(); } - PerformanceOverviewMetricsManager.recordPlanCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordPlanCost(System.nanoTime() - startTime); schedule(); } @@ -281,7 +284,7 @@ public class QueryExecution implements IQueryExecution { try { result = new Analyzer(context, partitionFetcher, schemaFetcher).analyze(statement); } finally { - PerformanceOverviewMetricsManager.recordAnalyzeCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordAnalyzeCost(System.nanoTime() - startTime); } return result; } @@ -309,7 +312,7 @@ public class QueryExecution implements IQueryExecution { syncInternalServiceClientManager, asyncInternalServiceClientManager); this.scheduler.start(); - PerformanceOverviewMetricsManager.recordScheduleCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleCost(System.nanoTime() - startTime); } // Use LogicalPlanner to do the logical query plan and logical optimization diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java index d848a9a45e..ccd650336d 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java @@ -23,11 +23,11 @@ import org.apache.iotdb.common.rpc.thrift.TAggregationType; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.db.constant.SqlConstant; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.template.TemplateQueryType; import org.apache.iotdb.db.metadata.utils.MetaFormatUtils; -import org.apache.iotdb.db.mpp.metric.PerformanceOverviewMetricsManager; import org.apache.iotdb.db.mpp.plan.expression.binary.GreaterEqualExpression; import org.apache.iotdb.db.mpp.plan.expression.binary.LessThanExpression; import org.apache.iotdb.db.mpp.plan.expression.binary.LogicAndExpression; @@ -108,7 +108,9 @@ import java.util.Map; /** Convert SQL and RPC requests to {@link Statement}. */ public class StatementGenerator { - // TODO @spricoder optimize the method adding metrics + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); + public static Statement createStatement(String sql, ZoneId zoneId) { return invokeParser(sql, zoneId); } @@ -146,7 +148,7 @@ public class StatementGenerator { queryStatement.setSelectComponent(selectComponent); queryStatement.setFromComponent(fromComponent); queryStatement.setWhereCondition(whereCondition); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return queryStatement; } @@ -180,7 +182,7 @@ public class StatementGenerator { lastQueryStatement.setSelectComponent(selectComponent); lastQueryStatement.setFromComponent(fromComponent); lastQueryStatement.setWhereCondition(whereCondition); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return lastQueryStatement; } @@ -236,7 +238,7 @@ public class StatementGenerator { whereCondition.setPredicate(predicate); queryStatement.setWhereCondition(whereCondition); } - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return queryStatement; } @@ -250,7 +252,7 @@ public class StatementGenerator { insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0])); insertStatement.setAligned(insertRecordReq.isAligned); insertStatement.fillValues(insertRecordReq.values); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; } @@ -266,7 +268,7 @@ public class StatementGenerator { insertStatement.setValues(insertRecordReq.getValues().toArray(new Object[0])); insertStatement.setNeedInferType(true); insertStatement.setAligned(insertRecordReq.isAligned); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; } @@ -295,7 +297,7 @@ public class StatementGenerator { } insertStatement.setDataTypes(dataTypes); insertStatement.setAligned(insertTabletReq.isAligned); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; } @@ -334,7 +336,7 @@ public class StatementGenerator { insertTabletStatementList.add(insertTabletStatement); } insertStatement.setInsertTabletStatementList(insertTabletStatementList); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; } @@ -358,7 +360,7 @@ public class StatementGenerator { insertRowStatementList.add(statement); } insertStatement.setInsertRowStatementList(insertRowStatementList); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; } @@ -384,7 +386,7 @@ public class StatementGenerator { insertRowStatementList.add(statement); } insertStatement.setInsertRowStatementList(insertRowStatementList); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; } @@ -409,7 +411,7 @@ public class StatementGenerator { insertRowStatementList.add(statement); } insertStatement.setInsertRowStatementList(insertRowStatementList); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; } @@ -436,7 +438,7 @@ public class StatementGenerator { insertRowStatementList.add(statement); } insertStatement.setInsertRowStatementList(insertRowStatementList); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return insertStatement; } @@ -447,7 +449,7 @@ public class StatementGenerator { DatabaseSchemaStatement statement = new DatabaseSchemaStatement(DatabaseSchemaStatement.DatabaseSchemaStatementType.CREATE); statement.setStorageGroupPath(parseStorageGroupRawString(storageGroup)); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -464,7 +466,7 @@ public class StatementGenerator { statement.setTags(req.tags); statement.setAttributes(req.attributes); statement.setAlias(req.measurementAlias); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -493,7 +495,7 @@ public class StatementGenerator { statement.setTagsList(req.tagsList); statement.setAttributesList(req.attributesList); statement.setAliasList(req.measurementAlias); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -526,7 +528,7 @@ public class StatementGenerator { statement.setTagsList(req.tagsList); statement.setAttributesList(req.attributesList); statement.setAliasList(req.measurementAliasList); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -538,7 +540,7 @@ public class StatementGenerator { parseStorageGroupRawString(path); } statement.setPrefixPath(storageGroups); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -553,7 +555,7 @@ public class StatementGenerator { statement.setPathList(pathList); statement.setDeleteStartTime(req.getStartTime()); statement.setDeleteEndTime(req.getEndTime()); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -644,7 +646,7 @@ public class StatementGenerator { CreateSchemaTemplateStatement statement = new CreateSchemaTemplateStatement( req.getName(), measurements, dataTypes, encodings, compressors, alignedPrefix.keySet()); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -669,7 +671,7 @@ public class StatementGenerator { default: break; } - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return result; } @@ -678,7 +680,7 @@ public class StatementGenerator { long startTime = System.nanoTime(); SetSchemaTemplateStatement statement = new SetSchemaTemplateStatement(req.getTemplateName(), new PartialPath(req.getPrefixPath())); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -688,14 +690,14 @@ public class StatementGenerator { UnsetSchemaTemplateStatement statement = new UnsetSchemaTemplateStatement( req.getTemplateName(), new PartialPath(req.getPrefixPath())); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } public static DropSchemaTemplateStatement createStatement(TSDropSchemaTemplateReq req) { final long startTime = System.nanoTime(); DropSchemaTemplateStatement statement = new DropSchemaTemplateStatement(req.getTemplateName()); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -707,7 +709,7 @@ public class StatementGenerator { pathPatternList.add(new PartialPath(pathPatternString)); } DeleteTimeSeriesStatement statement = new DeleteTimeSeriesStatement(pathPatternList); - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); return statement; } @@ -757,7 +759,7 @@ public class StatementGenerator { } return astVisitor.visit(tree); } finally { - PerformanceOverviewMetricsManager.recordParseCost(System.nanoTime() - startTime); + PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/AsyncSendPlanNodeHandler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/AsyncSendPlanNodeHandler.java index d0cb0e9679..2bd50a6c7e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/AsyncSendPlanNodeHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/AsyncSendPlanNodeHandler.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.mpp.plan.scheduler; -import org.apache.iotdb.db.mpp.metric.PerformanceOverviewMetricsManager; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.mpp.rpc.thrift.TSendPlanNodeResp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -33,6 +33,8 @@ public class AsyncSendPlanNodeHandler implements AsyncMethodCallback<TSendPlanNo private final AtomicLong pendingNumber; private final Map<Integer, TSendPlanNodeResp> instanceId2RespMap; private final long sendTime; + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); public AsyncSendPlanNodeHandler( int instanceId, @@ -49,7 +51,7 @@ public class AsyncSendPlanNodeHandler implements AsyncMethodCallback<TSendPlanNo public void onComplete(TSendPlanNodeResp tSendPlanNodeResp) { instanceId2RespMap.put(instanceId, tSendPlanNodeResp); if (pendingNumber.decrementAndGet() == 0) { - PerformanceOverviewMetricsManager.recordScheduleRemoteCost(System.nanoTime() - sendTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleRemoteCost(System.nanoTime() - sendTime); synchronized (pendingNumber) { pendingNumber.notifyAll(); } @@ -66,7 +68,7 @@ public class AsyncSendPlanNodeHandler implements AsyncMethodCallback<TSendPlanNo RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), errorMsg)); instanceId2RespMap.put(instanceId, resp); if (pendingNumber.decrementAndGet() == 0) { - PerformanceOverviewMetricsManager.recordScheduleRemoteCost(System.nanoTime() - sendTime); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleRemoteCost(System.nanoTime() - sendTime); synchronized (pendingNumber) { pendingNumber.notifyAll(); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java index 0719b86452..b990e0e4fb 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -26,13 +26,13 @@ import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException; import org.apache.iotdb.db.mpp.common.MPPQueryContext; import org.apache.iotdb.db.mpp.execution.executor.RegionExecutionResult; import org.apache.iotdb.db.mpp.execution.executor.RegionReadExecutor; import org.apache.iotdb.db.mpp.execution.executor.RegionWriteExecutor; -import org.apache.iotdb.db.mpp.metric.PerformanceOverviewMetricsManager; import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.analyze.QueryType; import org.apache.iotdb.db.mpp.plan.planner.plan.FragmentInstance; @@ -75,6 +75,8 @@ public class FragmentInstanceDispatcherImpl implements IFragInstanceDispatcher { asyncInternalServiceClientManager; private static final QueryMetricsManager QUERY_METRICS = QueryMetricsManager.getInstance(); + private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS = + PerformanceOverviewMetrics.getInstance(); public FragmentInstanceDispatcherImpl( QueryType type, @@ -194,7 +196,7 @@ public class FragmentInstanceDispatcherImpl implements IFragInstanceDispatcher { TSStatusCode.INTERNAL_SERVER_ERROR, "Unexpected errors: " + t.getMessage())); } } - PerformanceOverviewMetricsManager.recordScheduleLocalCost( + PERFORMANCE_OVERVIEW_METRICS.recordScheduleLocalCost( System.nanoTime() - localScheduleStartTime); // wait until remote dispatch done try { diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java index 453f4f9933..664b0c3a67 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java +++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.service.metrics; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.commons.service.metric.enums.PerformanceOverviewMetrics; import org.apache.iotdb.db.mpp.metric.DataExchangeCostMetricSet; import org.apache.iotdb.db.mpp.metric.DataExchangeCountMetricSet; import org.apache.iotdb.db.mpp.metric.DriverSchedulerMetricSet; @@ -51,5 +52,8 @@ public class DataNodeMetricsHelper { MetricService.getInstance().addMetricSet(new DataExchangeCostMetricSet()); MetricService.getInstance().addMetricSet(new DataExchangeCountMetricSet()); MetricService.getInstance().addMetricSet(new DriverSchedulerMetricSet()); + + // bind performance overview related metrics + MetricService.getInstance().addMetricSet(PerformanceOverviewMetrics.getInstance()); } }
