jon-wei commented on a change in pull request #8981: add query metrics for broker parallel merges, off by default URL: https://github.com/apache/incubator-druid/pull/8981#discussion_r353491280
########## File path: core/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java ########## @@ -1110,4 +1156,197 @@ RuntimeException getRuntimeException() return new RE(ex); } } + + /** + * Metrics for the execution of a {@link ParallelMergeCombiningSequence} on the {@link ForkJoinPool} + */ + public static class MergeCombineMetrics + { + private final int parallelism; + private final int inputSequences; + private final long inputRows; + private final long outputRows; + private final long taskCount; + private final long totalCpuTime; + + MergeCombineMetrics( + int parallelism, + int inputSequences, + long inputRows, + long outputRows, + long taskCount, + long totalCpuTime + ) + { + this.parallelism = parallelism; + this.inputSequences = inputSequences; + this.inputRows = inputRows; + this.outputRows = outputRows; + this.taskCount = taskCount; + this.totalCpuTime = totalCpuTime; + } + + /** + * Total number of layer 1 parallel tasks (+ 1 for total number of concurrent tasks for this query) + */ + public int getParallelism() + { + return parallelism; + } + + /** + * Total number of input {@link Sequence} processed by {@link ParallelMergeCombiningSequence} + */ + public long getInputSequences() + { + return inputSequences; + } + + /** + * Total number of input 'rows' processed by the {@link ParallelMergeCombiningSequence} + */ + public long getInputRows() + { + return inputRows; + } + + /** + * Total number of output 'rows' produced by merging and combining the set of input {@link Sequence}s + */ + public long getOutputRows() + { + return outputRows; + } + + /** + * Total number of {@link ForkJoinPool} tasks involved in executing the {@link ParallelMergeCombiningSequence}, + * including {@link MergeCombinePartitioningAction}, {@link PrepareMergeCombineInputsAction}, and + * {@link MergeCombineAction}. + */ + public long getTaskCount() + { + return taskCount; + } + + /** + * Total CPU time in nanoseconds during the 'hot loop' of doing actual merging and combining + * in {@link MergeCombineAction} + */ + public long getTotalCpuTime() + { + return totalCpuTime; + } + } + + /** + * Holder to accumlate metrics for all work done {@link ParallelMergeCombiningSequence}, containing layer 1 task + * metrics in {@link #partitionMetrics} and final merge task metrics in {@link #mergeMetrics}, in order to compute + * {@link MergeCombineMetrics} after the {@link ParallelMergeCombiningSequence} is completely consumed. + */ + static class MergeCombineMetricsAccumlator + { + List<MergeCombineActionMetricsAccumulator> partitionMetrics; + MergeCombineActionMetricsAccumulator mergeMetrics; + private final int inputSequences; + + MergeCombineMetricsAccumlator(int inputSequences) + { + this.inputSequences = inputSequences; + } + + void setMergeMetrics(MergeCombineActionMetricsAccumulator mergeMetrics) + { + this.mergeMetrics = mergeMetrics; + } + + void setPartitions(List<MergeCombineActionMetricsAccumulator> partitionMetrics) + { + this.partitionMetrics = partitionMetrics; + } + + MergeCombineMetrics build() + { + long numInputRows = 0; + long cpuTimeNanos = 0; + // 1 partition task, 1 layer 2 prepare merge inputs task, 1 layer 1 prepare merge inputs task for each partition Review comment: nit: suggest "one layer 2 prepare merge inputs task" or "1 layer two prepare merge inputs task" and similar, I was confused initially with all the numbers ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org