This is an automated email from the ASF dual-hosted git repository. eolivelli pushed a commit to branch branch-2.9 in repository https://gitbox.apache.org/repos/asf/pulsar.git
commit f99ff9cffbebe58f828a32acdf085a7bf12ebe34 Author: Matt Fleming <[email protected]> AuthorDate: Fri Nov 5 09:36:44 2021 +0000 [pulsar-perf] Write histogram files for consume command (#12569) * [pulsar-perf] Write histogram files for consume command * [pulsar-perf] Disable writing to histogram files by default Most users don't use the histogram files and instead opt for sending metrics to prometheus, etc, so there's no need to have this enabled by default. Instead, add a new --histogram-file parameter to pulsar-perf produce/consume which, when specified, dumps the contents of the internal histogram to the given filename. Previous behaviour can be achieved with the following options: $ pulsar-perf produce --histogram-file perf-producer-$(date +%s).hgrm * [pulsar-perf] Update docs with --histogram-file param (cherry picked from commit 48de2e251ee3ea38449d696402d0cc57c88ee3c7) --- .../pulsar/testclient/PerformanceConsumer.java | 22 +++++++++++++++++++ .../pulsar/testclient/PerformanceProducer.java | 25 +++++++++++++++------- site2/docs/performance-pulsar-perf.md | 4 +++- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index 52ba73e..db69778 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -26,6 +26,8 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.beust.jcommander.Parameters; import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.PrintStream; import java.nio.ByteBuffer; import java.text.DecimalFormat; import java.util.Collections; @@ -36,6 +38,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import org.HdrHistogram.Histogram; +import org.HdrHistogram.HistogramLogWriter; import org.HdrHistogram.Recorder; import org.apache.pulsar.client.api.ClientBuilder; import org.apache.pulsar.client.api.Consumer; @@ -185,6 +188,9 @@ public class PerformanceConsumer { @Parameter(names = {"-bw", "--busy-wait"}, description = "Enable Busy-Wait on the Pulsar client") public boolean enableBusyWait = false; + + @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file") + public String histogramFile = null; } public static void main(String[] args) throws Exception { @@ -406,7 +412,19 @@ public class PerformanceConsumer { long oldTime = System.nanoTime(); Histogram reportHistogram = null; + HistogramLogWriter histogramLogWriter = null; + + if (arguments.histogramFile != null) { + String statsFileName = arguments.histogramFile; + log.info("Dumping latency stats to {}", statsFileName); + + PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); + histogramLogWriter = new HistogramLogWriter(histogramLog); + // Some log header bits + histogramLogWriter.outputLogFormatVersion(); + histogramLogWriter.outputLegend(); + } while (true) { try { @@ -431,6 +449,10 @@ public class PerformanceConsumer { reportHistogram.getValueAtPercentile(99), reportHistogram.getValueAtPercentile(99.9), reportHistogram.getValueAtPercentile(99.99), reportHistogram.getMaxValue()); + if (histogramLogWriter != null) { + histogramLogWriter.outputIntervalHistogram(reportHistogram); + } + reportHistogram.reset(); oldTime = now; } diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 44b3c20..fee96a8 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -253,6 +253,9 @@ public class PerformanceProducer { @Parameter(names = {"-fc", "--format-class"}, description="Custom Formatter class name") public String formatterClass = "org.apache.pulsar.testclient.DefaultMessageFormatter"; + + @Parameter(names = { "--histogram-file" }, description = "HdrHistogram output file") + public String histogramFile = null; } public static void main(String[] args) throws Exception { @@ -436,16 +439,19 @@ public class PerformanceProducer { long oldTime = System.nanoTime(); Histogram reportHistogram = null; + HistogramLogWriter histogramLogWriter = null; - String statsFileName = "perf-producer-" + System.currentTimeMillis() + ".hgrm"; - log.info("Dumping latency stats to {}", statsFileName); + if (arguments.histogramFile != null) { + String statsFileName = arguments.histogramFile; + log.info("Dumping latency stats to {}", statsFileName); - PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); - HistogramLogWriter histogramLogWriter = new HistogramLogWriter(histogramLog); + PrintStream histogramLog = new PrintStream(new FileOutputStream(statsFileName), false); + histogramLogWriter = new HistogramLogWriter(histogramLog); - // Some log header bits - histogramLogWriter.outputLogFormatVersion(); - histogramLogWriter.outputLegend(); + // Some log header bits + histogramLogWriter.outputLogFormatVersion(); + histogramLogWriter.outputLegend(); + } while (true) { try { @@ -480,7 +486,10 @@ public class PerformanceProducer { dec.format(reportHistogram.getValueAtPercentile(99.99) / 1000.0), dec.format(reportHistogram.getMaxValue() / 1000.0)); - histogramLogWriter.outputIntervalHistogram(reportHistogram); + if (histogramLogWriter != null) { + histogramLogWriter.outputIntervalHistogram(reportHistogram); + } + reportHistogram.reset(); oldTime = now; diff --git a/site2/docs/performance-pulsar-perf.md b/site2/docs/performance-pulsar-perf.md index bb83115..c5aafaf 100644 --- a/site2/docs/performance-pulsar-perf.md +++ b/site2/docs/performance-pulsar-perf.md @@ -32,7 +32,7 @@ After the command is executed, the test data is continuously output on the Conso 19:54:44.336 [Thread-1] INFO org.apache.pulsar.testclient.PerformanceProducer - Aggregated latency stats --- Latency: mean: 3.383 ms - med: 3.293 - 95pct: 4.610 - 99pct: 5.059 - 99.9pct: 5.588 - 99.99pct: 5.837 - 99.999pct: 6.609 - Max: 6.609 ``` -From the above test data, you can get the throughput statistics and the write latency statistics. The aggregated statistics is printed when the Pulsar Perf is stopped. You can press **Ctrl**+**C** to stop the Pulsar Perf. After the Pulsar Perf is stopped, the [HdrHistogram](http://hdrhistogram.github.io/HdrHistogram/) formatted test result appears under your directory. The document looks like `perf-producer-1589370810837.hgrm`. You can also check the test result through [HdrHistogram Plo [...] +From the above test data, you can get the throughput statistics and the write latency statistics. The aggregated statistics is printed when the Pulsar Perf is stopped. You can press **Ctrl**+**C** to stop the Pulsar Perf. If you specify a filename with the `--histogram-file` parameter, a file with the [HdrHistogram](http://hdrhistogram.github.io/HdrHistogram/) formatted test result appears under your directory after Pulsar Perf is stopped. You can also check the test result through [HdrH [...] ### Configuration options for `pulsar-perf produce` @@ -61,6 +61,7 @@ The following table lists configuration options available for the `pulsar-perf p | format-class | Set the custom formatter class name. | org.apache.pulsar.testclient.DefaultMessageFormatter | | format-payload | Configure whether to format %i as a message index in the stream from producer and/or %t as the timestamp nanoseconds. | false | | help | Configure the help message. | false | +| histogram-file | HdrHistogram output file | N/A | | max-connections | Set the maximum number of TCP connections to a single broker. | 100 | | max-outstanding | Set the maximum number of outstanding messages. | 1000 | | max-outstanding-across-partitions | Set the maximum number of outstanding messages across partitions. | 50000 | @@ -131,6 +132,7 @@ The following table lists configuration options available for the `pulsar-perf c | encryption-key-name | Set the name of the public key used to encrypt the payload. | N/A | | encryption-key-value-file | Set the file which contains the public key used to encrypt the payload. | N/A | | help | Configure the help message. | false | +| histogram-file | HdrHistogram output file | N/A | | expire_time_incomplete_chunked_messages | Set the expiration time for incomplete chunk messages (in milliseconds). | 0 | | max-connections | Set the maximum number of TCP connections to a single broker. | 100 | | max_chunked_msg | Set the max pending chunk messages. | 0 |
