This is an automated email from the ASF dual-hosted git repository. penghui pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push: new b33aca9 [doc] Support generate document automatically for pulsar-perf (#10120) b33aca9 is described below commit b33aca9c36ab8f3c6b679400d7c31bf9e18052ba Author: linlinnn <linjunhuac...@163.com> AuthorDate: Mon Apr 19 22:42:36 2021 +0800 [doc] Support generate document automatically for pulsar-perf (#10120) Co-authored-by: linjunhua <lin...@jiguang.cn> --- bin/pulsar-perf | 4 + .../proxy/socket/client/PerformanceClient.java | 8 +- .../apache/pulsar/testclient/BrokerMonitor.java | 9 +- .../testclient/CmdGenerateDocumentation.java | 127 +++++++++++++++++++++ .../pulsar/testclient/LoadSimulationClient.java | 2 + .../testclient/LoadSimulationController.java | 9 +- .../pulsar/testclient/ManagedLedgerWriter.java | 3 +- .../pulsar/testclient/PerformanceConsumer.java | 9 +- .../pulsar/testclient/PerformanceProducer.java | 2 + .../pulsar/testclient/PerformanceReader.java | 2 + .../pulsar/testclient/GenerateDocumentionTest.java | 35 ++++++ 11 files changed, 195 insertions(+), 15 deletions(-) diff --git a/bin/pulsar-perf b/bin/pulsar-perf index b2a273f..9bc5c0c 100755 --- a/bin/pulsar-perf +++ b/bin/pulsar-perf @@ -93,6 +93,8 @@ where command is one of: simulation-client Run a simulation server acting as a Pulsar client simulation-controller Run a simulation controller to give commands to servers + gen-doc Generate documentation automatically. + help This help message or command is the full name of a class with a defined main() method. @@ -167,6 +169,8 @@ elif [ "$COMMAND" == "websocket-producer" ]; then exec $JAVA $OPTS org.apache.pulsar.proxy.socket.client.PerformanceClient "$@" elif [ "$COMMAND" == "managed-ledger" ]; then exec $JAVA $OPTS org.apache.pulsar.testclient.ManagedLedgerWriter "$@" +elif [ "$COMMAND" == "gen-doc" ]; then + exec $JAVA $OPTS org.apache.pulsar.testclient.CmdGenerateDocumentation "$@" else pulsar_help; fi diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/proxy/socket/client/PerformanceClient.java b/pulsar-testclient/src/main/java/org/apache/pulsar/proxy/socket/client/PerformanceClient.java index b046577..3902d5f 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/proxy/socket/client/PerformanceClient.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/proxy/socket/client/PerformanceClient.java @@ -21,6 +21,10 @@ package org.apache.pulsar.proxy.socket.client; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -54,9 +58,6 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.websocket.client.ClientUpgradeRequest; import org.eclipse.jetty.websocket.client.WebSocketClient; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; import com.google.common.util.concurrent.RateLimiter; import io.netty.util.concurrent.DefaultThreadFactory; @@ -68,6 +69,7 @@ public class PerformanceClient { private static final LongAdder bytesSent = new LongAdder(); private JCommander jc; + @Parameters(commandDescription = "Test pulsar websocket producer performance.") static class Arguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java index f0f9bf8..c1d400a 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/BrokerMonitor.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.testclient; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -27,7 +31,6 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import com.beust.jcommander.ParameterException; import org.apache.pulsar.broker.TimeAverageBrokerData; import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl; import org.apache.pulsar.policies.data.loadbalancer.LoadReport; @@ -41,8 +44,6 @@ import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; import com.google.gson.Gson; /** @@ -428,6 +429,8 @@ public class BrokerMonitor { } // JCommander arguments class. + @Parameters(commandDescription = "Monitors brokers and prints to the console information about their system resource usages," + + "\ntheir topic and bundle counts, their message rates, and other metrics.") private static class Arguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) boolean help; diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java new file mode 100644 index 0000000..a85ff70 --- /dev/null +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/CmdGenerateDocumentation.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterDescription; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; +import com.beust.jcommander.StringKey; +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CmdGenerateDocumentation { + + @Parameters(commandDescription = "Generate documentation automatically.") + static class Arguments { + + @Parameter(names = {"-h", "--help"}, description = "Help message", help = true) + boolean help; + + @Parameter(names = {"-n", "--command-names"}, description = "List of command names") + private List<String> commandNames = new ArrayList<>(); + + } + + public static void main(String[] args) throws Exception { + final Arguments arguments = new Arguments(); + final JCommander jc = new JCommander(arguments); + jc.setProgramName("pulsar-perf gen-doc"); + try { + jc.parse(args); + } catch (ParameterException e) { + System.out.println(e.getMessage()); + jc.usage(); + PerfClientUtils.exit(-1); + } + + if (arguments.help) { + jc.usage(); + PerfClientUtils.exit(-1); + } + + Map<String, Class<?>> cmdClassMap = new LinkedHashMap<>(); + cmdClassMap.put("produce", Class.forName("org.apache.pulsar.testclient.PerformanceProducer$Arguments")); + cmdClassMap.put("consume", Class.forName("org.apache.pulsar.testclient.PerformanceConsumer$Arguments")); + cmdClassMap.put("read", Class.forName("org.apache.pulsar.testclient.PerformanceReader$Arguments")); + cmdClassMap.put("monitor-brokers", Class.forName("org.apache.pulsar.testclient.BrokerMonitor$Arguments")); + cmdClassMap.put("simulation-client", Class.forName("org.apache.pulsar.testclient.LoadSimulationClient$MainArguments")); + cmdClassMap.put("simulation-controller", Class.forName("org.apache.pulsar.testclient.LoadSimulationController$MainArguments")); + cmdClassMap.put("websocket-producer", Class.forName("org.apache.pulsar.proxy.socket.client.PerformanceClient$Arguments")); + cmdClassMap.put("managed-ledger", Class.forName("org.apache.pulsar.testclient.ManagedLedgerWriter$Arguments")); + cmdClassMap.put("gen-doc", Class.forName("org.apache.pulsar.testclient.CmdGenerateDocumentation$Arguments")); + + for (Map.Entry<String, Class<?>> entry : cmdClassMap.entrySet()) { + String cmd = entry.getKey(); + Class<?> clazz = entry.getValue(); + Constructor<?> constructor = clazz.getDeclaredConstructor(); + constructor.setAccessible(true); + jc.addCommand(cmd, constructor.newInstance()); + } + + if (arguments.commandNames.size() == 0) { + for (Map.Entry<String, JCommander> cmd : jc.getCommands().entrySet()) { + generateDocument(cmd.getKey(), jc); + } + } else { + for (String commandName : arguments.commandNames) { + generateDocument(commandName, jc); + } + } + } + + private static String generateDocument(String module, JCommander parentCmd) { + StringBuilder sb = new StringBuilder(); + JCommander cmd = parentCmd.getCommands().get(module); + sb.append("------------\n\n"); + sb.append("# ").append(module).append("\n\n"); + sb.append("### Usage\n\n"); + sb.append("`$").append(module).append("`\n\n"); + sb.append("------------\n\n"); + sb.append(parentCmd.getUsageFormatter().getCommandDescription(module)).append("\n"); + sb.append("\n\n```bdocs-tab:example_shell\n") + .append("$ pulsar-perf ").append(module).append(" [options]") + .append("\n```"); + sb.append("\n\n"); + for (String s : cmd.getCommands().keySet()) { + sb.append("* `").append(s).append("`\n"); + } + sb.append("|Flag|Description|Default|\n"); + sb.append("|---|---|---|\n"); + // IKey is an internal interface and cannot be accessed directly, + // so the type needs to be erased and force cast to a subclass + Map descriptionMap = cmd.getDescriptions(); + descriptionMap.forEach((k, v) -> { + StringKey key = (StringKey) k; + ParameterDescription description = (ParameterDescription) v; + sb.append("| `").append(key.getName()) + .append("` | ").append(description.getDescription().replace("\n", " ")) + .append("|").append(description.getDefault()).append("|\n"); + + }); + System.out.println(sb.toString()); + return sb.toString(); + } +} diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationClient.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationClient.java index c7d67f1..675ca78 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationClient.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationClient.java @@ -21,6 +21,7 @@ package org.apache.pulsar.testclient; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; import com.google.common.util.concurrent.RateLimiter; import io.netty.util.concurrent.DefaultThreadFactory; @@ -171,6 +172,7 @@ public class LoadSimulationClient { } // JCommander arguments for starting a LoadSimulationClient. + @Parameters(commandDescription = "Simulate client load by maintaining producers and consumers for topics.") private static class MainArguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) boolean help; diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationController.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationController.java index 56db823..888d724 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationController.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/LoadSimulationController.java @@ -18,6 +18,10 @@ */ package org.apache.pulsar.testclient; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; import java.io.BufferedReader; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -52,10 +56,6 @@ import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; - /** * This class provides a shell for the user to dictate how simulation clients should incur load. */ @@ -84,6 +84,7 @@ public class LoadSimulationController { private static final ExecutorService threadPool = Executors.newCachedThreadPool(); // JCommander arguments for starting a controller via main. + @Parameters(commandDescription = "Provides a shell for the user to dictate how simulation clients should incur load.") private static class MainArguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) boolean help; diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ManagedLedgerWriter.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ManagedLedgerWriter.java index 978e1d8..990f486 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ManagedLedgerWriter.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/ManagedLedgerWriter.java @@ -23,9 +23,9 @@ import static java.util.concurrent.TimeUnit.NANOSECONDS; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.collect.Lists; import com.google.common.util.concurrent.RateLimiter; import io.netty.buffer.ByteBuf; @@ -72,6 +72,7 @@ public class ManagedLedgerWriter { private static Recorder recorder = new Recorder(TimeUnit.SECONDS.toMillis(120000), 5); private static Recorder cumulativeRecorder = new Recorder(TimeUnit.SECONDS.toMillis(120000), 5); + @Parameters(commandDescription = "Write directly on managed-ledgers") static class Arguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java index 0d8f902..ca6b07e 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceConsumer.java @@ -21,6 +21,10 @@ package org.apache.pulsar.testclient; import static org.apache.commons.lang3.StringUtils.isBlank; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; import java.io.FileInputStream; import java.text.DecimalFormat; import java.util.Collections; @@ -43,9 +47,6 @@ import org.apache.pulsar.common.naming.TopicName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.ParameterException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.collect.Lists; @@ -62,7 +63,7 @@ public class PerformanceConsumer { private static Recorder recorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5); private static Recorder cumulativeRecorder = new Recorder(TimeUnit.DAYS.toMillis(10), 5); - + @Parameters(commandDescription = "Test pulsar consumer performance.") static class Arguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java index 5ab4bb0..ece6853 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceProducer.java @@ -25,6 +25,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.collect.Lists; @@ -93,6 +94,7 @@ public class PerformanceProducer { private static Recorder recorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); private static Recorder cumulativeRecorder = new Recorder(TimeUnit.SECONDS.toMicros(120000), 5); + @Parameters(commandDescription = "Test pulsar producer performance.") static class Arguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) diff --git a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java index 11bdf02..423bf2c 100644 --- a/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java +++ b/pulsar-testclient/src/main/java/org/apache/pulsar/testclient/PerformanceReader.java @@ -23,6 +23,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; +import com.beust.jcommander.Parameters; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.collect.Lists; @@ -51,6 +52,7 @@ public class PerformanceReader { private static final LongAdder bytesReceived = new LongAdder(); private static final DecimalFormat dec = new DecimalFormat("0.000"); + @Parameters(commandDescription = "Test pulsar reader performance.") static class Arguments { @Parameter(names = { "-h", "--help" }, description = "Help message", help = true) diff --git a/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/GenerateDocumentionTest.java b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/GenerateDocumentionTest.java new file mode 100644 index 0000000..812979a --- /dev/null +++ b/pulsar-testclient/src/test/java/org/apache/pulsar/testclient/GenerateDocumentionTest.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.testclient; + +import org.junit.Test; + +public class GenerateDocumentionTest { + + @Test + public void testGenerateDocumention() throws Exception { + CmdGenerateDocumentation.main(new String[]{}); + } + + @Test + public void testSpecifyModuleName() throws Exception { + String[] args = new String[]{"-n", "produce", "-n", "consume"}; + CmdGenerateDocumentation.main(args); + } +} \ No newline at end of file