This is an automated email from the ASF dual-hosted git repository. xiangfu pushed a commit to branch new-pinot-admin-cmd in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 35d5d0a7da90440f9b0763446869a8784a3956c0 Author: Xiang Fu <[email protected]> AuthorDate: Sat Jan 4 23:55:02 2020 -0800 Update AddTableCommand to take schema file in order to creation pinot table in one line --- .../org/apache/pinot/tools/HybridQuickstart.java | 2 - .../java/org/apache/pinot/tools/Quickstart.java | 2 - .../org/apache/pinot/tools/RealtimeQuickStart.java | 2 - .../pinot/tools/admin/PinotAdministrator.java | 1 + .../pinot/tools/admin/command/AddTableCommand.java | 82 +++++++++++++++------- .../tools/admin/command/GenerateDataCommand.java | 2 +- .../tools/admin/command/QuickstartRunner.java | 11 +-- .../tools/query/comparison/ClusterStarter.java | 3 +- 8 files changed, 61 insertions(+), 44 deletions(-) diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java index 68717cb..aa33073 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java @@ -130,8 +130,6 @@ public class HybridQuickstart { runner.createServerTenantWith(1, 1, "airline"); printStatus(Color.YELLOW, "***** Creating a broker tenant with name 'airline_broker' *****"); runner.createBrokerTenantWith(2, "airline_broker"); - printStatus(Color.YELLOW, "***** Adding airlineStats schema *****"); - runner.addSchema(); printStatus(Color.YELLOW, "***** Adding airlineStats offline and realtime table *****"); runner.addTable(); printStatus(Color.YELLOW, "***** Building index segment for airlineStats *****"); diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java index d7d0bf5..9427b52 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java @@ -146,8 +146,6 @@ public class Quickstart { printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****"); runner.startAll(); - printStatus(Color.CYAN, "***** Adding baseballStats schema *****"); - runner.addSchema(); printStatus(Color.CYAN, "***** Adding baseballStats table *****"); runner.addTable(); printStatus(Color.CYAN, "***** Building index segment for baseballStats *****"); diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java index 1492355..82f76e2 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeQuickStart.java @@ -84,8 +84,6 @@ public class RealtimeQuickStart { printStatus(Color.CYAN, "***** Starting Zookeeper, controller, server and broker *****"); runner.startAll(); - printStatus(Color.CYAN, "***** Adding meetupRSVP schema *****"); - runner.addSchema(); printStatus(Color.CYAN, "***** Adding meetupRSVP table *****"); runner.addTable(); printStatus(Color.CYAN, "***** Starting meetup data stream and publishing to Kafka *****"); diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java index 9684cca..5fd2ccb 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/PinotAdministrator.java @@ -97,6 +97,7 @@ public class PinotAdministrator { @SubCommand(name = "ChangeTableState", impl = ChangeTableState.class), @SubCommand(name = "AddTenant", impl = AddTenantCommand.class), @SubCommand(name = "AddSchema", impl = AddSchemaCommand.class), + @SubCommand(name = "UpdateSchema", impl = AddSchemaCommand.class), @SubCommand(name = "UploadSegment", impl = UploadSegmentCommand.class), @SubCommand(name = "PostQuery", impl = PostQueryCommand.class), @SubCommand(name = "StopProcess", impl = StopProcessCommand.class), diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AddTableCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AddTableCommand.java index d035f10..4106e47 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AddTableCommand.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/AddTableCommand.java @@ -20,10 +20,13 @@ package org.apache.pinot.tools.admin.command; import com.fasterxml.jackson.databind.JsonNode; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; -import org.apache.pinot.spi.utils.JsonUtils; +import org.apache.pinot.common.utils.FileUploadDownloadClient; import org.apache.pinot.common.utils.NetUtil; import org.apache.pinot.controller.helix.ControllerRequestURLBuilder; +import org.apache.pinot.spi.data.Schema; +import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.tools.Command; import org.kohsuke.args4j.Option; import org.slf4j.Logger; @@ -37,8 +40,11 @@ import org.slf4j.LoggerFactory; public class AddTableCommand extends AbstractBaseAdminCommand implements Command { private static final Logger LOGGER = LoggerFactory.getLogger(AddTableCommand.class); - @Option(name = "-filePath", required = true, metaVar = "<string>", usage = "Path to the request.json file") - private String _filePath; + @Option(name = "-tableConfigFile", required = true, metaVar = "<string>", aliases = {"-tableConf", "-tableConfig", "-filePath"}, usage = "Path to table config file.") + private String _tableConfigFile; + + @Option(name = "-schemaFile", required = false, metaVar = "<string>", aliases = {"-schema"}, usage = "Path to table schema file.") + private String _schemaFile = null; @Option(name = "-controllerHost", required = false, metaVar = "<String>", usage = "host name for controller.") private String _controllerHost; @@ -61,29 +67,33 @@ public class AddTableCommand extends AbstractBaseAdminCommand implements Command @Override public String getName() { - return "AddTable"; + return "AddTableV2"; } @Override public String description() { - return "Add table specified in the json file, to the controller."; + return "Create Pinot table with table config and schema."; } @Override public String toString() { - String retString = ("AddTable -filePath " + _filePath + " -controllerHost " + _controllerHost + " -controllerPort " - + _controllerPort); - + String retString = + ("AddTable -tableConfigFile " + _tableConfigFile + " -schemaFile " + _schemaFile + " -controllerHost " + + _controllerHost + " -controllerPort " + _controllerPort); return ((_exec) ? (retString + " -exec") : retString); } @Override public void cleanup() { + } + public AddTableCommand setTableConfigFile(String tableConfigFile) { + _tableConfigFile = tableConfigFile; + return this; } - public AddTableCommand setFilePath(String filePath) { - _filePath = filePath; + public AddTableCommand setSchemaFile(String schemaFile) { + _schemaFile = schemaFile; return this; } @@ -102,33 +112,51 @@ public class AddTableCommand extends AbstractBaseAdminCommand implements Command return this; } - public boolean execute(JsonNode node) + public boolean sendTableCreationRequest(JsonNode node) throws IOException { - if (_controllerHost == null) { - _controllerHost = NetUtil.getHostAddress(); - } - _controllerAddress = "http://"; + _controllerHost + ":" + _controllerPort; + String res = AbstractBaseAdminCommand + .sendPostRequest(ControllerRequestURLBuilder.baseUrl(_controllerAddress).forTableCreate(), node.toString()); + LOGGER.info(res); + return res.contains("succesfully added"); + } + @Override + public boolean execute() + throws Exception { if (!_exec) { LOGGER.warn("Dry Running Command: " + toString()); LOGGER.warn("Use the -exec option to actually execute the command."); return true; } - LOGGER.info("Executing command: " + toString()); - String res = AbstractBaseAdminCommand - .sendPostRequest(ControllerRequestURLBuilder.baseUrl(_controllerAddress).forTableCreate(), node.toString()); + if (_controllerHost == null) { + _controllerHost = NetUtil.getHostAddress(); + } + _controllerAddress = "http://"; + _controllerHost + ":" + _controllerPort; - LOGGER.info(res); - if (res.contains("succesfully added")) { - return true; + LOGGER.info("Executing command: " + toString()); + // Backward compatible + if (_schemaFile != null) { + File schemaFile = new File(_schemaFile); + if (!schemaFile.exists()) { + throw new FileNotFoundException("schema file does not exist: " + _schemaFile); + } + + Schema schema = Schema.fromFile(schemaFile); + try (FileUploadDownloadClient fileUploadDownloadClient = new FileUploadDownloadClient()) { + fileUploadDownloadClient.addSchema( + FileUploadDownloadClient.getUploadSchemaHttpURI(_controllerHost, Integer.parseInt(_controllerPort)), + schema.getSchemaName(), schemaFile); + } catch (Exception e) { + LOGGER.error("Got Exception to upload Pinot Schema: " + schema.getSchemaName(), e); + throw e; + } + } + File tableConfigFile = new File(_tableConfigFile); + if (!tableConfigFile.exists()) { + throw new FileNotFoundException("table config file does not exist: " + _tableConfigFile); } - return false; - } - @Override - public boolean execute() - throws Exception { - return execute(JsonUtils.fileToJsonNode(new File(_filePath))); + return sendTableCreationRequest(JsonUtils.fileToJsonNode(tableConfigFile)); } } diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java index e46dcae..19ff1ce 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/GenerateDataCommand.java @@ -101,7 +101,7 @@ public class GenerateDataCommand extends AbstractBaseAdminCommand implements Com @Override public String description() { - return "Generate random data as per the provided scema"; + return "Generate random data as per the provided schema"; } @Override diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java index 11c860e..549ac52 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/QuickstartRunner.java @@ -153,18 +153,11 @@ public class QuickstartRunner { .setInstances(number).setRole(TenantRole.BROKER).setExecute(true).execute(); } - public void addSchema() - throws Exception { - for (QuickstartTableRequest request : _tableRequests) { - new AddSchemaCommand().setControllerPort(String.valueOf(_controllerPorts.get(0))) - .setSchemaFilePath(request.getSchemaFile().getAbsolutePath()).setExecute(true).execute(); - } - } - public void addTable() throws Exception { for (QuickstartTableRequest request : _tableRequests) { - new AddTableCommand().setFilePath(request.getTableRequestFile().getAbsolutePath()) + new AddTableCommand().setSchemaFile(request.getSchemaFile().getAbsolutePath()) + .setTableConfigFile(request.getTableRequestFile().getAbsolutePath()) .setControllerPort(String.valueOf(_controllerPorts.get(0))).setExecute(true).execute(); } } diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/query/comparison/ClusterStarter.java b/pinot-tools/src/main/java/org/apache/pinot/tools/query/comparison/ClusterStarter.java index bc7eb7a..b2840ab 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/query/comparison/ClusterStarter.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/query/comparison/ClusterStarter.java @@ -192,7 +192,8 @@ public class ClusterStarter { throws Exception { if (_tableConfigFile != null) { AddTableCommand addTableCommand = - new AddTableCommand().setControllerPort(_controllerPort).setFilePath(_tableConfigFile).setExecute(true); + new AddTableCommand().setControllerPort(_controllerPort) + .setSchemaFile(_schemaFileName).setTableConfigFile(_tableConfigFile).setExecute(true); addTableCommand.execute(); return; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
