HIVE-10623: Implement hive cli options using beeline functionality (Ferdinand via Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/03366392 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/03366392 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/03366392 Branch: refs/heads/master Commit: 03366392c6f17ae7688ad1646046af4c5b5b6792 Parents: 753b2b3 Author: Xuefu Zhang <xzh...@cloudera.com> Authored: Thu May 14 21:33:44 2015 -0700 Committer: Xuefu Zhang <xzh...@cloudera.com> Committed: Thu May 14 21:33:44 2015 -0700 ---------------------------------------------------------------------- .../java/org/apache/hive/beeline/BeeLine.java | 140 ++++++++++++++++- .../hive/beeline/cli/CliOptionsProcessor.java | 104 +++++++++++++ .../org/apache/hive/beeline/cli/HiveCli.java | 41 +++++ .../apache/hive/beeline/cli/TestHiveCli.java | 153 +++++++++++++++++++ beeline/src/test/resources/hive-site.xml | 37 +++++ 5 files changed, 468 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/03366392/beeline/src/java/org/apache/hive/beeline/BeeLine.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 0da15f6..4a82635 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -88,6 +88,8 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.hadoop.io.IOUtils; +import org.apache.hive.beeline.cli.CliOptionsProcessor; +import org.apache.hive.jdbc.Utils; /** * A console SQL shell with command completion. @@ -126,8 +128,10 @@ public class BeeLine implements Closeable { private ConsoleReader consoleReader; private List<String> batch = null; private final Reflector reflector; + private String dbName = null; private History history; + private boolean isBeeLine = true; private static final Options options = new Options(); @@ -491,10 +495,14 @@ public class BeeLine implements Closeable { public BeeLine() { + this(true); + } + + public BeeLine(boolean isBeeLine) { beeLineCommandCompleter = new BeeLineCommandCompleter(BeeLineCommandCompleter.getCompleters (this)); reflector = new Reflector(this); - + this.isBeeLine = isBeeLine; // attempt to dynamically load signal handler /* TODO disable signal handler try { @@ -508,7 +516,6 @@ public class BeeLine implements Closeable { */ } - DatabaseConnection getDatabaseConnection() { return getDatabaseConnections().current(); } @@ -633,7 +640,63 @@ public class BeeLine implements Closeable { super.processOption(arg, iter); } } + } + + int initArgsFromCliVars(String[] args) { + List<String> commands = Collections.emptyList(); + + CliOptionsProcessor optionsProcessor = new CliOptionsProcessor(); + if (!optionsProcessor.process(args)) { + return 1; + } + CommandLine commandLine = optionsProcessor.getCommandLine(); + + + Properties confProps = commandLine.getOptionProperties("hiveconf"); + for (String propKey : confProps.stringPropertyNames()) { + getOpts().getHiveConfVariables().put(propKey, confProps.getProperty(propKey)); + } + + Properties hiveVars = commandLine.getOptionProperties("define"); + for (String propKey : hiveVars.stringPropertyNames()) { + getOpts().getHiveConfVariables().put(propKey, hiveVars.getProperty(propKey)); + } + + Properties hiveVars2 = commandLine.getOptionProperties("hivevar"); + for (String propKey : hiveVars2.stringPropertyNames()) { + getOpts().getHiveConfVariables().put(propKey, hiveVars2.getProperty(propKey)); + } + + getOpts().setScriptFile(commandLine.getOptionValue("f")); + + dbName = commandLine.getOptionValue("database"); + getOpts().setVerbose(Boolean.valueOf(commandLine.getOptionValue("verbose"))); + getOpts().setSilent(Boolean.valueOf(commandLine.getOptionValue("slient"))); + + int code = 0; + if (commandLine.getOptionValues("e") != null) { + commands = Arrays.asList(commandLine.getOptionValues("e")); + } + if (!commands.isEmpty() && getOpts().getScriptFile() != null) { + System.err.println("The '-e' and '-f' options cannot be specified simultaneously"); + optionsProcessor.printCliUsage(); + return 1; + } + + if (!commands.isEmpty()) { + embeddedConnect(); + connectDBInEmbededMode(); + for (Iterator<String> i = commands.iterator(); i.hasNext(); ) { + String command = i.next().toString(); + debug(loc("executing-command", command)); + if (!dispatch(command)) { + code++; + } + } + exit = true; // execute and exit + } + return code; } int initArgs(String[] args) { @@ -687,7 +750,6 @@ public class BeeLine implements Closeable { commands = Arrays.asList(cl.getOptionValues('e')); } - // TODO: temporary disable this for easier debugging /* if (url == null) { @@ -755,9 +817,15 @@ public class BeeLine implements Closeable { } try { - int code = initArgs(args); - if (code != 0) { - return code; + if (isBeeLine) { + int code = initArgs(args); + if (code != 0) { + return code; + } + } else { + int code = initArgsFromCliVars(args); + if (code != 0) + return code; } if (getOpts().getScriptFile() != null) { @@ -788,6 +856,33 @@ public class BeeLine implements Closeable { return ERRNO_OK; } + private int embeddedConnect() { + if (!dispatch("!connect " + Utils.URL_PREFIX + " '' ''")) { + return ERRNO_OTHER; + } else { + return ERRNO_OK; + } + } + + private int connectDBInEmbededMode() { + if (dbName != null && !dbName.isEmpty()) { + if (!dispatch("use " + dbName + ";")) { + return ERRNO_OTHER; + } + } + return ERRNO_OK; + } + + public int defaultConnect(boolean exitOnError) { + if (embeddedConnect() != ERRNO_OK && exitOnError) { + return ERRNO_OTHER; + } + if (connectDBInEmbededMode() != ERRNO_OK && exitOnError) { + return ERRNO_OTHER; + } + return ERRNO_OK; + } + private int executeFile(String fileName) { FileInputStream initStream = null; try { @@ -805,13 +900,20 @@ public class BeeLine implements Closeable { private int execute(ConsoleReader reader, boolean exitOnError) { String line; + if (!isBeeLine) { + if (defaultConnect(exitOnError) != ERRNO_OK && exitOnError) { + return ERRNO_OTHER; + } + } while (!exit) { try { // Execute one instruction; terminate on executing a script if there is an error // in silent mode, prevent the query and prompt being echoed back to terminal line = (getOpts().isSilent() && getOpts().getScriptFile() != null) ? reader.readLine(null, ConsoleReader.NULL_MASK) : reader.readLine(getPrompt()); - + if (!isBeeLine) { + line = cliToBeelineCmd(line); + } if (!dispatch(line) && exitOnError) { return ERRNO_OTHER; } @@ -914,6 +1016,30 @@ public class BeeLine implements Closeable { output(loc("cmd-usage")); } + private String[] tokenizeCmd(String cmd) { + return cmd.split("\\s+"); + } + + public String cliToBeelineCmd(String cmd) { + if (cmd == null) + return null; + String cmd_trimmed = cmd.trim(); + String[] tokens = tokenizeCmd(cmd_trimmed); + + if (cmd_trimmed.equalsIgnoreCase("quit") || cmd_trimmed.equalsIgnoreCase("exit")) { + return null; + } else if (tokens[0].equalsIgnoreCase("source")) { + //TODO + return cmd; + } else if (cmd_trimmed.startsWith("!")) { + String shell_cmd = cmd_trimmed.substring(1); + return "!sh " + shell_cmd; + } else { // local mode + // command like dfs + return cmd; + } + } + /** * Dispatch the specified line to the appropriate {@link CommandHandler}. http://git-wip-us.apache.org/repos/asf/hive/blob/03366392/beeline/src/java/org/apache/hive/beeline/cli/CliOptionsProcessor.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/cli/CliOptionsProcessor.java b/beeline/src/java/org/apache/hive/beeline/cli/CliOptionsProcessor.java new file mode 100644 index 0000000..61c5ab6 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/cli/CliOptionsProcessor.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + +/** + * This class is used for parsing the options of Hive Cli + */ +public class CliOptionsProcessor { + private final Options options = new Options(); + private org.apache.commons.cli.CommandLine commandLine; + + public CliOptionsProcessor() { + // -database database + options.addOption(OptionBuilder.hasArg().withArgName("databasename").withLongOpt("database") + .withDescription("Specify the database to use").create()); + + // -e 'quoted-query-string' + options.addOption(OptionBuilder.hasArg().withArgName("quoted-query-string").withDescription + ("SQL from command line").create('e')); + + // -f <query-file> + options.addOption(OptionBuilder.hasArg().withArgName("filename").withDescription("SQL from " + + "files").create('f')); + + // -i <init-query-file> + options.addOption(OptionBuilder.hasArg().withArgName("filename").withDescription + ("Initialization SQL file").create('i')); + + // -hiveconf x=y + options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("property=value") + .withLongOpt("hiveconf").withDescription("Use value for given property").create()); + + // Substitution option -d, --define + options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("key=value") + .withLongOpt("define").withDescription("Variable subsitution to apply to hive commands. e" + + ".g. -d A=B or --define A=B").create('d')); + + // Substitution option --hivevar + options.addOption(OptionBuilder.withValueSeparator().hasArgs(2).withArgName("key=value") + .withLongOpt("hivevar").withDescription("Variable subsitution to apply to hive commands. " + + "e.g. --hivevar A=B").create()); + + // [-S|--silent] + options.addOption(new Option("S", "silent", false, "Silent mode in interactive shell")); + + // [-v|--verbose] + options.addOption(new Option("v", "verbose", false, "Verbose mode (echo executed SQL to the " + + "console)")); + + // [-H|--help] + options.addOption(new Option("H", "help", false, "Print help information")); + } + + public boolean process(String []argv){ + try { + commandLine = new GnuParser().parse(options, argv); + + if(commandLine.hasOption("help")){ + printCliUsage(); + return false; + } + } catch (ParseException e) { + System.err.println(e.getMessage()); + printCliUsage(); + return false; + } + return true; + } + + public void printCliUsage() { + new HelpFormatter().printHelp("hive", options); + } + + public CommandLine getCommandLine() { + return commandLine; + } + + public void setCommandLine(CommandLine commandLine) { + this.commandLine = commandLine; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/03366392/beeline/src/java/org/apache/hive/beeline/cli/HiveCli.java ---------------------------------------------------------------------- diff --git a/beeline/src/java/org/apache/hive/beeline/cli/HiveCli.java b/beeline/src/java/org/apache/hive/beeline/cli/HiveCli.java new file mode 100644 index 0000000..3e0da77 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/cli/HiveCli.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.cli; + +import org.apache.hive.beeline.BeeLine; +import org.apache.hive.jdbc.Utils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class HiveCli { + private BeeLine beeLine; + + public static void main(String[] args) throws IOException { + int status = new HiveCli().runWithArgs(args, null); + System.exit(status); + } + + public int runWithArgs(String[] cmd, InputStream inputStream) throws IOException { + beeLine = new BeeLine(false); + return beeLine.begin(cmd, inputStream); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/03366392/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java ---------------------------------------------------------------------- diff --git a/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java new file mode 100644 index 0000000..cc0b598 --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/cli/TestHiveCli.java @@ -0,0 +1,153 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.cli; + +import junit.framework.Assert; +import org.apache.commons.io.IOUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; + +public class TestHiveCli { + private static final Log LOG = LogFactory.getLog(TestHiveCli.class.getName()); + + final static String CMD = "create database if not exists test;\ncreate table if not exists test" + + ".testTbl(a " + + "" + "string, b string);\n"; + private HiveCli cli; + private OutputStream os; + private PrintStream ps; + private OutputStream errS; + private PrintStream errPs; + private File tmp = null; + + private void executeCMD(String[] args, String input, int retCode) { + InputStream inputStream = null; + int ret = 0; + try { + if (input != null) { + inputStream = IOUtils.toInputStream(input); + } + ret = cli.runWithArgs(args, inputStream); + } catch (IOException e) { + LOG.error("Failed to execute command due to the error: " + e); + } finally { + if (retCode != ret) { + LOG.error("Failed due to the error:" + errS.toString()); + Assert.fail("Supported return code is " + retCode + " while the actual is " + ret); + } + } + } + + private void verifyCMD(String CMD, String keywords, OutputStream os, String[] options, int + retCode) { + executeCMD(options, CMD, retCode); + String output = os.toString(); + Assert.assertTrue(output.contains(keywords)); + } + + @Test + public void testInValidCmd() { + verifyCMD("!lss\n", "Failed to execute lss", errS, null, 0); + } + + @Test + public void testHelp() { + verifyCMD(null, "usage: hive", os, new String[]{"-H"}, 1); + } + + @Test + public void testInvalidDatabaseOptions() { + verifyCMD("\nshow tables\nquit\n", "Database does not exist: invalidDB", errS, new + String[]{"--database", "invalidDB"}, 0); + } + + @Test + public void testDatabaseOptions() { + verifyCMD("\nshow tables;\nquit;", "testTbl", os, new String[]{"--database", "test"}, 0); + } + + @Test + public void testSqlFromCmd() { + verifyCMD(null, "", os, new String[]{"-e", "show databases;"}, 0); + } + + @Test + public void testSqlFromCmdWithDBName() { + verifyCMD(null, "testTbl", os, new String[]{"-e", "show tables;", "--database", "test"}, 0); + } + + @Test + public void testInvalidOptions() { + verifyCMD(null, "The '-e' and '-f' options cannot be specified simultaneously", errS, new + String[]{"-e", "show tables;", "-f", "path/to/file"}, 1); + } + + @Test + public void testInvalidOptions2() { + verifyCMD(null, "Unrecognized option: -k", errS, new String[]{"-k"}, 1); + } + + private void redirectOutputStream() { + // Setup output stream to redirect output to + os = new ByteArrayOutputStream(); + ps = new PrintStream(os); + errS = new ByteArrayOutputStream(); + errPs = new PrintStream(errS); + System.setOut(ps); + System.setErr(errPs); + } + + private void initFileFromFile() { + BufferedWriter bw = null; + try { + // create a tmp file + tmp = File.createTempFile("test", ".sql"); + bw = new BufferedWriter(new FileWriter(tmp)); + bw.write(CMD); + } catch (IOException e) { + LOG.error("Failed to write tmp file due to the exception: " + e); + } finally { + IOUtils.closeQuietly(bw); + } + executeCMD(new String[]{"-f", "\"" + tmp.getAbsolutePath() + "\""}, null, 0); + } + + @Before + public void setup() { + cli = new HiveCli(); + redirectOutputStream(); + initFileFromFile(); + } + + @After + public void tearDown() { + tmp.delete(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/03366392/beeline/src/test/resources/hive-site.xml ---------------------------------------------------------------------- diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml new file mode 100644 index 0000000..d2df03c --- /dev/null +++ b/beeline/src/test/resources/hive-site.xml @@ -0,0 +1,37 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <property> + <name>hive.in.test</name> + <value>true</value> + <description>Internal marker for test. Used for masking env-dependent values</description> + </property> + <property> + <name>javax.jdo.option.ConnectionURL</name> + <value>jdbc:derby:;databaseName=${test.tmp.dir}/metastore_db;create=true</value> + <description>JDBC connect string for a JDBC metastore</description> + </property> + <property> + <!-- this should eventually be deprecated since the metastore should supply this --> + <name>hive.metastore.warehouse.dir</name> + <value>${test.tmp.dir}/warehouse</value> + <description></description> + </property> +</configuration>
