HIVE-16115: Stop printing progress info from operation logs with beeline progress bar (Anishek Agarwal reviewed by Vaibhav Gumashta)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ba4f6e7b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ba4f6e7b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ba4f6e7b Branch: refs/heads/hive-14535 Commit: ba4f6e7b193ef701ee426fc2b271350c05d4fb5c Parents: 37ed5aa Author: Vaibhav Gumashta <[email protected]> Authored: Mon Mar 13 00:41:02 2017 -0700 Committer: Vaibhav Gumashta <[email protected]> Committed: Mon Mar 13 00:41:02 2017 -0700 ---------------------------------------------------------------------- itests/hive-unit/pom.xml | 12 - .../hive/beeline/TestBeeLineWithArgs.java | 313 ++++++++++++------- .../ql/exec/tez/monitoring/RenderStrategy.java | 22 +- .../org/apache/hive/service/cli/CLIService.java | 8 +- 4 files changed, 218 insertions(+), 137 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ba4f6e7b/itests/hive-unit/pom.xml ---------------------------------------------------------------------- diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index 6a190d1..789192b 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -138,12 +138,6 @@ </dependency> <dependency> <groupId>org.apache.hive</groupId> - <artifactId>hive-jdbc</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> <artifactId>hive-metastore</artifactId> <version>${project.version}</version> <classifier>tests</classifier> @@ -382,12 +376,6 @@ <optional>true</optional> </dependency> <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-api</artifactId> - <version>${hadoop.version}</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.curator</groupId> <artifactId>curator-test</artifactId> <version>${curator.version}</version> http://git-wip-us.apache.org/repos/asf/hive/blob/ba4f6e7b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java index 42ef280..650c4b7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/TestBeeLineWithArgs.java @@ -20,6 +20,7 @@ package org.apache.hive.beeline; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.ByteArrayOutputStream; @@ -34,12 +35,17 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.google.common.base.Function; +import com.google.common.collect.Lists; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -76,6 +82,7 @@ public class TestBeeLineWithArgs { argList.add(userName); return argList; } + /** * Start up a local Hive Server 2 for these tests */ @@ -119,7 +126,7 @@ public class TestBeeLineWithArgs { try { stmt.execute("drop table " + tableName); } catch (Exception ex) { - fail(ex.toString()); + fail(ex.toString() + " " + ExceptionUtils.getStackTrace(ex)); } // create table @@ -150,7 +157,8 @@ public class TestBeeLineWithArgs { * @return The stderr and stdout from running the script * @throws Throwable */ - private String testCommandLineScript(List<String> argList, InputStream inputStream, OutStream streamType) + private static String testCommandLineScript(List<String> argList, InputStream inputStream, + OutStream streamType) throws Throwable { BeeLine beeLine = new BeeLine(); ByteArrayOutputStream os = new ByteArrayOutputStream(); @@ -177,7 +185,7 @@ public class TestBeeLineWithArgs { * Attempt to execute a simple script file with the -f and -i option to * BeeLine to test for presence of an expected pattern in the output (stdout * or stderr), fail if not found. Print PASSED or FAILED - * + * * @param expectedRegex * Text to look for in command output (stdout) * @param shouldMatch @@ -185,9 +193,11 @@ public class TestBeeLineWithArgs { * @throws Exception * on command execution error */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List<String> argList) throws Throwable { - testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, OutStream.OUT); + private void testScriptFile(String scriptText, List<String> argList, String expectedRegex, + boolean shouldMatch) throws Throwable { + testScriptFile(scriptText, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch)) + ); } /** @@ -195,33 +205,40 @@ public class TestBeeLineWithArgs { * to BeeLine to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED - * @param expectedRegex Text to look for in command output (stdout) - * @param shouldMatch true if the pattern should be found, false if it should not * @param argList arguments * @param outType output stream type + * @param expectedRegex Text to look for in command output (stdout) + * @param shouldMatch true if the pattern should be found, false if it should not * @throws Throwable */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List<String> argList, OutStream outType) throws Throwable { - testScriptFile(scriptText, expectedRegex, shouldMatch, argList, true, true, outType); + private void testScriptFile(String scriptText, List<String> argList, OutStream outType, + String expectedRegex, boolean shouldMatch) throws Throwable { + testScriptFile(scriptText, argList, outType, + Collections.singletonList(new Tuple<>(expectedRegex, shouldMatch)) + ); } - + + private void testScriptFile(String scriptText, List<String> argList, OutStream streamType, + List<Tuple<String>> expectedMatches) throws Throwable { + testScriptFile(scriptText, argList, streamType, expectedMatches, + Arrays.asList(Modes.values())); + } + /** * Attempt to execute a simple script file with the -f or -i option * to BeeLine (or both) to test for presence of an expected pattern * in the output (stdout or stderr), fail if not found. * Print PASSED or FAILED - * @param expectedRegex Text to look for in command output/error - * @param shouldMatch true if the pattern should be found, false if it should not - * @param testScript Whether we should test -f - * @param testInit Whether we should test -i + * @param scriptText script to test the output for + * @param argList arguments to be passed to the script file to execute and produce output * @param streamType Whether match should be done against STDERR or STDOUT + * @param expectedMatches List of Tuple's defining the pattern to match and result of matching + * @param modes testing modes we have to run the script as * @throws Exception on command execution error */ - private void testScriptFile(String scriptText, String expectedRegex, - boolean shouldMatch, List<String> argList, - boolean testScript, boolean testInit, OutStream streamType) throws Throwable { - + private void testScriptFile(String scriptText, List<String> argList, + OutStream streamType, List<Tuple<String>> expectedMatches, List<Modes> modes) + throws Throwable { // Put the script content in a temp file File scriptFile = File.createTempFile(this.getClass().getSimpleName(), "temp"); System.out.println("script file is " + scriptFile.getAbsolutePath()); @@ -230,43 +247,59 @@ public class TestBeeLineWithArgs { os.print(scriptText); os.close(); - Pattern expectedPattern = Pattern.compile(".*" + expectedRegex + ".*", Pattern.DOTALL); - if (testScript) { - List<String> copy = new ArrayList<String>(argList); - copy.add("-f"); - copy.add(scriptFile.getAbsolutePath()); - - String output = testCommandLineScript(copy, null, streamType); - - Matcher m = expectedPattern.matcher(output); - boolean matches = m.matches(); - if (shouldMatch != matches) { - //failed - fail("Output" + output + " should" + (shouldMatch ? "" : " not") + - " contain " + expectedRegex); + List<Tuple<Pattern>> patternsToBeMatched = Lists.transform(expectedMatches, + new Function<Tuple<String>, Tuple<Pattern>>() { + @Override + public Tuple<Pattern> apply(Tuple<String> tuple) { + return new Tuple<>( + Pattern.compile(".*" + tuple.pattern + ".*", Pattern.DOTALL), + tuple.shouldMatch + ); + } + }); + + for (Modes mode : modes) { + String output = mode.output(scriptFile, argList, streamType); + for (Tuple<Pattern> patternToMatch : patternsToBeMatched) { + Matcher m = patternToMatch.pattern.matcher(output); + boolean matches = m.matches(); + if (patternToMatch.shouldMatch != matches) { + //failed + fail("Output" + output + " should" + (patternToMatch.shouldMatch ? "" : " not") + + " contain " + patternToMatch.pattern.pattern()); + } } } + scriptFile.delete(); + } - // Not all scripts can be used as init scripts, so we parameterize. - // (scripts that test !connect, for eg., since -i runs after connects) - // So, we keep this optional. Most tests should leave this as true, however. - if (testInit) { - List<String> copy = new ArrayList<String>(argList); - copy.add("-i"); - copy.add(scriptFile.getAbsolutePath()); - - String output = testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType); - Matcher m = expectedPattern.matcher(output); - boolean matches = m.matches(); - if (shouldMatch != matches) { - //failed - fail("Output" + output + " should" + (shouldMatch ? "" : " not") + - " contain " + expectedRegex); + /* + We are testing for both type of modes always so not passing that as a parameter for now + */ + enum Modes { + INIT { + @Override + String output(File scriptFile, List<String> argList, OutStream streamType) throws Throwable { + List<String> copy = new ArrayList<>(argList); + copy.add("-i"); + copy.add(scriptFile.getAbsolutePath()); + return testCommandLineScript(copy, new StringBufferInputStream("!quit\n"), streamType); } - } - scriptFile.delete(); + }, SCRIPT { + @Override + String output(File scriptFile, List<String> argList, OutStream streamType) throws Throwable { + List<String> copy = new ArrayList<>(argList); + copy.add("-f"); + copy.add(scriptFile.getAbsolutePath()); + return testCommandLineScript(copy, null, streamType); + } + }; + + abstract String output(File scriptFile, List<String> argList, OutStream streamType) + throws Throwable; } + /** * Attempt to execute the enclosed query with the -e option to BeeLine * Test for presence of an expected pattern @@ -302,7 +335,7 @@ public class TestBeeLineWithArgs { final String SCRIPT_TEXT = " -- comment has spaces and tabs before it\n # comment has spaces and tabs before it\n"; final String EXPECTED_PATTERN = "cannot recognize input near '<EOF>'"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false); } /** @@ -316,7 +349,7 @@ public class TestBeeLineWithArgs { final String SCRIPT_TEXT = "show databases;\n"; final String EXPECTED_PATTERN = " default "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -328,7 +361,7 @@ public class TestBeeLineWithArgs { final String SCRIPT_TEXT = "show databases;\nshow tables;"; final String EXPECTED_PATTERN = " testbeelinetable1 "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -344,7 +377,7 @@ public class TestBeeLineWithArgs { argList.add("DUMMY_TBL=dummy"); final String SCRIPT_TEXT = "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};"; final String EXPECTED_PATTERN = "dummy"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -355,7 +388,7 @@ public class TestBeeLineWithArgs { final String SCRIPT_TEXT = "create table ${hiveconf:test.hive.table.name} (d int);\nshow tables;\n" + " drop table ${hiveconf:test.hive.table.name};\n"; final String EXPECTED_PATTERN = "dummy"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -384,7 +417,7 @@ public class TestBeeLineWithArgs { + "(${hiveconf:COLUMN_NAME} ${hiveconf:COLUMN_TYPE});" + "\nshow tables;\n drop ${OBJECT} ${TABLE_NAME};\n"; final String EXPECTED_PATTERN = "dummy2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -397,7 +430,7 @@ public class TestBeeLineWithArgs { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "select * from abcdefg01;\nshow databases;\n"; final String EXPECTED_PATTERN = " default "; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, false); } @Test @@ -406,7 +439,7 @@ public class TestBeeLineWithArgs { final String SCRIPT_TEXT = "CREATE\tTABLE IF NOT EXISTS testTabInScriptFile\n(id\tint);\nSHOW TABLES;" + "\ndrop table testTabInScriptFile"; final String EXPECTED_PATTERN = "testTabInScriptFile"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -414,7 +447,11 @@ public class TestBeeLineWithArgs { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!sh echo \"hello world.\" > hw.txt\n!sh cat hw.txt\n!rm hw.txt"; final String EXPECTED_PATTERN = "hello world"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT) + ); } /** @@ -426,7 +463,7 @@ public class TestBeeLineWithArgs { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } /** @@ -438,14 +475,14 @@ public class TestBeeLineWithArgs { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + "!set nullemptystring false\n select null from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "NULL"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } @Test public void testGetVariableValue() throws Throwable { final String SCRIPT_TEXT = "set env:TERM;"; final String EXPECTED_PATTERN = "env:TERM"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL())); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), EXPECTED_PATTERN, true); } /** @@ -457,13 +494,13 @@ public class TestBeeLineWithArgs { @Test public void testNullEmpty() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; + "!set nullemptystring true\n select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "abc,,def"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv2"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -477,7 +514,7 @@ public class TestBeeLineWithArgs { argList.add("--delimiterForDSV=;"); final String EXPECTED_PATTERN = "1;NULL;defg;ab\"c;1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -490,7 +527,7 @@ public class TestBeeLineWithArgs { argList.add("--outputformat=tsv2"); final String EXPECTED_PATTERN = "1\tNULL\tdefg\tab\"c\t1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -503,7 +540,7 @@ public class TestBeeLineWithArgs { argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c\'\t'1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -517,7 +554,7 @@ public class TestBeeLineWithArgs { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV,"false"); final String EXPECTED_PATTERN = "1\tNULL\tdefg\t\"ab\"\"c\"\t\"\"\"aa\"\"\"\t1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -532,7 +569,7 @@ public class TestBeeLineWithArgs { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "'1'\t'NULL'\t'defg'\t'ab\"c'\t'\"aa\"'\t'1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -547,7 +584,7 @@ public class TestBeeLineWithArgs { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1,NULL,defg,\"ab\"\"c\",\"\"\"aa\"\"\",1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -562,7 +599,7 @@ public class TestBeeLineWithArgs { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c','\"aa\"','1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -578,7 +615,7 @@ public class TestBeeLineWithArgs { System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "false"); final String EXPECTED_PATTERN = "1;NULL;defg;\"ab\"\"c\";\"\"\"aa\"\"\";1.0"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); System.setProperty(SeparatedValuesOutputFormat.DISABLE_QUOTING_FOR_SV, "true"); } @@ -593,7 +630,7 @@ public class TestBeeLineWithArgs { argList.add("--outputformat=tsv"); final String EXPECTED_PATTERN = "Format tsv is deprecated, please use tsv2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } /** @@ -607,7 +644,8 @@ public class TestBeeLineWithArgs { argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "Format csv is deprecated, please use csv2"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, true, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true))); } /** @@ -619,10 +657,9 @@ public class TestBeeLineWithArgs { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=csv"); final String EXPECTED_PATTERN = "'1','NULL','defg','ab\"c\','1.0'"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } - private String getFormatTestQuery() { return "set hive.support.concurrency = false;\n" + "select 1, null, 'defg', 'ab\"c', 1.0D from " + tableName + " limit 1 ;\n"; @@ -642,14 +679,14 @@ public class TestBeeLineWithArgs { @Test public void testNullEmptyCmdArg() throws Throwable { final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; + "select 'abc',null,'def' from " + tableName + " limit 1 ;\n"; final String EXPECTED_PATTERN = "'abc','','def'"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--nullemptystring=true"); argList.add("--outputformat=csv"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -694,12 +731,12 @@ public class TestBeeLineWithArgs { beeLine.runCommands( new String[] {"!typeinfo"} ); String output = os.toString("UTF8"); Assert.assertFalse( output.contains("java.lang.NullPointerException") ); - Assert.assertTrue( output.contains("No current connection") ); + assertTrue(output.contains("No current connection")); beeLine.runCommands( new String[] {"!nativesql"} ); output = os.toString("UTF8"); Assert.assertFalse( output.contains("java.lang.NullPointerException") ); - Assert.assertTrue( output.contains("No current connection") ); + assertTrue(output.contains("No current connection")); System.out.println(">>> PASSED " + "testNPE" ); } @@ -709,21 +746,21 @@ public class TestBeeLineWithArgs { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL() + "#D_TBL=dummy_t"); final String SCRIPT_TEXT = "create table ${D_TBL} (d int);\nshow tables;\ndrop table ${D_TBL};\n"; final String EXPECTED_PATTERN = "dummy_t"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testEmbeddedBeelineConnection() throws Throwable{ String embeddedJdbcURL = Utils.URL_PREFIX+"/Default"; List<String> argList = getBaseArgs(embeddedJdbcURL); - argList.add("--hivevar"); + argList.add("--hivevar"); argList.add("DUMMY_TBL=embedded_table"); // Set to non-zk lock manager to avoid trying to connect to zookeeper final String SCRIPT_TEXT = "set hive.lock.manager=org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager;\n" + - "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n"; + "create table ${DUMMY_TBL} (d int);\nshow tables;\n drop table ${DUMMY_TBL};\n"; final String EXPECTED_PATTERN = "embedded_table"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -732,12 +769,19 @@ public class TestBeeLineWithArgs { */ @Test public void testQueryProgress() throws Throwable { - final String SCRIPT_TEXT = "set hive.support.concurrency = false;\n" + - "select count(*) from " + tableName + ";\n"; + final String SCRIPT_TEXT = + "set hive.support.concurrency = false;\n" + + "set hive.server2.logging.operation.level=execution;\n" + + "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information - final String EXPECTED_PATTERN = "Number of reducers determined to be.*ELAPSED TIME"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()), - OutStream.ERR); + final String EXPECTED_PATTERN = "ELAPSED TIME"; + final String UNEXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)"; + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + Arrays.asList( + new Tuple<>(EXPECTED_PATTERN, true), + new Tuple<>(UNEXPECTED_PATTERN, false) + ) + ); } /** @@ -757,8 +801,9 @@ public class TestBeeLineWithArgs { "select count(*) from " + tableName + ";\n"; // Check for part of log message as well as part of progress information final String EXPECTED_PATTERN = "Number of reducers determined to be."; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, getBaseArgs(miniHS2.getBaseJdbcURL()), - OutStream.ERR); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + EXPECTED_PATTERN, true + ); } /** @@ -771,7 +816,23 @@ public class TestBeeLineWithArgs { "!set silent true\n" + "select count(*) from " + tableName + ";\n"; final String EXPECTED_PATTERN = "Executing command"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, false, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR); + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + EXPECTED_PATTERN, false); + } + + @Test + public void testQueryProgressWithHiveServer2ProgressBarDisabled() + throws Throwable { + final String SCRIPT_TEXT = + "set hive.support.concurrency = false;\nset hive.server2.in.place.progress=false;\n" + + "select count(*) from " + tableName + ";\n"; + // Check for part of log message as well as part of progress information + final String EXPECTED_PATTERN = "(?=Reducer 2\\:).*(?=Map 1\\:)"; + testScriptFile(SCRIPT_TEXT, getBaseArgs(miniHS2.getBaseJdbcURL()), OutStream.ERR, + Arrays.asList( + new Tuple<>(EXPECTED_PATTERN, true), + new Tuple<>("ELAPSED TIME", false)) + ); } @Test @@ -780,10 +841,10 @@ public class TestBeeLineWithArgs { +"(key int);show tables; --multicommands in one line"; final String EXPECTED_PATTERN = " multicmdtbl "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table multiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -804,10 +865,10 @@ public class TestBeeLineWithArgs { + "(key int);show tables; --one command in multiple lines"; final String EXPECTED_PATTERN = " multicmdtbl "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -817,10 +878,10 @@ public class TestBeeLineWithArgs { + " TERMINATED BY '\\n';show tables; --one command in multiple lines"; final String EXPECTED_PATTERN = " multicmdtbl "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); final String SCRIPT_TEXT_DROP = "drop table\nmultiCmdTbl;show tables;"; - testScriptFile(SCRIPT_TEXT_DROP, EXPECTED_PATTERN, false, argList); + testScriptFile(SCRIPT_TEXT_DROP, argList, EXPECTED_PATTERN, false); } @Test @@ -847,7 +908,7 @@ public class TestBeeLineWithArgs { + "set a=1;\nselect count(*) from embeddedBeelineOutputs;\n" + "drop table embeddedBeelineOutputs;\n"; final String EXPECTED_PATTERN = "Stage-1 map ="; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); } @Test @@ -855,7 +916,7 @@ public class TestBeeLineWithArgs { List<String> argList = getBaseArgs(miniHS2.getJdbcURL("default", "sess_var_list?var1=value1")); final String SCRIPT_TEXT = "set var1"; final String EXPECTED_PATTERN = "var1=value1"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -865,7 +926,7 @@ public class TestBeeLineWithArgs { @Test public void testBeelineConnectEnvVar() throws Throwable { final String jdbcUrl = miniHS2.getBaseJdbcURL(); - List<String> argList = new ArrayList<String>(); + List<String> argList = new ArrayList<>(); argList.add("-u"); argList.add("blue"); argList.add("-d"); @@ -892,7 +953,9 @@ public class TestBeeLineWithArgs { }; BeeLineOpts.setEnv(newEnv); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } /** @@ -904,11 +967,13 @@ public class TestBeeLineWithArgs { List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); final String SCRIPT_TEXT = "!close\n" + - "!reconnect\n\n\n" + - "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n"; + "!reconnect\n\n\n" + + "create table reconnecttest (d int);\nshow tables;\ndrop table reconnecttest;\n"; final String EXPECTED_PATTERN = "reconnecttest"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } @@ -921,14 +986,14 @@ public class TestBeeLineWithArgs { @Test public void testConnectionWithURLParams() throws Throwable { final String EXPECTED_PATTERN = " hivetest "; - List<String> argList = new ArrayList<String>(); + List<String> argList = new ArrayList<>(); argList.add("-d"); argList.add(BeeLine.BEELINE_DEFAULT_JDBC_DRIVER); argList.add("-u"); argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive"); String SCRIPT_TEXT = "select current_user();"; - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -937,10 +1002,10 @@ public class TestBeeLineWithArgs { @Test public void testQueryNonEscapedSemiColon() throws Throwable { String SCRIPT_TEXT = "drop table if exists nonEscapedSemiColon;create table nonEscapedSemiColon " - + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;"; + + "(key int) ROW FORMAT DELIMITED FIELDS TERMINATED BY ';';show tables;"; final String EXPECTED_PATTERN = " nonEscapedSemiColon "; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test @@ -949,7 +1014,7 @@ public class TestBeeLineWithArgs { final String EXPECTED_PATTERN = ";\t';'\t\";\"\t';\t;'\t\";\t;\""; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--outputformat=tsv2"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } /** @@ -961,21 +1026,23 @@ public class TestBeeLineWithArgs { @Test public void testShowDbInPrompt() throws Throwable { final String EXPECTED_PATTERN = " \\(default\\)>"; - List<String> argList = new ArrayList<String>(); + List<String> argList = new ArrayList<>(); argList.add("--showDbInPrompt"); argList.add("-u"); argList.add(miniHS2.getBaseJdbcURL() + ";user=hivetest;password=hive"); String SCRIPT_TEXT = "select current_user();"; - testScriptFile( SCRIPT_TEXT, EXPECTED_PATTERN, true, argList); + testScriptFile(SCRIPT_TEXT, argList, EXPECTED_PATTERN, true); } @Test public void testBeelineShellCommandWithoutConn() throws Throwable { - List<String> argList = new ArrayList<String>(); + List<String> argList = new ArrayList<>(); final String SCRIPT_TEXT = "!sh echo hello world"; final String EXPECTED_PATTERN = "hello world"; - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, true, false, OutStream.OUT); + testScriptFile(SCRIPT_TEXT, argList, OutStream.OUT, + Collections.singletonList(new Tuple<>(EXPECTED_PATTERN, true)), + Collections.singletonList(Modes.SCRIPT)); } /** @@ -985,11 +1052,21 @@ public class TestBeeLineWithArgs { @Test public void testBeelineWithForce() throws Throwable { final String SCRIPT_TEXT = "drop table does_not_exist;\ncreate table incomplete_syntax(a, string, );\n " - + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n " - + "desc new_table;\n"; + + "drop table if exists new_table;\n create table new_table(foo int, bar string);\n " + + "desc new_table;\n"; final String EXPECTED_PATTERN = "2 rows selected"; List<String> argList = getBaseArgs(miniHS2.getBaseJdbcURL()); argList.add("--force"); - testScriptFile(SCRIPT_TEXT, EXPECTED_PATTERN, true, argList, OutStream.ERR); + testScriptFile(SCRIPT_TEXT, argList, OutStream.ERR, EXPECTED_PATTERN, true); + } + + private static class Tuple<K> { + final K pattern; + final boolean shouldMatch; + + Tuple(K pattern, boolean shouldMatch) { + this.pattern = pattern; + this.shouldMatch = shouldMatch; + } } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba4f6e7b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java index bb9a5e7..2535b10 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/RenderStrategy.java @@ -2,10 +2,13 @@ package org.apache.hadoop.hive.ql.exec.tez.monitoring; import org.apache.hadoop.hive.common.log.InPlaceUpdate; import org.apache.hadoop.hive.common.log.ProgressMonitor; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.tez.dag.api.client.DAGStatus; import org.apache.tez.dag.api.client.Progress; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.StringWriter; import java.util.Map; @@ -48,10 +51,17 @@ class RenderStrategy { || System.currentTimeMillis() >= lastPrintTime + PRINT_INTERVAL; } + /* + This is used to print the progress information as pure text , a sample is as below: + Map 1: 0/1 Reducer 2: 0/1 + Map 1: 0(+1)/1 Reducer 2: 0/1 + Map 1: 1/1 Reducer 2: 0(+1)/1 + Map 1: 1/1 Reducer 2: 1/1 + */ private String getReport(Map<String, Progress> progressMap) { StringWriter reportBuffer = new StringWriter(); - SortedSet<String> keys = new TreeSet<String>(progressMap.keySet()); + SortedSet<String> keys = new TreeSet<>(progressMap.keySet()); for (String s : keys) { Progress progress = progressMap.get(s); final int complete = progress.getSucceededTaskCount(); @@ -109,7 +119,9 @@ class RenderStrategy { * same information to beeline client when requested. */ static class LogToFileFunction extends BaseUpdateFunction { - + private static final Logger LOGGER = LoggerFactory.getLogger(LogToFileFunction.class); + private boolean hiveServer2InPlaceProgressEnabled = + SessionState.get().getConf().getBoolVar(HiveConf.ConfVars.HIVE_SERVER2_INPLACE_PROGRESS); LogToFileFunction(TezJobMonitor monitor) { super(monitor); } @@ -121,7 +133,11 @@ class RenderStrategy { @Override public void renderReport(String report) { - monitor.console.printInfo(report); + if (hiveServer2InPlaceProgressEnabled) { + LOGGER.info(report); + } else { + monitor.console.printInfo(report); + } } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba4f6e7b/service/src/java/org/apache/hive/service/cli/CLIService.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java b/service/src/java/org/apache/hive/service/cli/CLIService.java index 5bb4ee0..689b948 100644 --- a/service/src/java/org/apache/hive/service/cli/CLIService.java +++ b/service/src/java/org/apache/hive/service/cli/CLIService.java @@ -442,8 +442,8 @@ public class CLIService extends CompositeService implements ICLIService { * we block for a duration determined by a step function, before we return * However, if the background operation is complete, we return immediately. */ + HiveConf conf = operation.getParentSession().getHiveConf(); if (operation.shouldRunAsync()) { - HiveConf conf = operation.getParentSession().getHiveConf(); long maxTimeout = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_SERVER2_LONG_POLLING_TIMEOUT, TimeUnit.MILLISECONDS); @@ -472,7 +472,7 @@ public class CLIService extends CompositeService implements ICLIService { } OperationStatus opStatus = operation.getStatus(); LOG.debug(opHandle + ": getOperationStatus()"); - opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation)); + opStatus.setJobProgressUpdate(progressUpdateLog(getProgressUpdate, operation, conf)); return opStatus; } @@ -482,8 +482,8 @@ public class CLIService extends CompositeService implements ICLIService { } private static final long PROGRESS_MAX_WAIT_NS = 30 * 1000000000l; - private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation) { - if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(hiveConf) + private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation, HiveConf conf) { + if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(conf) || !OperationType.EXECUTE_STATEMENT.equals(operation.getType())) { return new JobProgressUpdate(ProgressMonitor.NULL); }
