TAJO-813: CLI should support comment character with multi-line query. (Hyoungjun Kim via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/8e523c12 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/8e523c12 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/8e523c12 Branch: refs/heads/window_function Commit: 8e523c12e50ff660b09b4d6fa289ea617e08fbfa Parents: eb4e54a Author: Hyunsik Choi <[email protected]> Authored: Thu May 8 20:15:53 2014 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Thu May 8 20:15:53 2014 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../java/org/apache/tajo/cli/ParsedResult.java | 10 +- .../java/org/apache/tajo/cli/SimpleParser.java | 179 ++++++++++++++++--- .../main/java/org/apache/tajo/cli/TajoCli.java | 6 +- .../org/apache/tajo/jdbc/TajoResultSetBase.java | 2 +- .../java/org/apache/tajo/QueryTestCaseBase.java | 8 +- .../org/apache/tajo/cli/TestSimpleParser.java | 120 +++++++++++-- .../apache/tajo/engine/eval/ExprTestBase.java | 4 +- 8 files changed, 278 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 3c5237e..0620893 100644 --- a/CHANGES +++ b/CHANGES @@ -31,6 +31,9 @@ Release 0.9.0 - unreleased BUG FIXES + TAJO-813: CLI should support comment character with multi-line query. + (Hyoungjun Kim via hyunsik) + TAJO-800: CLI's meta command should be aware "TABLE_NAME" style. (Hyoungjun Kim via hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java b/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java index fb89678..001aded 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java @@ -26,22 +26,28 @@ public class ParsedResult { } private final StatementType type; + private final String historyStatement; private final String statement; - public ParsedResult(StatementType type, String statement) { + public ParsedResult(StatementType type, String statement, String historyStatement) { this.type = type; this.statement = statement; + this.historyStatement = historyStatement; } public StatementType getType() { return type; } + public String getHistoryStatement() { + return historyStatement.trim(); + } + public String getStatement() { return statement.trim(); } public String toString() { - return "(" + type.name() + ") " + statement; + return "(" + type.name() + ") " + historyStatement; } } http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java b/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java index 61f6d2a..afb8a59 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java @@ -36,7 +36,6 @@ public class SimpleParser { META, // Meta Command STATEMENT, // Statement WITHIN_QUOTE, // Within Quote - COMMENT, INVALID, // Invalid Statement STATEMENT_EOS, // End State (End of Statement) META_EOS // End State (End of Statement) @@ -44,7 +43,16 @@ public class SimpleParser { ParsingState state = START_STATE; int lineNum; - StringBuilder appender = new StringBuilder(); + + /** + * It will be used to store a query statement into Jline history. + * the query statement for history does not include unnecessary white spaces and new line. + */ + private StringBuilder historyAppender = new StringBuilder(); + /** + * It will be used to submit a query statement to the TajoMaster. It just contains a raw query statement string. + */ + private StringBuilder rawAppender = new StringBuilder(); public static final ParsingState START_STATE = ParsingState.TOK_START; @@ -80,6 +88,12 @@ public class SimpleParser { int idx = 0; char [] chars = str.toCharArray(); + // if parsing continues, it means that the previous line is broken by '\n'. + // So, we should add new line to rawAppender. + if (isStatementContinue()) { + rawAppender.append("\n"); + } + while(idx < str.length()) { // initialization for new statement @@ -107,30 +121,31 @@ public class SimpleParser { //////////////////////////// while (state != ParsingState.META_EOS && idx < chars.length) { char character = chars[idx++]; - if (Character.isWhitespace(character)) { - // skip - } else if (isEndOfMeta(character)) { + + if (isEndOfMeta(character)) { state = ParsingState.META_EOS; + } else if (Character.isWhitespace(character)) { + // skip } } if (state == ParsingState.META_EOS) { - appender.append(str.subSequence(lineStartIdx, idx - 1).toString()); + historyAppender.append(str.subSequence(lineStartIdx, idx - 1).toString()); + appendToRawStatement(str.subSequence(lineStartIdx, idx - 1).toString(), true); } else { - appender.append(str.subSequence(lineStartIdx, idx).toString()); + historyAppender.append(str.subSequence(lineStartIdx, idx).toString()); + appendToRawStatement(str.subSequence(lineStartIdx, idx).toString(), true); } - } else if (isCommentStart(chars[idx])) { - idx++; - while (!isLineEnd(chars[idx]) && idx < chars.length) { - idx++; - } + } else if (isInlineCommentStart(chars, idx)) { + idx = consumeInlineComment(chars, idx); + appendToRawStatement(str.subSequence(lineStartIdx, idx).toString(), true); + ///////////////////////////////// // TOK_START -> STATEMENT // or TOK_STATEMENT -> STATEMENT //////////////////////////////// } else if (isStatementContinue() || isStatementStart(chars[idx])) { - int endIdx = 0; if (!isStatementContinue()) { // TOK_START -> STATEMENT state = ParsingState.STATEMENT; } @@ -138,9 +153,16 @@ public class SimpleParser { while (!isTerminateState(state) && idx < chars.length) { char character = chars[idx++]; + /////////////////////////////////////////////////////// + // in-statement loop BEGIN + /////////////////////////////////////////////////////// if (isEndOfStatement(character)) { state = ParsingState.STATEMENT_EOS; - endIdx = idx - 1; + + } else if (state == ParsingState.STATEMENT && character == '\n') { + appendToBothStatements(chars, lineStartIdx, idx, 1); // omit new line chacter '\n' from history statement + lineStartIdx = idx; + } else if (state == ParsingState.STATEMENT && character == '\'') { // TOK_STATEMENT -> WITHIN_QUOTE state = ParsingState.WITHIN_QUOTE; @@ -149,7 +171,21 @@ public class SimpleParser { } else { continue; } + + + // idx points the characters followed by the current character. So, we should use 'idx - 1' + // in order to point the current character. + } else if (state == ParsingState.STATEMENT && idx < chars.length && isInlineCommentStart(chars, idx - 1)) { + idx++; + appendToBothStatements(chars, lineStartIdx, idx, 2); // omit two dash characters '--' from history statement + int commentStartIdx = idx; + idx = consumeInlineComment(chars, idx); + appendToRawStatement(str.subSequence(commentStartIdx, idx).toString(), true); + lineStartIdx = idx; } + /////////////////////////////////////////////////////// + // in-statement loop END + /////////////////////////////////////////////////////// if (state == ParsingState.WITHIN_QUOTE) { while(idx < chars.length) { @@ -168,14 +204,17 @@ public class SimpleParser { } } - if (state == ParsingState.STATEMENT_EOS) { - appender.append(str.subSequence(lineStartIdx, endIdx).toString()); + // After all characters are consumed + + if (state == ParsingState.STATEMENT_EOS) { // If one query statement is terminated + appendToBothStatements(chars, lineStartIdx, idx - 1); // skip semicolon (;) } else { - appender.append(str.subSequence(lineStartIdx, idx).toString()); + appendToBothStatements(chars, lineStartIdx, idx); - // if it is not within quote and there is no space between lines, add a space. - if (state == ParsingState.STATEMENT && (appender.charAt(appender.length() - 1) != ' ')) { - appender.append(" "); + // if it is not within quote and there is no space between lines, adds a space. + if (state == ParsingState.STATEMENT && (historyAppender.charAt(historyAppender.length() - 1) != ' ')) { + historyAppender.append(" "); + rawAppender.append("\n"); } } } else { // skip unknown character @@ -189,6 +228,65 @@ public class SimpleParser { return statements; } + /** + * Append the range of characters into a given StringBuilder instance. + * + * @param chars Characters + * @param fromIdx start character index + * @param toIdx end character index + */ + private void appendToStatement(StringBuilder builder, char[] chars, int fromIdx, int toIdx) { + builder.append(chars, fromIdx, toIdx - fromIdx); + } + + /** + * Append the range of characters into both history and raw appenders. It omits the number of characters specified by + * <code>omitCharNums</code>. + * + * + * @param chars Characters + * @param fromIdx start character index + * @param toIdx end character index + * @param omitCharNums how many characters will be omitted from history statement + */ + private void appendToBothStatements(char[] chars, int fromIdx, int toIdx, int omitCharNums) { + appendToStatement(historyAppender, chars, fromIdx, toIdx - omitCharNums); + if (historyAppender.charAt(historyAppender.length() - 1) != ' ') { + historyAppender.append(" "); + } + appendToStatement(rawAppender, chars, fromIdx, toIdx); + } + + /** + * Append the range of characters into both history and raw appenders. + * + * + * @param chars Characters + * @param fromIdx start character index + * @param toIdx end character index + */ + private void appendToBothStatements(char[] chars, int fromIdx, int toIdx) { + historyAppender.append(chars, fromIdx, toIdx - fromIdx); + rawAppender.append(chars, fromIdx, toIdx - fromIdx); + } + + private int consumeInlineComment(char [] chars, int currentIdx) { + currentIdx++; + while (currentIdx < chars.length && !isNewLine(chars[currentIdx])) { + currentIdx++; + } + return currentIdx; + } + + private void appendToRawStatement(String str, boolean addLF) { + if (!str.isEmpty() && !"\n".equals(str) && + rawAppender.length() > 0 && addLF && rawAppender.charAt(rawAppender.length() - 1) != '\n') { + rawAppender.append(str); + } else { + rawAppender.append(str); + } + } + private static boolean isEndOfMeta(char character) { return character == ';' || character == '\n'; } @@ -197,11 +295,21 @@ public class SimpleParser { return character == ';'; } - private boolean isCommentStart(char character) { - return state == ParsingState.TOK_START && character == '-'; + /** + * It checks if inline comment '--' begins. + * @param chars + * @param idx + * @return + */ + private boolean isInlineCommentStart(char[] chars, int idx) { + if (idx >= chars.length - 1) { + return false; + } + return (state == ParsingState.STATEMENT || state == ParsingState.TOK_START) && + (chars[idx] == '-' && chars[idx + 1] == '-'); } - private boolean isLineEnd(char character) { + private boolean isNewLine(char character) { return character == '\n'; } @@ -213,6 +321,13 @@ public class SimpleParser { return state == ParsingState.WITHIN_QUOTE || state == ParsingState.STATEMENT; } + /** + * process all parsed statements so far and return a list of parsed results. + * + * @param endOfFile TRUE if the end of file. + * @return the list of parsed results, each of result contains one query statement or meta command. + * @throws InvalidStatementException + */ private List<ParsedResult> doProcessEndOfStatement(boolean endOfFile) throws InvalidStatementException { List<ParsedResult> parsedResults = new ArrayList<ParsedResult>(); String errorMessage = ""; @@ -228,24 +343,32 @@ public class SimpleParser { } if (isTerminateState(state)) { - String statement = appender.toString(); + String historyStatement = historyAppender.toString(); + String rawStatement = rawAppender.toString(); if (state == ParsingState.META_EOS) { - parsedResults.add(new ParsedResult(META, statement)); + parsedResults.add(new ParsedResult(META, rawStatement, historyStatement)); state = ParsingState.TOK_START; } else if (state == ParsingState.STATEMENT_EOS) { - parsedResults.add(new ParsedResult(STATEMENT, statement)); + parsedResults.add(new ParsedResult(STATEMENT, rawStatement, historyStatement)); } else { throw new InvalidStatementException("ERROR: " + errorMessage); } // reset all states - appender.delete(0, appender.length()); + historyAppender.delete(0, historyAppender.length()); + rawAppender.delete(0, rawAppender.length()); state = START_STATE; } return parsedResults; } + /** + * It manually triggers the end of file. + * + * @return the list of parsed results, each of result contains one query statement or meta command. + * @throws InvalidStatementException + */ public List<ParsedResult> EOF() throws InvalidStatementException { return doProcessEndOfStatement(true); } @@ -259,6 +382,6 @@ public class SimpleParser { } public String toString() { - return "[" + state.name() + "]: " + appender.toString(); + return "[" + state.name() + "]: " + historyAppender.toString(); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java index 2f9d5cf..7489351 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java @@ -305,7 +305,7 @@ public class TajoCli { if (parsedResults.size() > 0) { for (ParsedResult parsed : parsedResults) { - history.addStatement(parsed.getStatement() + (parsed.getType() == STATEMENT ? ";":"")); + history.addStatement(parsed.getHistoryStatement() + (parsed.getType() == STATEMENT ? ";":"")); } } executeParsedResults(parsedResults); @@ -471,11 +471,11 @@ public class TajoCli { private void printUsage() { HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp( "tsql [options] [database]", options ); + formatter.printHelp("tsql [options] [database]", options); } private void printInvalidCommand(String command) { - sout.println("Invalid command " + command +". Try \\? for help."); + sout.println("Invalid command " + command + ". Try \\? for help."); } public void close() { http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java index b3c53e1..4c307b3 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java @@ -493,7 +493,7 @@ public abstract class TajoResultSetBase implements ResultSet { @Override public Statement getStatement() throws SQLException { - throw new SQLFeatureNotSupportedException("getStatement not supported"); + throw new SQLFeatureNotSupportedException("getHistoryStatement not supported"); } @Override http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index bf5891a..3843c58 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -253,7 +253,7 @@ public class QueryTestCaseBase { if (parsedResults.size() > 1) { assertNotNull("This script \"" + queryFileName + "\" includes two or more queries"); } - ResultSet result = client.executeQueryAndGetResult(parsedResults.get(0).getStatement()); + ResultSet result = client.executeQueryAndGetResult(parsedResults.get(0).getHistoryStatement()); assertNotNull("Query succeeded test", result); return result; } @@ -470,13 +470,13 @@ public class QueryTestCaseBase { for (ParsedResult parsedResult : parsedResults) { // parse a statement - Expr expr = sqlParser.parse(parsedResult.getStatement()); + Expr expr = sqlParser.parse(parsedResult.getHistoryStatement()); assertNotNull(ddlFilePath + " cannot be parsed", expr); if (expr.getType() == OpType.CreateTable) { CreateTable createTable = (CreateTable) expr; String tableName = createTable.getTableName(); - assertTrue("Table [" + tableName + "] creation is failed.", client.updateQuery(parsedResult.getStatement())); + assertTrue("Table [" + tableName + "] creation is failed.", client.updateQuery(parsedResult.getHistoryStatement())); TableDesc createdTable = client.getTableDesc(tableName); String createdTableName = createdTable.getName(); @@ -491,7 +491,7 @@ public class QueryTestCaseBase { String tableName = dropTable.getTableName(); assertTrue("table '" + tableName + "' existence check", client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName))); - assertTrue("table drop is failed.", client.updateQuery(parsedResult.getStatement())); + assertTrue("table drop is failed.", client.updateQuery(parsedResult.getHistoryStatement())); assertFalse("table '" + tableName + "' dropped check", client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName))); if (isLocalTable) { http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java index 9c02b65..5b5057b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java +++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java @@ -47,68 +47,151 @@ public class TestSimpleParser { List<ParsedResult> res1 = SimpleParser.parseScript("\\d"); assertEquals(1, res1.size()); assertEquals(ParsedResult.StatementType.META, res1.get(0).getType()); - assertEquals("\\d", res1.get(0).getStatement()); + assertEquals("\\d", res1.get(0).getHistoryStatement()); List<ParsedResult> res2 = SimpleParser.parseScript("\\d;\\c;\\f;"); assertEquals(3, res2.size()); assertEquals(ParsedResult.StatementType.META, res2.get(0).getType()); - assertEquals("\\d", res2.get(0).getStatement()); + assertEquals("\\d", res2.get(0).getHistoryStatement()); assertEquals(ParsedResult.StatementType.META, res2.get(1).getType()); - assertEquals("\\c", res2.get(1).getStatement()); + assertEquals("\\c", res2.get(1).getHistoryStatement()); assertEquals(ParsedResult.StatementType.META, res2.get(2).getType()); - assertEquals("\\f", res2.get(2).getStatement()); + assertEquals("\\f", res2.get(2).getHistoryStatement()); List<ParsedResult> res3 = SimpleParser.parseScript("\n\t\t \\d;\n\\c;\t\t\\f ;"); assertEquals(3, res3.size()); assertEquals(ParsedResult.StatementType.META, res3.get(0).getType()); - assertEquals("\\d", res3.get(0).getStatement()); + assertEquals("\\d", res3.get(0).getHistoryStatement()); assertEquals(ParsedResult.StatementType.META, res3.get(1).getType()); - assertEquals("\\c", res3.get(1).getStatement()); + assertEquals("\\c", res3.get(1).getHistoryStatement()); assertEquals(ParsedResult.StatementType.META, res3.get(2).getType()); - assertEquals("\\f", res3.get(2).getStatement()); + assertEquals("\\f", res3.get(2).getHistoryStatement()); List<ParsedResult> res4 = SimpleParser.parseScript("\\\td;"); assertEquals(1, res4.size()); - assertEquals("\\\td", res4.get(0).getStatement()); + assertEquals("\\\td", res4.get(0).getHistoryStatement()); } @Test - public final void testStatements() throws InvalidStatementException { + public final void testParseScript() throws InvalidStatementException { List<ParsedResult> res1 = SimpleParser.parseScript("select * from test;"); assertEquals(1, res1.size()); assertEquals(ParsedResult.StatementType.STATEMENT, res1.get(0).getType()); assertEquals("select * from test", res1.get(0).getStatement()); + assertEquals("select * from test", res1.get(0).getHistoryStatement()); List<ParsedResult> res2 = SimpleParser.parseScript("select * from test;"); assertEquals(1, res2.size()); assertEquals(ParsedResult.StatementType.STATEMENT, res2.get(0).getType()); assertEquals("select * from test", res2.get(0).getStatement()); + assertEquals("select * from test", res2.get(0).getHistoryStatement()); List<ParsedResult> res3 = SimpleParser.parseScript("select * from test1;select * from test2;"); assertEquals(2, res3.size()); assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(0).getType()); assertEquals("select * from test1", res3.get(0).getStatement()); + assertEquals("select * from test1", res3.get(0).getHistoryStatement()); assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(1).getType()); assertEquals("select * from test2", res3.get(1).getStatement()); + assertEquals("select * from test2", res3.get(1).getHistoryStatement()); List<ParsedResult> res4 = SimpleParser.parseScript("\t\t\n\rselect * from \ntest1;select * from test2\n;"); assertEquals(2, res4.size()); assertEquals(ParsedResult.StatementType.STATEMENT, res4.get(0).getType()); assertEquals("select * from \ntest1", res4.get(0).getStatement()); + assertEquals("select * from test1", res4.get(0).getHistoryStatement()); assertEquals(ParsedResult.StatementType.STATEMENT, res4.get(1).getType()); assertEquals("select * from test2", res4.get(1).getStatement()); + assertEquals("select * from test2", res4.get(1).getHistoryStatement()); List<ParsedResult> res5 = SimpleParser.parseScript("\t\t\n\rselect * from \ntest1;\\d test;select * from test2;\n\nselect 1;"); assertEquals(4, res5.size()); assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(0).getType()); assertEquals("select * from \ntest1", res5.get(0).getStatement()); + assertEquals("select * from test1", res5.get(0).getHistoryStatement()); assertEquals(ParsedResult.StatementType.META, res5.get(1).getType()); assertEquals("\\d test", res5.get(1).getStatement()); assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(2).getType()); assertEquals("select * from test2", res5.get(2).getStatement()); + assertEquals("select * from test2", res5.get(2).getHistoryStatement()); assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(3).getType()); assertEquals("select 1", res5.get(3).getStatement()); + assertEquals("select 1", res5.get(3).getHistoryStatement()); + + List<ParsedResult> res6 = + SimpleParser.parseScript("select * from \n--test1; select * from test2;\ntest3;"); + assertEquals(1, res6.size()); + assertEquals("select * from test3", res6.get(0).getHistoryStatement()); + assertEquals("select * from \n--test1; select * from test2;\ntest3", res6.get(0).getStatement()); + + List<ParsedResult> res7 = + SimpleParser.parseScript("select * from --test1; select * from test2;\ntest3;"); + assertEquals(1, res7.size()); + assertEquals("select * from test3", res7.get(0).getHistoryStatement()); + assertEquals("select * from --test1; select * from test2;\ntest3", res7.get(0).getStatement()); + + List<ParsedResult> res8 = SimpleParser.parseScript("\\d test\nselect * \n--from test1;\nfrom test2;\\d test2;"); + assertEquals(3, res8.size()); + assertEquals(ParsedResult.StatementType.META, res8.get(0).getType()); + assertEquals("\\d test", res8.get(0).getStatement()); + assertEquals("\\d test", res8.get(0).getHistoryStatement()); + assertEquals(ParsedResult.StatementType.STATEMENT, res8.get(1).getType()); + assertEquals("select * \n--from test1;\nfrom test2", res8.get(1).getStatement()); + assertEquals("select * from test2", res8.get(1).getHistoryStatement()); + assertEquals(ParsedResult.StatementType.META, res8.get(2).getType()); + assertEquals("\\d test2", res8.get(2).getStatement()); + assertEquals("\\d test2", res8.get(2).getHistoryStatement()); + } + + @Test + public final void testParseLines() throws InvalidStatementException { + SimpleParser simpleParser = new SimpleParser(); + List<ParsedResult> res1 = null; + + res1 = simpleParser.parseLines("select * from test1; select * from test2;"); + assertEquals(2, res1.size()); + assertEquals("select * from test1", res1.get(0).getStatement()); + assertEquals("select * from test2", res1.get(1).getStatement()); + assertEquals("select * from test1", res1.get(0).getHistoryStatement()); + assertEquals("select * from test2", res1.get(1).getHistoryStatement()); + + simpleParser = new SimpleParser(); + res1 = simpleParser.parseLines("select * from "); + assertEquals(0, res1.size()); + res1 = simpleParser.parseLines("test1; select * from test2;"); + assertEquals(2, res1.size()); + assertEquals("select * from \ntest1", res1.get(0).getStatement()); + assertEquals("select * from test2", res1.get(1).getStatement()); + assertEquals("select * from test1", res1.get(0).getHistoryStatement()); + assertEquals("select * from test2", res1.get(1).getHistoryStatement()); + + // select * from + // --test1; select * from test2; + // test3; + simpleParser = new SimpleParser(); + res1 = simpleParser.parseLines("select * from "); + assertEquals(0, res1.size()); + res1 = simpleParser.parseLines("--test1; select * from test2;"); + assertEquals(0, res1.size()); + res1 = simpleParser.parseLines("test3;"); + assertEquals(1, res1.size()); + assertEquals("select * from test3", res1.get(0).getHistoryStatement()); + assertEquals("select * from \n--test1; select * from test2;\ntest3", res1.get(0).getStatement()); + + + // select * from + // test1 --select * from test2; + // where col1 = '123'; + simpleParser = new SimpleParser(); + res1 = simpleParser.parseLines("select * from "); + assertEquals(0, res1.size()); + res1 = simpleParser.parseLines("test1 --select * from test2;"); + assertEquals(0, res1.size()); + res1 = simpleParser.parseLines("where col1 = '123';"); + assertEquals(1, res1.size()); + assertEquals("select * from test1 where col1 = '123'", res1.get(0).getHistoryStatement()); + assertEquals("select * from \ntest1 --select * from test2;\nwhere col1 = '123'", res1.get(0).getStatement()); } @Test @@ -116,13 +199,21 @@ public class TestSimpleParser { List<ParsedResult> res1 = SimpleParser.parseScript("select '\n;' from test;"); assertEquals(1, res1.size()); assertEquals(ParsedResult.StatementType.STATEMENT, res1.get(0).getType()); + assertEquals("select '\n;' from test", res1.get(0).getHistoryStatement()); assertEquals("select '\n;' from test", res1.get(0).getStatement()); List<ParsedResult> res2 = SimpleParser.parseScript("select 'abc\nbbc\nddf' from test;"); assertEquals(1, res2.size()); assertEquals(ParsedResult.StatementType.STATEMENT, res2.get(0).getType()); + assertEquals("select 'abc\nbbc\nddf' from test", res2.get(0).getHistoryStatement()); assertEquals("select 'abc\nbbc\nddf' from test", res2.get(0).getStatement()); + List<ParsedResult> res3 = SimpleParser.parseScript("select '--test', \n'--test2' from test"); + assertEquals(1, res3.size()); + assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(0).getType()); + assertEquals("select '--test', '--test2' from test", res3.get(0).getHistoryStatement()); + assertEquals("select '--test', \n'--test2' from test", res3.get(0).getStatement()); + try { SimpleParser.parseScript("select 'abc"); assertTrue(false); @@ -144,7 +235,8 @@ public class TestSimpleParser { assertEquals(0, result2.size()); List<ParsedResult> result3 = parser.EOF(); assertEquals(1, result3.size()); - assertEquals(lines[0] + lines[1], result3.get(0).getStatement()); + assertEquals(lines[0] + lines[1], result3.get(0).getHistoryStatement()); + assertEquals(lines[0] + "\n" + lines[1], result3.get(0).getStatement()); } @Test @@ -158,8 +250,8 @@ public class TestSimpleParser { assertEquals(0, result1.size()); List<ParsedResult> result2 = parser.parseLines(lines[1]); assertEquals(2, result2.size()); - assertEquals("select abc, 'bbc' from test", result2.get(0).getStatement()); - assertEquals("select * from test3", result2.get(1).getStatement()); + assertEquals("select abc, 'bbc' from test", result2.get(0).getHistoryStatement()); + assertEquals("select * from test3", result2.get(1).getHistoryStatement()); } @Test @@ -173,7 +265,7 @@ public class TestSimpleParser { assertEquals(0, result1.size()); List<ParsedResult> result2 = parser.parseLines(lines[1]); assertEquals(2, result2.size()); - assertEquals("select abc, 'bbc' from test", result2.get(0).getStatement()); - assertEquals("select * from test3", result2.get(1).getStatement()); + assertEquals("select abc, 'bbc' from test", result2.get(0).getHistoryStatement()); + assertEquals("select * from test3", result2.get(1).getHistoryStatement()); } } http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 979bee2..bab6ec7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -105,7 +105,7 @@ public class ExprTestBase { if (parsedResults.size() > 1) { throw new RuntimeException("this query includes two or more statements."); } - Expr expr = analyzer.parse(parsedResults.get(0).getStatement()); + Expr expr = analyzer.parse(parsedResults.get(0).getHistoryStatement()); VerificationState state = new VerificationState(); preLogicalPlanVerifier.verify(session, state, expr); if (state.getErrorMessages().size() > 0) { @@ -124,7 +124,7 @@ public class ExprTestBase { Target [] targets = plan.getRootBlock().getRawTargets(); if (targets == null) { - throw new PlanningException("Wrong query statement or query plan: " + parsedResults.get(0).getStatement()); + throw new PlanningException("Wrong query statement or query plan: " + parsedResults.get(0).getHistoryStatement()); } for (Target t : targets) { assertJsonSerDer(t.getEvalTree());
