[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49777022
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala
---
@@ -17,36 +17,157 @@
package org.apache.spark.sql.catalyst
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias,
UnresolvedAttribute, UnresolvedFunction}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{OneRowRelation,
Project}
+import org.apache.spark.unsafe.types.CalendarInterval
class CatalystQlSuite extends PlanTest {
val parser = new CatalystQl()
+ test("test case insensitive") {
+val result = Project(UnresolvedAlias(Literal(1)):: Nil, OneRowRelation)
+assert(result === parser.parsePlan("seLect 1"))
+assert(result === parser.parsePlan("select 1"))
+assert(result === parser.parsePlan("SELECT 1"))
+ }
+
+ test("test NOT operator with comparison operations") {
+val parsed = parser.parsePlan("SELECT NOT TRUE > TRUE")
+val expected = Project(
+ UnresolvedAlias(
+Not(
+ GreaterThan(Literal(true), Literal(true)))
+ ) :: Nil,
+ OneRowRelation)
+comparePlans(parsed, expected)
+ }
+
+ test("support hive interval literal") {
+def checkInterval(sql: String, result: CalendarInterval): Unit = {
+ val parsed = parser.parsePlan(sql)
+ val expected = Project(
+UnresolvedAlias(
+ Literal(result)
+) :: Nil,
+OneRowRelation)
+ comparePlans(parsed, expected)
+}
+
+def checkYearMonth(lit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' YEAR TO MONTH",
+CalendarInterval.fromYearMonthString(lit))
+}
+
+def checkDayTime(lit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' DAY TO SECOND",
+CalendarInterval.fromDayTimeString(lit))
+}
+
+def checkSingleUnit(lit: String, unit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' $unit",
+CalendarInterval.fromSingleUnitString(unit, lit))
+}
+
+checkYearMonth("123-10")
+checkYearMonth("496-0")
+checkYearMonth("-2-3")
+checkYearMonth("-123-0")
+
+checkDayTime("99 11:22:33.123456789")
+checkDayTime("-99 11:22:33.123456789")
+checkDayTime("10 9:8:7.123456789")
+checkDayTime("1 0:0:0")
+checkDayTime("-1 0:0:0")
+checkDayTime("1 0:0:1")
+
+for (unit <- Seq("year", "month", "day", "hour", "minute", "second")) {
+ checkSingleUnit("7", unit)
+ checkSingleUnit("-7", unit)
+ checkSingleUnit("0", unit)
+}
+
+checkSingleUnit("13.123456789", "second")
+checkSingleUnit("-13.123456789", "second")
+ }
+
+ test("support scientific notation") {
+def assertRight(input: String, output: Double): Unit = {
+ val parsed = parser.parsePlan("SELECT " + input)
+ val expected = Project(
+UnresolvedAlias(
+ Literal(output)
+) :: Nil,
+OneRowRelation)
+ comparePlans(parsed, expected)
+}
+
+assertRight("9.0e1", 90)
+assertRight("0.9e+2", 90)
+assertRight("900e-1", 90)
+assertRight("900.0E-1", 90)
+assertRight("9.e+1", 90)
+
+intercept[AnalysisException](parser.parsePlan("SELECT .e3"))
+ }
+
+ test("parse expressions") {
+compareExpressions(
+ parser.parseExpression("prinln('hello', 'world')"),
+ UnresolvedFunction(
+"prinln", Literal("hello") :: Literal("world") :: Nil, false))
+
+compareExpressions(
+ parser.parseExpression("1 + r.r As q"),
+ Alias(Add(Literal(1), UnresolvedAttribute("r.r")), "q")())
+
+compareExpressions(
+ parser.parseExpression("1 - f('o', o(bar))"),
+ Subtract(Literal(1),
+UnresolvedFunction("f",
+ Literal("o") ::
+ UnresolvedFunction("o", UnresolvedAttribute("bar") :: Nil,
false) ::
+ Nil, false)))
+ }
+
+ test("table identifier") {
+assert(TableIdentifier("q") === parser.parseTableIdentifier("q"))
+assert(TableIdentifier("q", Some("d")) ===
parser.parseTableIdentifier("d.q"))
+intercept[AnalysisException](parser.parseTableIdentifier(""))
+// TODO parser swallows third identifier.
+// intercept[AnalysisException](parser.parseTableIdentifier("d.q.g"))
--- End diff --
yea
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user hvanhovell commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49776238
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala
---
@@ -17,36 +17,157 @@
package org.apache.spark.sql.catalyst
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias,
UnresolvedAttribute, UnresolvedFunction}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{OneRowRelation,
Project}
+import org.apache.spark.unsafe.types.CalendarInterval
class CatalystQlSuite extends PlanTest {
val parser = new CatalystQl()
+ test("test case insensitive") {
+val result = Project(UnresolvedAlias(Literal(1)):: Nil, OneRowRelation)
+assert(result === parser.parsePlan("seLect 1"))
+assert(result === parser.parsePlan("select 1"))
+assert(result === parser.parsePlan("SELECT 1"))
+ }
+
+ test("test NOT operator with comparison operations") {
+val parsed = parser.parsePlan("SELECT NOT TRUE > TRUE")
+val expected = Project(
+ UnresolvedAlias(
+Not(
+ GreaterThan(Literal(true), Literal(true)))
+ ) :: Nil,
+ OneRowRelation)
+comparePlans(parsed, expected)
+ }
+
+ test("support hive interval literal") {
+def checkInterval(sql: String, result: CalendarInterval): Unit = {
+ val parsed = parser.parsePlan(sql)
+ val expected = Project(
+UnresolvedAlias(
+ Literal(result)
+) :: Nil,
+OneRowRelation)
+ comparePlans(parsed, expected)
+}
+
+def checkYearMonth(lit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' YEAR TO MONTH",
+CalendarInterval.fromYearMonthString(lit))
+}
+
+def checkDayTime(lit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' DAY TO SECOND",
+CalendarInterval.fromDayTimeString(lit))
+}
+
+def checkSingleUnit(lit: String, unit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' $unit",
+CalendarInterval.fromSingleUnitString(unit, lit))
+}
+
+checkYearMonth("123-10")
+checkYearMonth("496-0")
+checkYearMonth("-2-3")
+checkYearMonth("-123-0")
+
+checkDayTime("99 11:22:33.123456789")
+checkDayTime("-99 11:22:33.123456789")
+checkDayTime("10 9:8:7.123456789")
+checkDayTime("1 0:0:0")
+checkDayTime("-1 0:0:0")
+checkDayTime("1 0:0:1")
+
+for (unit <- Seq("year", "month", "day", "hour", "minute", "second")) {
+ checkSingleUnit("7", unit)
+ checkSingleUnit("-7", unit)
+ checkSingleUnit("0", unit)
+}
+
+checkSingleUnit("13.123456789", "second")
+checkSingleUnit("-13.123456789", "second")
+ }
+
+ test("support scientific notation") {
+def assertRight(input: String, output: Double): Unit = {
+ val parsed = parser.parsePlan("SELECT " + input)
+ val expected = Project(
+UnresolvedAlias(
+ Literal(output)
+) :: Nil,
+OneRowRelation)
+ comparePlans(parsed, expected)
+}
+
+assertRight("9.0e1", 90)
+assertRight("0.9e+2", 90)
+assertRight("900e-1", 90)
+assertRight("900.0E-1", 90)
+assertRight("9.e+1", 90)
+
+intercept[AnalysisException](parser.parsePlan("SELECT .e3"))
+ }
+
+ test("parse expressions") {
+compareExpressions(
+ parser.parseExpression("prinln('hello', 'world')"),
+ UnresolvedFunction(
+"prinln", Literal("hello") :: Literal("world") :: Nil, false))
+
+compareExpressions(
+ parser.parseExpression("1 + r.r As q"),
+ Alias(Add(Literal(1), UnresolvedAttribute("r.r")), "q")())
+
+compareExpressions(
+ parser.parseExpression("1 - f('o', o(bar))"),
+ Subtract(Literal(1),
+UnresolvedFunction("f",
+ Literal("o") ::
+ UnresolvedFunction("o", UnresolvedAttribute("bar") :: Nil,
false) ::
+ Nil, false)))
+ }
+
+ test("table identifier") {
+assert(TableIdentifier("q") === parser.parseTableIdentifier("q"))
+assert(TableIdentifier("q", Some("d")) ===
parser.parseTableIdentifier("d.q"))
+intercept[AnalysisException](parser.parseTableIdentifier(""))
+// TODO parser swallows third identifier.
+// intercept[AnalysisException](parser.parseTableIdentifier("d.q.g"))
--- End diff --
I
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49524882
--- Diff:
sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/CatalystQlSuite.scala
---
@@ -17,36 +17,157 @@
package org.apache.spark.sql.catalyst
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias,
UnresolvedAttribute, UnresolvedFunction}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{OneRowRelation,
Project}
+import org.apache.spark.unsafe.types.CalendarInterval
class CatalystQlSuite extends PlanTest {
val parser = new CatalystQl()
+ test("test case insensitive") {
+val result = Project(UnresolvedAlias(Literal(1)):: Nil, OneRowRelation)
+assert(result === parser.parsePlan("seLect 1"))
+assert(result === parser.parsePlan("select 1"))
+assert(result === parser.parsePlan("SELECT 1"))
+ }
+
+ test("test NOT operator with comparison operations") {
+val parsed = parser.parsePlan("SELECT NOT TRUE > TRUE")
+val expected = Project(
+ UnresolvedAlias(
+Not(
+ GreaterThan(Literal(true), Literal(true)))
+ ) :: Nil,
+ OneRowRelation)
+comparePlans(parsed, expected)
+ }
+
+ test("support hive interval literal") {
+def checkInterval(sql: String, result: CalendarInterval): Unit = {
+ val parsed = parser.parsePlan(sql)
+ val expected = Project(
+UnresolvedAlias(
+ Literal(result)
+) :: Nil,
+OneRowRelation)
+ comparePlans(parsed, expected)
+}
+
+def checkYearMonth(lit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' YEAR TO MONTH",
+CalendarInterval.fromYearMonthString(lit))
+}
+
+def checkDayTime(lit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' DAY TO SECOND",
+CalendarInterval.fromDayTimeString(lit))
+}
+
+def checkSingleUnit(lit: String, unit: String): Unit = {
+ checkInterval(
+s"SELECT INTERVAL '$lit' $unit",
+CalendarInterval.fromSingleUnitString(unit, lit))
+}
+
+checkYearMonth("123-10")
+checkYearMonth("496-0")
+checkYearMonth("-2-3")
+checkYearMonth("-123-0")
+
+checkDayTime("99 11:22:33.123456789")
+checkDayTime("-99 11:22:33.123456789")
+checkDayTime("10 9:8:7.123456789")
+checkDayTime("1 0:0:0")
+checkDayTime("-1 0:0:0")
+checkDayTime("1 0:0:1")
+
+for (unit <- Seq("year", "month", "day", "hour", "minute", "second")) {
+ checkSingleUnit("7", unit)
+ checkSingleUnit("-7", unit)
+ checkSingleUnit("0", unit)
+}
+
+checkSingleUnit("13.123456789", "second")
+checkSingleUnit("-13.123456789", "second")
+ }
+
+ test("support scientific notation") {
+def assertRight(input: String, output: Double): Unit = {
+ val parsed = parser.parsePlan("SELECT " + input)
+ val expected = Project(
+UnresolvedAlias(
+ Literal(output)
+) :: Nil,
+OneRowRelation)
+ comparePlans(parsed, expected)
+}
+
+assertRight("9.0e1", 90)
+assertRight("0.9e+2", 90)
+assertRight("900e-1", 90)
+assertRight("900.0E-1", 90)
+assertRight("9.e+1", 90)
+
+intercept[AnalysisException](parser.parsePlan("SELECT .e3"))
+ }
+
+ test("parse expressions") {
+compareExpressions(
+ parser.parseExpression("prinln('hello', 'world')"),
+ UnresolvedFunction(
+"prinln", Literal("hello") :: Literal("world") :: Nil, false))
+
+compareExpressions(
+ parser.parseExpression("1 + r.r As q"),
+ Alias(Add(Literal(1), UnresolvedAttribute("r.r")), "q")())
+
+compareExpressions(
+ parser.parseExpression("1 - f('o', o(bar))"),
+ Subtract(Literal(1),
+UnresolvedFunction("f",
+ Literal("o") ::
+ UnresolvedFunction("o", UnresolvedAttribute("bar") :: Nil,
false) ::
+ Nil, false)))
+ }
+
+ test("table identifier") {
+assert(TableIdentifier("q") === parser.parseTableIdentifier("q"))
+assert(TableIdentifier("q", Some("d")) ===
parser.parseTableIdentifier("d.q"))
+intercept[AnalysisException](parser.parseTableIdentifier(""))
+// TODO parser swallows third identifier.
+// intercept[AnalysisException](parser.parseTableIdentifier("d.q.g"))
--- End diff --
are
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user asfgit closed the pull request at: https://github.com/apache/spark/pull/10649 --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user rxin commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170742248 I'm going to merge this. @cloud-fan it would be great for you to still take a look post-hoc and @hvanhovell can address feedback in his next pull request. Thanks. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170742128 Merged build finished. Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170742130 Test PASSed. Refer to this link for build results (access rights to CI server needed): https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/49184/ Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170741917 **[Test build #49184 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/49184/consoleFull)** for PR 10649 at commit [`81588d2`](https://github.com/apache/spark/commit/81588d2d6a4a583c477a35019d0b86a727a9ae13). * This patch passes all tests. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170714313 **[Test build #49184 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/49184/consoleFull)** for PR 10649 at commit [`81588d2`](https://github.com/apache/spark/commit/81588d2d6a4a583c477a35019d0b86a727a9ae13). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170710543 Merged build finished. Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170710545 Test FAILed. Refer to this link for build results (access rights to CI server needed): https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/49180/ Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user yhuai commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170709769 @cloud-fan can you also take a look? --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49386798
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala ---
@@ -41,16 +47,9 @@ private[sql] class CatalystQl(val conf: ParserConf =
SimpleParserConf()) {
}
}
-
- /**
- * Returns the AST for the given SQL string.
- */
- protected def getAst(sql: String): ASTNode = ParseDriver.parse(sql, conf)
-
- /** Creates LogicalPlan for a given HiveQL string. */
- def createPlan(sql: String): LogicalPlan = {
+ protected def safeParse[T](sql: String, ast: ASTNode)(toResult: ASTNode
=> T): T = {
--- End diff --
there is a extra space before the def.
can you also add some doc explaining what this function does (e.g. what
"safe" means)?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user nongli commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49374680
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala ---
@@ -58,26 +57,39 @@ private[sql] class CatalystQl(val conf: ParserConf =
SimpleParserConf()) {
throw new AnalysisException(e.getMessage)
case e: NotImplementedError =>
throw new AnalysisException(
- s"""
- |Unsupported language features in query: $sql
- |${getAst(sql).treeString}
+ s"""== Unsupported language features in query ==
+ |== SQL ==
+ |$sql
+ |== AST ==
+ |${ast.treeString}
+ |== Error ==
|$e
+ |== Stacktrace ==
|${e.getStackTrace.head}
""".stripMargin)
}
}
- protected def createPlan(sql: String, tree: ASTNode): LogicalPlan =
nodeToPlan(tree)
-
- def parseDdl(ddl: String): Seq[Attribute] = {
-val tree = getAst(ddl)
-assert(tree.text == "TOK_CREATETABLE", "Only CREATE TABLE supported.")
-val tableOps = tree.children
-val colList = tableOps
- .find(_.text == "TOK_TABCOLLIST")
- .getOrElse(sys.error("No columnList!"))
-
-colList.children.map(nodeToAttribute)
+ /** Creates LogicalPlan for a given SQL string. */
+ def parsePlan(sql: String): LogicalPlan =
+safeParse(sql, ParseDriver.parsePlan(sql, conf))(nodeToPlan)
+
+ /** Creates Expression for a given SQL string. */
+ def parseExpression(sql: String): Expression =
+safeParse(sql, ParseDriver.parseExpression(sql,
conf))(selExprNodeToExpr(_).get)
+
+ /** Creates Expression for a given SQL string. */
--- End diff --
Expression -> TableIdentifier
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170097976 Merged build finished. Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170097979 Test PASSed. Refer to this link for build results (access rights to CI server needed): https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/49022/ Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170097654 **[Test build #49022 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/49022/consoleFull)** for PR 10649 at commit [`bc0e298`](https://github.com/apache/spark/commit/bc0e298ebfd06b3182d561b2456b5bc55fa23fd4). * This patch passes all tests. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-170070628 **[Test build #49022 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/49022/consoleFull)** for PR 10649 at commit [`bc0e298`](https://github.com/apache/spark/commit/bc0e298ebfd06b3182d561b2456b5bc55fa23fd4). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169928806 Test PASSed. Refer to this link for build results (access rights to CI server needed): https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/49005/ Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169928804 Merged build finished. Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169928634 **[Test build #49005 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/49005/consoleFull)** for PR 10649 at commit [`b070bf9`](https://github.com/apache/spark/commit/b070bf9b3af9ad61913a3caa7c571eeea61588a9). * This patch passes all tests. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169911391 **[Test build #49005 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/49005/consoleFull)** for PR 10649 at commit [`b070bf9`](https://github.com/apache/spark/commit/b070bf9b3af9ad61913a3caa7c571eeea61588a9). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169846245 **[Test build #48983 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/48983/consoleFull)** for PR 10649 at commit [`c2b35b7`](https://github.com/apache/spark/commit/c2b35b7efdd80ab4930b46a437bb9289c87b5206). * This patch **fails Spark unit tests**. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169846362 Merged build finished. Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169846366 Test FAILed. Refer to this link for build results (access rights to CI server needed): https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/48983/ Test FAILed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169838962 **[Test build #48983 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/48983/consoleFull)** for PR 10649 at commit [`c2b35b7`](https://github.com/apache/spark/commit/c2b35b7efdd80ab4930b46a437bb9289c87b5206). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169821749 Merged build finished. Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user AmplabJenkins commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169821750 Test PASSed. Refer to this link for build results (access rights to CI server needed): https://amplab.cs.berkeley.edu/jenkins//job/SparkPullRequestBuilder/48965/ Test PASSed. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169821476 **[Test build #48965 has finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/48965/consoleFull)** for PR 10649 at commit [`7f37d81`](https://github.com/apache/spark/commit/7f37d81a1a50ffa82aac63141c9cc62db65eb26f). * This patch passes all tests. * This patch merges cleanly. * This patch adds no public classes. --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49128910
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala ---
@@ -58,26 +51,35 @@ private[sql] class CatalystQl(val conf: ParserConf =
SimpleParserConf()) {
throw new AnalysisException(e.getMessage)
case e: NotImplementedError =>
throw new AnalysisException(
- s"""
- |Unsupported language features in query: $sql
- |${getAst(sql).treeString}
+ s"""== Unsupported language features in query ==
+ |== SQL ==
+ |$sql
+ |== AST ==
+ |${ast.treeString}
+ |== Error ==
|$e
+ |== Stacktrace ==
|${e.getStackTrace.head}
""".stripMargin)
}
}
- protected def createPlan(sql: String, tree: ASTNode): LogicalPlan =
nodeToPlan(tree)
-
- def parseDdl(ddl: String): Seq[Attribute] = {
-val tree = getAst(ddl)
-assert(tree.text == "TOK_CREATETABLE", "Only CREATE TABLE supported.")
-val tableOps = tree.children
-val colList = tableOps
- .find(_.text == "TOK_TABCOLLIST")
- .getOrElse(sys.error("No columnList!"))
-
-colList.children.map(nodeToAttribute)
+ /** Creates LogicalPlan for a given SQL string. */
+ def createPlan(sql: String): LogicalPlan =
--- End diff --
but no big deal.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49128521
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala ---
@@ -58,26 +51,35 @@ private[sql] class CatalystQl(val conf: ParserConf =
SimpleParserConf()) {
throw new AnalysisException(e.getMessage)
case e: NotImplementedError =>
throw new AnalysisException(
- s"""
- |Unsupported language features in query: $sql
- |${getAst(sql).treeString}
+ s"""== Unsupported language features in query ==
+ |== SQL ==
+ |$sql
+ |== AST ==
+ |${ast.treeString}
+ |== Error ==
|$e
+ |== Stacktrace ==
|${e.getStackTrace.head}
""".stripMargin)
}
}
- protected def createPlan(sql: String, tree: ASTNode): LogicalPlan =
nodeToPlan(tree)
-
- def parseDdl(ddl: String): Seq[Attribute] = {
-val tree = getAst(ddl)
-assert(tree.text == "TOK_CREATETABLE", "Only CREATE TABLE supported.")
-val tableOps = tree.children
-val colList = tableOps
- .find(_.text == "TOK_TABCOLLIST")
- .getOrElse(sys.error("No columnList!"))
-
-colList.children.map(nodeToAttribute)
+ /** Creates LogicalPlan for a given SQL string. */
+ def createPlan(sql: String): LogicalPlan =
--- End diff --
FWIW, I like the old name "parseXXX" better.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49128237
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
---
@@ -28,7 +28,15 @@ import org.apache.spark.sql.AnalysisException
* This is based on Hive's org.apache.hadoop.hive.ql.parse.ParseDriver
*/
object ParseDriver extends Logging {
- def parse(command: String, conf: ParserConf): ASTNode = {
+ def parsePlan(command: String, conf: ParserConf): ASTNode =
parse(command, conf) { parser =>
+parser.statement().getTree
+ }
+
+ def parseExpression(command: String, conf: ParserConf): ASTNode =
parse(command, conf) { parser =>
+parser.expression().getTree
+ }
+
+ def parse(command: String, conf: ParserConf)(toTree: SparkSqlParser =>
CommonTree): ASTNode = {
--- End diff --
should this be private?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49128191
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/ParseDriver.scala
---
@@ -28,7 +28,15 @@ import org.apache.spark.sql.AnalysisException
* This is based on Hive's org.apache.hadoop.hive.ql.parse.ParseDriver
*/
object ParseDriver extends Logging {
- def parse(command: String, conf: ParserConf): ASTNode = {
+ def parsePlan(command: String, conf: ParserConf): ASTNode =
parse(command, conf) { parser =>
--- End diff --
can you add some function doc for these 3 functions, and also cover what
the differences are?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/10649#discussion_r49125905
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystQl.scala ---
@@ -41,16 +41,9 @@ private[sql] class CatalystQl(val conf: ParserConf =
SimpleParserConf()) {
}
}
-
- /**
- * Returns the AST for the given SQL string.
- */
- protected def getAst(sql: String): ASTNode = ParseDriver.parse(sql, conf)
-
- /** Creates LogicalPlan for a given HiveQL string. */
- def createPlan(sql: String): LogicalPlan = {
+ protected def safeParse[T](sql: String, ast: ASTNode, toResult: ASTNode
=> T): T = {
--- End diff --
use multi parameter list?
```scala
protected def safeParse[T](sql: String, ast: ASTNode)(toResult: ASTNode =>
T): T
```
then you can call it in a slightly nicer way
```scala
safeParse(sql, ParseDriver.parseExpression(sql, conf)) { ast =>
...
}
```
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
Github user SparkQA commented on the pull request: https://github.com/apache/spark/pull/10649#issuecomment-169797907 **[Test build #48965 has started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/48965/consoleFull)** for PR 10649 at commit [`7f37d81`](https://github.com/apache/spark/commit/7f37d81a1a50ffa82aac63141c9cc62db65eb26f). --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request: [SPARK-12576][SQL] Enable expression parsing i...
GitHub user hvanhovell opened a pull request: https://github.com/apache/spark/pull/10649 [SPARK-12576][SQL] Enable expression parsing in CatalystQl The PR allows us to use the new SQL parser to parse SQL expressions such as: ```1 + sin(x*x)``` We enable this functionality in this PR, but we will not start using this actively yet. This will be done as soon as we have reached grammar parity with the existing parser stack. cc @rxin You can merge this pull request into a Git repository by running: $ git pull https://github.com/hvanhovell/spark SPARK-12576 Alternatively you can review and apply these changes as the patch at: https://github.com/apache/spark/pull/10649.patch To close this pull request, make a commit to your master/trunk branch with (at least) the following in the commit message: This closes #10649 commit c15ae2909ff11352fbde2e23167253e118ab05d8 Author: Herman van Hovell Date: 2016-01-07T17:38:16Z Enable Expression Parsing in CatalysQl commit cd7f8ec616a667717ab31b45a42967e8286f057e Author: Herman van Hovell Date: 2016-01-07T17:38:47Z Enable Expression Parsing in CatalysQl commit 682df131ee6f2e89d9d46a21fe79d1b06d8fa54a Author: Herman van Hovell Date: 2016-01-07T17:45:15Z Merge remote-tracking branch 'spark/master' into SPARK-12576 commit 7f37d81a1a50ffa82aac63141c9cc62db65eb26f Author: Herman van Hovell Date: 2016-01-07T19:39:13Z Add tests --- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
