[ https://issues.apache.org/jira/browse/SPARK-9374?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Yi Zhou updated SPARK-9374: --------------------------- Summary: [Spark SQL] UDFUnixTimeStamp throw out erorr of "AnalysisException: nondeterministic expressions are only allowed in Project or Filter" during the spark sql parse phase (was: [Spark SQL] Throw out erorr of "AnalysisException: nondeterministic expressions are only allowed in Project or Filter" during the spark sql parse phase) > [Spark SQL] UDFUnixTimeStamp throw out erorr of "AnalysisException: > nondeterministic expressions are only allowed in Project or Filter" during > the spark sql parse phase > ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ > > Key: SPARK-9374 > URL: https://issues.apache.org/jira/browse/SPARK-9374 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 1.5.0 > Reporter: Yi Zhou > Priority: Blocker > > #################Spark SQL Query######## > INSERT INTO TABLE TEST_QUERY_0_result > SELECT w_state, i_item_id, > SUM( > CASE WHEN (unix_timestamp(d_date,'yyyy-MM-dd') < > unix_timestamp('2001-03-16','yyyy-MM-dd')) > THEN ws_sales_price - COALESCE(wr_refunded_cash,0) > ELSE 0.0 END > ) AS sales_before, > SUM( > CASE WHEN (unix_timestamp(d_date,'yyyy-MM-dd') >= > unix_timestamp('2001-03-16','yyyy-MM-dd')) > THEN ws_sales_price - coalesce(wr_refunded_cash,0) > ELSE 0.0 END > ) AS sales_after > FROM ( > SELECT * > FROM web_sales ws > LEFT OUTER JOIN web_returns wr ON (ws.ws_order_number = wr.wr_order_number > AND ws.ws_item_sk = wr.wr_item_sk) > ) a1 > JOIN item i ON a1.ws_item_sk = i.i_item_sk > JOIN warehouse w ON a1.ws_warehouse_sk = w.w_warehouse_sk > JOIN date_dim d ON a1.ws_sold_date_sk = d.d_date_sk > AND unix_timestamp(d.d_date, 'yyyy-MM-dd') >= unix_timestamp('2001-03-16', > 'yyyy-MM-dd') - 30*24*60*60 --subtract 30 days in seconds > AND unix_timestamp(d.d_date, 'yyyy-MM-dd') <= unix_timestamp('2001-03-16', > 'yyyy-MM-dd') + 30*24*60*60 --add 30 days in seconds > GROUP BY w_state,i_item_id > CLUSTER BY w_state,i_item_id > ############Error Message################## > org.apache.spark.sql.AnalysisException: nondeterministic expressions are only > allowed in Project or Filter, found: > (((ws_sold_date_sk = d_date_sk) && > (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date,yyyy-MM-dd) > >= > (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd) > - CAST((((30 * 24) * 60) * 60), LongType)))) && > (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date,yyyy-MM-dd) > <= > (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd) > + CAST((((30 * 24) * 60) * 60), LongType)))) > in operator Join Inner, Some((((ws_sold_date_sk#289L = d_date_sk#383L) && > (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date#385,yyyy-MM-dd) > >= > (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd) > - CAST((((30 * 24) * 60) * 60), LongType)))) && > (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(d_date#385,yyyy-MM-dd) > <= > (HiveGenericUDF#org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnixTimeStamp(2001-03-16,yyyy-MM-dd) > + CAST((((30 * 24) * 60) * 60), LongType))))) > ; > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.failAnalysis(CheckAnalysis.scala:37) > at > org.apache.spark.sql.catalyst.analysis.Analyzer.failAnalysis(Analyzer.scala:43) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:148) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:49) > at > org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:103) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102) > at scala.collection.immutable.List.foreach(List.scala:318) > at > org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102) > at scala.collection.immutable.List.foreach(List.scala:318) > at > org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102) > at scala.collection.immutable.List.foreach(List.scala:318) > at > org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102) > at > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:102) > at scala.collection.immutable.List.foreach(List.scala:318) > at > org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:102) > at > org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:49) > at > org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:43) > at > org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:976) > at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:131) > at org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:52) > at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:792) > at > org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:61) > at > org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:284) > at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:423) > at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:359) > at > org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:456) > at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:466) > at > org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:163) > at > org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at > org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:666) > at > org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:178) > at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:203) > at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:118) > at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org