Hi Udara, as Sachith explained, this is a flaw in spark and we will send a PR to them. but unfortunately, this would be resolved in the next DAS release together with a spark upgrade.
best On Tue, Feb 23, 2016 at 9:34 AM, Sachith Withana <[email protected]> wrote: > I'm afraid it's not. > > This is due to the fact that it's not supported in the Spark version that > we have now. > > Thanks, > Sachith > > On Tue, Feb 23, 2016 at 1:14 AM, Udara Rathnayake <[email protected]> wrote: > >> Hi Niranda, >> >> Noticed same issue in DAS 3.0.1. Is this fixed now? >> >> Thanks! >> >> On Tue, Jan 5, 2016 at 11:12 AM, Niranda Perera <[email protected]> wrote: >> >>> Hi Udara, >>> >>> Yes, this is a known issue / limitation in the current implementation. >>> You would have to pass a dummy param because udf0 implementation was not >>> available by the time we released. >>> >>> Best >>> On Jan 5, 2016 21:37, "Udara Rathnayake" <[email protected]> wrote: >>> >>>> Hi, >>>> >>>> When I try to call a UDF without any parameter, getting following >>>> error[1]. Let's assume my UDF is convertDate(). But if I try the same with >>>> a parameter like convertDate(null) it works. >>>> >>>> Any Idea? Noticed that we have TimeNowUDF[2] sample, do we need to use >>>> "now(null)" within a spark query? >>>> >>>> >>>> [1] >>>> >>>> TID: [-1] [] [2016-01-05 10:45:51,744] ERROR >>>> {org.wso2.carbon.ntask.core.impl.TaskQuartzJobAdapter} - Error in >>>> executing task: >>>> org.apache.spark.sql.UDFRegistration$$anonfun$register$24$$anonfun$apply$1 >>>> cannot be cast to scala.Function0 >>>> {org.wso2.carbon.ntask.core.impl.TaskQuartzJobAdapter} >>>> java.lang.ClassCastException: >>>> org.apache.spark.sql.UDFRegistration$$anonfun$register$24$$anonfun$apply$1 >>>> cannot be cast to scala.Function0 >>>> at >>>> org.apache.spark.sql.catalyst.expressions.ScalaUdf.<init>(ScalaUdf.scala:61) >>>> at >>>> org.apache.spark.sql.UDFRegistration$$anonfun$register$24.apply(UDFRegistration.scala:408) >>>> at >>>> org.apache.spark.sql.UDFRegistration$$anonfun$register$24.apply(UDFRegistration.scala:408) >>>> at >>>> org.apache.spark.sql.catalyst.analysis.SimpleFunctionRegistry.lookupFunction(FunctionRegistry.scala:57) >>>> at >>>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$13$$anonfun$applyOrElse$5.applyOrElse(Analyzer.scala:465) >>>> at >>>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$13$$anonfun$applyOrElse$5.applyOrElse(Analyzer.scala:463) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:222) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:222) >>>> at >>>> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:51) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:221) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:242) >>>> at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) >>>> at scala.collection.Iterator$class.foreach(Iterator.scala:727) >>>> at >>>> scala.collection.AbstractIterator.foreach(Iterator.scala:1157) >>>> at >>>> scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) >>>> at >>>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) >>>> at >>>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) >>>> at scala.collection.TraversableOnce$class.to >>>> (TraversableOnce.scala:273) >>>> at scala.collection.AbstractIterator.to(Iterator.scala:1157) >>>> at >>>> scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) >>>> at >>>> scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157) >>>> at >>>> scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252) >>>> at >>>> scala.collection.AbstractIterator.toArray(Iterator.scala:1157) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transformChildrenDown(TreeNode.scala:272) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:227) >>>> at org.apache.spark.sql.catalyst.plans.QueryPlan.org >>>> $apache$spark$sql$catalyst$plans$QueryPlan$$transformExpressionDown$1(QueryPlan.scala:75) >>>> at >>>> org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$1.apply(QueryPlan.scala:85) >>>> at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) >>>> at scala.collection.Iterator$class.foreach(Iterator.scala:727) >>>> at >>>> scala.collection.AbstractIterator.foreach(Iterator.scala:1157) >>>> at >>>> scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) >>>> at >>>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) >>>> at >>>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) >>>> at scala.collection.TraversableOnce$class.to >>>> (TraversableOnce.scala:273) >>>> at scala.collection.AbstractIterator.to(Iterator.scala:1157) >>>> at >>>> scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) >>>> at >>>> scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157) >>>> at >>>> scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252) >>>> at >>>> scala.collection.AbstractIterator.toArray(Iterator.scala:1157) >>>> at >>>> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsDown(QueryPlan.scala:94) >>>> at >>>> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressions(QueryPlan.scala:64) >>>> at >>>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$13.applyOrElse(Analyzer.scala:463) >>>> at >>>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$$anonfun$apply$13.applyOrElse(Analyzer.scala:461) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:222) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:222) >>>> at >>>> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:51) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:221) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:242) >>>> at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) >>>> at scala.collection.Iterator$class.foreach(Iterator.scala:727) >>>> at >>>> scala.collection.AbstractIterator.foreach(Iterator.scala:1157) >>>> at >>>> scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) >>>> at >>>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) >>>> at >>>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) >>>> at scala.collection.TraversableOnce$class.to >>>> (TraversableOnce.scala:273) >>>> at scala.collection.AbstractIterator.to(Iterator.scala:1157) >>>> at >>>> scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) >>>> at >>>> scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157) >>>> at >>>> scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252) >>>> at >>>> scala.collection.AbstractIterator.toArray(Iterator.scala:1157) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transformChildrenDown(TreeNode.scala:272) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:227) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:242) >>>> at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) >>>> at scala.collection.Iterator$class.foreach(Iterator.scala:727) >>>> at >>>> scala.collection.AbstractIterator.foreach(Iterator.scala:1157) >>>> at >>>> scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) >>>> at >>>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) >>>> at >>>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) >>>> at scala.collection.TraversableOnce$class.to >>>> (TraversableOnce.scala:273) >>>> at scala.collection.AbstractIterator.to(Iterator.scala:1157) >>>> at >>>> scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) >>>> at >>>> scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157) >>>> at >>>> scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252) >>>> at >>>> scala.collection.AbstractIterator.toArray(Iterator.scala:1157) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transformChildrenDown(TreeNode.scala:272) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:227) >>>> at >>>> org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:212) >>>> at >>>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$.apply(Analyzer.scala:461) >>>> at >>>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveFunctions$.apply(Analyzer.scala:460) >>>> at >>>> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:61) >>>> at >>>> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:59) >>>> at >>>> scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:111) >>>> at scala.collection.immutable.List.foldLeft(List.scala:84) >>>> at >>>> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:59) >>>> at >>>> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:51) >>>> at scala.collection.immutable.List.foreach(List.scala:318) >>>> at >>>> org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:51) >>>> at >>>> org.apache.spark.sql.SQLContext$QueryExecution.analyzed$lzycompute(SQLContext.scala:933) >>>> at >>>> org.apache.spark.sql.SQLContext$QueryExecution.analyzed(SQLContext.scala:933) >>>> at >>>> org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:931) >>>> at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:131) >>>> at org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:51) >>>> at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:755) >>>> at >>>> org.wso2.carbon.analytics.spark.core.internal.SparkAnalyticsExecutor.executeQueryLocal(SparkAnalyticsExecutor.java:710) >>>> at >>>> org.wso2.carbon.analytics.spark.core.internal.SparkAnalyticsExecutor.executeQuery(SparkAnalyticsExecutor.java:692) >>>> at >>>> org.wso2.carbon.analytics.spark.core.CarbonAnalyticsProcessorService.executeQuery(CarbonAnalyticsProcessorService.java:199) >>>> at >>>> org.wso2.carbon.analytics.spark.core.CarbonAnalyticsProcessorService.executeScript(CarbonAnalyticsProcessorService.java:149) >>>> at >>>> org.wso2.carbon.analytics.spark.core.AnalyticsTask.execute(AnalyticsTask.java:57) >>>> at >>>> org.wso2.carbon.ntask.core.impl.TaskQuartzJobAdapter.execute(TaskQuartzJobAdapter.java:67) >>>> at org.quartz.core.JobRunShell.run(JobRunShell.java:213) >>>> at >>>> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) >>>> at java.util.concurrent.FutureTask.run(FutureTask.java:266) >>>> at >>>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) >>>> at >>>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) >>>> at java.lang.Thread.run(Thread.java:745) >>>> >>>> [2] >>>> https://github.com/wso2/carbon-analytics/blob/master/components/analytics-processors/org.wso2.carbon.analytics.spark.core/src/main/java/org/wso2/carbon/analytics/spark/core/udf/defaults/TimeNowUDF.java >>>> >>>> -- >>>> Regards, >>>> UdaraR >>>> >>> >> >> >> -- >> Regards, >> UdaraR >> > > > > -- > Sachith Withana > Software Engineer; WSO2 Inc.; http://wso2.com > E-mail: sachith AT wso2.com > M: +94715518127 > Linked-In: <http://goog_416592669> > https://lk.linkedin.com/in/sachithwithana > -- *Niranda Perera* Software Engineer, WSO2 Inc. Mobile: +94-71-554-8430 Twitter: @n1r44 <https://twitter.com/N1R44> https://pythagoreanscript.wordpress.com/
_______________________________________________ Dev mailing list [email protected] http://wso2.org/cgi-bin/mailman/listinfo/dev
