Lishu Liu created SPARK-5632:
--------------------------------
Summary: not able to resolve dot('.') in field name
Key: SPARK-5632
URL: https://issues.apache.org/jira/browse/SPARK-5632
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 1.2.0
Environment: Spark cluster: EC2 m1.small + Spark 1.2.0
Cassandra cluster: EC2 m3.xlarge + Cassandra 2.1.2
Reporter: Lishu Liu
My cassandra table task_trace has a field sm.result which contains dot in the
name. So SQL tried to look up sm instead of full name 'sm.result'.
Here is my code:
scala> import org.apache.spark.sql.cassandra.CassandraSQLContext
scala> val cc = new CassandraSQLContext(sc)
scala> val task_trace = cc.jsonFile("/task_trace.json")
scala> task_trace.registerTempTable("task_trace")
scala> cc.setKeyspace("cerberus_data_v4")
scala> val res = cc.sql("SELECT received_datetime, task_body.cerberus_id,
task_body.sm.result FROM task_trace WHERE task_id =
'fff7304e-9984-4b45-b10c-0423a96745ce'")
res: org.apache.spark.sql.SchemaRDD =
SchemaRDD[57] at RDD at SchemaRDD.scala:108
== Query Plan ==
== Physical Plan ==
java.lang.RuntimeException: No such struct field sm in cerberus_batch_id,
cerberus_id, couponId, coupon_code, created, description, domain, expires,
message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots,
sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, validity
The full schema look like this:
scala> task_trace.printSchema()
root
|-- received_datetime: long (nullable = true)
|-- task_body: struct (nullable = true)
| |-- cerberus_batch_id: string (nullable = true)
| |-- cerberus_id: string (nullable = true)
| |-- couponId: integer (nullable = true)
| |-- coupon_code: string (nullable = true)
| |-- created: string (nullable = true)
| |-- description: string (nullable = true)
| |-- domain: string (nullable = true)
| |-- expires: string (nullable = true)
| |-- message_id: string (nullable = true)
| |-- neverShowAfter: string (nullable = true)
| |-- neverShowBefore: string (nullable = true)
| |-- offerTitle: string (nullable = true)
| |-- screenshots: array (nullable = true)
| | |-- element: string (containsNull = false)
| |-- sm.result: struct (nullable = true)
| | |-- cerberus_batch_id: string (nullable = true)
| | |-- cerberus_id: string (nullable = true)
| | |-- code: string (nullable = true)
| | |-- couponId: integer (nullable = true)
| | |-- created: string (nullable = true)
| | |-- description: string (nullable = true)
| | |-- domain: string (nullable = true)
| | |-- expires: string (nullable = true)
| | |-- message_id: string (nullable = true)
| | |-- neverShowAfter: string (nullable = true)
| | |-- neverShowBefore: string (nullable = true)
| | |-- offerTitle: string (nullable = true)
| | |-- result: struct (nullable = true)
| | | |-- post: struct (nullable = true)
| | | | |-- alchemy_out_of_stock: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: boolean (nullable = true)
| | | | |-- meta: struct (nullable = true)
| | | | | |-- None_tx_value: array (nullable = true)
| | | | | | |-- element: string (containsNull = false)
| | | | | |-- exceptions: array (nullable = true)
| | | | | | |-- element: string (containsNull = false)
| | | | | |-- no_input_value: array (nullable = true)
| | | | | | |-- element: string (containsNull = false)
| | | | | |-- not_mapped: array (nullable = true)
| | | | | | |-- element: string (containsNull = false)
| | | | | |-- not_transformed: array (nullable = true)
| | | | | | |-- element: array (containsNull = false)
| | | | | | | |-- element: string (containsNull = false)
| | | | |-- now_price_checkout: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: double (nullable = true)
| | | | |-- shipping_price: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: double (nullable = true)
| | | | |-- tax: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: double (nullable = true)
| | | | |-- total: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: double (nullable = true)
| | | |-- pre: struct (nullable = true)
| | | | |-- alchemy_out_of_stock: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: boolean (nullable = true)
| | | | |-- meta: struct (nullable = true)
| | | | | |-- None_tx_value: array (nullable = true)
| | | | | | |-- element: string (containsNull = false)
| | | | | |-- exceptions: array (nullable = true)
| | | | | | |-- element: string (containsNull = false)
| | | | | |-- no_input_value: array (nullable = true)
| | | | | | |-- element: string (containsNull = false)
| | | | | |-- not_mapped: array (nullable = true)
| | | | | | |-- element: string (containsNull = false)
| | | | | |-- not_transformed: array (nullable = true)
| | | | | | |-- element: array (containsNull = false)
| | | | | | | |-- element: string (containsNull = false)
| | | | |-- now_price_checkout: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: double (nullable = true)
| | | | |-- shipping_price: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: double (nullable = true)
| | | | |-- tax: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: double (nullable = true)
| | | | |-- total: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: double (nullable = true)
| | |-- startDate: string (nullable = true)
| | |-- status: string (nullable = true)
| | |-- task_id: string (nullable = true)
| | |-- url: string (nullable = true)
| | |-- uuid: string (nullable = true)
| | |-- validationDateTime: string (nullable = true)
| | |-- validity: string (nullable = true)
| |-- sm.task: struct (nullable = true)
| | |-- cerberus_batch_id: string (nullable = true)
| | |-- cerberus_id: string (nullable = true)
| | |-- couponId: integer (nullable = true)
| | |-- coupon_code: string (nullable = true)
| | |-- created: string (nullable = true)
| | |-- description: string (nullable = true)
| | |-- domain: string (nullable = true)
| | |-- expires: string (nullable = true)
| | |-- message_id: string (nullable = true)
| | |-- neverShowAfter: string (nullable = true)
| | |-- neverShowBefore: string (nullable = true)
| | |-- offerTitle: string (nullable = true)
| | |-- result: struct (nullable = true)
| | | |-- post: struct (nullable = true)
| | | | |-- final_price: struct (nullable = true)
| | | | | |-- ci: integer (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- merchant_exception: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: boolean (nullable = true)
| | | | |-- now_price_checkout: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- orig_price: struct (nullable = true)
| | | | | |-- ci: integer (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- out_of_stock: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: boolean (nullable = true)
| | | | |-- sale_price: struct (nullable = true)
| | | | | |-- ci: integer (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- shipping_price: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- tax: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- total: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | |-- pre: struct (nullable = true)
| | | | |-- final_price: struct (nullable = true)
| | | | | |-- ci: integer (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- merchant_exception: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: boolean (nullable = true)
| | | | |-- now_price_checkout: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- orig_price: struct (nullable = true)
| | | | | |-- ci: integer (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- out_of_stock: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: boolean (nullable = true)
| | | | |-- sale_price: struct (nullable = true)
| | | | | |-- ci: integer (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- shipping_price: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- tax: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: string (nullable = true)
| | | | |-- total: struct (nullable = true)
| | | | | |-- ci: double (nullable = true)
| | | | | |-- value: string (nullable = true)
| | |-- screenshots: array (nullable = true)
| | | |-- element: string (containsNull = false)
| | |-- startDate: string (nullable = true)
| | |-- status: string (nullable = true)
| | |-- task_id: string (nullable = true)
| | |-- url: string (nullable = true)
| | |-- uuid: string (nullable = true)
| | |-- validationDateTime: string (nullable = true)
| | |-- validity: string (nullable = true)
| |-- startDate: string (nullable = true)
| |-- task_id: string (nullable = true)
| |-- url: string (nullable = true)
| |-- uuid: string (nullable = true)
| |-- validationDateTime: string (nullable = true)
| |-- validity: string (nullable = true)
|-- task_id: string (nullable = true)
|-- task_step: integer (nullable = true)
|-- task_type: integer (nullable = true)
I also tried val sqlContext = new org.apache.spark.sql.SQLContext(sc), but it
give the same exception.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
