[ 
https://issues.apache.org/jira/browse/SPARK-5632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Reynold Xin updated SPARK-5632:
-------------------------------
             Priority: Blocker  (was: Major)
     Target Version/s: 1.4.0
    Affects Version/s: 1.3.0

> not able to resolve dot('.') in field name
> ------------------------------------------
>
>                 Key: SPARK-5632
>                 URL: https://issues.apache.org/jira/browse/SPARK-5632
>             Project: Spark
>          Issue Type: Bug
>          Components: SQL
>    Affects Versions: 1.2.0, 1.3.0
>         Environment: Spark cluster: EC2 m1.small + Spark 1.2.0
> Cassandra cluster: EC2 m3.xlarge + Cassandra 2.1.2
>            Reporter: Lishu Liu
>            Priority: Blocker
>
> My cassandra table task_trace has a field sm.result which contains dot in the 
> name. So SQL tried to look up sm instead of full name 'sm.result'. 
> Here is my code: 
> {code}
> scala> import org.apache.spark.sql.cassandra.CassandraSQLContext
> scala> val cc = new CassandraSQLContext(sc)
> scala> val task_trace = cc.jsonFile("/task_trace.json")
> scala> task_trace.registerTempTable("task_trace")
> scala> cc.setKeyspace("cerberus_data_v4")
> scala> val res = cc.sql("SELECT received_datetime, task_body.cerberus_id, 
> task_body.sm.result FROM task_trace WHERE task_id = 
> 'fff7304e-9984-4b45-b10c-0423a96745ce'")
> res: org.apache.spark.sql.SchemaRDD = 
> SchemaRDD[57] at RDD at SchemaRDD.scala:108
> == Query Plan ==
> == Physical Plan ==
> java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, 
> cerberus_id, couponId, coupon_code, created, description, domain, expires, 
> message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, 
> sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, 
> validity
> {code}
> The full schema look like this:
> {code}
> scala> task_trace.printSchema()
> root
>  \|-- received_datetime: long (nullable = true)
>  \|-- task_body: struct (nullable = true)
>  \|    \|-- cerberus_batch_id: string (nullable = true)
>  \|    \|-- cerberus_id: string (nullable = true)
>  \|    \|-- couponId: integer (nullable = true)
>  \|    \|-- coupon_code: string (nullable = true)
>  \|    \|-- created: string (nullable = true)
>  \|    \|-- description: string (nullable = true)
>  \|    \|-- domain: string (nullable = true)
>  \|    \|-- expires: string (nullable = true)
>  \|    \|-- message_id: string (nullable = true)
>  \|    \|-- neverShowAfter: string (nullable = true)
>  \|    \|-- neverShowBefore: string (nullable = true)
>  \|    \|-- offerTitle: string (nullable = true)
>  \|    \|-- screenshots: array (nullable = true)
>  \|    \|    \|-- element: string (containsNull = false)
>  \|    \|-- sm.result: struct (nullable = true)
>  \|    \|    \|-- cerberus_batch_id: string (nullable = true)
>  \|    \|    \|-- cerberus_id: string (nullable = true)
>  \|    \|    \|-- code: string (nullable = true)
>  \|    \|    \|-- couponId: integer (nullable = true)
>  \|    \|    \|-- created: string (nullable = true)
>  \|    \|    \|-- description: string (nullable = true)
>  \|    \|    \|-- domain: string (nullable = true)
>  \|    \|    \|-- expires: string (nullable = true)
>  \|    \|    \|-- message_id: string (nullable = true)
>  \|    \|    \|-- neverShowAfter: string (nullable = true)
>  \|    \|    \|-- neverShowBefore: string (nullable = true)
>  \|    \|    \|-- offerTitle: string (nullable = true)
>  \|    \|    \|-- result: struct (nullable = true)
>  \|    \|    \|    \|-- post: struct (nullable = true)
>  \|    \|    \|    \|    \|-- alchemy_out_of_stock: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
>  \|    \|    \|    \|    \|-- meta: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- None_tx_value: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|-- exceptions: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|-- no_input_value: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|-- not_mapped: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|-- not_transformed: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: array (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull 
> = false)
>  \|    \|    \|    \|    \|-- now_price_checkout: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
>  \|    \|    \|    \|    \|-- shipping_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
>  \|    \|    \|    \|    \|-- tax: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
>  \|    \|    \|    \|    \|-- total: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
>  \|    \|    \|    \|-- pre: struct (nullable = true)
>  \|    \|    \|    \|    \|-- alchemy_out_of_stock: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
>  \|    \|    \|    \|    \|-- meta: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- None_tx_value: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|-- exceptions: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|-- no_input_value: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|-- not_mapped: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|-- not_transformed: array (nullable = true)
>  \|    \|    \|    \|    \|    \|    \|-- element: array (containsNull = 
> false)
>  \|    \|    \|    \|    \|    \|    \|    \|-- element: string (containsNull 
> = false)
>  \|    \|    \|    \|    \|-- now_price_checkout: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
>  \|    \|    \|    \|    \|-- shipping_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
>  \|    \|    \|    \|    \|-- tax: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
>  \|    \|    \|    \|    \|-- total: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: double (nullable = true)
>  \|    \|    \|-- startDate: string (nullable = true)
>  \|    \|    \|-- status: string (nullable = true)
>  \|    \|    \|-- task_id: string (nullable = true)
>  \|    \|    \|-- url: string (nullable = true)
>  \|    \|    \|-- uuid: string (nullable = true)
>  \|    \|    \|-- validationDateTime: string (nullable = true)
>  \|    \|    \|-- validity: string (nullable = true)
>  \|    \|-- sm.task: struct (nullable = true)
>  \|    \|    \|-- cerberus_batch_id: string (nullable = true)
>  \|    \|    \|-- cerberus_id: string (nullable = true)
>  \|    \|    \|-- couponId: integer (nullable = true)
>  \|    \|    \|-- coupon_code: string (nullable = true)
>  \|    \|    \|-- created: string (nullable = true)
>  \|    \|    \|-- description: string (nullable = true)
>  \|    \|    \|-- domain: string (nullable = true)
>  \|    \|    \|-- expires: string (nullable = true)
>  \|    \|    \|-- message_id: string (nullable = true)
>  \|    \|    \|-- neverShowAfter: string (nullable = true)
>  \|    \|    \|-- neverShowBefore: string (nullable = true)
>  \|    \|    \|-- offerTitle: string (nullable = true)
>  \|    \|    \|-- result: struct (nullable = true)
>  \|    \|    \|    \|-- post: struct (nullable = true)
>  \|    \|    \|    \|    \|-- final_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- merchant_exception: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
>  \|    \|    \|    \|    \|-- now_price_checkout: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- orig_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- out_of_stock: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
>  \|    \|    \|    \|    \|-- sale_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- shipping_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- tax: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- total: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|-- pre: struct (nullable = true)
>  \|    \|    \|    \|    \|-- final_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- merchant_exception: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
>  \|    \|    \|    \|    \|-- now_price_checkout: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- orig_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- out_of_stock: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: boolean (nullable = true)
>  \|    \|    \|    \|    \|-- sale_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: integer (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- shipping_price: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- tax: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|    \|    \|-- total: struct (nullable = true)
>  \|    \|    \|    \|    \|    \|-- ci: double (nullable = true)
>  \|    \|    \|    \|    \|    \|-- value: string (nullable = true)
>  \|    \|    \|-- screenshots: array (nullable = true)
>  \|    \|    \|    \|-- element: string (containsNull = false)
>  \|    \|    \|-- startDate: string (nullable = true)
>  \|    \|    \|-- status: string (nullable = true)
>  \|    \|    \|-- task_id: string (nullable = true)
>  \|    \|    \|-- url: string (nullable = true)
>  \|    \|    \|-- uuid: string (nullable = true)
>  \|    \|    \|-- validationDateTime: string (nullable = true)
>  \|    \|    \|-- validity: string (nullable = true)
>  \|    \|-- startDate: string (nullable = true)
>  \|    \|-- task_id: string (nullable = true)
>  \|    \|-- url: string (nullable = true)
>  \|    \|-- uuid: string (nullable = true)
>  \|    \|-- validationDateTime: string (nullable = true)
>  \|    \|-- validity: string (nullable = true)
>  \|-- task_id: string (nullable = true)
>  \|-- task_step: integer (nullable = true)
>  \|-- task_type: integer (nullable = true)
> {code}
> I also tried val sqlContext = new org.apache.spark.sql.SQLContext(sc), but it 
> give the same exception. 



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org

Reply via email to