[jira] [Updated] (SPARK-5632) not able to resolve dot('.') in field name

[Sean Owen (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SPARK-5632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Sean Owen updated SPARK-5632:
-
Assignee: Wenchen Fan

 not able to resolve dot('.') in field name
 --

 Key: SPARK-5632
 URL: https://issues.apache.org/jira/browse/SPARK-5632
 Project: Spark
  Issue Type: Sub-task
  Components: SQL
Affects Versions: 1.2.0, 1.3.0
 Environment: Spark cluster: EC2 m1.small + Spark 1.2.0
 Cassandra cluster: EC2 m3.xlarge + Cassandra 2.1.2
Reporter: Lishu Liu
Assignee: Wenchen Fan
Priority: Blocker
 Fix For: 1.4.0


 My cassandra table task_trace has a field sm.result which contains dot in the 
 name. So SQL tried to look up sm instead of full name 'sm.result'. 
 Here is my code: 
 {code}
 scala import org.apache.spark.sql.cassandra.CassandraSQLContext
 scala val cc = new CassandraSQLContext(sc)
 scala val task_trace = cc.jsonFile(/task_trace.json)
 scala task_trace.registerTempTable(task_trace)
 scala cc.setKeyspace(cerberus_data_v4)
 scala val res = cc.sql(SELECT received_datetime, task_body.cerberus_id, 
 task_body.sm.result FROM task_trace WHERE task_id = 
 'fff7304e-9984-4b45-b10c-0423a96745ce')
 res: org.apache.spark.sql.SchemaRDD = 
 SchemaRDD[57] at RDD at SchemaRDD.scala:108
 == Query Plan ==
 == Physical Plan ==
 java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, 
 cerberus_id, couponId, coupon_code, created, description, domain, expires, 
 message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, 
 sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, 
 validity
 {code}
 The full schema look like this:
 {code}
 scala task_trace.printSchema()
 root
  \|-- received_datetime: long (nullable = true)
  \|-- task_body: struct (nullable = true)
  \|\|-- cerberus_batch_id: string (nullable = true)
  \|\|-- cerberus_id: string (nullable = true)
  \|\|-- couponId: integer (nullable = true)
  \|\|-- coupon_code: string (nullable = true)
  \|\|-- created: string (nullable = true)
  \|\|-- description: string (nullable = true)
  \|\|-- domain: string (nullable = true)
  \|\|-- expires: string (nullable = true)
  \|\|-- message_id: string (nullable = true)
  \|\|-- neverShowAfter: string (nullable = true)
  \|\|-- neverShowBefore: string (nullable = true)
  \|\|-- offerTitle: string (nullable = true)
  \|\|-- screenshots: array (nullable = true)
  \|\|\|-- element: string (containsNull = false)
  \|\|-- sm.result: struct (nullable = true)
  \|\|\|-- cerberus_batch_id: string (nullable = true)
  \|\|\|-- cerberus_id: string (nullable = true)
  \|\|\|-- code: string (nullable = true)
  \|\|\|-- couponId: integer (nullable = true)
  \|\|\|-- created: string (nullable = true)
  \|\|\|-- description: string (nullable = true)
  \|\|\|-- domain: string (nullable = true)
  \|\|\|-- expires: string (nullable = true)
  \|\|\|-- message_id: string (nullable = true)
  \|\|\|-- neverShowAfter: string (nullable = true)
  \|\|\|-- neverShowBefore: string (nullable = true)
  \|\|\|-- offerTitle: string (nullable = true)
  \|\|\|-- result: struct (nullable = true)
  \|\|\|\|-- post: struct (nullable = true)
  \|\|\|\|\|-- alchemy_out_of_stock: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double (nullable = true)
  \|\|\|\|\|\|-- value: boolean (nullable = true)
  \|\|\|\|\|-- meta: struct (nullable = true)
  \|\|\|\|\|\|-- None_tx_value: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- exceptions: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- no_input_value: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- not_mapped: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- not_transformed: array (nullable = true)
  \|\|\|\|\|\|\|-- element: array (containsNull = 
 false)
  \|\|\|\|\|\|\|\|-- element: string (containsNull 
 = false)
  \|\|\|\|\|-- now_price_checkout: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double (nullable = true)
  \|\|\|\|\|\|-- value: double (nullable = true)
  \|\|\|\|\|-- shipping_price: struct (nullable = true)
  \|\|\|\|\|   

[jira] [Updated] (SPARK-5632) not able to resolve dot('.') in field name

[Patrick Wendell (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SPARK-5632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Patrick Wendell updated SPARK-5632:
---
Fix Version/s: 1.4.0

 not able to resolve dot('.') in field name
 --

 Key: SPARK-5632
 URL: https://issues.apache.org/jira/browse/SPARK-5632
 Project: Spark
  Issue Type: Sub-task
  Components: SQL
Affects Versions: 1.2.0, 1.3.0
 Environment: Spark cluster: EC2 m1.small + Spark 1.2.0
 Cassandra cluster: EC2 m3.xlarge + Cassandra 2.1.2
Reporter: Lishu Liu
Priority: Blocker
 Fix For: 1.4.0


 My cassandra table task_trace has a field sm.result which contains dot in the 
 name. So SQL tried to look up sm instead of full name 'sm.result'. 
 Here is my code: 
 {code}
 scala import org.apache.spark.sql.cassandra.CassandraSQLContext
 scala val cc = new CassandraSQLContext(sc)
 scala val task_trace = cc.jsonFile(/task_trace.json)
 scala task_trace.registerTempTable(task_trace)
 scala cc.setKeyspace(cerberus_data_v4)
 scala val res = cc.sql(SELECT received_datetime, task_body.cerberus_id, 
 task_body.sm.result FROM task_trace WHERE task_id = 
 'fff7304e-9984-4b45-b10c-0423a96745ce')
 res: org.apache.spark.sql.SchemaRDD = 
 SchemaRDD[57] at RDD at SchemaRDD.scala:108
 == Query Plan ==
 == Physical Plan ==
 java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, 
 cerberus_id, couponId, coupon_code, created, description, domain, expires, 
 message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, 
 sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, 
 validity
 {code}
 The full schema look like this:
 {code}
 scala task_trace.printSchema()
 root
  \|-- received_datetime: long (nullable = true)
  \|-- task_body: struct (nullable = true)
  \|\|-- cerberus_batch_id: string (nullable = true)
  \|\|-- cerberus_id: string (nullable = true)
  \|\|-- couponId: integer (nullable = true)
  \|\|-- coupon_code: string (nullable = true)
  \|\|-- created: string (nullable = true)
  \|\|-- description: string (nullable = true)
  \|\|-- domain: string (nullable = true)
  \|\|-- expires: string (nullable = true)
  \|\|-- message_id: string (nullable = true)
  \|\|-- neverShowAfter: string (nullable = true)
  \|\|-- neverShowBefore: string (nullable = true)
  \|\|-- offerTitle: string (nullable = true)
  \|\|-- screenshots: array (nullable = true)
  \|\|\|-- element: string (containsNull = false)
  \|\|-- sm.result: struct (nullable = true)
  \|\|\|-- cerberus_batch_id: string (nullable = true)
  \|\|\|-- cerberus_id: string (nullable = true)
  \|\|\|-- code: string (nullable = true)
  \|\|\|-- couponId: integer (nullable = true)
  \|\|\|-- created: string (nullable = true)
  \|\|\|-- description: string (nullable = true)
  \|\|\|-- domain: string (nullable = true)
  \|\|\|-- expires: string (nullable = true)
  \|\|\|-- message_id: string (nullable = true)
  \|\|\|-- neverShowAfter: string (nullable = true)
  \|\|\|-- neverShowBefore: string (nullable = true)
  \|\|\|-- offerTitle: string (nullable = true)
  \|\|\|-- result: struct (nullable = true)
  \|\|\|\|-- post: struct (nullable = true)
  \|\|\|\|\|-- alchemy_out_of_stock: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double (nullable = true)
  \|\|\|\|\|\|-- value: boolean (nullable = true)
  \|\|\|\|\|-- meta: struct (nullable = true)
  \|\|\|\|\|\|-- None_tx_value: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- exceptions: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- no_input_value: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- not_mapped: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- not_transformed: array (nullable = true)
  \|\|\|\|\|\|\|-- element: array (containsNull = 
 false)
  \|\|\|\|\|\|\|\|-- element: string (containsNull 
 = false)
  \|\|\|\|\|-- now_price_checkout: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double (nullable = true)
  \|\|\|\|\|\|-- value: double (nullable = true)
  \|\|\|\|\|-- shipping_price: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double 

[jira] [Updated] (SPARK-5632) not able to resolve dot('.') in field name

[Reynold Xin (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SPARK-5632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Reynold Xin updated SPARK-5632:
---
Issue Type: Sub-task  (was: Bug)
Parent: SPARK-6116

 not able to resolve dot('.') in field name
 --

 Key: SPARK-5632
 URL: https://issues.apache.org/jira/browse/SPARK-5632
 Project: Spark
  Issue Type: Sub-task
  Components: SQL
Affects Versions: 1.2.0, 1.3.0
 Environment: Spark cluster: EC2 m1.small + Spark 1.2.0
 Cassandra cluster: EC2 m3.xlarge + Cassandra 2.1.2
Reporter: Lishu Liu
Priority: Blocker

 My cassandra table task_trace has a field sm.result which contains dot in the 
 name. So SQL tried to look up sm instead of full name 'sm.result'. 
 Here is my code: 
 {code}
 scala import org.apache.spark.sql.cassandra.CassandraSQLContext
 scala val cc = new CassandraSQLContext(sc)
 scala val task_trace = cc.jsonFile(/task_trace.json)
 scala task_trace.registerTempTable(task_trace)
 scala cc.setKeyspace(cerberus_data_v4)
 scala val res = cc.sql(SELECT received_datetime, task_body.cerberus_id, 
 task_body.sm.result FROM task_trace WHERE task_id = 
 'fff7304e-9984-4b45-b10c-0423a96745ce')
 res: org.apache.spark.sql.SchemaRDD = 
 SchemaRDD[57] at RDD at SchemaRDD.scala:108
 == Query Plan ==
 == Physical Plan ==
 java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, 
 cerberus_id, couponId, coupon_code, created, description, domain, expires, 
 message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, 
 sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, 
 validity
 {code}
 The full schema look like this:
 {code}
 scala task_trace.printSchema()
 root
  \|-- received_datetime: long (nullable = true)
  \|-- task_body: struct (nullable = true)
  \|\|-- cerberus_batch_id: string (nullable = true)
  \|\|-- cerberus_id: string (nullable = true)
  \|\|-- couponId: integer (nullable = true)
  \|\|-- coupon_code: string (nullable = true)
  \|\|-- created: string (nullable = true)
  \|\|-- description: string (nullable = true)
  \|\|-- domain: string (nullable = true)
  \|\|-- expires: string (nullable = true)
  \|\|-- message_id: string (nullable = true)
  \|\|-- neverShowAfter: string (nullable = true)
  \|\|-- neverShowBefore: string (nullable = true)
  \|\|-- offerTitle: string (nullable = true)
  \|\|-- screenshots: array (nullable = true)
  \|\|\|-- element: string (containsNull = false)
  \|\|-- sm.result: struct (nullable = true)
  \|\|\|-- cerberus_batch_id: string (nullable = true)
  \|\|\|-- cerberus_id: string (nullable = true)
  \|\|\|-- code: string (nullable = true)
  \|\|\|-- couponId: integer (nullable = true)
  \|\|\|-- created: string (nullable = true)
  \|\|\|-- description: string (nullable = true)
  \|\|\|-- domain: string (nullable = true)
  \|\|\|-- expires: string (nullable = true)
  \|\|\|-- message_id: string (nullable = true)
  \|\|\|-- neverShowAfter: string (nullable = true)
  \|\|\|-- neverShowBefore: string (nullable = true)
  \|\|\|-- offerTitle: string (nullable = true)
  \|\|\|-- result: struct (nullable = true)
  \|\|\|\|-- post: struct (nullable = true)
  \|\|\|\|\|-- alchemy_out_of_stock: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double (nullable = true)
  \|\|\|\|\|\|-- value: boolean (nullable = true)
  \|\|\|\|\|-- meta: struct (nullable = true)
  \|\|\|\|\|\|-- None_tx_value: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- exceptions: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- no_input_value: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- not_mapped: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- not_transformed: array (nullable = true)
  \|\|\|\|\|\|\|-- element: array (containsNull = 
 false)
  \|\|\|\|\|\|\|\|-- element: string (containsNull 
 = false)
  \|\|\|\|\|-- now_price_checkout: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double (nullable = true)
  \|\|\|\|\|\|-- value: double (nullable = true)
  \|\|\|\|\|-- shipping_price: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double 

[jira] [Updated] (SPARK-5632) not able to resolve dot('.') in field name

[Reynold Xin (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SPARK-5632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Reynold Xin updated SPARK-5632:
---
 Priority: Blocker  (was: Major)
 Target Version/s: 1.4.0
Affects Version/s: 1.3.0

 not able to resolve dot('.') in field name
 --

 Key: SPARK-5632
 URL: https://issues.apache.org/jira/browse/SPARK-5632
 Project: Spark
  Issue Type: Bug
  Components: SQL
Affects Versions: 1.2.0, 1.3.0
 Environment: Spark cluster: EC2 m1.small + Spark 1.2.0
 Cassandra cluster: EC2 m3.xlarge + Cassandra 2.1.2
Reporter: Lishu Liu
Priority: Blocker

 My cassandra table task_trace has a field sm.result which contains dot in the 
 name. So SQL tried to look up sm instead of full name 'sm.result'. 
 Here is my code: 
 {code}
 scala import org.apache.spark.sql.cassandra.CassandraSQLContext
 scala val cc = new CassandraSQLContext(sc)
 scala val task_trace = cc.jsonFile(/task_trace.json)
 scala task_trace.registerTempTable(task_trace)
 scala cc.setKeyspace(cerberus_data_v4)
 scala val res = cc.sql(SELECT received_datetime, task_body.cerberus_id, 
 task_body.sm.result FROM task_trace WHERE task_id = 
 'fff7304e-9984-4b45-b10c-0423a96745ce')
 res: org.apache.spark.sql.SchemaRDD = 
 SchemaRDD[57] at RDD at SchemaRDD.scala:108
 == Query Plan ==
 == Physical Plan ==
 java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, 
 cerberus_id, couponId, coupon_code, created, description, domain, expires, 
 message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, 
 sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, 
 validity
 {code}
 The full schema look like this:
 {code}
 scala task_trace.printSchema()
 root
  \|-- received_datetime: long (nullable = true)
  \|-- task_body: struct (nullable = true)
  \|\|-- cerberus_batch_id: string (nullable = true)
  \|\|-- cerberus_id: string (nullable = true)
  \|\|-- couponId: integer (nullable = true)
  \|\|-- coupon_code: string (nullable = true)
  \|\|-- created: string (nullable = true)
  \|\|-- description: string (nullable = true)
  \|\|-- domain: string (nullable = true)
  \|\|-- expires: string (nullable = true)
  \|\|-- message_id: string (nullable = true)
  \|\|-- neverShowAfter: string (nullable = true)
  \|\|-- neverShowBefore: string (nullable = true)
  \|\|-- offerTitle: string (nullable = true)
  \|\|-- screenshots: array (nullable = true)
  \|\|\|-- element: string (containsNull = false)
  \|\|-- sm.result: struct (nullable = true)
  \|\|\|-- cerberus_batch_id: string (nullable = true)
  \|\|\|-- cerberus_id: string (nullable = true)
  \|\|\|-- code: string (nullable = true)
  \|\|\|-- couponId: integer (nullable = true)
  \|\|\|-- created: string (nullable = true)
  \|\|\|-- description: string (nullable = true)
  \|\|\|-- domain: string (nullable = true)
  \|\|\|-- expires: string (nullable = true)
  \|\|\|-- message_id: string (nullable = true)
  \|\|\|-- neverShowAfter: string (nullable = true)
  \|\|\|-- neverShowBefore: string (nullable = true)
  \|\|\|-- offerTitle: string (nullable = true)
  \|\|\|-- result: struct (nullable = true)
  \|\|\|\|-- post: struct (nullable = true)
  \|\|\|\|\|-- alchemy_out_of_stock: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double (nullable = true)
  \|\|\|\|\|\|-- value: boolean (nullable = true)
  \|\|\|\|\|-- meta: struct (nullable = true)
  \|\|\|\|\|\|-- None_tx_value: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- exceptions: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- no_input_value: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- not_mapped: array (nullable = true)
  \|\|\|\|\|\|\|-- element: string (containsNull = 
 false)
  \|\|\|\|\|\|-- not_transformed: array (nullable = true)
  \|\|\|\|\|\|\|-- element: array (containsNull = 
 false)
  \|\|\|\|\|\|\|\|-- element: string (containsNull 
 = false)
  \|\|\|\|\|-- now_price_checkout: struct (nullable = true)
  \|\|\|\|\|\|-- ci: double (nullable = true)
  \|\|\|\|\|\|-- value: double (nullable = true)
  \|\|\|\|\|-- shipping_price: struct (nullable = true)
  \|\|\|  

[jira] [Updated] (SPARK-5632) not able to resolve dot('.') in field name

[Reynold Xin (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SPARK-5632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Reynold Xin updated SPARK-5632:
---
Description: 
My cassandra table task_trace has a field sm.result which contains dot in the 
name. So SQL tried to look up sm instead of full name 'sm.result'. 

Here is my code: 
{code}
scala import org.apache.spark.sql.cassandra.CassandraSQLContext
scala val cc = new CassandraSQLContext(sc)
scala val task_trace = cc.jsonFile(/task_trace.json)
scala task_trace.registerTempTable(task_trace)
scala cc.setKeyspace(cerberus_data_v4)
scala val res = cc.sql(SELECT received_datetime, task_body.cerberus_id, 
task_body.sm.result FROM task_trace WHERE task_id = 
'fff7304e-9984-4b45-b10c-0423a96745ce')
res: org.apache.spark.sql.SchemaRDD = 
SchemaRDD[57] at RDD at SchemaRDD.scala:108
== Query Plan ==
== Physical Plan ==
java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, 
cerberus_id, couponId, coupon_code, created, description, domain, expires, 
message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, 
sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, validity
{code}

The full schema look like this:
{code}
scala task_trace.printSchema()
root
 \|-- received_datetime: long (nullable = true)
 \|-- task_body: struct (nullable = true)
 \|\|-- cerberus_batch_id: string (nullable = true)
 \|\|-- cerberus_id: string (nullable = true)
 \|\|-- couponId: integer (nullable = true)
 \|\|-- coupon_code: string (nullable = true)
 \|\|-- created: string (nullable = true)
 \|\|-- description: string (nullable = true)
 \|\|-- domain: string (nullable = true)
 \|\|-- expires: string (nullable = true)
 \|\|-- message_id: string (nullable = true)
 \|\|-- neverShowAfter: string (nullable = true)
 \|\|-- neverShowBefore: string (nullable = true)
 \|\|-- offerTitle: string (nullable = true)
 \|\|-- screenshots: array (nullable = true)
 \|\|\|-- element: string (containsNull = false)
 \|\|-- sm.result: struct (nullable = true)
 \|\|\|-- cerberus_batch_id: string (nullable = true)
 \|\|\|-- cerberus_id: string (nullable = true)
 \|\|\|-- code: string (nullable = true)
 \|\|\|-- couponId: integer (nullable = true)
 \|\|\|-- created: string (nullable = true)
 \|\|\|-- description: string (nullable = true)
 \|\|\|-- domain: string (nullable = true)
 \|\|\|-- expires: string (nullable = true)
 \|\|\|-- message_id: string (nullable = true)
 \|\|\|-- neverShowAfter: string (nullable = true)
 \|\|\|-- neverShowBefore: string (nullable = true)
 \|\|\|-- offerTitle: string (nullable = true)
 \|\|\|-- result: struct (nullable = true)
 \|\|\|\|-- post: struct (nullable = true)
 \|\|\|\|\|-- alchemy_out_of_stock: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: boolean (nullable = true)
 \|\|\|\|\|-- meta: struct (nullable = true)
 \|\|\|\|\|\|-- None_tx_value: array (nullable = true)
 \|\|\|\|\|\|\|-- element: string (containsNull = false)
 \|\|\|\|\|\|-- exceptions: array (nullable = true)
 \|\|\|\|\|\|\|-- element: string (containsNull = false)
 \|\|\|\|\|\|-- no_input_value: array (nullable = true)
 \|\|\|\|\|\|\|-- element: string (containsNull = false)
 \|\|\|\|\|\|-- not_mapped: array (nullable = true)
 \|\|\|\|\|\|\|-- element: string (containsNull = false)
 \|\|\|\|\|\|-- not_transformed: array (nullable = true)
 \|\|\|\|\|\|\|-- element: array (containsNull = false)
 \|\|\|\|\|\|\|\|-- element: string (containsNull = 
false)
 \|\|\|\|\|-- now_price_checkout: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: double (nullable = true)
 \|\|\|\|\|-- shipping_price: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: double (nullable = true)
 \|\|\|\|\|-- tax: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: double (nullable = true)
 \|\|\|\|\|-- total: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: double (nullable = true)
 \|\|\|\|-- pre: struct (nullable = true)
 \|\|\|\|\|-- alchemy_out_of_stock: struct (nullable = true)
 \|\|\|\|

[jira] [Updated] (SPARK-5632) not able to resolve dot('.') in field name

[Lishu Liu (JIRA)]

 [ 
https://issues.apache.org/jira/browse/SPARK-5632?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Lishu Liu updated SPARK-5632:
-
Description: 
My cassandra table task_trace has a field sm.result which contains dot in the 
name. So SQL tried to look up sm instead of full name 'sm.result'. 

Here is my code: 
scala import org.apache.spark.sql.cassandra.CassandraSQLContext
scala val cc = new CassandraSQLContext(sc)
scala val task_trace = cc.jsonFile(/task_trace.json)
scala task_trace.registerTempTable(task_trace)
scala cc.setKeyspace(cerberus_data_v4)
scala val res = cc.sql(SELECT received_datetime, task_body.cerberus_id, 
task_body.sm.result FROM task_trace WHERE task_id = 
'fff7304e-9984-4b45-b10c-0423a96745ce')
res: org.apache.spark.sql.SchemaRDD = 
SchemaRDD[57] at RDD at SchemaRDD.scala:108
== Query Plan ==
== Physical Plan ==
java.lang.RuntimeException: No such struct field sm in cerberus_batch_id, 
cerberus_id, couponId, coupon_code, created, description, domain, expires, 
message_id, neverShowAfter, neverShowBefore, offerTitle, screenshots, 
sm.result, sm.task, startDate, task_id, url, uuid, validationDateTime, validity

The full schema look like this:
scala task_trace.printSchema()
root
 \|-- received_datetime: long (nullable = true)
 \|-- task_body: struct (nullable = true)
 \|\|-- cerberus_batch_id: string (nullable = true)
 \|\|-- cerberus_id: string (nullable = true)
 \|\|-- couponId: integer (nullable = true)
 \|\|-- coupon_code: string (nullable = true)
 \|\|-- created: string (nullable = true)
 \|\|-- description: string (nullable = true)
 \|\|-- domain: string (nullable = true)
 \|\|-- expires: string (nullable = true)
 \|\|-- message_id: string (nullable = true)
 \|\|-- neverShowAfter: string (nullable = true)
 \|\|-- neverShowBefore: string (nullable = true)
 \|\|-- offerTitle: string (nullable = true)
 \|\|-- screenshots: array (nullable = true)
 \|\|\|-- element: string (containsNull = false)
 \|\|-- sm.result: struct (nullable = true)
 \|\|\|-- cerberus_batch_id: string (nullable = true)
 \|\|\|-- cerberus_id: string (nullable = true)
 \|\|\|-- code: string (nullable = true)
 \|\|\|-- couponId: integer (nullable = true)
 \|\|\|-- created: string (nullable = true)
 \|\|\|-- description: string (nullable = true)
 \|\|\|-- domain: string (nullable = true)
 \|\|\|-- expires: string (nullable = true)
 \|\|\|-- message_id: string (nullable = true)
 \|\|\|-- neverShowAfter: string (nullable = true)
 \|\|\|-- neverShowBefore: string (nullable = true)
 \|\|\|-- offerTitle: string (nullable = true)
 \|\|\|-- result: struct (nullable = true)
 \|\|\|\|-- post: struct (nullable = true)
 \|\|\|\|\|-- alchemy_out_of_stock: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: boolean (nullable = true)
 \|\|\|\|\|-- meta: struct (nullable = true)
 \|\|\|\|\|\|-- None_tx_value: array (nullable = true)
 \|\|\|\|\|\|\|-- element: string (containsNull = false)
 \|\|\|\|\|\|-- exceptions: array (nullable = true)
 \|\|\|\|\|\|\|-- element: string (containsNull = false)
 \|\|\|\|\|\|-- no_input_value: array (nullable = true)
 \|\|\|\|\|\|\|-- element: string (containsNull = false)
 \|\|\|\|\|\|-- not_mapped: array (nullable = true)
 \|\|\|\|\|\|\|-- element: string (containsNull = false)
 \|\|\|\|\|\|-- not_transformed: array (nullable = true)
 \|\|\|\|\|\|\|-- element: array (containsNull = false)
 \|\|\|\|\|\|\|\|-- element: string (containsNull = 
false)
 \|\|\|\|\|-- now_price_checkout: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: double (nullable = true)
 \|\|\|\|\|-- shipping_price: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: double (nullable = true)
 \|\|\|\|\|-- tax: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: double (nullable = true)
 \|\|\|\|\|-- total: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double (nullable = true)
 \|\|\|\|\|\|-- value: double (nullable = true)
 \|\|\|\|-- pre: struct (nullable = true)
 \|\|\|\|\|-- alchemy_out_of_stock: struct (nullable = true)
 \|\|\|\|\|\|-- ci: double