1. scala> import org.apache.spark.sql.hive.HiveContext
2. import org.apache.spark.sql.hive.HiveContext
3.
4. scala> import org.apache.spark.sql.hive.orc._
5. import org.apache.spark.sql.hive.orc._
6.
7. scala> import org.apache.spark.sql.types.{StructType, StructField,
StringType, IntegerType};
8. import org.apache.spark.sql.types.{StructType, StructField, StringType
, IntegerType}
9.
10. scala> val hiveContext = new org.apache.spark.sql.hive.HiveContext(sc
)
11. 15/12/21 02:06:24 WARN SparkConf: The configuration key
'spark.yarn.applicationMaster.waitTries' has been deprecated as of Spark
1.3 and and may be r
12. emoved in the future. Please use the new key 'spark.yarn.am.waitTime'
instead.
13. 15/12/21 02:06:24 INFO HiveContext: Initializing execution hive,
version 0.13.1
14. hiveContext: org.apache.spark.sql.hive.HiveContext = org.apache.spark
.sql.hive.HiveContext@74cba4b
15.
16.
17. scala> val customSchema = StructType(Seq(StructField("year",
IntegerType, true),StructField("make", StringType, true),StructField(
"model", StringType
18. , true),StructField("comment", StringType, true),StructField("blank",
StringType, true)))
19. customSchema: org.apache.spark.sql.types.StructType = StructType(
StructField(year,IntegerType,true), StructField(make,StringType,true),
StructField(m
20. odel,StringType,true), StructField(comment,StringType,true),
StructField(blank,StringType,true))
21.
22. scala> val customSchema = (new StructType).add("year", IntegerType,
true).add("make", StringType, true).add("model", StringType, true).add(
"comment",
23. StringType, true).add("blank", StringType, true)
24. :24: error: not enough arguments for constructor StructType: (fields:
Array[org.apache.spark.sql.types.StructField])org.apache.spark.sql.typ
25. es.StructType.
26. Unspecified value parameter fields.
27. val customSchema = (new StructType).add("year", IntegerType, true).
add("make", StringType, true).add("model", StringType, true).add(
"comment",
28. StringType, true).add("blank", StringType, true)
