Hi,
I am using spark 1.3.1. I tried to insert (a new partition) into an
existing partitioned hive table, but got ArrayIndexOutOfBoundsException.
Below is a code snippet and the debug log. Any suggestions please.
+++++++++++++++++++++++++++++++++++++++++++++
case class Record4Dim(key: String, date: Int, hh: Int, x: Int, y: Int,
z: Int, height: Float,
u: Float , v: Float, w: Float, ph: Float, phb: Float,
t: Float, p: Float, pb: Float, qvapor: Float, qgraup: Float,
qnice: Float, qnrain: Float, tke_pbl: Float, el_pbl: Float)
def flatKeyFromWrf(x: (String, (Map[String,Float], Float))): Record4Dim
= { .... }
....
val varWHeightFlatRDD =
varWHeightRDD.map(FlatMapUtilClass().flatKeyFromWrf).toDF()
varWHeightFlatRDD.registerTempTable("table_4Dim")
for (zz <- 1 to 51)
hiveContext.sql("INSERT OVERWRITE table 4dim partition (zone=" +
ZONE + ",z=" + zz + ",year=" + YEAR + ",month=" + MONTH + ") " +
"select date, hh, x, y, height, u, v, w, ph, phb, t, pb, pb,
qvapor, qgraup, qnice, tke_pbl, el_pbl from table_4Dim where z=" + zz);
+++++++++++++++++++++++++++++++++++++++++++++
15/06/01 21:07:20 DEBUG YarnHistoryService: Enqueue [1433192840040]:
SparkListenerTaskEnd(4,0,ResultTask,ExceptionFailure(java.lang.ArrayIndexOutOfBoundsException,18,[Ljava.lang.StackTraceElement;@5783ce22,java.lang.ArrayIndexOutOfBoundsException:
18
at
org.apache.spark.sql.catalyst.expressions.GenericRow.isNullAt(rows.scala:79)
at
org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$org$apache$spark$sql$hive$execution$InsertIntoHiveTable$$writeToFile$1$1.apply(InsertIntoHiveTable.scala:103)
at
org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$org$apache$spark$sql$hive$execution$InsertIntoHiveTable$$writeToFile$1$1.apply(InsertIntoHiveTable.scala:100)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at
org.apache.spark.sql.hive.execution.InsertIntoHiveTable.org$apache$spark$sql$hive$execution$InsertIntoHiveTable$$writeToFile$1(InsertIntoHiveTable.scala:100)
at
org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$saveAsHiveFile$3.apply(InsertIntoHiveTable.scala:82)
at
org.apache.spark.sql.hive.execution.InsertIntoHiveTable$$anonfun$saveAsHiveFile$3.apply(InsertIntoHiveTable.scala:82)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)
at org.apache.spark.scheduler.Task.run(Task.scala:64)
at
org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:203)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Best,
Patcharee
---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscr...@spark.apache.org
For additional commands, e-mail: user-h...@spark.apache.org
