[
https://issues.apache.org/jira/browse/SPARK-37196?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sergey Yavorski updated SPARK-37196:
Description:
Still reproducible in Spark 3.0.1.
+*How to reproduce:*+
*SHELL*
> export SPARK_MAJOR_VERSION=3
*SPARK PART 1*
> spark-shell --master local
SPARK_MAJOR_VERSION is set to 3, using Spark3
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use
setLogLevel(newLevel).
21/10/25 11:43:58 WARN Utils: Service 'SparkUI' could not bind on port 4040.
Attempting port 4041.
21/10/25 11:43:58 WARN Utils: Service 'SparkUI' could not bind on port 4041.
Attempting port 4042.
Spark context Web UI available at :4042
Spark context available as 'sc' (master = local, app id = local-1635151438558).
Spark session available as 'spark'.
Welcome to
__
/ __/__ ___ _/ /__
_\ \/ _ \/ _ `/ __/ '_/
/___/ .__/\_,_/_/ /_/\_\ version 3.0.1
/_/
Using Scala version 2.12.10 (OpenJDK 64-Bit Server VM, Java 1.8.0_302)
Type in expressions to have them evaluated.
Type :help for more information.
scala> val df = spark.sql("select 'dummy' as name,
100010.7010 as value")
df: org.apache.spark.sql.DataFrame = [name: string, value: decimal(38,16)]
scala> df.write.mode("Overwrite").parquet("test.parquet")
*HIVE*
hive> create external table test_precision(name string, value Decimal(18,6))
STORED AS PARQUET LOCATION 'test';
OK
Time taken: 0.067 seconds
*SPARK PART 2*
scala> spark.conf.set("spark.sql.hive.convertMetastoreParquet","false");
scala> val df_hive = spark.sql("select * from test_precision");
df_hive: org.apache.spark.sql.DataFrame = [name: string, value: decimal(18,6)]
scala> df_hive.show;
21/10/25 12:04:51 ERROR Executor: Exception in task 0.0 in stage 2.0 (TID 2)/ 1]
*{color:red}java.lang.NullPointerException
at
org.apache.spark.sql.hive.HiveShim$.toCatalystDecimal(HiveShim.scala:106){color}*
at
org.apache.spark.sql.hive.HadoopTableReader$.$anonfun$fillObject$13(TableReader.scala:465)
at
org.apache.spark.sql.hive.HadoopTableReader$.$anonfun$fillObject$13$adapted(TableReader.scala:464)
at
org.apache.spark.sql.hive.HadoopTableReader$.$anonfun$fillObject$18(TableReader.scala:493)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:459)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:459)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown
Source)
at
org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at
org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:729)
at
org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:340)
at
org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:872)
at
org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:872)
at
org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:313)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
at org.apache.spark.scheduler.Task.run(Task.scala:127)
at
org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446)
at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21/10/25 12:04:51 WARN TaskSetManager: Lost task 0.0 in stage 2.0 (TID 2,
tklis-kappd0005.dev.df.sbrf.ru, executor driver): java.lang.NullPointerException
at
org.apache.spark.sql.hive.HiveShim$.toCatalystDecimal(HiveShim.scala:106)
at
org.apache.spark.sql.hive.HadoopTableReader$.$anonfun$fillObject$13(TableReader.scala:465)
at
org.apache.spark.sql.hive.HadoopTableReader$.$anonfun$fillObject$13$adapted(TableReader.scala:464)
at
org.apache.spark.sql.hive.HadoopTableReader$.$anonfun$fillObject$18(TableReader.scala:493)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:459)
at scala.collection.Iterator$$anon$10.next(Iterator.scala:459)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown
Source)
at
org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
at
