[jira] [Updated] (SPARK-37420) Oracle JDBC - java.lang.ArithmeticException: Decimal precision 49 exceeds max precision 38
[ https://issues.apache.org/jira/browse/SPARK-37420?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] t oo updated SPARK-37420: - Description: reading oracle jdbc is not working as expected, i thought a simple df show should work. {code:java} /usr/local/bin/pyspark --driver-class-path "/home/user/extra_jar_spark/*" --jars "/home/user/extra_jar_spark/*" jdbc2DF = spark.read \ .format("jdbc") \ .option("url", "jdbc:oracle:thin:@redact") \ .option("driver", "oracle.jdbc.OracleDriver") \ .option("dbtable", "s.t") \ .option("user", "redact") \ .option("password", "redact") \ .option("fetchsize", 1) \ .load() jdbc2DF.printSchema() root |-- ID: decimal(38,10) (nullable = true) |-- OBJECT_VERSION_NUMBER: decimal(9,0) (nullable = true) |-- START_DATE: timestamp (nullable = true) |-- END_DATE: timestamp (nullable = true) |-- CREATED_BY: decimal(15,0) (nullable = true) |-- CREATION_DATE: timestamp (nullable = true) |-- LAST_UPDATED_BY: decimal(15,0) (nullable = true) |-- LAST_UPDATE_DATE: timestamp (nullable = true) |-- LAST_UPDATE_LOGIN: decimal(15,0) (nullable = true) |-- CONTINGENCY: string (nullable = true) |-- CONTINGENCY_ID: decimal(38,10) (nullable = true) jdbc2DF.show() 21/11/20 23:42:00 ERROR Executor: Exception in task 0.0 in stage 2.0 (TID 2) java.lang.ArithmeticException: Decimal precision 49 exceeds max precision 38 at org.apache.spark.sql.errors.QueryExecutionErrors$.decimalPrecisionExceedsMaxPrecisionError(QueryExecutionErrors.scala:847) at org.apache.spark.sql.types.Decimal.set(Decimal.scala:123) at org.apache.spark.sql.types.Decimal$.apply(Decimal.scala:572) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$4(JdbcUtils.scala:418) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.nullSafeConvert(JdbcUtils.scala:546) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$3(JdbcUtils.scala:418) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$3$adapted(JdbcUtils.scala:416) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anon$1.getNext(JdbcUtils.scala:367) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anon$1.getNext(JdbcUtils.scala:349) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37) at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:31) at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:759) at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:349) at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898) at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21/11/20 23:42:00 WARN TaskSetManager: Lost task 0.0 in stage 2.0 (TID 2) (localhost executor driver): java.lang.ArithmeticException: Decimal precision 49 exceeds max precision 38 at org.apache.spark.sql.errors.QueryExecutionErrors$.decimalPrecisionExceedsMaxPrecisionError(QueryExecutionErrors.scala:847) at org.apache.spark.sql.types.Decimal.set(Decimal.scala:123) at org.apache.spark.sql.types.Decimal$.apply(Decimal.scala:572) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$4(JdbcUtils.scala:418) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.nullSafeConvert(JdbcUtils.scala:546) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$3(JdbcUtils.scala:418) at
[jira] [Updated] (SPARK-37420) Oracle JDBC - java.lang.ArithmeticException: Decimal precision 49 exceeds max precision 38
[ https://issues.apache.org/jira/browse/SPARK-37420?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] t oo updated SPARK-37420: - Description: reading oracle jdbc is not working as expected, i thought a simple df show should work. {code:java} /usr/local/bin/pyspark --driver-class-path "/home/user/extra_jar_spark/*" --jars "/home/user/extra_jar_spark/*" jdbc2DF = spark.read \ .format("jdbc") \ .option("url", "jdbc:oracle:thin:@redact") \ .option("driver", "oracle.jdbc.OracleDriver") \ .option("dbtable", "s.t") \ .option("user", "redact") \ .option("password", "redact") \ .option("fetchsize", 1) \ .load() jdbc2DF.printSchema() root |-- ID: decimal(38,10) (nullable = true) |-- OBJECT_VERSION_NUMBER: decimal(9,0) (nullable = true) |-- START_DATE: timestamp (nullable = true) |-- END_DATE: timestamp (nullable = true) |-- CREATED_BY: decimal(15,0) (nullable = true) |-- CREATION_DATE: timestamp (nullable = true) |-- LAST_UPDATED_BY: decimal(15,0) (nullable = true) |-- LAST_UPDATE_DATE: timestamp (nullable = true) |-- LAST_UPDATE_LOGIN: decimal(15,0) (nullable = true) |-- CONTINGENCY: string (nullable = true) |-- CONTINGENCY_ID: decimal(38,10) (nullable = true) jdbc2DF.show() 21/11/20 23:42:00 ERROR Executor: Exception in task 0.0 in stage 2.0 (TID 2) java.lang.ArithmeticException: Decimal precision 49 exceeds max precision 38 at org.apache.spark.sql.errors.QueryExecutionErrors$.decimalPrecisionExceedsMaxPrecisionError(QueryExecutionErrors.scala:847) at org.apache.spark.sql.types.Decimal.set(Decimal.scala:123) at org.apache.spark.sql.types.Decimal$.apply(Decimal.scala:572) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$4(JdbcUtils.scala:418) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.nullSafeConvert(JdbcUtils.scala:546) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$3(JdbcUtils.scala:418) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$3$adapted(JdbcUtils.scala:416) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anon$1.getNext(JdbcUtils.scala:367) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anon$1.getNext(JdbcUtils.scala:349) at org.apache.spark.util.NextIterator.hasNext(NextIterator.scala:73) at org.apache.spark.InterruptibleIterator.hasNext(InterruptibleIterator.scala:37) at org.apache.spark.util.CompletionIterator.hasNext(CompletionIterator.scala:31) at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExec$$anon$1.hasNext(WholeStageCodegenExec.scala:759) at org.apache.spark.sql.execution.SparkPlan.$anonfun$getByteArrayRdd$1(SparkPlan.scala:349) at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2(RDD.scala:898) at org.apache.spark.rdd.RDD.$anonfun$mapPartitionsInternal$2$adapted(RDD.scala:898) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:373) at org.apache.spark.rdd.RDD.iterator(RDD.scala:337) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:131) at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:506) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1462) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:509) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21/11/20 23:42:00 WARN TaskSetManager: Lost task 0.0 in stage 2.0 (TID 2) (localhost executor driver): java.lang.ArithmeticException: Decimal precision 49 exceeds max precision 38 at org.apache.spark.sql.errors.QueryExecutionErrors$.decimalPrecisionExceedsMaxPrecisionError(QueryExecutionErrors.scala:847) at org.apache.spark.sql.types.Decimal.set(Decimal.scala:123) at org.apache.spark.sql.types.Decimal$.apply(Decimal.scala:572) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$4(JdbcUtils.scala:418) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.nullSafeConvert(JdbcUtils.scala:546) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$.$anonfun$makeGetter$3(JdbcUtils.scala:418) at