yjphhw opened a new issue, #57463:
URL: https://github.com/apache/spark/issues/57463
In spark, the Hive derby jar in pom.xml is
<derby.version>10.16.1.1</derby.version>
The derby jar version is lower than current Hive 4.2.0 use, which is
10.17.1.0.
Because of version lag, the spark 4.2.0 can not load Hive tables, give
errors when run `spark-sql` as follows:
`26/07/22 11:09:47 ERROR Datastore: Error thrown executing ALTER TABLE
TAB_COL_STATS ADD "TABLE_NAME" VARCHAR(256) NOT NULL : In an ALTER TABLE
statement, the column 'TABLE_NAME' has been specified as NOT NULL and either
the DEFAULT clause was not specified or was specified as DEFAULT NULL.
java.sql.SQLSyntaxErrorException: In an ALTER TABLE statement, the column
'TABLE_NAME' has been specified as NOT NULL and either the DEFAULT clause was
not specified or was specified as DEFAULT NULL.
at
org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(SQLExceptionFactory.java:103)
at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Util.java:230)
at
org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(TransactionResourceImpl.java:431)
at
org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(TransactionResourceImpl.java:360)
at
org.apache.derby.impl.jdbc.EmbedConnection.handleException(EmbedConnection.java:2400)
at
org.apache.derby.impl.jdbc.ConnectionChild.handleException(ConnectionChild.java:86)
at
org.apache.derby.impl.jdbc.EmbedStatement.execute(EmbedStatement.java:697)
at
org.apache.derby.impl.jdbc.EmbedStatement.execute(EmbedStatement.java:637)
at
org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
at
org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
at
org.datanucleus.store.rdbms.table.AbstractTable.executeDdlStatement(AbstractTable.java:879)
at
org.datanucleus.store.rdbms.table.AbstractTable.executeDdlStatementList(AbstractTable.java:830)
at
org.datanucleus.store.rdbms.table.TableImpl.validateColumns(TableImpl.java:257)
at
org.datanucleus.store.rdbms.RDBMSStoreManager$ClassAdder.performTablesValidation(RDBMSStoreManager.java:3398)
at
org.datanucleus.store.rdbms.RDBMSStoreManager$ClassAdder.run(RDBMSStoreManager.java:2896)
at
org.datanucleus.store.rdbms.AbstractSchemaTransaction.execute(AbstractSchemaTransaction.java:119)
at
org.datanucleus.store.rdbms.RDBMSStoreManager.manageClasses(RDBMSStoreManager.java:1627)
at
org.datanucleus.store.rdbms.RDBMSStoreManager.getDatastoreClass(RDBMSStoreManager.java:672)
at
org.datanucleus.store.rdbms.query.RDBMSQueryUtils.getStatementForCandidates(RDBMSQueryUtils.java:425)
at
org.datanucleus.store.rdbms.query.JDOQLQuery.compileQueryFull(JDOQLQuery.java:865)
at
org.datanucleus.store.rdbms.query.JDOQLQuery.compileInternal(JDOQLQuery.java:347)
at org.datanucleus.store.query.Query.executeQuery(Query.java:1816)
at org.datanucleus.store.query.Query.executeWithArray(Query.java:1744)
at org.datanucleus.store.query.Query.execute(Query.java:1726)
at org.datanucleus.api.jdo.JDOQuery.executeInternal(JDOQuery.java:374)
at org.datanucleus.api.jdo.JDOQuery.execute(JDOQuery.java:216)
at
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.ensureDbInit(MetaStoreDirectSql.java:184)
at
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.<init>(MetaStoreDirectSql.java:144)
at
org.apache.hadoop.hive.metastore.ObjectStore.initializeHelper(ObjectStore.java:411)
at
org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:343)
at
org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:304)
at
org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:77)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:159)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:126)
at
org.apache.hadoop.hive.metastore.RawStoreProxy.<init>(RawStoreProxy.java:58)
at
org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:67)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStoreForConf(HiveMetaStore.java:629)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMSForConf(HiveMetaStore.java:595)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:589)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:656)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:432)
at
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:148)
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107)
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:79)
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:92)
at
org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6903)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:163)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:70)
at
java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
at
java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
at
org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1740)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:83)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:133)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)
at
org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3607)
at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3659)
at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3639)
at org.apache.hadoop.hive.ql.metadata.Hive.getDatabase(Hive.java:1563)
at
org.apache.hadoop.hive.ql.metadata.Hive.databaseExists(Hive.java:1552)
at
org.apache.spark.sql.hive.client.Shim_v2_0.databaseExists(HiveShim.scala:919)
at
org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$databaseExists$1(HiveClientImpl.scala:411)
at
scala.runtime.java8.JFunction0$mcZ$sp.apply(JFunction0$mcZ$sp.scala:17)
at
org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:296)
at
org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:239)
at
org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:238)
at
org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:276)
at
org.apache.spark.sql.hive.client.HiveClientImpl.databaseExists(HiveClientImpl.scala:411)
at
org.apache.spark.sql.hive.HiveExternalCatalog.$anonfun$databaseExists$1(HiveExternalCatalog.scala:197)
at
scala.runtime.java8.JFunction0$mcZ$sp.apply(JFunction0$mcZ$sp.scala:17)
at
org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:105)
at
org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:197)
at
org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:155)
at
org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:149)
at
org.apache.spark.sql.hive.thriftserver.SparkSQLEnv$.init(SparkSQLEnv.scala:76)
at
org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.<init>(SparkSQLCLIDriver.scala:388)
at
org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:154)
at
org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)
at
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at
org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:1033)
at
org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:203)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:226)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:95)
at
org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1171)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1180)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: ERROR 42601: In an ALTER TABLE statement, the column 'TABLE_NAME'
has been specified as NOT NULL and either the DEFAULT clause was not specified
or was specified as DEFAULT NULL.
at
org.apache.derby.shared.common.error.StandardException.newException(StandardException.java:299)
at
org.apache.derby.shared.common.error.StandardException.newException(StandardException.java:294)
at
org.apache.derby.impl.sql.compile.ColumnDefinitionNode.bindAndValidateDefault(ColumnDefinitionNode.java:484)
at
org.apache.derby.impl.sql.compile.TableElementList.validate(TableElementList.java:201)
at
org.apache.derby.impl.sql.compile.AlterTableNode.bindStatement(AlterTableNode.java:383)
at
org.apache.derby.impl.sql.GenericStatement.prepMinion(GenericStatement.java:401)
at
org.apache.derby.impl.sql.GenericStatement.prepare(GenericStatement.java:99)
at
org.apache.derby.impl.sql.conn.GenericLanguageConnectionContext.prepareInternalStatement(GenericLanguageConnectionContext.java:1114)
at
org.apache.derby.impl.jdbc.EmbedStatement.execute(EmbedStatement.java:689)
... 83 more
26/07/22 11:09:47 ERROR Datastore: An exception was thrown while
adding/validating class(es) : In an ALTER TABLE statement, the column
'TABLE_NAME' has been specified as NOT NULL and either the DEFAULT clause was
not specified or was specified as DEFAULT NULL.
java.sql.SQLSyntaxErrorException: In an ALTER TABLE statement, the column
'TABLE_NAME' has been specified as NOT NULL and either the DEFAULT clause was
not specified or was specified as DEFAULT NULL.
at
org.apache.derby.impl.jdbc.SQLExceptionFactory.getSQLException(SQLExceptionFactory.java:103)
at org.apache.derby.impl.jdbc.Util.generateCsSQLException(Util.java:230)
at
org.apache.derby.impl.jdbc.TransactionResourceImpl.wrapInSQLException(TransactionResourceImpl.java:431)
at
org.apache.derby.impl.jdbc.TransactionResourceImpl.handleException(TransactionResourceImpl.java:360)
at
org.apache.derby.impl.jdbc.EmbedConnection.handleException(EmbedConnection.java:2400)
at
org.apache.derby.impl.jdbc.ConnectionChild.handleException(ConnectionChild.java:86)
at
org.apache.derby.impl.jdbc.EmbedStatement.execute(EmbedStatement.java:697)
at
org.apache.derby.impl.jdbc.EmbedStatement.execute(EmbedStatement.java:637)
at
org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
at
org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
at
org.datanucleus.store.rdbms.table.AbstractTable.executeDdlStatement(AbstractTable.java:879)
at
org.datanucleus.store.rdbms.table.AbstractTable.executeDdlStatementList(AbstractTable.java:830)
at
org.datanucleus.store.rdbms.table.TableImpl.validateColumns(TableImpl.java:257)
at
org.datanucleus.store.rdbms.RDBMSStoreManager$ClassAdder.performTablesValidation(RDBMSStoreManager.java:3398)
at
org.datanucleus.store.rdbms.RDBMSStoreManager$ClassAdder.run(RDBMSStoreManager.java:2896)
at
org.datanucleus.store.rdbms.AbstractSchemaTransaction.execute(AbstractSchemaTransaction.java:119)
at
org.datanucleus.store.rdbms.RDBMSStoreManager.manageClasses(RDBMSStoreManager.java:1627)
at
org.datanucleus.store.rdbms.RDBMSStoreManager.getDatastoreClass(RDBMSStoreManager.java:672)
at
org.datanucleus.store.rdbms.query.RDBMSQueryUtils.getStatementForCandidates(RDBMSQueryUtils.java:425)
at
org.datanucleus.store.rdbms.query.JDOQLQuery.compileQueryFull(JDOQLQuery.java:865)
at
org.datanucleus.store.rdbms.query.JDOQLQuery.compileInternal(JDOQLQuery.java:347)
at org.datanucleus.store.query.Query.executeQuery(Query.java:1816)
at org.datanucleus.store.query.Query.executeWithArray(Query.java:1744)
at org.datanucleus.store.query.Query.execute(Query.java:1726)
at org.datanucleus.api.jdo.JDOQuery.executeInternal(JDOQuery.java:374)
at org.datanucleus.api.jdo.JDOQuery.execute(JDOQuery.java:216)
at
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.ensureDbInit(MetaStoreDirectSql.java:184)
at
org.apache.hadoop.hive.metastore.MetaStoreDirectSql.<init>(MetaStoreDirectSql.java:144)
at
org.apache.hadoop.hive.metastore.ObjectStore.initializeHelper(ObjectStore.java:411)
at
org.apache.hadoop.hive.metastore.ObjectStore.initialize(ObjectStore.java:343)
at
org.apache.hadoop.hive.metastore.ObjectStore.setConf(ObjectStore.java:304)
at
org.apache.hadoop.util.ReflectionUtils.setConf(ReflectionUtils.java:77)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:159)
at
org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:126)
at
org.apache.hadoop.hive.metastore.RawStoreProxy.<init>(RawStoreProxy.java:58)
at
org.apache.hadoop.hive.metastore.RawStoreProxy.getProxy(RawStoreProxy.java:67)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.newRawStoreForConf(HiveMetaStore.java:629)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMSForConf(HiveMetaStore.java:595)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.getMS(HiveMetaStore.java:589)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:656)
at
org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:432)
at
java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:148)
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:107)
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:79)
at
org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:92)
at
org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6903)
at
org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:163)
at
org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:70)
at
java.base/jdk.internal.reflect.DirectConstructorHandleAccessor.newInstance(DirectConstructorHandleAccessor.java:62)
at
java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:502)
at
java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:486)
at
org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1740)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:83)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:133)
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:104)
at
org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3607)
`
Althoug derby is not used in production, but for beginer learning is also
important.
So for consitent, I propose to change
<derby.version>10.16.1.1</derby.version> to
<derby.version>10.17.0</derby.version>, which can eliminate the above errors.
thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
