Qingsheng Ren created FLINK-29218:
-------------------------------------
Summary: ADD JAR syntax could not work with Hive catalog in SQL
client
Key: FLINK-29218
URL: https://issues.apache.org/jira/browse/FLINK-29218
Project: Flink
Issue Type: Bug
Components: Connectors / Hive, Table SQL / API
Affects Versions: 1.16.0
Reporter: Qingsheng Ren
Assignee: dalongliu
ADD JAR syntax is not working for adding Hive / Hadoop dependencies into SQL
client.
To reproduce the problem:
# Place Hive connector and Hadoop JAR outside {{{}lib{}}}, and add them into
the session using {{ADD JAR}} syntax.
# Create a Hive catalog using {{CREATE CATALOG}}
Exception thrown by SQL client:
{code:java}
2022-09-07 15:23:15,737 WARN org.apache.flink.table.client.cli.CliClient
[] - Could not execute SQL statement.
org.apache.flink.table.client.gateway.SqlExecutionException: Could not execute
SQL statement.
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:208)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.executeOperation(CliClient.java:634)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.callOperation(CliClient.java:468)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.executeOperation(CliClient.java:371)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.getAndExecuteStatements(CliClient.java:328)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.executeInteractive(CliClient.java:279)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.executeInInteractiveMode(CliClient.java:227)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at org.apache.flink.table.client.SqlClient.openCli(SqlClient.java:151)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at org.apache.flink.table.client.SqlClient.start(SqlClient.java:95)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at org.apache.flink.table.client.SqlClient.startClient(SqlClient.java:187)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at org.apache.flink.table.client.SqlClient.main(SqlClient.java:161)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
Caused by: org.apache.flink.table.api.ValidationException: Unable to create
catalog 'myhive'.Catalog options are:
'hive-conf-dir'='file:///Users/renqs/Workspaces/flink/flink-master/build-target/opt/hive-conf'
'type'='hive'
at
org.apache.flink.table.factories.FactoryUtil.createCatalog(FactoryUtil.java:438)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.createCatalog(TableEnvironmentImpl.java:1423)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:1165)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:206)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
... 10 more
Caused by: org.apache.flink.table.catalog.exceptions.CatalogException: Failed
to load hive-site.xml from specified
path:file:/Users/renqs/Workspaces/flink/flink-master/build-target/opt/hive-conf/hive-site.xml
at
org.apache.flink.table.catalog.hive.HiveCatalog.createHiveConf(HiveCatalog.java:273)
~[?:?]
at
org.apache.flink.table.catalog.hive.HiveCatalog.<init>(HiveCatalog.java:184)
~[?:?]
at
org.apache.flink.table.catalog.hive.factories.HiveCatalogFactory.createCatalog(HiveCatalogFactory.java:76)
~[?:?]
at
org.apache.flink.table.factories.FactoryUtil.createCatalog(FactoryUtil.java:435)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.createCatalog(TableEnvironmentImpl.java:1423)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:1165)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:206)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
... 10 more
Caused by: java.io.IOException: No FileSystem for scheme: file
at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2799)
~[?:?]
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2810)
~[?:?]
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:100) ~[?:?]
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2849)
~[?:?]
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2831) ~[?:?]
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:389) ~[?:?]
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:356) ~[?:?]
at
org.apache.flink.table.catalog.hive.HiveCatalog.createHiveConf(HiveCatalog.java:268)
~[?:?]
at
org.apache.flink.table.catalog.hive.HiveCatalog.<init>(HiveCatalog.java:184)
~[?:?]
at
org.apache.flink.table.catalog.hive.factories.HiveCatalogFactory.createCatalog(HiveCatalogFactory.java:76)
~[?:?]
at
org.apache.flink.table.factories.FactoryUtil.createCatalog(FactoryUtil.java:435)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.createCatalog(TableEnvironmentImpl.java:1423)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:1165)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:206)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
... 10 more
{code}
Another test:
# Put Hadoop JAR inside {{{}lib{}}}, and Hive connector outside, then add Hive
connector JAR into the session with {{ADD JAR}} syntax
# Create a Hive catalog using {{CREATE CATALOG}}
Exception thrown by SQL client:
{code:java}
2022-09-07 15:29:57,362 WARN org.apache.flink.table.client.cli.CliClient
[] - Could not execute SQL statement.
org.apache.flink.table.client.gateway.SqlExecutionException: Could not execute
SQL statement.
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:208)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.executeOperation(CliClient.java:634)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.callOperation(CliClient.java:468)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.executeOperation(CliClient.java:371)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.getAndExecuteStatements(CliClient.java:328)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.executeInteractive(CliClient.java:279)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.cli.CliClient.executeInInteractiveMode(CliClient.java:227)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at org.apache.flink.table.client.SqlClient.openCli(SqlClient.java:151)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at org.apache.flink.table.client.SqlClient.start(SqlClient.java:95)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at org.apache.flink.table.client.SqlClient.startClient(SqlClient.java:187)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at org.apache.flink.table.client.SqlClient.main(SqlClient.java:161)
[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
Caused by: org.apache.flink.table.api.ValidationException: Could not execute
CREATE CATALOG: (catalogName: [myhive], properties:
[{hive-conf-dir=file:///Users/renqs/Workspaces/flink/flink-master/build-target/opt/hive-conf,
type=hive}])
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.createCatalog(TableEnvironmentImpl.java:1432)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:1165)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:206)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
... 10 more
Caused by: org.apache.flink.table.catalog.exceptions.CatalogException: Failed
to create Hive Metastore client
at
org.apache.flink.table.catalog.hive.client.HiveShimV230.getHiveMetastoreClient(HiveShimV230.java:74)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.createMetastoreClient(HiveMetastoreClientWrapper.java:283)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.<init>(HiveMetastoreClientWrapper.java:84)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.<init>(HiveMetastoreClientWrapper.java:74)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientFactory.create(HiveMetastoreClientFactory.java:32)
~[?:?]
at
org.apache.flink.table.catalog.hive.HiveCatalog.open(HiveCatalog.java:300)
~[?:?]
at
org.apache.flink.table.catalog.CatalogManager.registerCatalog(CatalogManager.java:211)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.createCatalog(TableEnvironmentImpl.java:1428)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:1165)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:206)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
... 10 more
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_292]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_292]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_292]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_292]
at
org.apache.flink.table.catalog.hive.client.HiveShimV230.getHiveMetastoreClient(HiveShimV230.java:72)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.createMetastoreClient(HiveMetastoreClientWrapper.java:283)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.<init>(HiveMetastoreClientWrapper.java:84)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.<init>(HiveMetastoreClientWrapper.java:74)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientFactory.create(HiveMetastoreClientFactory.java:32)
~[?:?]
at
org.apache.flink.table.catalog.hive.HiveCatalog.open(HiveCatalog.java:300)
~[?:?]
at
org.apache.flink.table.catalog.CatalogManager.registerCatalog(CatalogManager.java:211)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.createCatalog(TableEnvironmentImpl.java:1428)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:1165)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:206)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
... 10 more
Caused by: org.apache.hadoop.hive.metastore.api.MetaException:
org.apache.hadoop.hive.metastore.HiveMetaStoreClient class not found
at
org.apache.hadoop.hive.metastore.MetaStoreUtils.getClass(MetaStoreUtils.java:1710)
~[?:?]
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:131)
~[?:?]
at
org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:89)
~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_292]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_292]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_292]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_292]
at
org.apache.flink.table.catalog.hive.client.HiveShimV230.getHiveMetastoreClient(HiveShimV230.java:72)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.createMetastoreClient(HiveMetastoreClientWrapper.java:283)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.<init>(HiveMetastoreClientWrapper.java:84)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper.<init>(HiveMetastoreClientWrapper.java:74)
~[?:?]
at
org.apache.flink.table.catalog.hive.client.HiveMetastoreClientFactory.create(HiveMetastoreClientFactory.java:32)
~[?:?]
at
org.apache.flink.table.catalog.hive.HiveCatalog.open(HiveCatalog.java:300)
~[?:?]
at
org.apache.flink.table.catalog.CatalogManager.registerCatalog(CatalogManager.java:211)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.createCatalog(TableEnvironmentImpl.java:1428)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.api.internal.TableEnvironmentImpl.executeInternal(TableEnvironmentImpl.java:1165)
~[flink-table-api-java-uber-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
at
org.apache.flink.table.client.gateway.local.LocalExecutor.executeOperation(LocalExecutor.java:206)
~[flink-sql-client-1.16-SNAPSHOT.jar:1.16-SNAPSHOT]
... 10 more {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
