nrg4878 commented on code in PR #4720: URL: https://github.com/apache/hive/pull/4720#discussion_r1330464405
########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/HiveJDBCConnectorProvider.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.dataconnector.jdbc; + +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.DataConnector; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class HiveJDBCConnectorProvider extends AbstractJDBCConnectorProvider { + private static Logger LOG = LoggerFactory.getLogger(HiveJDBCConnectorProvider.class); + + private static final String DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver".intern(); + + public HiveJDBCConnectorProvider(String dbName, DataConnector dataConn) { + super(dbName, dataConn, DRIVER_CLASS); + } + + /** + * Returns a list of all table names from the remote database. + * @return List A collection of all the table names, null if there are no tables. + * @throws MetaException To indicate any failures with executing this API + */ + @Override protected ResultSet fetchTableNames() throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getTables(scoped_db, null, null, new String[] { "TABLE" }); + } catch (SQLException sqle) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + sqle.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + sqle); + } + return rs; + } + + /** + * Fetch a single table with the given name, returns a Hive Table object from the remote database + * @return Table A Table object for the matching table, null otherwise. + * @throws MetaException To indicate any failures with executing this API + * @param tableName + */ + @Override public ResultSet fetchTableMetadata(String tableName) throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getColumns(null, scoped_db, tableName, null); + } catch (Exception ex) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + ex.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + ex); + } + return rs; +} + + @Override protected String getCatalogName() { + return null; + } + + @Override protected String getDatabaseName() { + return scoped_db; + } + + protected String getDataType(String dbDataType, int size) { + String mappedType = super.getDataType(dbDataType, size); + if (!mappedType.equalsIgnoreCase(ColumnType.VOID_TYPE_NAME)) { + return mappedType; + } + + // map any db specific types here. + switch (dbDataType.toLowerCase()) + { + case "string": + mappedType = ColumnType.STRING_TYPE_NAME; + break; + default: + mappedType = ColumnType.VOID_TYPE_NAME; + break; + } Review Comment: Actually, this reminds of some pending work that I needed to do for complex datatypes. Thanks for the reminder. We will have more cases here. ########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/HiveJDBCConnectorProvider.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.dataconnector.jdbc; + +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.DataConnector; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class HiveJDBCConnectorProvider extends AbstractJDBCConnectorProvider { + private static Logger LOG = LoggerFactory.getLogger(HiveJDBCConnectorProvider.class); Review Comment: fixed ########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/HiveJDBCConnectorProvider.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.dataconnector.jdbc; + +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.DataConnector; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class HiveJDBCConnectorProvider extends AbstractJDBCConnectorProvider { + private static Logger LOG = LoggerFactory.getLogger(HiveJDBCConnectorProvider.class); + + private static final String DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver".intern(); + + public HiveJDBCConnectorProvider(String dbName, DataConnector dataConn) { + super(dbName, dataConn, DRIVER_CLASS); + } + + /** + * Returns a list of all table names from the remote database. + * @return List A collection of all the table names, null if there are no tables. + * @throws MetaException To indicate any failures with executing this API + */ + @Override protected ResultSet fetchTableNames() throws MetaException { + ResultSet rs = null; Review Comment: we can use 2 new variables for catalog/schema and schema/db and use those or introduce 2 addional methods that subclasses have to implement getSchema(), getDatabase(). How did you want me to refactor? IMHO, this is the reason we use abstraction so subclasses can provide their implementation. ########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/HiveJDBCConnectorProvider.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.dataconnector.jdbc; + +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.DataConnector; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class HiveJDBCConnectorProvider extends AbstractJDBCConnectorProvider { + private static Logger LOG = LoggerFactory.getLogger(HiveJDBCConnectorProvider.class); + + private static final String DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver".intern(); + + public HiveJDBCConnectorProvider(String dbName, DataConnector dataConn) { + super(dbName, dataConn, DRIVER_CLASS); + } + + /** + * Returns a list of all table names from the remote database. + * @return List A collection of all the table names, null if there are no tables. + * @throws MetaException To indicate any failures with executing this API + */ + @Override protected ResultSet fetchTableNames() throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getTables(scoped_db, null, null, new String[] { "TABLE" }); + } catch (SQLException sqle) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + sqle.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + sqle); + } + return rs; + } + + /** + * Fetch a single table with the given name, returns a Hive Table object from the remote database + * @return Table A Table object for the matching table, null otherwise. + * @throws MetaException To indicate any failures with executing this API + * @param tableName + */ + @Override public ResultSet fetchTableMetadata(String tableName) throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getColumns(null, scoped_db, tableName, null); + } catch (Exception ex) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + ex.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + ex); + } + return rs; +} + + @Override protected String getCatalogName() { + return null; + } + + @Override protected String getDatabaseName() { + return scoped_db; + } + + protected String getDataType(String dbDataType, int size) { + String mappedType = super.getDataType(dbDataType, size); + if (!mappedType.equalsIgnoreCase(ColumnType.VOID_TYPE_NAME)) { + return mappedType; + } + + // map any db specific types here. + switch (dbDataType.toLowerCase()) + { + case "string": + mappedType = ColumnType.STRING_TYPE_NAME; + break; + default: + mappedType = ColumnType.VOID_TYPE_NAME; + break; + } + return mappedType; + } + + @Override protected String getDatasourceType() { return "HIVE"; } Review Comment: ACK ########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/HiveJDBCConnectorProvider.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.dataconnector.jdbc; + +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.DataConnector; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class HiveJDBCConnectorProvider extends AbstractJDBCConnectorProvider { + private static Logger LOG = LoggerFactory.getLogger(HiveJDBCConnectorProvider.class); + + private static final String DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver".intern(); + + public HiveJDBCConnectorProvider(String dbName, DataConnector dataConn) { + super(dbName, dataConn, DRIVER_CLASS); + } + + /** + * Returns a list of all table names from the remote database. + * @return List A collection of all the table names, null if there are no tables. + * @throws MetaException To indicate any failures with executing this API + */ + @Override protected ResultSet fetchTableNames() throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getTables(scoped_db, null, null, new String[] { "TABLE" }); + } catch (SQLException sqle) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + sqle.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + sqle); + } + return rs; + } + + /** + * Fetch a single table with the given name, returns a Hive Table object from the remote database + * @return Table A Table object for the matching table, null otherwise. + * @throws MetaException To indicate any failures with executing this API + * @param tableName + */ + @Override public ResultSet fetchTableMetadata(String tableName) throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getColumns(null, scoped_db, tableName, null); + } catch (Exception ex) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + ex.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + ex); + } + return rs; +} + + @Override protected String getCatalogName() { + return null; + } + + @Override protected String getDatabaseName() { + return scoped_db; Review Comment: not sure if I understood correctly but I think you are asking if the database we are mapping is already a "mapped/remote" database on the remote HS2 right? I believe the GetTablesOperation implementation uses a different API that looks for tables registered in HMS for this DB. In this case, it would be empty. Although the idea is neat, I wasnt sure if this would be a good idea. This will need additional work because we have to retain the original set of properties returned without overwriting them (like the URL, type, credentials etc). Not sure what other problems this could pose other than having cyclical references. A simple workaround could be to map that end database directly using a local connector. Hope this makes sense. ########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/HiveJDBCConnectorProvider.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.dataconnector.jdbc; + +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.DataConnector; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class HiveJDBCConnectorProvider extends AbstractJDBCConnectorProvider { + private static Logger LOG = LoggerFactory.getLogger(HiveJDBCConnectorProvider.class); + + private static final String DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver".intern(); Review Comment: old habit. let me read thru ########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/HiveJDBCConnectorProvider.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.dataconnector.jdbc; + +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.DataConnector; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class HiveJDBCConnectorProvider extends AbstractJDBCConnectorProvider { + private static Logger LOG = LoggerFactory.getLogger(HiveJDBCConnectorProvider.class); + + private static final String DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver".intern(); + + public HiveJDBCConnectorProvider(String dbName, DataConnector dataConn) { + super(dbName, dataConn, DRIVER_CLASS); + } + + /** + * Returns a list of all table names from the remote database. + * @return List A collection of all the table names, null if there are no tables. + * @throws MetaException To indicate any failures with executing this API + */ + @Override protected ResultSet fetchTableNames() throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getTables(scoped_db, null, null, new String[] { "TABLE" }); + } catch (SQLException sqle) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + sqle.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + sqle); + } + return rs; + } + + /** + * Fetch a single table with the given name, returns a Hive Table object from the remote database + * @return Table A Table object for the matching table, null otherwise. + * @throws MetaException To indicate any failures with executing this API + * @param tableName + */ + @Override public ResultSet fetchTableMetadata(String tableName) throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getColumns(null, scoped_db, tableName, null); + } catch (Exception ex) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + ex.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + ex); + } + return rs; +} + + @Override protected String getCatalogName() { + return null; + } + + @Override protected String getDatabaseName() { + return scoped_db; + } + + protected String getDataType(String dbDataType, int size) { Review Comment: Missed during porting from CDH code. ########## standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/jdbc/HiveJDBCConnectorProvider.java: ########## @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.dataconnector.jdbc; + +import org.apache.hadoop.hive.metastore.ColumnType; +import org.apache.hadoop.hive.metastore.api.DataConnector; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class HiveJDBCConnectorProvider extends AbstractJDBCConnectorProvider { + private static Logger LOG = LoggerFactory.getLogger(HiveJDBCConnectorProvider.class); + + private static final String DRIVER_CLASS = "org.apache.hive.jdbc.HiveDriver".intern(); + + public HiveJDBCConnectorProvider(String dbName, DataConnector dataConn) { + super(dbName, dataConn, DRIVER_CLASS); + } + + /** + * Returns a list of all table names from the remote database. + * @return List A collection of all the table names, null if there are no tables. + * @throws MetaException To indicate any failures with executing this API + */ + @Override protected ResultSet fetchTableNames() throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getTables(scoped_db, null, null, new String[] { "TABLE" }); + } catch (SQLException sqle) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + sqle.getMessage()); + throw new MetaException("Could not retrieve table names from remote datasource, cause:" + sqle); + } + return rs; + } + + /** + * Fetch a single table with the given name, returns a Hive Table object from the remote database + * @return Table A Table object for the matching table, null otherwise. + * @throws MetaException To indicate any failures with executing this API + * @param tableName + */ + @Override public ResultSet fetchTableMetadata(String tableName) throws MetaException { + ResultSet rs = null; + try { + rs = getConnection().getMetaData().getColumns(null, scoped_db, tableName, null); + } catch (Exception ex) { + LOG.warn("Could not retrieve table names from remote datasource, cause:" + ex.getMessage()); Review Comment: ACK -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
