This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 18f34e75da0 HIVE-28460: Determine the database type once the
PersistenceManagerFactory created (#5474) (Zhihua Deng, reviewed by Dmitriy
Fingerman, Denys Kuzmenko)
18f34e75da0 is described below
commit 18f34e75da0141d37d9a8f1cef4f7f64ba09fadb
Author: dengzh <[email protected]>
AuthorDate: Tue Nov 5 09:17:15 2024 +0800
HIVE-28460: Determine the database type once the PersistenceManagerFactory
created (#5474) (Zhihua Deng, reviewed by Dmitriy Fingerman, Denys Kuzmenko)
---
.../hadoop/hive/metastore/DatabaseProduct.java | 19 +++++++++++++
.../hadoop/hive/metastore/MetaStoreDirectSql.java | 23 +--------------
.../apache/hadoop/hive/metastore/ObjectStore.java | 23 ++-------------
.../hive/metastore/PersistenceManagerProvider.java | 33 ++++++++++++++++++++--
.../hadoop/hive/metastore/txn/TxnHandler.java | 25 +++-------------
5 files changed, 56 insertions(+), 67 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
index 7a5f2059589..2b012b9a9bb 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
+import java.sql.Connection;
import java.sql.SQLException;
import java.sql.SQLIntegrityConstraintViolationException;
import java.sql.SQLTransactionRollbackException;
@@ -44,6 +45,8 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import javax.sql.DataSource;
+
/** Database product inferred via JDBC. Encapsulates all SQL logic associated
with
* the database product.
* This class is a singleton, which is instantiated the first time
@@ -70,6 +73,8 @@ public class DatabaseProduct implements Configurable {
private static DatabaseProduct theDatabaseProduct;
Configuration myConf;
+
+ private String productName;
/**
* Protected constructor for singleton class
*/
@@ -83,6 +88,16 @@ public class DatabaseProduct implements Configurable {
public static final String ORACLE_NAME = "oracle";
public static final String UNDEFINED_NAME = "other";
+ public static DatabaseProduct determineDatabaseProduct(DataSource connPool,
+ Configuration conf) {
+ try (Connection conn = connPool.getConnection()) {
+ String s = conn.getMetaData().getDatabaseProductName();
+ return determineDatabaseProduct(s, conf);
+ } catch (SQLException e) {
+ throw new IllegalStateException("Unable to get database product name",
e);
+ }
+ }
+
/**
* Determine the database product type
* @param productName string to defer database connection
@@ -144,6 +159,7 @@ public class DatabaseProduct implements Configurable {
}
theDatabaseProduct.dbType = dbt;
+ theDatabaseProduct.productName = productName;
}
}
return theDatabaseProduct;
@@ -817,4 +833,7 @@ public class DatabaseProduct implements Configurable {
return dbType == DbType.DERBY || dbType == DbType.ORACLE;
}
+ public String getProductName() {
+ return productName;
+ }
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 35079df61d3..a1ada03b7c2 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -193,11 +193,7 @@ class MetaStoreDirectSql {
this.pm = pm;
this.conf = conf;
this.schema = schema;
- DatabaseProduct dbType = null;
-
- dbType = DatabaseProduct.determineDatabaseProduct(getProductName(pm),
conf);
-
- this.dbType = dbType;
+ this.dbType = PersistenceManagerProvider.getDatabaseProduct();
int batchSize = MetastoreConf.getIntVar(conf,
ConfVars.DIRECT_SQL_PARTITION_BATCH_SIZE);
this.directSqlInsertPart = new DirectSqlInsertPart(pm, dbType, batchSize);
if (batchSize == DETECT_BATCHING) {
@@ -272,23 +268,6 @@ class MetaStoreDirectSql {
+ "\"" + tblName + "\"";
}
-
- public MetaStoreDirectSql(PersistenceManager pm, Configuration conf) {
- this(pm, conf, "");
- }
-
- static String getProductName(PersistenceManager pm) {
- JDOConnection jdoConn = pm.getDataStoreConnection();
- try {
- return
((Connection)jdoConn.getNativeConnection()).getMetaData().getDatabaseProductName();
- } catch (Throwable t) {
- LOG.warn("Error retrieving product name", t);
- return null;
- } finally {
- jdoConn.close(); // We must release the connection before we call other
pm methods.
- }
- }
-
private boolean ensureDbInit() {
Transaction tx = pm.currentTransaction();
boolean doCommit = false;
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index c41d4d0087e..314f78fd1d5 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -434,12 +434,10 @@ public class ObjectStore implements RawStore,
Configurable {
LOG.info("RawStore: {}, with PersistenceManager: {}" +
" created in the thread with id: {}", this, pm,
Thread.currentThread().getId());
- String productName = MetaStoreDirectSql.getProductName(pm);
- sqlGenerator = new
SQLGenerator(DatabaseProduct.determineDatabaseProduct(productName, conf), conf);
-
isInitialized = pm != null;
if (isInitialized) {
- dbType = determineDatabaseProduct();
+ dbType = PersistenceManagerProvider.getDatabaseProduct();
+ sqlGenerator = new SQLGenerator(dbType, conf);
expressionProxy = PartFilterExprUtil.createExpressionProxy(conf);
if (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL)) {
String schema =
PersistenceManagerProvider.getProperty("javax.jdo.mapping.Schema");
@@ -460,22 +458,6 @@ public class ObjectStore implements RawStore, Configurable
{
return propertyStore;
}
- private DatabaseProduct determineDatabaseProduct() {
- return DatabaseProduct.determineDatabaseProduct(getProductName(pm),
conf);
- }
-
- private static String getProductName(PersistenceManager pm) {
- JDOConnection jdoConn = pm.getDataStoreConnection();
- try {
- return
((Connection)jdoConn.getNativeConnection()).getMetaData().getDatabaseProductName();
- } catch (Throwable t) {
- LOG.warn("Error retrieving product name", t);
- return null;
- } finally {
- jdoConn.close(); // We must release the connection before we call other
pm methods.
- }
- }
-
/**
* Configure SSL encryption to the database store.
*
@@ -4449,7 +4431,6 @@ public class ObjectStore implements RawStore,
Configurable {
boolean isConfigEnabled = MetastoreConf.getBoolVar(getConf(),
ConfVars.TRY_DIRECT_SQL)
&& (MetastoreConf.getBoolVar(getConf(), ConfVars.TRY_DIRECT_SQL_DDL)
|| !isInTxn);
if (isConfigEnabled && directSql == null) {
- dbType = determineDatabaseProduct();
directSql = new MetaStoreDirectSql(pm, getConf(), "");
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
index 723ece51a36..9afc7168aaa 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/PersistenceManagerProvider.java
@@ -83,6 +83,7 @@ import static
org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.COMPA
public class PersistenceManagerProvider {
private static PersistenceManagerFactory pmf;
private static PersistenceManagerFactory compactorPmf;
+ private static DatabaseProduct databaseProduct;
private static Properties prop;
private static final ReentrantReadWriteLock pmfLock = new
ReentrantReadWriteLock();
private static final Lock pmfReadLock = pmfLock.readLock();
@@ -271,6 +272,10 @@ public class PersistenceManagerProvider {
if (compactorPmf == null && useCompactorPool) {
compactorPmf = retry(() -> initPMF(conf, true));
}
+ if (databaseProduct == null) {
+ String url = MetastoreConf.getVar(conf,
MetastoreConf.ConfVars.CONNECT_URL_KEY);
+ databaseProduct = DatabaseProduct.determineDatabaseProduct(url,
conf);
+ }
}
// downgrade by acquiring read lock before releasing write lock
pmfReadLock.lock();
@@ -288,7 +293,7 @@ public class PersistenceManagerProvider {
private static PersistenceManagerFactory initPMF(Configuration conf, boolean
forCompactor) {
DataSourceProvider dsp =
DataSourceProviderFactory.tryGetDataSourceProviderOrNull(conf);
- PersistenceManagerFactory pmf;
+ PersistenceManagerFactory pmf = null;
// Any preexisting datanucleus property should be passed along
Map<Object, Object> dsProp = new HashMap<>(prop);
@@ -301,15 +306,18 @@ public class PersistenceManagerProvider {
if (dsp == null) {
pmf = JDOHelper.getPersistenceManagerFactory(dsProp);
} else {
+ DataSource ds = null;
+ DataSource ds2 = null;
String sourceName = forCompactor ? "objectstore-compactor" :
"objectstore";
try (DataSourceProvider.DataSourceNameConfigurator configurator =
new DataSourceProvider.DataSourceNameConfigurator(conf,
sourceName)) {
- DataSource ds = (maxPoolSize > 0) ? dsp.create(conf, maxPoolSize) :
dsp.create(conf);
+ ds = (maxPoolSize > 0) ? dsp.create(conf, maxPoolSize) :
dsp.create(conf);
+ databaseProduct = DatabaseProduct.determineDatabaseProduct(ds, conf);
// The secondary connection factory is used for schema generation, and
for value generation operations.
// We should use a different pool for the secondary connection factory
to avoid resource starvation.
// Since DataNucleus uses locks for schema generation and value
generation, 2 connections should be sufficient.
configurator.resetName(sourceName + "-secondary");
- DataSource ds2 = dsp.create(conf, /* maxPoolSize */ 2);
+ ds2 = dsp.create(conf, /* maxPoolSize */ 2);
dsProp.put(PropertyNames.PROPERTY_CONNECTION_FACTORY, ds);
dsProp.put(PropertyNames.PROPERTY_CONNECTION_FACTORY2, ds2);
dsProp.put(ConfVars.MANAGER_FACTORY_CLASS.getVarname(),
@@ -319,8 +327,18 @@ public class PersistenceManagerProvider {
LOG.warn("Could not create PersistenceManagerFactory using "
+ "connection pool properties, will fall back", e);
pmf = JDOHelper.getPersistenceManagerFactory(prop);
+ } finally {
+ if (pmf == null && ds instanceof AutoCloseable) {
+ try (AutoCloseable close1 = (AutoCloseable) ds;
+ AutoCloseable close2 = (AutoCloseable) ds2 ) {
+ LOG.debug("Trying to close the dangling datasource as the pmf is
null");
+ } catch (Exception e) {
+ LOG.warn("Failed to close the DataSource", e);
+ }
+ }
}
}
+ assert pmf != null;
DataStoreCache dsc = pmf.getDataStoreCache();
if (dsc != null) {
String objTypes = MetastoreConf.getVar(conf, ConfVars.CACHE_PINOBJTYPES);
@@ -499,6 +517,14 @@ public class PersistenceManagerProvider {
}
}
+ public static DatabaseProduct getDatabaseProduct() {
+ if (databaseProduct == null) {
+ throw new IllegalStateException(
+ "Cannot determine the database product. PersistenceManagerFactory
has not been initialized yet");
+ }
+ return databaseProduct;
+ }
+
/**
* Properties specified in hive-default.xml override the properties specified
* in jpox.properties.
@@ -625,6 +651,7 @@ public class PersistenceManagerProvider {
LOG.warn("Exception retry limit reached, not retrying any
longer.", e);
} else {
LOG.debug("Non-retriable exception.", e);
+ break;
}
ex = e;
}
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index f9283364546..6e002da6ad9 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -292,11 +292,10 @@ public abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
}
}
if (dbProduct == null) {
- try (Connection dbConn =
getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPool)) {
- determineDatabaseProduct(dbConn);
- } catch (SQLException e) {
- LOG.error("Unable to determine database product", e);
- throw new RuntimeException(e);
+ dbProduct = DatabaseProduct.determineDatabaseProduct(connPool,
conf);
+ if (dbProduct.isUNDEFINED()) {
+ String msg = "Unrecognized database product name <" +
dbProduct.getProductName() + ">";
+ throw new IllegalStateException(msg);
}
}
if (sqlGenerator == null) {
@@ -1065,22 +1064,6 @@ public abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
(ResultSet rs, int rowNum) -> rs.getTimestamp(1));
}
- private void determineDatabaseProduct(Connection conn) {
- try {
- String s = conn.getMetaData().getDatabaseProductName();
- dbProduct = DatabaseProduct.determineDatabaseProduct(s, conf);
- if (dbProduct.isUNDEFINED()) {
- String msg = "Unrecognized database product name <" + s + ">";
- LOG.error(msg);
- throw new IllegalStateException(msg);
- }
- } catch (SQLException e) {
- String msg = "Unable to get database product name";
- LOG.error(msg, e);
- throw new IllegalStateException(msg, e);
- }
- }
-
private void initJdbcResource() {
if (jdbcResource == null) {
jdbcResource = new MultiDataSourceJdbcResource(dbProduct, conf,
sqlGenerator);
