sankarh commented on a change in pull request #1527:
URL: https://github.com/apache/hive/pull/1527#discussion_r506285109
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
##########
@@ -749,6 +797,52 @@ public void
refreshUniqueConstraints(List<SQLUniqueConstraint> constraints) {
}
}
+ public void refreshDefaultConstraints(List<SQLDefaultConstraint>
constraints) {
+ Map<String, SQLDefaultConstraint> newConstraints = new
ConcurrentHashMap<>();
+ try {
+ tableLock.writeLock().lock();
+ int size = 0;
+ for (SQLDefaultConstraint constraint : constraints) {
+ if
(compareAndSetMemberCacheUpdated(MemberName.DEFAULT_CONSTRAINT_CACHE, true,
false)) {
+ LOG.debug("Skipping default constraint cache update for table: " +
getTable().getTableName()
+ + "; the default constraint are already refreshed.");
+ return;
+ }
+ newConstraints.put(constraint.getDc_name().toLowerCase(),
constraint);
+ size += getObjectSize(SQLUniqueConstraint.class, constraint);
+ }
+ defaultConstraintCache = newConstraints;
+ updateMemberSize(MemberName.DEFAULT_CONSTRAINT_CACHE, size,
SizeMode.Snapshot);
+ LOG.debug("Default constraints refresh in cache was successful for
{}.{}.{}", this.getTable().getCatName(),
+ this.getTable().getDbName(), this.getTable().getTableName());
+ } finally {
+ tableLock.writeLock().unlock();
+ }
+ }
+
+ public void refreshCheckConstraints(List<SQLCheckConstraint> constraints) {
+ Map<String, SQLCheckConstraint> newConstraints = new
ConcurrentHashMap<>();
+ try {
+ tableLock.writeLock().lock();
+ int size = 0;
+ for (SQLCheckConstraint constraint : constraints) {
+ if
(compareAndSetMemberCacheUpdated(MemberName.CHECK_CONSTRAINT_CACHE, true,
false)) {
+ LOG.debug("Skipping check constraint cache update for table: " +
getTable().getTableName()
+ + "; the check constraint are already refreshed.");
+ return;
+ }
+ newConstraints.put(constraint.getDc_name().toLowerCase(),
constraint);
+ size += getObjectSize(SQLCheckConstraint.class, constraint);
+ }
+ checkConstraintCache = newConstraints;
+ updateMemberSize(MemberName.CHECK_CONSTRAINT_CACHE, size,
SizeMode.Snapshot);
+ LOG.debug("Unique constraints refresh in cache was successful for
{}.{}.{}", this.getTable().getCatName(),
Review comment:
Copy paste error: "Check constraints .."
##########
File path:
itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java
##########
@@ -314,160 +308,129 @@ public void testConstraintsForUpdateUsingEvents()
throws Exception {
hmsHandler.create_database(db);
db = rawStore.getDatabase(DEFAULT_CATALOG_NAME, dbName);
- String foreignDbName = "test_table_ops_foreign";
- Database foreignDb = createTestDb(foreignDbName, dbOwner);
- hmsHandler.create_database(foreignDb);
- foreignDb = rawStore.getDatabase(DEFAULT_CATALOG_NAME, foreignDbName);
// Add a table via rawStore
+ String parentTableName = "ftbl";
String tblName = "tbl";
String tblOwner = "user1";
FieldSchema col1 = new FieldSchema("col1", "int", "integer column");
FieldSchema col2 = new FieldSchema("col2", "string", "string column");
+ FieldSchema col3 = new FieldSchema("col3", "int", "integer column");
List<FieldSchema> cols = new ArrayList<FieldSchema>();
cols.add(col1);
cols.add(col2);
+ cols.add(col3);
List<FieldSchema> ptnCols = new ArrayList<FieldSchema>();
+ Table parentTable = createTestTbl(dbName, parentTableName, tblOwner, cols,
ptnCols);
Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols);
- String foreignTblName = "ftbl";
- Table foreignTbl = createTestTbl(foreignDbName, foreignTblName, tblOwner,
cols, ptnCols);
-
- SQLPrimaryKey key = new SQLPrimaryKey(dbName, tblName, col1.getName(), 1,
"pk1",
- false, false, false);
- SQLUniqueConstraint uC = new SQLUniqueConstraint(DEFAULT_CATALOG_NAME,
dbName, tblName,
- col1.getName(), 2, "uc1", false, false, false);
- SQLNotNullConstraint nN = new SQLNotNullConstraint(DEFAULT_CATALOG_NAME,
dbName, tblName,
- col1.getName(), "nn1", false, false, false);
- SQLForeignKey foreignKey = new SQLForeignKey(key.getTable_db(),
key.getTable_name(), key.getColumn_name(),
- foreignDbName, foreignTblName, key.getColumn_name(), 2, 1,2,
- "fk1", key.getPk_name(), false, false, false);
-
- hmsHandler.create_table_with_constraints(tbl,
- Arrays.asList(key), null, Arrays.asList(uC), Arrays.asList(nN),
null, null);
- hmsHandler.create_table_with_constraints(foreignTbl, null,
Arrays.asList(foreignKey),
- null, null, null, null);
+
+ // Constraints for parent Table
+ List<SQLPrimaryKey> parentPkBase =
+ Arrays.asList(new SQLPrimaryKey(dbName, parentTableName,
col1.getName(), 1, "parentpk1", false, false, false));
+
+ // Constraints for table
+ List<SQLPrimaryKey> pkBase =
+ Arrays.asList(new SQLPrimaryKey(dbName, tblName, col1.getName(), 1,
"pk1", false, false, false));
+ List<SQLUniqueConstraint> ucBase = Arrays.asList(
+ new SQLUniqueConstraint(DEFAULT_CATALOG_NAME, dbName, tblName,
col1.getName(), 2, "uc1", false, false, false));
+ List<SQLNotNullConstraint> nnBase = Arrays.asList(
+ new SQLNotNullConstraint(DEFAULT_CATALOG_NAME, dbName, tblName,
col1.getName(), "nn1", false, false, false));
+ List<SQLDefaultConstraint> dcBase = Arrays.asList(
+ new SQLDefaultConstraint(DEFAULT_CATALOG_NAME, tbl.getDbName(),
tbl.getTableName(), col2.getName(), "1", "dc1",
+ false, false, false));
+ List<SQLCheckConstraint> ccBase = Arrays.asList(
+ new SQLCheckConstraint(DEFAULT_CATALOG_NAME, tbl.getDbName(),
tbl.getTableName(), col2.getName(), "1", "cc1",
+ false, false, false));
+ List<SQLForeignKey> fkBase = Arrays.asList(
+ new SQLForeignKey(parentPkBase.get(0).getTable_db(),
parentPkBase.get(0).getTable_name(),
+ parentPkBase.get(0).getColumn_name(), dbName, tblName,
col3.getName(), 2, 1, 2, "fk1",
+ parentPkBase.get(0).getPk_name(), false, false, false));
+
+ // Create table and parent table
+ hmsHandler.create_table_with_constraints(parentTable, parentPkBase, null,
null, null, null, null);
+ hmsHandler.create_table_with_constraints(tbl, pkBase, fkBase, ucBase,
nnBase, dcBase, ccBase);
tbl = rawStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName);
- foreignTbl = rawStore.getTable(DEFAULT_CATALOG_NAME, foreignDbName,
foreignTblName);
+ parentTable = rawStore.getTable(DEFAULT_CATALOG_NAME, dbName,
parentTableName);
// Read database, table via CachedStore
- Database dbRead= sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME,
dbName);
+ Database dbRead = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME,
dbName);
Assert.assertEquals(db, dbRead);
+
+ // Read table via CachedStore
Table tblRead = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME,
dbName, tblName);
+ Table parentTableRead =
sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbName, parentTableName);
compareTables(tblRead, tbl);
+ compareTables(parentTableRead, parentTable);
- Table foreignTblRead = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME,
foreignDbName, foreignTblName);
- compareTables(foreignTblRead, foreignTbl);
-
- List<SQLPrimaryKey> keys = rawStore.getPrimaryKeys(DEFAULT_CATALOG_NAME,
dbName, tblName);
- List<SQLPrimaryKey> keysRead =
sharedCache.listCachedPrimaryKeys(DEFAULT_CATALOG_NAME, dbName, tblName);
- assertsForPrimarkaryKey(keysRead, 1, 0, keys.get(0));
-
- List<SQLNotNullConstraint> nNs =
rawStore.getNotNullConstraints(DEFAULT_CATALOG_NAME, dbName, tblName);
- List<SQLNotNullConstraint> nNsRead =
sharedCache.listCachedNotNullConstraints(DEFAULT_CATALOG_NAME, dbName, tblName);
- assertsForNotNullConstraints(nNsRead, 1, 0, nNs.get(0));
-
- List<SQLUniqueConstraint> uns =
rawStore.getUniqueConstraints(DEFAULT_CATALOG_NAME, dbName, tblName);
- List<SQLUniqueConstraint> unsRead =
sharedCache.listCachedUniqueConstraint(DEFAULT_CATALOG_NAME, dbName, tblName);
- assertsForUniqueConstraints(unsRead, 1, 0, uns.get(0));
-
- List<SQLForeignKey> fks = rawStore.getForeignKeys(DEFAULT_CATALOG_NAME,
dbName, tblName, foreignDbName, foreignTblName);
- List<SQLForeignKey> fksRead =
sharedCache.listCachedForeignKeys(DEFAULT_CATALOG_NAME, foreignDbName,
- foreignTblName, dbName, tblName);
- assertsForForeignKey(fksRead, 1, 0, fks.get(0));
-
- fksRead = sharedCache.listCachedForeignKeys(DEFAULT_CATALOG_NAME,
foreignDbName, foreignTblName,
- dbName, foreignTblName);
- Assert.assertEquals(fksRead.size(), 0);
- fksRead = sharedCache.listCachedForeignKeys(DEFAULT_CATALOG_NAME,
foreignDbName, foreignTblName,
- foreignDbName, tblName);
- Assert.assertEquals(fksRead.size(), 0);
- fksRead = sharedCache.listCachedForeignKeys(DEFAULT_CATALOG_NAME,
foreignDbName, foreignTblName,
- foreignDbName, foreignTblName);
- Assert.assertEquals(fksRead.size(), 0);
-
- fksRead = sharedCache.listCachedForeignKeys(DEFAULT_CATALOG_NAME,
foreignDbName, foreignTblName,
- null, null);
- Assert.assertEquals(fksRead.size(), 1);
-
- // Dropping the constraint
- DropConstraintRequest dropConstraintRequest = new
DropConstraintRequest(foreignDbName, foreignTblName, foreignKey.getFk_name());
+ // Validating constraint values from CachedStore with rawStore for table
+ assertRawStoreAndCachedStoreConstraint(DEFAULT_CATALOG_NAME, dbName,
tblName);
+
+ // Validating constraint values from CachedStore with rawStore for parent
table
+ assertRawStoreAndCachedStoreConstraint(DEFAULT_CATALOG_NAME, dbName,
parentTableName);
+
+ // Dropping all the constraint
+ DropConstraintRequest dropConstraintRequest =
+ new DropConstraintRequest(dbName, tblName, fkBase.get(0).getFk_name());
+ hmsHandler.drop_constraint(dropConstraintRequest);
+ dropConstraintRequest = new DropConstraintRequest(dbName, tblName,
pkBase.get(0).getPk_name());
+ hmsHandler.drop_constraint(dropConstraintRequest);
+ dropConstraintRequest = new DropConstraintRequest(dbName, tblName,
nnBase.get(0).getNn_name());
hmsHandler.drop_constraint(dropConstraintRequest);
- dropConstraintRequest = new DropConstraintRequest(dbName, tblName,
key.getPk_name());
+ dropConstraintRequest = new DropConstraintRequest(dbName, tblName,
ucBase.get(0).getUk_name());
hmsHandler.drop_constraint(dropConstraintRequest);
- dropConstraintRequest = new DropConstraintRequest(dbName, tblName,
nN.getNn_name());
+ dropConstraintRequest = new DropConstraintRequest(dbName, tblName,
dcBase.get(0).getDc_name());
hmsHandler.drop_constraint(dropConstraintRequest);
- dropConstraintRequest = new DropConstraintRequest(dbName, tblName,
uC.getUk_name());
+ dropConstraintRequest = new DropConstraintRequest(dbName, tblName,
ccBase.get(0).getDc_name());
+ hmsHandler.drop_constraint(dropConstraintRequest);
+ dropConstraintRequest = new DropConstraintRequest(dbName, parentTableName,
parentPkBase.get(0).getPk_name());
hmsHandler.drop_constraint(dropConstraintRequest);
- keys = sharedCache.listCachedPrimaryKeys(DEFAULT_CATALOG_NAME, dbName,
tblName);
- nNs = sharedCache.listCachedNotNullConstraints(DEFAULT_CATALOG_NAME,
dbName, tblName);
- uns = sharedCache.listCachedUniqueConstraint(DEFAULT_CATALOG_NAME, dbName,
tblName);
- fksRead = sharedCache.listCachedForeignKeys(DEFAULT_CATALOG_NAME,
foreignDbName, foreignTblName, dbName, tblName);
- Assert.assertEquals(keys.size(), 0);
- Assert.assertEquals(nNs.size(), 0);
- Assert.assertEquals(uns.size(), 0);
- Assert.assertEquals(fksRead.size(), 0);
+ // Validate cache store constraint is dropped
+ Assert
+
.assertTrue(CollectionUtils.isEmpty(sharedCache.listCachedPrimaryKeys(DEFAULT_CATALOG_NAME,
dbName, tblName)));
+ Assert.assertTrue(
+
CollectionUtils.isEmpty(sharedCache.listCachedNotNullConstraints(DEFAULT_CATALOG_NAME,
dbName, tblName)));
+ Assert.assertTrue(
+
CollectionUtils.isEmpty(sharedCache.listCachedUniqueConstraint(DEFAULT_CATALOG_NAME,
dbName, tblName)));
+ Assert.assertTrue(
+
CollectionUtils.isEmpty(sharedCache.listCachedDefaultConstraint(DEFAULT_CATALOG_NAME,
dbName, tblName)));
+ Assert.assertTrue(
+
CollectionUtils.isEmpty(sharedCache.listCachedCheckConstraint(DEFAULT_CATALOG_NAME,
dbName, tblName)));
+ Assert.assertTrue(
+
CollectionUtils.isEmpty(sharedCache.listCachedForeignKeys(DEFAULT_CATALOG_NAME,
dbName, tblName, null, null)));
// Adding keys back
- AddPrimaryKeyRequest req = new AddPrimaryKeyRequest(Arrays.asList(key));
- hmsHandler.add_primary_key(req);
- keys = sharedCache.listCachedPrimaryKeys(DEFAULT_CATALOG_NAME, dbName,
tblName);
- assertsForPrimarkaryKey(keys, 1, 0, key);
-
- AddUniqueConstraintRequest uniqueConstraintRequest = new
AddUniqueConstraintRequest(Arrays.asList(uC));
- hmsHandler.add_unique_constraint(uniqueConstraintRequest);
- uns = sharedCache.listCachedUniqueConstraint(DEFAULT_CATALOG_NAME, dbName,
tblName);
- assertsForUniqueConstraints(uns, 1, 0, uC);
-
- AddNotNullConstraintRequest notNullConstraintRequest = new
AddNotNullConstraintRequest(Arrays.asList(nN));
- hmsHandler.add_not_null_constraint(notNullConstraintRequest);
- nNs = sharedCache.listCachedNotNullConstraints(DEFAULT_CATALOG_NAME,
dbName, tblName);
- assertsForNotNullConstraints(nNs, 1, 0, nN);
-
- AddForeignKeyRequest foreignKeyRequest = new
AddForeignKeyRequest(Arrays.asList(foreignKey));
- hmsHandler.add_foreign_key(foreignKeyRequest);
- fksRead = sharedCache.listCachedForeignKeys(DEFAULT_CATALOG_NAME,
foreignDbName, foreignTblName, dbName, tblName);
- assertsForForeignKey(fksRead, 1, 0, foreignKey);
+ hmsHandler.add_primary_key(new AddPrimaryKeyRequest(parentPkBase));
+ hmsHandler.add_primary_key(new AddPrimaryKeyRequest(pkBase));
+ hmsHandler.add_unique_constraint(new AddUniqueConstraintRequest(ucBase));
+ hmsHandler.add_not_null_constraint(new
AddNotNullConstraintRequest(nnBase));
+ hmsHandler.add_foreign_key(new AddForeignKeyRequest(fkBase));
+ hmsHandler.add_default_constraint(new AddDefaultConstraintRequest(dcBase));
+ hmsHandler.add_check_constraint(new AddCheckConstraintRequest(ccBase));
+
+ // Validating constraint values from Cache with rawStore
+ assertRawStoreAndCachedStoreConstraint(DEFAULT_CATALOG_NAME, dbName,
tblName);
sharedCache.getDatabaseCache().clear();
sharedCache.clearTableCache();
sharedCache.getSdCache().clear();
}
- private void assertsForPrimarkaryKey(List<SQLPrimaryKey> keys, int size, int
ele, SQLPrimaryKey key) {
- Assert.assertEquals(keys.size(), size);
- Assert.assertEquals(keys.get(ele).getPk_name(), key.getPk_name());
- Assert.assertEquals(keys.get(ele).getColumn_name(), key.getColumn_name());
- Assert.assertEquals(keys.get(ele).getTable_name(), key.getTable_name());
- Assert.assertEquals(keys.get(ele).getTable_db(), key.getTable_db());
- }
-
- private void assertsForForeignKey(List<SQLForeignKey> keys, int size, int
ele, SQLForeignKey key) {
- Assert.assertEquals(keys.size(), size);
- Assert.assertEquals(keys.get(ele).getPk_name(), key.getPk_name());
- Assert.assertEquals(keys.get(ele).getFk_name(), key.getFk_name());
- Assert.assertEquals(keys.get(ele).getFktable_db(), key.getFktable_db());
- Assert.assertEquals(keys.get(ele).getFktable_name(),
key.getFktable_name());
- Assert.assertEquals(keys.get(ele).getPktable_db(), key.getPktable_db());
- Assert.assertEquals(keys.get(ele).getPktable_name(),
key.getPktable_name());
- Assert.assertEquals(keys.get(ele).getPkcolumn_name(),
key.getPkcolumn_name());
- Assert.assertEquals(keys.get(ele).getFkcolumn_name(),
key.getFkcolumn_name());
- }
-
- private void assertsForNotNullConstraints(List<SQLNotNullConstraint> nns,
int size, int ele, SQLNotNullConstraint nN) {
- Assert.assertEquals(nns.size(), size);
- Assert.assertEquals(nns.get(ele).getNn_name(), nN.getNn_name());
- Assert.assertEquals(nns.get(ele).getColumn_name(), nN.getColumn_name());
- Assert.assertEquals(nns.get(ele).getTable_name(), nN.getTable_name());
- Assert.assertEquals(nns.get(ele).getTable_db(), nN.getTable_db());
- }
-
- private void assertsForUniqueConstraints(List<SQLUniqueConstraint> uks, int
size, int ele, SQLUniqueConstraint uk) {
- Assert.assertEquals(uks.size(), size);
- Assert.assertEquals(uks.get(ele).getUk_name(), uk.getUk_name());
- Assert.assertEquals(uks.get(ele).getColumn_name(), uk.getColumn_name());
- Assert.assertEquals(uks.get(ele).getTable_name(), uk.getTable_name());
- Assert.assertEquals(uks.get(ele).getTable_db(), uk.getTable_db());
+ public void assertRawStoreAndCachedStoreConstraint(String catName, String
dbName, String tblName)
+ throws MetaException, NoSuchObjectException {
+ SQLAllTableConstraints rawStoreConstraints =
rawStore.getAllTableConstraints(catName, dbName, tblName);
+ List<SQLPrimaryKey> primaryKeys =
sharedCache.listCachedPrimaryKeys(catName, dbName, tblName);
+ List<SQLNotNullConstraint> notNullConstraints =
sharedCache.listCachedNotNullConstraints(catName, dbName, tblName);
+ List<SQLUniqueConstraint> uniqueConstraints =
sharedCache.listCachedUniqueConstraint(catName, dbName, tblName);
+ List<SQLDefaultConstraint> defaultConstraints =
sharedCache.listCachedDefaultConstraint(catName, dbName, tblName);
+ List<SQLCheckConstraint> checkConstraints =
sharedCache.listCachedCheckConstraint(catName, dbName, tblName);
+ List<SQLForeignKey> foreignKeys =
sharedCache.listCachedForeignKeys(catName, dbName, tblName, null, null);
+ Assert.assertEquals(rawStoreConstraints.getPrimaryKeys(),primaryKeys);
Review comment:
nit: ass space after comma.
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
##########
@@ -909,82 +906,137 @@ private void updateTableColStats(RawStore rawStore,
String catName, String dbNam
}
private void updateTableForeignKeys(RawStore rawStore, String catName,
String dbName, String tblName) {
+ catName = StringUtils.normalizeIdentifier(catName);
+ dbName = StringUtils.normalizeIdentifier(dbName);
+ tblName = StringUtils.normalizeIdentifier(tblName);
LOG.debug("CachedStore: updating cached foreign keys objects for
catalog: {}, database: {}, table: {}", catName,
- dbName, tblName);
+ dbName, tblName);
List<SQLForeignKey> fks = null;
try {
Deadline.startTimer("getForeignKeys");
fks = rawStore.getForeignKeys(catName, null, null, dbName, tblName);
Deadline.stopTimer();
} catch (MetaException e) {
- LOG.info("Updating CachedStore: unable to update foreign keys of
catalog: " + catName + ", database: "
- + dbName + ", table: " + tblName, e);
+ LOG.info("Updating CachedStore: unable to update foreign keys of
catalog: " + catName + ", database: " + dbName
+ + ", table: " + tblName, e);
}
if (fks != null) {
-
sharedCache.refreshForeignKeysInCache(StringUtils.normalizeIdentifier(catName),
- StringUtils.normalizeIdentifier(dbName),
StringUtils.normalizeIdentifier(tblName), fks);
- LOG.debug("CachedStore: updated cached foreign keys objects for
catalog: {}, database: {}, table: {}",
- catName, dbName, tblName);
+ sharedCache.refreshForeignKeysInCache(catName, dbName, tblName, fks);
+ LOG.debug("CachedStore: updated cached foreign keys objects for
catalog: {}, database: {}, table: {}", catName,
+ dbName, tblName);
}
}
private void updateTableNotNullConstraints(RawStore rawStore, String
catName, String dbName, String tblName) {
+ catName = StringUtils.normalizeIdentifier(catName);
+ dbName = StringUtils.normalizeIdentifier(dbName);
+ tblName = StringUtils.normalizeIdentifier(tblName);
LOG.debug("CachedStore: updating cached not null constraints for
catalog: {}, database: {}, table: {}", catName,
- dbName, tblName);
+ dbName, tblName);
List<SQLNotNullConstraint> nns = null;
try {
Deadline.startTimer("getNotNullConstraints");
nns = rawStore.getNotNullConstraints(catName, dbName, tblName);
Deadline.stopTimer();
} catch (MetaException e) {
LOG.info("Updating CachedStore: unable to update not null constraints
of catalog: " + catName + ", database: "
- + dbName + ", table: " + tblName, e);
+ + dbName + ", table: " + tblName, e);
}
if (nns != null) {
-
sharedCache.refreshNotNullConstraintsInCache(StringUtils.normalizeIdentifier(catName),
- StringUtils.normalizeIdentifier(dbName),
StringUtils.normalizeIdentifier(tblName), nns);
- LOG.debug("CachedStore: updated cached not null constraints for
catalog: {}, database: {}, table: {}",
- catName, dbName, tblName);
+ sharedCache.refreshNotNullConstraintsInCache(catName, dbName, tblName,
nns);
+ LOG.debug("CachedStore: updated cached not null constraints for
catalog: {}, database: {}, table: {}", catName,
+ dbName, tblName);
}
}
private void updateTableUniqueConstraints(RawStore rawStore, String
catName, String dbName, String tblName) {
+ catName = StringUtils.normalizeIdentifier(catName);
+ dbName = StringUtils.normalizeIdentifier(dbName);
+ tblName = StringUtils.normalizeIdentifier(tblName);
LOG.debug("CachedStore: updating cached unique constraints for catalog:
{}, database: {}, table: {}", catName,
- dbName, tblName);
+ dbName, tblName);
List<SQLUniqueConstraint> ucs = null;
try {
Deadline.startTimer("getUniqueConstraints");
ucs = rawStore.getUniqueConstraints(catName, dbName, tblName);
Deadline.stopTimer();
} catch (MetaException e) {
- LOG.info("Updating CachedStore: unable to update unique constraints of
catalog: " + catName + ", database: "
- + dbName + ", table: " + tblName, e);
+ LOG.info(
+ "Updating CachedStore: unable to update unique constraints of
catalog: " + catName + ", database: " + dbName
+ + ", table: " + tblName, e);
}
if (ucs != null) {
-
sharedCache.refreshUniqueConstraintsInCache(StringUtils.normalizeIdentifier(catName),
- StringUtils.normalizeIdentifier(dbName),
StringUtils.normalizeIdentifier(tblName), ucs);
- LOG.debug("CachedStore: updated cached unique constraints for catalog:
{}, database: {}, table: {}",
- catName, dbName, tblName);
+ sharedCache.refreshUniqueConstraintsInCache(catName, dbName, tblName,
ucs);
+ LOG.debug("CachedStore: updated cached unique constraints for catalog:
{}, database: {}, table: {}", catName,
+ dbName, tblName);
}
}
private void updateTablePrimaryKeys(RawStore rawStore, String catName,
String dbName, String tblName) {
+ catName = StringUtils.normalizeIdentifier(catName);
+ dbName = StringUtils.normalizeIdentifier(dbName);
+ tblName = StringUtils.normalizeIdentifier(tblName);
LOG.debug("CachedStore: updating cached primary keys objects for
catalog: {}, database: {}, table: {}", catName,
- dbName, tblName);
+ dbName, tblName);
List<SQLPrimaryKey> pks = null;
try {
Deadline.startTimer("getPrimaryKeys");
pks = rawStore.getPrimaryKeys(catName, dbName, tblName);
Deadline.stopTimer();
} catch (MetaException e) {
- LOG.info("Updating CachedStore: unable to update primary keys of
catalog: " + catName + ", database: "
- + dbName + ", table: " + tblName, e);
+ LOG.info("Updating CachedStore: unable to update primary keys of
catalog: " + catName + ", database: " + dbName
+ + ", table: " + tblName, e);
}
if (pks != null) {
-
sharedCache.refreshPrimaryKeysInCache(StringUtils.normalizeIdentifier(catName),
- StringUtils.normalizeIdentifier(dbName),
StringUtils.normalizeIdentifier(tblName), pks);
- LOG.debug("CachedStore: updated cached primary keys objects for
catalog: {}, database: {}, table: {}",
+ sharedCache.refreshPrimaryKeysInCache(catName, dbName, tblName, pks);
+ LOG.debug("CachedStore: updated cached primary keys objects for
catalog: {}, database: {}, table: {}", catName,
+ dbName, tblName);
+ }
+ }
+
+ private void updateTableDefaultConstraints(RawStore rawStore, String
catName, String dbName, String tblName) {
+ catName = StringUtils.normalizeIdentifier(catName);
+ dbName = StringUtils.normalizeIdentifier(dbName);
+ tblName = StringUtils.normalizeIdentifier(tblName);
+ LOG.debug("CachedStore: updating cached default Constraint objects for
catalog: {}, database: {}, table: {}",
catName, dbName, tblName);
+ List<SQLDefaultConstraint> dc = null;
+ try {
+ Deadline.startTimer("getDefaultConstraints");
+ dc = rawStore.getDefaultConstraints(catName, dbName, tblName);
+ Deadline.stopTimer();
+ } catch (MetaException e) {
+ LOG.info(
+ "Updating CachedStore: unable to update default Constraint of
catalog: " + catName + ", database: " + dbName
+ + ", table: " + tblName, e);
+ }
+ if (dc != null) {
+ sharedCache.refreshDefaultConstraintsInCache(catName, dbName, tblName,
dc);
+ LOG.debug("CachedStore: updated cached default constraint objects for
catalog: {}, database: {}, table: {}",
+ catName, dbName, tblName);
+ }
+ }
+
+ private void updateTableCheckConstraints(RawStore rawStore, String
catName, String dbName, String tblName) {
+ catName = StringUtils.normalizeIdentifier(catName);
+ dbName = StringUtils.normalizeIdentifier(dbName);
+ tblName = StringUtils.normalizeIdentifier(tblName);
+ LOG.debug("CachedStore: updating cached check constraint objects for
catalog: {}, database: {}, table: {}",
+ catName, dbName, tblName);
+ List<SQLCheckConstraint> cc = null;
+ try {
+ Deadline.startTimer("getCheckConstraint");
Review comment:
Name can match the method name "getCheckConstraints"
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
##########
@@ -749,6 +797,52 @@ public void
refreshUniqueConstraints(List<SQLUniqueConstraint> constraints) {
}
}
+ public void refreshDefaultConstraints(List<SQLDefaultConstraint>
constraints) {
+ Map<String, SQLDefaultConstraint> newConstraints = new
ConcurrentHashMap<>();
+ try {
+ tableLock.writeLock().lock();
+ int size = 0;
+ for (SQLDefaultConstraint constraint : constraints) {
+ if
(compareAndSetMemberCacheUpdated(MemberName.DEFAULT_CONSTRAINT_CACHE, true,
false)) {
+ LOG.debug("Skipping default constraint cache update for table: " +
getTable().getTableName()
+ + "; the default constraint are already refreshed.");
+ return;
+ }
+ newConstraints.put(constraint.getDc_name().toLowerCase(),
constraint);
+ size += getObjectSize(SQLUniqueConstraint.class, constraint);
Review comment:
Copy paste error: Should use SQLDefaultConstraint.class.
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/TableCacheObjects.java
##########
@@ -3,59 +3,21 @@
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
-import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint;
-import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
-import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint;
+import org.apache.hadoop.hive.metastore.api.SQLAllTableConstraints;
import java.util.List;
/*
* Holder class for table objects like partitions, statistics, constraints etc.
*/
public class TableCacheObjects {
- private List<SQLPrimaryKey> primaryKeys;
- private List<SQLForeignKey> foreignKeys;
- private List<SQLNotNullConstraint> notNullConstraints;
- private List<SQLUniqueConstraint> uniqueConstraints;
+ private SQLAllTableConstraints tableConstraints;
Review comment:
What happens if constraints are partially cached due to space
constraint? Will CachedStore calls rawStore or returns empty list?
##########
File path:
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java
##########
@@ -1669,33 +1763,47 @@ public boolean populateTableInCache(Table table,
TableCacheObjects cacheObjects)
tblWrapper.setMemberCacheUpdated(MemberName.PARTITION_COL_STATS_CACHE,
false);
tblWrapper.setMemberCacheUpdated(MemberName.AGGR_COL_STATS_CACHE, false);
- if (cacheObjects.getPrimaryKeys() != null) {
- if(!tblWrapper.cachePrimaryKeys(cacheObjects.getPrimaryKeys(), true)) {
+ if
(CollectionUtils.isNotEmpty(cacheObjects.getTableConstraints().getPrimaryKeys()))
{
+ if
(!tblWrapper.cachePrimaryKeys(cacheObjects.getTableConstraints().getPrimaryKeys(),
true)) {
+ return false;
+ }
+ }
+ tblWrapper.setMemberCacheUpdated(MemberName.PRIMARY_KEY_CACHE, false);
+
+ if
(CollectionUtils.isNotEmpty(cacheObjects.getTableConstraints().getForeignKeys()))
{
Review comment:
Can store cacheObjects.getTableConstraints() in local variable and use
it instead of multiple calls.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
