http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index de94172..3cea4b2 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -19,6 +19,9 @@ package org.apache.hadoop.hive.metastore; import static org.apache.commons.lang.StringUtils.join; +import static org.apache.hadoop.hive.metastore.Warehouse.getCatalogQualifiedDbName; +import static org.apache.hadoop.hive.metastore.Warehouse.getCatalogQualifiedTableName; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; import java.io.IOException; @@ -69,6 +72,7 @@ import javax.sql.DataSource; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -79,6 +83,8 @@ import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.metastore.MetaStoreDirectSql.SqlFilterForPushdown; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -156,6 +162,7 @@ import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider; import org.apache.hadoop.hive.metastore.datasource.DataSourceProviderFactory; import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; +import org.apache.hadoop.hive.metastore.model.MCatalog; import org.apache.hadoop.hive.metastore.model.MColumnDescriptor; import org.apache.hadoop.hive.metastore.model.MConstraint; import org.apache.hadoop.hive.metastore.model.MCreationMetadata; @@ -203,6 +210,7 @@ import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; +import org.apache.thrift.TDeserializer; import org.apache.thrift.TException; import org.datanucleus.AbstractNucleusContext; import org.datanucleus.ClassLoaderResolver; @@ -792,9 +800,127 @@ public class ObjectStore implements RawStore, Configurable { } @Override + public void createCatalog(Catalog cat) throws MetaException { + LOG.debug("Creating catalog " + cat.getName()); + boolean committed = false; + MCatalog mCat = catToMCat(cat); + try { + openTransaction(); + pm.makePersistent(mCat); + committed = commitTransaction(); + } finally { + if (!committed) rollbackTransaction(); + } + } + + @Override + public void alterCatalog(String catName, Catalog cat) + throws MetaException, InvalidOperationException { + if (!cat.getName().equals(catName)) { + throw new InvalidOperationException("You cannot change a catalog's name"); + } + boolean committed = false; + try { + MCatalog mCat = getMCatalog(catName); + if (org.apache.commons.lang.StringUtils.isNotBlank(cat.getLocationUri())) { + mCat.setLocationUri(cat.getLocationUri()); + } + if (org.apache.commons.lang.StringUtils.isNotBlank(cat.getDescription())) { + mCat.setDescription(cat.getDescription()); + } + openTransaction(); + pm.makePersistent(mCat); + committed = commitTransaction(); + } finally { + if (!committed) rollbackTransaction(); + } + } + + @Override + public Catalog getCatalog(String catalogName) throws NoSuchObjectException, MetaException { + LOG.debug("Fetching catalog " + catalogName); + MCatalog mCat = getMCatalog(catalogName); + if (mCat == null) throw new NoSuchObjectException("No catalog " + catalogName); + return mCatToCat(mCat); + } + + @Override + public List<String> getCatalogs() throws MetaException { + LOG.debug("Fetching all catalog names"); + boolean commited = false; + List<String> catalogs = null; + + String queryStr = "select name from org.apache.hadoop.hive.metastore.model.MCatalog"; + Query query = null; + + openTransaction(); + try { + query = pm.newQuery(queryStr); + query.setResult("name"); + catalogs = new ArrayList<>((Collection<String>) query.execute()); + commited = commitTransaction(); + } finally { + rollbackAndCleanup(commited, query); + } + Collections.sort(catalogs); + return catalogs; + } + + @Override + public void dropCatalog(String catalogName) throws NoSuchObjectException, MetaException { + LOG.debug("Dropping catalog " + catalogName); + boolean committed = false; + try { + openTransaction(); + MCatalog mCat = getMCatalog(catalogName); + pm.retrieve(mCat); + if (mCat == null) throw new NoSuchObjectException("No catalog " + catalogName); + pm.deletePersistent(mCat); + committed = commitTransaction(); + } finally { + if (!committed) rollbackTransaction(); + } + } + + private MCatalog getMCatalog(String catalogName) throws MetaException { + boolean committed = false; + Query query = null; + try { + openTransaction(); + catalogName = normalizeIdentifier(catalogName); + query = pm.newQuery(MCatalog.class, "name == catname"); + query.declareParameters("java.lang.String catname"); + query.setUnique(true); + MCatalog mCat = (MCatalog)query.execute(catalogName); + pm.retrieve(mCat); + committed = commitTransaction(); + return mCat; + } finally { + rollbackAndCleanup(committed, query); + } + } + + private MCatalog catToMCat(Catalog cat) { + MCatalog mCat = new MCatalog(); + mCat.setName(normalizeIdentifier(cat.getName())); + if (cat.isSetDescription()) mCat.setDescription(cat.getDescription()); + mCat.setLocationUri(cat.getLocationUri()); + return mCat; + } + + private Catalog mCatToCat(MCatalog mCat) { + Catalog cat = new Catalog(mCat.getName(), mCat.getLocationUri()); + if (mCat.getDescription() != null) cat.setDescription(mCat.getDescription()); + return cat; + } + + @Override public void createDatabase(Database db) throws InvalidObjectException, MetaException { boolean commited = false; MDatabase mdb = new MDatabase(); + assert db.getCatalogName() != null; + mdb.setCatalogName(normalizeIdentifier(db.getCatalogName())); + assert mdb.getCatalogName() != null; mdb.setName(db.getName().toLowerCase()); mdb.setLocationUri(db.getLocationUri()); mdb.setDescription(db.getDescription()); @@ -814,34 +940,35 @@ public class ObjectStore implements RawStore, Configurable { } @SuppressWarnings("nls") - private MDatabase getMDatabase(String name) throws NoSuchObjectException { + private MDatabase getMDatabase(String catName, String name) throws NoSuchObjectException { MDatabase mdb = null; boolean commited = false; Query query = null; try { openTransaction(); name = normalizeIdentifier(name); - query = pm.newQuery(MDatabase.class, "name == dbname"); - query.declareParameters("java.lang.String dbname"); + catName = normalizeIdentifier(catName); + query = pm.newQuery(MDatabase.class, "name == dbname && catalogName == catname"); + query.declareParameters("java.lang.String dbname, java.lang.String catname"); query.setUnique(true); - mdb = (MDatabase) query.execute(name); + mdb = (MDatabase) query.execute(name, catName); pm.retrieve(mdb); commited = commitTransaction(); } finally { rollbackAndCleanup(commited, query); } if (mdb == null) { - throw new NoSuchObjectException("There is no database named " + name); + throw new NoSuchObjectException("There is no database " + catName + "." + name); } return mdb; } @Override - public Database getDatabase(String name) throws NoSuchObjectException { + public Database getDatabase(String catalogName, String name) throws NoSuchObjectException { MetaException ex = null; Database db = null; try { - db = getDatabaseInternal(name); + db = getDatabaseInternal(catalogName, name); } catch (MetaException e) { // Signature restriction to NSOE, and NSOE being a flat exception prevents us from // setting the cause of the NSOE as the MetaException. We should not lose the info @@ -850,32 +977,34 @@ public class ObjectStore implements RawStore, Configurable { ex = e; } if (db == null) { - LOG.warn("Failed to get database {}, returning NoSuchObjectException", name, ex); + LOG.warn("Failed to get database {}.{}, returning NoSuchObjectException", + catalogName, name, ex); throw new NoSuchObjectException(name + (ex == null ? "" : (": " + ex.getMessage()))); } return db; } - public Database getDatabaseInternal(String name) throws MetaException, NoSuchObjectException { - return new GetDbHelper(name, true, true) { + public Database getDatabaseInternal(String catalogName, String name) + throws MetaException, NoSuchObjectException { + return new GetDbHelper(catalogName, name, true, true) { @Override protected Database getSqlResult(GetHelper<Database> ctx) throws MetaException { - return directSql.getDatabase(dbName); + return directSql.getDatabase(catalogName, dbName); } @Override protected Database getJdoResult(GetHelper<Database> ctx) throws MetaException, NoSuchObjectException { - return getJDODatabase(dbName); + return getJDODatabase(catalogName, dbName); } }.run(false); } - public Database getJDODatabase(String name) throws NoSuchObjectException { + public Database getJDODatabase(String catName, String name) throws NoSuchObjectException { MDatabase mdb = null; boolean commited = false; try { openTransaction(); - mdb = getMDatabase(name); + mdb = getMDatabase(catName, name); commited = commitTransaction(); } finally { if (!commited) { @@ -891,6 +1020,7 @@ public class ObjectStore implements RawStore, Configurable { String type = org.apache.commons.lang.StringUtils.defaultIfBlank(mdb.getOwnerType(), null); PrincipalType principalType = (type == null) ? null : PrincipalType.valueOf(type); db.setOwnerType(principalType); + db.setCatalogName(catName); return db; } @@ -903,13 +1033,13 @@ public class ObjectStore implements RawStore, Configurable { * @throws NoSuchObjectException */ @Override - public boolean alterDatabase(String dbName, Database db) + public boolean alterDatabase(String catName, String dbName, Database db) throws MetaException, NoSuchObjectException { MDatabase mdb = null; boolean committed = false; try { - mdb = getMDatabase(dbName); + mdb = getMDatabase(catName, dbName); mdb.setParameters(db.getParameters()); mdb.setOwnerName(db.getOwnerName()); if (db.getOwnerType() != null) { @@ -934,19 +1064,21 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public boolean dropDatabase(String dbname) throws NoSuchObjectException, MetaException { + public boolean dropDatabase(String catName, String dbname) + throws NoSuchObjectException, MetaException { boolean success = false; - LOG.info("Dropping database {} along with all tables", dbname); + LOG.info("Dropping database {}.{} along with all tables", catName, dbname); dbname = normalizeIdentifier(dbname); + catName = normalizeIdentifier(catName); QueryWrapper queryWrapper = new QueryWrapper(); try { openTransaction(); // then drop the database - MDatabase db = getMDatabase(dbname); + MDatabase db = getMDatabase(catName, dbname); pm.retrieve(db); if (db != null) { - List<MDBPrivilege> dbGrants = this.listDatabaseGrants(dbname, queryWrapper); + List<MDBPrivilege> dbGrants = this.listDatabaseGrants(catName, dbname, queryWrapper); if (CollectionUtils.isNotEmpty(dbGrants)) { pm.deletePersistentAll(dbGrants); } @@ -960,9 +1092,9 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<String> getDatabases(String pattern) throws MetaException { + public List<String> getDatabases(String catName, String pattern) throws MetaException { if (pattern == null || pattern.equals("*")) { - return getAllDatabases(); + return getAllDatabases(catName); } boolean commited = false; List<String> databases = null; @@ -974,6 +1106,7 @@ public class ObjectStore implements RawStore, Configurable { String[] subpatterns = pattern.trim().split("\\|"); StringBuilder filterBuilder = new StringBuilder(); List<String> parameterVals = new ArrayList<>(subpatterns.length); + appendSimpleCondition(filterBuilder, "catalogName", new String[] {catName}, parameterVals); appendPatternCondition(filterBuilder, "name", subpatterns, parameterVals); query = pm.newQuery(MDatabase.class, filterBuilder.toString()); query.setResult("name"); @@ -988,18 +1121,20 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<String> getAllDatabases() throws MetaException { + public List<String> getAllDatabases(String catName) throws MetaException { boolean commited = false; List<String> databases = null; - String queryStr = "select name from org.apache.hadoop.hive.metastore.model.MDatabase"; Query query = null; + catName = normalizeIdentifier(catName); openTransaction(); try { - query = pm.newQuery(queryStr); + query = pm.newQuery("select name from org.apache.hadoop.hive.metastore.model.MDatabase " + + "where catalogName == catname"); + query.declareParameters("java.lang.String catname"); query.setResult("name"); - databases = new ArrayList<>((Collection<String>) query.execute()); + databases = new ArrayList<>((Collection<String>) query.execute(catName)); commited = commitTransaction(); } finally { rollbackAndCleanup(commited, query); @@ -1112,12 +1247,25 @@ public class ObjectStore implements RawStore, Configurable { // Add constraints. // We need not do a deep retrieval of the Table Column Descriptor while persisting the // constraints since this transaction involving create table is not yet committed. - List<String> constraintNames = addForeignKeys(foreignKeys, false, primaryKeys, uniqueConstraints); - constraintNames.addAll(addPrimaryKeys(primaryKeys, false)); - constraintNames.addAll(addUniqueConstraints(uniqueConstraints, false)); - constraintNames.addAll(addNotNullConstraints(notNullConstraints, false)); - constraintNames.addAll(addDefaultConstraints(defaultConstraints, false)); - constraintNames.addAll(addCheckConstraints(checkConstraints, false)); + List<String> constraintNames = new ArrayList<>(); + if (foreignKeys != null) { + constraintNames.addAll(addForeignKeys(foreignKeys, false, primaryKeys, uniqueConstraints)); + } + if (primaryKeys != null) { + constraintNames.addAll(addPrimaryKeys(primaryKeys, false)); + } + if (uniqueConstraints != null) { + constraintNames.addAll(addUniqueConstraints(uniqueConstraints, false)); + } + if (notNullConstraints != null) { + constraintNames.addAll(addNotNullConstraints(notNullConstraints, false)); + } + if (defaultConstraints != null) { + constraintNames.addAll(addDefaultConstraints(defaultConstraints, false)); + } + if (checkConstraints != null) { + constraintNames.addAll(addCheckConstraints(checkConstraints, false)); + } success = commitTransaction(); return constraintNames; } finally { @@ -1205,47 +1353,47 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public boolean dropTable(String dbName, String tableName) throws MetaException, - NoSuchObjectException, InvalidObjectException, InvalidInputException { + public boolean dropTable(String catName, String dbName, String tableName) + throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { boolean materializedView = false; boolean success = false; try { openTransaction(); - MTable tbl = getMTable(dbName, tableName); + MTable tbl = getMTable(catName, dbName, tableName); pm.retrieve(tbl); if (tbl != null) { materializedView = TableType.MATERIALIZED_VIEW.toString().equals(tbl.getTableType()); // first remove all the grants - List<MTablePrivilege> tabGrants = listAllTableGrants(dbName, tableName); + List<MTablePrivilege> tabGrants = listAllTableGrants(catName, dbName, tableName); if (CollectionUtils.isNotEmpty(tabGrants)) { pm.deletePersistentAll(tabGrants); } - List<MTableColumnPrivilege> tblColGrants = listTableAllColumnGrants(dbName, + List<MTableColumnPrivilege> tblColGrants = listTableAllColumnGrants(catName, dbName, tableName); if (CollectionUtils.isNotEmpty(tblColGrants)) { pm.deletePersistentAll(tblColGrants); } - List<MPartitionPrivilege> partGrants = this.listTableAllPartitionGrants(dbName, tableName); + List<MPartitionPrivilege> partGrants = this.listTableAllPartitionGrants(catName, dbName, tableName); if (CollectionUtils.isNotEmpty(partGrants)) { pm.deletePersistentAll(partGrants); } - List<MPartitionColumnPrivilege> partColGrants = listTableAllPartitionColumnGrants(dbName, + List<MPartitionColumnPrivilege> partColGrants = listTableAllPartitionColumnGrants(catName, dbName, tableName); if (CollectionUtils.isNotEmpty(partColGrants)) { pm.deletePersistentAll(partColGrants); } // delete column statistics if present try { - deleteTableColumnStatistics(dbName, tableName, null); + deleteTableColumnStatistics(catName, dbName, tableName, null); } catch (NoSuchObjectException e) { - LOG.info("Found no table level column statistics associated with db {}" + - " table {} record to delete", dbName, tableName); + LOG.info("Found no table level column statistics associated with {} to delete", + getCatalogQualifiedTableName(catName, dbName, tableName)); } List<MConstraint> tabConstraints = listAllTableConstraintsWithOptionalConstraintName( - dbName, tableName, null); + catName, dbName, tableName, null); if (CollectionUtils.isNotEmpty(tabConstraints)) { pm.deletePersistentAll(tabConstraints); } @@ -1253,7 +1401,7 @@ public class ObjectStore implements RawStore, Configurable { preDropStorageDescriptor(tbl.getSd()); if (materializedView) { - dropCreationMetadata( + dropCreationMetadata(tbl.getDatabase().getCatalogName(), tbl.getDatabase().getName(), tbl.getTableName()); } @@ -1273,14 +1421,14 @@ public class ObjectStore implements RawStore, Configurable { return success; } - private boolean dropCreationMetadata(String dbName, String tableName) throws MetaException, + private boolean dropCreationMetadata(String catName, String dbName, String tableName) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { boolean success = false; dbName = normalizeIdentifier(dbName); tableName = normalizeIdentifier(tableName); try { openTransaction(); - MCreationMetadata mcm = getCreationMetadata(dbName, tableName); + MCreationMetadata mcm = getCreationMetadata(catName, dbName, tableName); pm.retrieve(mcm); if (mcm != null) { pm.deletePersistentAll(mcm); @@ -1294,8 +1442,9 @@ public class ObjectStore implements RawStore, Configurable { return success; } - private List<MConstraint> listAllTableConstraintsWithOptionalConstraintName - (String dbName, String tableName, String constraintname) { + private List<MConstraint> listAllTableConstraintsWithOptionalConstraintName( + String catName, String dbName, String tableName, String constraintname) { + catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tableName = normalizeIdentifier(tableName); constraintname = constraintname!=null?normalizeIdentifier(constraintname):null; @@ -1305,19 +1454,21 @@ public class ObjectStore implements RawStore, Configurable { try { query = pm.newQuery("select constraintName from org.apache.hadoop.hive.metastore.model.MConstraint where " - + "((parentTable.tableName == ptblname && parentTable.database.name == pdbname) || " - + "(childTable != null && childTable.tableName == ctblname && " - + "childTable.database.name == cdbname)) " + (constraintname != null ? - " && constraintName == constraintname" : "")); + + "((parentTable.tableName == ptblname && parentTable.database.name == pdbname && " + + "parentTable.database.catalogName == pcatname) || " + + "(childTable != null && childTable.tableName == ctblname &&" + + "childTable.database.name == cdbname && childTable.database.catalogName == ccatname)) " + + (constraintname != null ? " && constraintName == constraintname" : "")); query.declareParameters("java.lang.String ptblname, java.lang.String pdbname," - + "java.lang.String ctblname, java.lang.String cdbname" + + + "java.lang.String pcatname, java.lang.String ctblname, java.lang.String cdbname," + + "java.lang.String ccatname" + (constraintname != null ? ", java.lang.String constraintname" : "")); Collection<?> constraintNamesColl = constraintname != null ? ((Collection<?>) query. - executeWithArray(tableName, dbName, tableName, dbName, constraintname)): + executeWithArray(tableName, dbName, catName, tableName, dbName, catName, constraintname)): ((Collection<?>) query. - executeWithArray(tableName, dbName, tableName, dbName)); + executeWithArray(tableName, dbName, catName, tableName, dbName, catName)); for (Iterator<?> i = constraintNamesColl.iterator(); i.hasNext();) { String currName = (String) i.next(); constraintNames.add(currName); @@ -1340,16 +1491,16 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public Table getTable(String dbName, String tableName) throws MetaException { + public Table getTable(String catName, String dbName, String tableName) throws MetaException { boolean commited = false; Table tbl = null; try { openTransaction(); - tbl = convertToTable(getMTable(dbName, tableName)); + tbl = convertToTable(getMTable(catName, dbName, tableName)); // Retrieve creation metadata if needed if (tbl != null && TableType.MATERIALIZED_VIEW.toString().equals(tbl.getTableType())) { tbl.setCreationMetadata( - convertToCreationMetadata(getCreationMetadata(dbName, tableName))); + convertToCreationMetadata(getCreationMetadata(catName, dbName, tableName))); } commited = commitTransaction(); } finally { @@ -1361,40 +1512,46 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<String> getTables(String dbName, String pattern) throws MetaException { - return getTables(dbName, pattern, null); + public List<String> getTables(String catName, String dbName, String pattern) + throws MetaException { + return getTables(catName, dbName, pattern, null); } @Override - public List<String> getTables(String dbName, String pattern, TableType tableType) throws MetaException { + public List<String> getTables(String catName, String dbName, String pattern, TableType tableType) + throws MetaException { try { // We only support pattern matching via jdo since pattern matching in Java // might be different than the one used by the metastore backends - return getTablesInternal(dbName, pattern, tableType, (pattern == null || pattern.equals(".*")), true); + return getTablesInternal(catName, dbName, pattern, tableType, + (pattern == null || pattern.equals(".*")), true); } catch (NoSuchObjectException e) { throw new MetaException(ExceptionUtils.getStackTrace(e)); } } - protected List<String> getTablesInternal(String dbName, String pattern, TableType tableType, - boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { + protected List<String> getTablesInternal(String catName, String dbName, String pattern, + TableType tableType, boolean allowSql, boolean allowJdo) + throws MetaException, NoSuchObjectException { final String db_name = normalizeIdentifier(dbName); - return new GetListHelper<String>(dbName, null, allowSql, allowJdo) { + final String cat_name = normalizeIdentifier(catName); + return new GetListHelper<String>(cat_name, dbName, null, allowSql, allowJdo) { @Override protected List<String> getSqlResult(GetHelper<List<String>> ctx) throws MetaException { - return directSql.getTables(db_name, tableType); + return directSql.getTables(cat_name, db_name, tableType); } @Override protected List<String> getJdoResult(GetHelper<List<String>> ctx) throws MetaException, NoSuchObjectException { - return getTablesInternalViaJdo(db_name, pattern, tableType); + return getTablesInternalViaJdo(cat_name, db_name, pattern, tableType); } }.run(false); } - private List<String> getTablesInternalViaJdo(String dbName, String pattern, TableType tableType) throws MetaException { + private List<String> getTablesInternalViaJdo(String catName, String dbName, String pattern, + TableType tableType) throws MetaException { boolean commited = false; Query query = null; List<String> tbls = null; @@ -1407,6 +1564,7 @@ public class ObjectStore implements RawStore, Configurable { StringBuilder filterBuilder = new StringBuilder(); //adds database.name == dbName to the filter appendSimpleCondition(filterBuilder, "database.name", new String[] {dbName}, parameterVals); + appendSimpleCondition(filterBuilder, "database.catalogName", new String[] {catName}, parameterVals); if(pattern != null) { appendPatternCondition(filterBuilder, "tableName", pattern, parameterVals); } @@ -1427,21 +1585,23 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<String> getMaterializedViewsForRewriting(String dbName) + public List<String> getMaterializedViewsForRewriting(String catName, String dbName) throws MetaException, NoSuchObjectException { final String db_name = normalizeIdentifier(dbName); + catName = normalizeIdentifier(catName); boolean commited = false; Query<?> query = null; List<String> tbls = null; try { openTransaction(); dbName = normalizeIdentifier(dbName); - query = pm.newQuery(MTable.class, "database.name == db && tableType == tt" - + " && rewriteEnabled == re"); - query.declareParameters("java.lang.String db, java.lang.String tt, boolean re"); + query = pm.newQuery(MTable.class, + "database.name == db && database.catalogName == cat && tableType == tt && rewriteEnabled == re"); + query.declareParameters( + "java.lang.String db, java.lang.String cat, java.lang.String tt, boolean re"); query.setResult("tableName"); - Collection<String> names = (Collection<String>) query.execute( - db_name, TableType.MATERIALIZED_VIEW.toString(), true); + Collection<String> names = (Collection<String>) query.executeWithArray( + db_name, catName, TableType.MATERIALIZED_VIEW.toString(), true); tbls = new ArrayList<>(names); commited = commitTransaction(); } finally { @@ -1483,8 +1643,8 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes) - throws MetaException { + public List<TableMeta> getTableMeta(String catName, String dbNames, String tableNames, + List<String> tableTypes) throws MetaException { boolean commited = false; Query query = null; @@ -1495,6 +1655,7 @@ public class ObjectStore implements RawStore, Configurable { // patterns StringBuilder filterBuilder = new StringBuilder(); List<String> parameterVals = new ArrayList<>(); + appendSimpleCondition(filterBuilder, "database.catalogName", new String[] {catName}, parameterVals); if (dbNames != null && !dbNames.equals("*")) { appendPatternCondition(filterBuilder, "database.name", dbNames, parameterVals); } @@ -1505,6 +1666,10 @@ public class ObjectStore implements RawStore, Configurable { appendSimpleCondition(filterBuilder, "tableType", tableTypes.toArray(new String[0]), parameterVals); } + if (LOG.isDebugEnabled()) { + LOG.debug("getTableMeta with filter " + filterBuilder.toString() + " params: " + + StringUtils.join(parameterVals, ", ")); + } query = pm.newQuery(MTable.class, filterBuilder.toString()); Collection<MTable> tables = (Collection<MTable>) query.executeWithArray(parameterVals.toArray(new String[parameterVals.size()])); for (MTable table : tables) { @@ -1563,8 +1728,8 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<String> getAllTables(String dbName) throws MetaException { - return getTables(dbName, ".*"); + public List<String> getAllTables(String catName, String dbName) throws MetaException { + return getTables(catName, dbName, ".*"); } class AttachedMTableInfo { @@ -1579,19 +1744,25 @@ public class ObjectStore implements RawStore, Configurable { } } - private AttachedMTableInfo getMTable(String db, String table, boolean retrieveCD) { + private AttachedMTableInfo getMTable(String catName, String db, String table, + boolean retrieveCD) { AttachedMTableInfo nmtbl = new AttachedMTableInfo(); MTable mtbl = null; boolean commited = false; Query query = null; try { openTransaction(); + catName = normalizeIdentifier(catName); db = normalizeIdentifier(db); table = normalizeIdentifier(table); - query = pm.newQuery(MTable.class, "tableName == table && database.name == db"); - query.declareParameters("java.lang.String table, java.lang.String db"); + query = pm.newQuery(MTable.class, + "tableName == table && database.name == db && database.catalogName == catname"); + query.declareParameters( + "java.lang.String table, java.lang.String db, java.lang.String catname"); query.setUnique(true); - mtbl = (MTable) query.execute(table, db); + LOG.debug("Executing getMTable for " + + getCatalogQualifiedTableName(catName, db, table)); + mtbl = (MTable) query.execute(table, db, catName); pm.retrieve(mtbl); // Retrieving CD can be expensive and unnecessary, so do it only when required. if (mtbl != null && retrieveCD) { @@ -1607,17 +1778,17 @@ public class ObjectStore implements RawStore, Configurable { return nmtbl; } - private MCreationMetadata getCreationMetadata(String dbName, String tblName) { + private MCreationMetadata getCreationMetadata(String catName, String dbName, String tblName) { boolean commited = false; MCreationMetadata mcm = null; Query query = null; try { openTransaction(); query = pm.newQuery( - MCreationMetadata.class, "tblName == table && dbName == db"); - query.declareParameters("java.lang.String table, java.lang.String db"); + MCreationMetadata.class, "tblName == table && dbName == db && catalogName == cat"); + query.declareParameters("java.lang.String table, java.lang.String db, java.lang.String cat"); query.setUnique(true); - mcm = (MCreationMetadata) query.execute(tblName, dbName); + mcm = (MCreationMetadata) query.execute(tblName, dbName, catName); pm.retrieve(mcm); commited = commitTransaction(); } finally { @@ -1626,14 +1797,14 @@ public class ObjectStore implements RawStore, Configurable { return mcm; } - private MTable getMTable(String db, String table) { - AttachedMTableInfo nmtbl = getMTable(db, table, false); + private MTable getMTable(String catName, String db, String table) { + AttachedMTableInfo nmtbl = getMTable(catName, db, table, false); return nmtbl.mtbl; } @Override - public List<Table> getTableObjectsByName(String db, List<String> tbl_names) throws MetaException, - UnknownDBException { + public List<Table> getTableObjectsByName(String catName, String db, List<String> tbl_names) + throws MetaException, UnknownDBException { List<Table> tables = new ArrayList<>(); boolean committed = false; Query dbExistsQuery = null; @@ -1641,25 +1812,31 @@ public class ObjectStore implements RawStore, Configurable { try { openTransaction(); db = normalizeIdentifier(db); - dbExistsQuery = pm.newQuery(MDatabase.class, "name == db"); - dbExistsQuery.declareParameters("java.lang.String db"); - dbExistsQuery.setUnique(true); - dbExistsQuery.setResult("name"); - String dbNameIfExists = (String) dbExistsQuery.execute(db); - if (org.apache.commons.lang.StringUtils.isEmpty(dbNameIfExists)) { - throw new UnknownDBException("Could not find database " + db); - } + catName = normalizeIdentifier(catName); List<String> lowered_tbl_names = new ArrayList<>(tbl_names.size()); for (String t : tbl_names) { lowered_tbl_names.add(normalizeIdentifier(t)); } query = pm.newQuery(MTable.class); - query.setFilter("database.name == db && tbl_names.contains(tableName)"); - query.declareParameters("java.lang.String db, java.util.Collection tbl_names"); - Collection mtables = (Collection) query.execute(db, lowered_tbl_names); - for (Iterator iter = mtables.iterator(); iter.hasNext();) { - tables.add(convertToTable((MTable) iter.next())); + query.setFilter("database.name == db && database.catalogName == cat && tbl_names.contains(tableName)"); + query.declareParameters("java.lang.String db, java.lang.String cat, java.util.Collection tbl_names"); + Collection mtables = (Collection) query.execute(db, catName, lowered_tbl_names); + if (mtables == null || mtables.isEmpty()) { + // Need to differentiate between an unmatched pattern and a non-existent database + dbExistsQuery = pm.newQuery(MDatabase.class, "name == db && catalogName == cat"); + dbExistsQuery.declareParameters("java.lang.String db, java.lang.String cat"); + dbExistsQuery.setUnique(true); + dbExistsQuery.setResult("name"); + String dbNameIfExists = (String) dbExistsQuery.execute(db, catName); + if (org.apache.commons.lang.StringUtils.isEmpty(dbNameIfExists)) { + throw new UnknownDBException("Could not find database " + + getCatalogQualifiedDbName(catName, db)); + } + } else { + for (Iterator iter = mtables.iterator(); iter.hasNext(); ) { + tables.add(convertToTable((MTable) iter.next())); + } } committed = commitTransaction(); } finally { @@ -1703,6 +1880,7 @@ public class ObjectStore implements RawStore, Configurable { convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()), mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType); t.setRewriteEnabled(mtbl.isRewriteEnabled()); + t.setCatName(mtbl.getDatabase().getCatalogName()); return t; } @@ -1712,12 +1890,13 @@ public class ObjectStore implements RawStore, Configurable { return null; } MDatabase mdb = null; + String catName = tbl.isSetCatName() ? tbl.getCatName() : getDefaultCatalog(conf); try { - mdb = getMDatabase(tbl.getDbName()); + mdb = getMDatabase(catName, tbl.getDbName()); } catch (NoSuchObjectException e) { LOG.error("Could not convert to MTable", e); - throw new InvalidObjectException("Database " + tbl.getDbName() - + " doesn't exist."); + throw new InvalidObjectException("Database " + + getCatalogQualifiedDbName(catName, tbl.getDbName()) + " doesn't exist."); } // If the table has property EXTERNAL set, update table type @@ -1965,9 +2144,9 @@ public class ObjectStore implements RawStore, Configurable { Set<MTable> tablesUsed = new HashSet<>(); for (String fullyQualifiedName : m.getTablesUsed()) { String[] names = fullyQualifiedName.split("\\."); - tablesUsed.add(getMTable(names[0], names[1], false).mtbl); + tablesUsed.add(getMTable(m.getCatName(), names[0], names[1], false).mtbl); } - return new MCreationMetadata(m.getDbName(), m.getTblName(), + return new MCreationMetadata(m.getCatName(), m.getDbName(), m.getTblName(), tablesUsed, m.getValidTxnList()); } @@ -1982,7 +2161,7 @@ public class ObjectStore implements RawStore, Configurable { Warehouse.getQualifiedName( mtbl.getDatabase().getName(), mtbl.getTableName())); } - CreationMetadata r = new CreationMetadata( + CreationMetadata r = new CreationMetadata(s.getCatalogName(), s.getDbName(), s.getTblName(), tablesUsed); if (s.getTxnList() != null) { r.setValidTxnList(s.getTxnList()); @@ -1991,17 +2170,17 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public boolean addPartitions(String dbName, String tblName, List<Partition> parts) + public boolean addPartitions(String catName, String dbName, String tblName, List<Partition> parts) throws InvalidObjectException, MetaException { boolean success = false; openTransaction(); try { List<MTablePrivilege> tabGrants = null; List<MTableColumnPrivilege> tabColumnGrants = null; - MTable table = this.getMTable(dbName, tblName); + MTable table = this.getMTable(catName, dbName, tblName); if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { - tabGrants = this.listAllTableGrants(dbName, tblName); - tabColumnGrants = this.listTableAllColumnGrants(dbName, tblName); + tabGrants = this.listAllTableGrants(catName, dbName, tblName); + tabColumnGrants = this.listTableAllColumnGrants(catName, dbName, tblName); } List<Object> toPersist = new ArrayList<>(); for (Partition part : parts) { @@ -2046,7 +2225,7 @@ public class ObjectStore implements RawStore, Configurable { Partition part, boolean ifNotExists) throws MetaException { MetaStoreUtils.validatePartitionNameCharacters(part.getValues(), partitionValidationPattern); - boolean doesExist = doesPartitionExist( + boolean doesExist = doesPartitionExist(part.getCatName(), part.getDbName(), part.getTableName(), part.getValues()); if (doesExist && !ifNotExists) { throw new MetaException("Partition already exists: " + part); @@ -2055,7 +2234,7 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public boolean addPartitions(String dbName, String tblName, + public boolean addPartitions(String catName, String dbName, String tblName, PartitionSpecProxy partitionSpec, boolean ifNotExists) throws InvalidObjectException, MetaException { boolean success = false; @@ -2063,10 +2242,10 @@ public class ObjectStore implements RawStore, Configurable { try { List<MTablePrivilege> tabGrants = null; List<MTableColumnPrivilege> tabColumnGrants = null; - MTable table = this.getMTable(dbName, tblName); + MTable table = this.getMTable(catName, dbName, tblName); if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { - tabGrants = this.listAllTableGrants(dbName, tblName); - tabColumnGrants = this.listTableAllColumnGrants(dbName, tblName); + tabGrants = this.listAllTableGrants(catName, dbName, tblName); + tabColumnGrants = this.listTableAllColumnGrants(catName, dbName, tblName); } if (!partitionSpec.getTableName().equals(tblName) || !partitionSpec.getDbName().equals(dbName)) { @@ -2117,14 +2296,14 @@ public class ObjectStore implements RawStore, Configurable { boolean success = false; boolean commited = false; try { - MTable table = this.getMTable(part.getDbName(), part.getTableName()); + String catName = part.isSetCatName() ? part.getCatName() : getDefaultCatalog(conf); + MTable table = this.getMTable(catName, part.getDbName(), part.getTableName()); List<MTablePrivilege> tabGrants = null; List<MTableColumnPrivilege> tabColumnGrants = null; if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { - tabGrants = this.listAllTableGrants(part - .getDbName(), part.getTableName()); + tabGrants = this.listAllTableGrants(catName, part.getDbName(), part.getTableName()); tabColumnGrants = this.listTableAllColumnGrants( - part.getDbName(), part.getTableName()); + catName, part.getDbName(), part.getTableName()); } openTransaction(); MPartition mpart = convertToMPart(part, true); @@ -2167,10 +2346,10 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public Partition getPartition(String dbName, String tableName, + public Partition getPartition(String catName, String dbName, String tableName, List<String> part_vals) throws NoSuchObjectException, MetaException { openTransaction(); - Partition part = convertToPart(getMPartition(dbName, tableName, part_vals)); + Partition part = convertToPart(getMPartition(catName, dbName, tableName, part_vals)); commitTransaction(); if(part == null) { throw new NoSuchObjectException("partition values=" @@ -2180,7 +2359,7 @@ public class ObjectStore implements RawStore, Configurable { return part; } - private MPartition getMPartition(String dbName, String tableName, List<String> part_vals) + private MPartition getMPartition(String catName, String dbName, String tableName, List<String> part_vals) throws MetaException { List<MPartition> mparts = null; MPartition ret = null; @@ -2188,9 +2367,10 @@ public class ObjectStore implements RawStore, Configurable { Query query = null; try { openTransaction(); + catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tableName = normalizeIdentifier(tableName); - MTable mtbl = getMTable(dbName, tableName); + MTable mtbl = getMTable(catName, dbName, tableName); if (mtbl == null) { commited = commitTransaction(); return null; @@ -2201,9 +2381,11 @@ public class ObjectStore implements RawStore, Configurable { Warehouse.makePartName(convertToFieldSchemas(mtbl.getPartitionKeys()), part_vals); query = pm.newQuery(MPartition.class, - "table.tableName == t1 && table.database.name == t2 && partitionName == t3"); - query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); - mparts = (List<MPartition>) query.execute(tableName, dbName, name); + "table.tableName == t1 && table.database.name == t2 && partitionName == t3 " + + " && table.database.catalogName == t4"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, " + + "java.lang.String t4"); + mparts = (List<MPartition>) query.executeWithArray(tableName, dbName, name, catName); pm.retrieveAll(mparts); commited = commitTransaction(); // We need to compare partition name with requested name since some DBs @@ -2246,7 +2428,7 @@ public class ObjectStore implements RawStore, Configurable { if (part == null) { return null; } - MTable mt = getMTable(part.getDbName(), part.getTableName()); + MTable mt = getMTable(part.getCatName(), part.getDbName(), part.getTableName()); if (mt == null) { throw new InvalidObjectException( "Partition doesn't have a valid table or database name"); @@ -2277,30 +2459,34 @@ public class ObjectStore implements RawStore, Configurable { if (mpart == null) { return null; } - return new Partition(convertList(mpart.getValues()), mpart.getTable().getDatabase() + Partition p = new Partition(convertList(mpart.getValues()), mpart.getTable().getDatabase() .getName(), mpart.getTable().getTableName(), mpart.getCreateTime(), mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd()), convertMap(mpart.getParameters())); + p.setCatName(mpart.getTable().getDatabase().getCatalogName()); + return p; } - private Partition convertToPart(String dbName, String tblName, MPartition mpart) + private Partition convertToPart(String catName, String dbName, String tblName, MPartition mpart) throws MetaException { if (mpart == null) { return null; } - return new Partition(convertList(mpart.getValues()), dbName, tblName, + Partition p = new Partition(convertList(mpart.getValues()), dbName, tblName, mpart.getCreateTime(), mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd(), false), convertMap(mpart.getParameters())); + p.setCatName(catName); + return p; } @Override - public boolean dropPartition(String dbName, String tableName, + public boolean dropPartition(String catName, String dbName, String tableName, List<String> part_vals) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { boolean success = false; try { openTransaction(); - MPartition part = getMPartition(dbName, tableName, part_vals); + MPartition part = getMPartition(catName, dbName, tableName, part_vals); dropPartitionCommon(part); success = commitTransaction(); } finally { @@ -2312,7 +2498,7 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public void dropPartitions(String dbName, String tblName, List<String> partNames) + public void dropPartitions(String catName, String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException { if (CollectionUtils.isEmpty(partNames)) { return; @@ -2321,15 +2507,15 @@ public class ObjectStore implements RawStore, Configurable { openTransaction(); try { // Delete all things. - dropPartitionGrantsNoTxn(dbName, tblName, partNames); - dropPartitionAllColumnGrantsNoTxn(dbName, tblName, partNames); - dropPartitionColumnStatisticsNoTxn(dbName, tblName, partNames); + dropPartitionGrantsNoTxn(catName, dbName, tblName, partNames); + dropPartitionAllColumnGrantsNoTxn(catName, dbName, tblName, partNames); + dropPartitionColumnStatisticsNoTxn(catName, dbName, tblName, partNames); // CDs are reused; go thry partition SDs, detach all CDs from SDs, then remove unused CDs. - for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(dbName, tblName, partNames)) { + for (MColumnDescriptor mcd : detachCdsFromSdsNoTxn(catName, dbName, tblName, partNames)) { removeUnusedColumnDescriptor(mcd); } - dropPartitionsNoTxn(dbName, tblName, partNames); + dropPartitionsNoTxn(catName, dbName, tblName, partNames); if (!(success = commitTransaction())) { throw new MetaException("Failed to drop partitions"); // Should not happen? } @@ -2343,12 +2529,6 @@ public class ObjectStore implements RawStore, Configurable { /** * Drop an MPartition and cascade deletes (e.g., delete partition privilege grants, * drop the storage descriptor cleanly, etc.) - * @param part - the MPartition to drop - * @return whether the transaction committed successfully - * @throws InvalidInputException - * @throws InvalidObjectException - * @throws MetaException - * @throws NoSuchObjectException */ private boolean dropPartitionCommon(MPartition part) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { @@ -2364,6 +2544,7 @@ public class ObjectStore implements RawStore, Configurable { String partName = FileUtils.makePartName(colNames, part.getValues()); List<MPartitionPrivilege> partGrants = listPartitionGrants( + part.getTable().getDatabase().getCatalogName(), part.getTable().getDatabase().getName(), part.getTable().getTableName(), Lists.newArrayList(partName)); @@ -2373,6 +2554,7 @@ public class ObjectStore implements RawStore, Configurable { } List<MPartitionColumnPrivilege> partColumnGrants = listPartitionAllColumnGrants( + part.getTable().getDatabase().getCatalogName(), part.getTable().getDatabase().getName(), part.getTable().getTableName(), Lists.newArrayList(partName)); @@ -2380,12 +2562,13 @@ public class ObjectStore implements RawStore, Configurable { pm.deletePersistentAll(partColumnGrants); } + String catName = part.getTable().getDatabase().getCatalogName(); String dbName = part.getTable().getDatabase().getName(); String tableName = part.getTable().getTableName(); // delete partition level column stats if it exists try { - deletePartitionColumnStatistics(dbName, tableName, partName, part.getValues(), null); + deletePartitionColumnStatistics(catName, dbName, tableName, partName, part.getValues(), null); } catch (NoSuchObjectException e) { LOG.info("No column statistics records found to delete"); } @@ -2403,26 +2586,26 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<Partition> getPartitions( - String dbName, String tableName, int maxParts) throws MetaException, NoSuchObjectException { - return getPartitionsInternal(dbName, tableName, maxParts, true, true); + public List<Partition> getPartitions(String catName, String dbName, String tableName, + int maxParts) throws MetaException, NoSuchObjectException { + return getPartitionsInternal(catName, dbName, tableName, maxParts, true, true); } - protected List<Partition> getPartitionsInternal( - String dbName, String tblName, final int maxParts, boolean allowSql, boolean allowJdo) + protected List<Partition> getPartitionsInternal(String catName, String dbName, String tblName, + final int maxParts, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { - return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) { + return new GetListHelper<Partition>(catName, dbName, tblName, allowSql, allowJdo) { @Override protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException { Integer max = (maxParts < 0) ? null : maxParts; - return directSql.getPartitions(dbName, tblName, max); + return directSql.getPartitions(catName, dbName, tblName, max); } @Override protected List<Partition> getJdoResult( GetHelper<List<Partition>> ctx) throws MetaException { QueryWrapper queryWrapper = new QueryWrapper(); try { - return convertToParts(listMPartitions(dbName, tblName, maxParts, queryWrapper)); + return convertToParts(listMPartitions(catName, dbName, tblName, maxParts, queryWrapper)); } finally { queryWrapper.close(); } @@ -2431,7 +2614,7 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<Partition> getPartitionsWithAuth(String dbName, String tblName, + public List<Partition> getPartitionsWithAuth(String catName, String dbName, String tblName, short max, String userName, List<String> groupNames) throws MetaException, InvalidObjectException { boolean success = false; @@ -2439,7 +2622,7 @@ public class ObjectStore implements RawStore, Configurable { try { openTransaction(); - List<MPartition> mparts = listMPartitions(dbName, tblName, max, queryWrapper); + List<MPartition> mparts = listMPartitions(catName, dbName, tblName, max, queryWrapper); List<Partition> parts = new ArrayList<>(mparts.size()); if (CollectionUtils.isNotEmpty(mparts)) { for (MPartition mpart : mparts) { @@ -2450,7 +2633,7 @@ public class ObjectStore implements RawStore, Configurable { if ("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl .getPartitionKeys()), part.getValues()); - PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(dbName, + PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(catName, dbName, tblName, partName, userName, groupNames); part.setPrivileges(partAuth); } @@ -2464,13 +2647,13 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public Partition getPartitionWithAuth(String dbName, String tblName, + public Partition getPartitionWithAuth(String catName, String dbName, String tblName, List<String> partVals, String user_name, List<String> group_names) throws NoSuchObjectException, MetaException, InvalidObjectException { boolean success = false; try { openTransaction(); - MPartition mpart = getMPartition(dbName, tblName, partVals); + MPartition mpart = getMPartition(catName, dbName, tblName, partVals); if (mpart == null) { commitTransaction(); throw new NoSuchObjectException("partition values=" @@ -2482,7 +2665,7 @@ public class ObjectStore implements RawStore, Configurable { if ("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl .getPartitionKeys()), partVals); - PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(dbName, + PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(catName, dbName, tblName, partName, user_name, group_names); part.setPrivileges(partAuth); } @@ -2515,11 +2698,11 @@ public class ObjectStore implements RawStore, Configurable { return dest; } - private List<Partition> convertToParts(String dbName, String tblName, List<MPartition> mparts) + private List<Partition> convertToParts(String catName, String dbName, String tblName, List<MPartition> mparts) throws MetaException { List<Partition> parts = new ArrayList<>(mparts.size()); for (MPartition mp : mparts) { - parts.add(convertToPart(dbName, tblName, mp)); + parts.add(convertToPart(catName, dbName, tblName, mp)); Deadline.checkTimeout(); } return parts; @@ -2527,14 +2710,14 @@ public class ObjectStore implements RawStore, Configurable { // TODO:pc implement max @Override - public List<String> listPartitionNames(String dbName, String tableName, + public List<String> listPartitionNames(String catName, String dbName, String tableName, short max) throws MetaException { List<String> pns = null; boolean success = false; try { openTransaction(); LOG.debug("Executing getPartitionNames"); - pns = getPartitionNamesNoTxn(dbName, tableName, max); + pns = getPartitionNamesNoTxn(catName, dbName, tableName, max); success = commitTransaction(); } finally { if (!success) { @@ -2584,21 +2767,24 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public PartitionValuesResponse listPartitionValues(String dbName, String tableName, List<FieldSchema> cols, - boolean applyDistinct, String filter, boolean ascending, - List<FieldSchema> order, long maxParts) throws MetaException { + public PartitionValuesResponse listPartitionValues(String catName, String dbName, + String tableName, List<FieldSchema> cols, + boolean applyDistinct, String filter, + boolean ascending, List<FieldSchema> order, + long maxParts) throws MetaException { + catName = normalizeIdentifier(catName); dbName = dbName.toLowerCase().trim(); tableName = tableName.toLowerCase().trim(); try { if (filter == null || filter.isEmpty()) { - PartitionValuesResponse response = - getDistinctValuesForPartitionsNoTxn(dbName, tableName, cols, applyDistinct, ascending, maxParts); + PartitionValuesResponse response = getDistinctValuesForPartitionsNoTxn(catName, dbName, + tableName, cols, applyDistinct, maxParts); LOG.info("Number of records fetched: {}", response.getPartitionValues().size()); return response; } else { PartitionValuesResponse response = - extractPartitionNamesByFilter(dbName, tableName, filter, cols, ascending, applyDistinct, maxParts); + extractPartitionNamesByFilter(catName, dbName, tableName, filter, cols, ascending, maxParts); if (response != null && response.getPartitionValues() != null) { LOG.info("Number of records fetched with filter: {}", response.getPartitionValues().size()); } @@ -2607,31 +2793,33 @@ public class ObjectStore implements RawStore, Configurable { } catch (Exception t) { LOG.error("Exception in ORM", t); throw new MetaException("Error retrieving partition values: " + t); - } finally { } } - private PartitionValuesResponse extractPartitionNamesByFilter(String dbName, String tableName, String filter, - List<FieldSchema> cols, boolean ascending, boolean applyDistinct, long maxParts) + private PartitionValuesResponse extractPartitionNamesByFilter( + String catName, String dbName, String tableName, String filter, List<FieldSchema> cols, + boolean ascending, long maxParts) throws MetaException, NoSuchObjectException { - LOG.info("Database: {} Table: {} filter: \"{}\" cols: {}", dbName, tableName, filter, cols); + LOG.info("Table: {} filter: \"{}\" cols: {}", + getCatalogQualifiedTableName(catName, dbName, tableName), filter, cols); List<String> partitionNames = null; List<Partition> partitions = null; - Table tbl = getTable(dbName, tableName); + Table tbl = getTable(catName, dbName, tableName); try { // Get partitions by name - ascending or descending - partitionNames = getPartitionNamesByFilter(dbName, tableName, filter, ascending, maxParts); + partitionNames = getPartitionNamesByFilter(catName, dbName, tableName, filter, ascending, + maxParts); } catch (MetaException e) { LOG.warn("Querying by partition names failed, trying out with partition objects, filter: {}", filter); } if (partitionNames == null) { - partitions = getPartitionsByFilter(dbName, tableName, filter, (short) maxParts); + partitions = getPartitionsByFilter(catName, dbName, tableName, filter, (short) maxParts); } if (partitions != null) { - partitionNames = new ArrayList<String>(partitions.size()); + partitionNames = new ArrayList<>(partitions.size()); for (Partition partition : partitions) { // Check for NULL's just to be safe if (tbl.getPartitionKeys() != null && partition.getValues() != null) { @@ -2642,7 +2830,7 @@ public class ObjectStore implements RawStore, Configurable { if (partitionNames == null && partitions == null) { throw new MetaException("Cannot obtain list of partitions by filter:\"" + filter + - "\" for " + dbName + ":" + tableName); + "\" for " + getCatalogQualifiedTableName(catName, dbName, tableName)); } if (!ascending) { @@ -2651,7 +2839,7 @@ public class ObjectStore implements RawStore, Configurable { // Return proper response PartitionValuesResponse response = new PartitionValuesResponse(); - response.setPartitionValues(new ArrayList<PartitionValuesRow>(partitionNames.size())); + response.setPartitionValues(new ArrayList<>(partitionNames.size())); LOG.info("Converting responses to Partition values for items: {}", partitionNames.size()); for (String partName : partitionNames) { ArrayList<String> vals = new ArrayList<String>(Collections.nCopies(tbl.getPartitionKeys().size(), null)); @@ -2665,26 +2853,27 @@ public class ObjectStore implements RawStore, Configurable { return response; } - private List<String> getPartitionNamesByFilter(String dbName, String tableName, + private List<String> getPartitionNamesByFilter(String catName, String dbName, String tableName, String filter, boolean ascending, long maxParts) throws MetaException { boolean success = false; - List<String> partNames = new ArrayList<String>(); + List<String> partNames = new ArrayList<>(); try { openTransaction(); LOG.debug("Executing getPartitionNamesByFilter"); + catName = normalizeIdentifier(catName); dbName = dbName.toLowerCase(); tableName = tableName.toLowerCase(); - MTable mtable = getMTable(dbName, tableName); + MTable mtable = getMTable(catName, dbName, tableName); if( mtable == null ) { // To be consistent with the behavior of listPartitionNames, if the // table or db does not exist, we return an empty list return partNames; } - Map<String, Object> params = new HashMap<String, Object>(); - String queryFilterString = makeQueryFilterString(dbName, mtable, filter, params); + Map<String, Object> params = new HashMap<>(); + String queryFilterString = makeQueryFilterString(catName, dbName, mtable, filter, params); Query query = pm.newQuery( "select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " + "where " + queryFilterString); @@ -2723,15 +2912,16 @@ public class ObjectStore implements RawStore, Configurable { return partNames; } - private PartitionValuesResponse getDistinctValuesForPartitionsNoTxn(String dbName, String tableName, List<FieldSchema> cols, - boolean applyDistinct, boolean ascending, long maxParts) + private PartitionValuesResponse getDistinctValuesForPartitionsNoTxn( + String catName, String dbName, String tableName, List<FieldSchema> cols, + boolean applyDistinct, long maxParts) throws MetaException { - try { openTransaction(); Query q = pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " - + "where table.database.name == t1 && table.tableName == t2 "); - q.declareParameters("java.lang.String t1, java.lang.String t2"); + + "where table.database.name == t1 && table.database.catalogName == t2 && " + + "table.tableName == t3 "); + q.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); // TODO: Ordering seems to affect the distinctness, needs checking, disabling. /* @@ -2748,7 +2938,7 @@ public class ObjectStore implements RawStore, Configurable { if (applyDistinct) { partValuesSelect.append("DISTINCT "); } - List<FieldSchema> partitionKeys = getTable(dbName, tableName).getPartitionKeys(); + List<FieldSchema> partitionKeys = getTable(catName, dbName, tableName).getPartitionKeys(); for (FieldSchema key : cols) { partValuesSelect.append(extractPartitionKey(key, partitionKeys)).append(", "); } @@ -2757,9 +2947,9 @@ public class ObjectStore implements RawStore, Configurable { q.setResult(partValuesSelect.toString()); PartitionValuesResponse response = new PartitionValuesResponse(); - response.setPartitionValues(new ArrayList<PartitionValuesRow>()); + response.setPartitionValues(new ArrayList<>()); if (cols.size() > 1) { - List<Object[]> results = (List<Object[]>) q.execute(dbName, tableName); + List<Object[]> results = (List<Object[]>) q.execute(dbName, catName, tableName); for (Object[] row : results) { PartitionValuesRow rowResponse = new PartitionValuesRow(); for (Object columnValue : row) { @@ -2768,7 +2958,7 @@ public class ObjectStore implements RawStore, Configurable { response.addToPartitionValues(rowResponse); } } else { - List<Object> results = (List<Object>) q.execute(dbName, tableName); + List<Object> results = (List<Object>) q.execute(dbName, catName, tableName); for (Object row : results) { PartitionValuesRow rowResponse = new PartitionValuesRow(); rowResponse.addToRow((String) row); @@ -2782,24 +2972,25 @@ public class ObjectStore implements RawStore, Configurable { } } - private List<String> getPartitionNamesNoTxn(String dbName, String tableName, short max) { + private List<String> getPartitionNamesNoTxn(String catName, String dbName, String tableName, short max) { List<String> pns = new ArrayList<>(); if (max == 0) { return pns; } + catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tableName = normalizeIdentifier(tableName); Query query = pm.newQuery("select partitionName from org.apache.hadoop.hive.metastore.model.MPartition " - + "where table.database.name == t1 && table.tableName == t2 " + + "where table.database.name == t1 && table.tableName == t2 && table.database.catalogName == t3 " + "order by partitionName asc"); - query.declareParameters("java.lang.String t1, java.lang.String t2"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setResult("partitionName"); if (max > 0) { query.setRange(0, max); } - Collection<String> names = (Collection<String>) query.execute(dbName, tableName); + Collection<String> names = (Collection<String>) query.execute(dbName, tableName, catName); pns.addAll(names); if (query != null) { @@ -2823,14 +3014,16 @@ public class ObjectStore implements RawStore, Configurable { * you want results for. E.g., if resultsCol is partitionName, the Collection * has types of String, and if resultsCol is null, the types are MPartition. */ - private Collection getPartitionPsQueryResults(String dbName, String tableName, + private Collection getPartitionPsQueryResults(String catName, String dbName, String tableName, List<String> part_vals, short max_parts, String resultsCol, QueryWrapper queryWrapper) throws MetaException, NoSuchObjectException { + catName = normalizeIdentifier(catName); dbName = normalizeIdentifier(dbName); tableName = normalizeIdentifier(tableName); - Table table = getTable(dbName, tableName); + Table table = getTable(catName, dbName, tableName); if (table == null) { - throw new NoSuchObjectException(dbName + "." + tableName + " table not found"); + throw new NoSuchObjectException(getCatalogQualifiedTableName(catName, dbName, tableName) + + " table not found"); } List<FieldSchema> partCols = table.getPartitionKeys(); int numPartKeys = partCols.size(); @@ -2851,10 +3044,11 @@ public class ObjectStore implements RawStore, Configurable { } Query query = queryWrapper.query = pm.newQuery(MPartition.class); StringBuilder queryFilter = new StringBuilder("table.database.name == dbName"); + queryFilter.append(" && table.database.catalogName == catName"); queryFilter.append(" && table.tableName == tableName"); queryFilter.append(" && partitionName.matches(partialRegex)"); query.setFilter(queryFilter.toString()); - query.declareParameters("java.lang.String dbName, " + query.declareParameters("java.lang.String dbName, java.lang.String catName, " + "java.lang.String tableName, java.lang.String partialRegex"); if (max_parts >= 0) { // User specified a row limit, set it on the Query @@ -2864,11 +3058,11 @@ public class ObjectStore implements RawStore, Configurable { query.setResult(resultsCol); } - return (Collection) query.execute(dbName, tableName, partNameMatcher); + return (Collection) query.executeWithArray(dbName, catName, tableName, partNameMatcher); } @Override - public List<Partition> listPartitionsPsWithAuth(String db_name, String tbl_name, + public List<Partition> listPartitionsPsWithAuth(String catName, String db_name, String tbl_name, List<String> part_vals, short max_parts, String userName, List<String> groupNames) throws MetaException, InvalidObjectException, NoSuchObjectException { List<Partition> partitions = new ArrayList<>(); @@ -2878,9 +3072,9 @@ public class ObjectStore implements RawStore, Configurable { try { openTransaction(); LOG.debug("executing listPartitionNamesPsWithAuth"); - Collection parts = getPartitionPsQueryResults(db_name, tbl_name, + Collection parts = getPartitionPsQueryResults(catName, db_name, tbl_name, part_vals, max_parts, null, queryWrapper); - MTable mtbl = getMTable(db_name, tbl_name); + MTable mtbl = getMTable(catName, db_name, tbl_name); for (Object o : parts) { Partition part = convertToPart((MPartition) o); //set auth privileges @@ -2888,7 +3082,7 @@ public class ObjectStore implements RawStore, Configurable { "TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl .getPartitionKeys()), part.getValues()); - PrincipalPrivilegeSet partAuth = getPartitionPrivilegeSet(db_name, + PrincipalPrivilegeSet partAuth = getPartitionPrivilegeSet(catName, db_name, tbl_name, partName, userName, groupNames); part.setPrivileges(partAuth); } @@ -2902,7 +3096,7 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<String> listPartitionNamesPs(String dbName, String tableName, + public List<String> listPartitionNamesPs(String catName, String dbName, String tableName, List<String> part_vals, short max_parts) throws MetaException, NoSuchObjectException { List<String> partitionNames = new ArrayList<>(); boolean success = false; @@ -2911,7 +3105,7 @@ public class ObjectStore implements RawStore, Configurable { try { openTransaction(); LOG.debug("Executing listPartitionNamesPs"); - Collection<String> names = getPartitionPsQueryResults(dbName, tableName, + Collection<String> names = getPartitionPsQueryResults(catName, dbName, tableName, part_vals, max_parts, "partitionName", queryWrapper); partitionNames.addAll(names); success = commitTransaction(); @@ -2922,7 +3116,8 @@ public class ObjectStore implements RawStore, Configurable { } // TODO:pc implement max - private List<MPartition> listMPartitions(String dbName, String tableName, int max, QueryWrapper queryWrapper) { + private List<MPartition> listMPartitions(String catName, String dbName, String tableName, + int max, QueryWrapper queryWrapper) { boolean success = false; List<MPartition> mparts = null; try { @@ -2930,13 +3125,14 @@ public class ObjectStore implements RawStore, Configurable { LOG.debug("Executing listMPartitions"); dbName = normalizeIdentifier(dbName); tableName = normalizeIdentifier(tableName); - Query query = queryWrapper.query = pm.newQuery(MPartition.class, "table.tableName == t1 && table.database.name == t2"); - query.declareParameters("java.lang.String t1, java.lang.String t2"); + Query query = queryWrapper.query = pm.newQuery(MPartition.class, + "table.tableName == t1 && table.database.name == t2 && table.database.catalogName == t3"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3"); query.setOrdering("partitionName ascending"); if (max > 0) { query.setRange(0, max); } - mparts = (List<MPartition>) query.execute(tableName, dbName); + mparts = (List<MPartition>) query.execute(tableName, dbName, catName); LOG.debug("Done executing query for listMPartitions"); pm.retrieveAll(mparts); success = commitTransaction(); @@ -2950,41 +3146,43 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<Partition> getPartitionsByNames(String dbName, String tblName, + public List<Partition> getPartitionsByNames(String catName, String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException { - return getPartitionsByNamesInternal(dbName, tblName, partNames, true, true); + return getPartitionsByNamesInternal(catName, dbName, tblName, partNames, true, true); } - protected List<Partition> getPartitionsByNamesInternal(String dbName, String tblName, - final List<String> partNames, boolean allowSql, boolean allowJdo) + protected List<Partition> getPartitionsByNamesInternal(String catName, String dbName, + String tblName, + final List<String> partNames, + boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { - return new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) { + return new GetListHelper<Partition>(catName, dbName, tblName, allowSql, allowJdo) { @Override protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException { - return directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames); + return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, partNames); } @Override protected List<Partition> getJdoResult( GetHelper<List<Partition>> ctx) throws MetaException, NoSuchObjectException { - return getPartitionsViaOrmFilter(dbName, tblName, partNames); + return getPartitionsViaOrmFilter(catName, dbName, tblName, partNames); } }.run(false); } @Override - public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr, + public boolean getPartitionsByExpr(String catName, String dbName, String tblName, byte[] expr, String defaultPartitionName, short maxParts, List<Partition> result) throws TException { return getPartitionsByExprInternal( - dbName, tblName, expr, defaultPartitionName, maxParts, result, true, true); + catName, dbName, tblName, expr, defaultPartitionName, maxParts, result, true, true); } - protected boolean getPartitionsByExprInternal(String dbName, String tblName, final byte[] expr, + protected boolean getPartitionsByExprInternal(String catName, String dbName, String tblName, final byte[] expr, final String defaultPartitionName, final short maxParts, List<Partition> result, boolean allowSql, boolean allowJdo) throws TException { assert result != null; final ExpressionTree exprTree = PartFilterExprUtil.makeExpressionTree(expressionProxy, expr); final AtomicBoolean hasUnknownPartitions = new AtomicBoolean(false); - result.addAll(new GetListHelper<Partition>(dbName, tblName, allowSql, allowJdo) { + result.addAll(new GetListHelper<Partition>(catName, dbName, tblName, allowSql, allowJdo) { @Override protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException { // If we have some sort of expression tree, try SQL filter pushdown. @@ -2999,7 +3197,7 @@ public class ObjectStore implements RawStore, Configurable { List<String> partNames = new LinkedList<>(); hasUnknownPartitions.set(getPartitionNamesPrunedByExprNoTxn( ctx.getTable(), expr, defaultPartitionName, maxParts, partNames)); - return directSql.getPartitionsViaSqlFilter(dbName, tblName, partNames); + return directSql.getPartitionsViaSqlFilter(catName, dbName, tblName, partNames); } @Override @@ -3015,7 +3213,7 @@ public class ObjectStore implements RawStore, Configurable { List<String> partNames = new ArrayList<>(); hasUnknownPartitions.set(getPartitionNamesPrunedByExprNoTxn( ctx.getTable(), expr, defaultPartitionName, maxParts, partNames)); - result = getPartitionsViaOrmFilter(dbName, tblName, partNames); + result = getPartitionsViaOrmFilter(catName, dbName, tblName, partNames); } return result; } @@ -3036,7 +3234,7 @@ public class ObjectStore implements RawStore, Configurable { */ private boolean getPartitionNamesPrunedByExprNoTxn(Table table, byte[] expr, String defaultPartName, short maxParts, List<String> result) throws MetaException { - result.addAll(getPartitionNamesNoTxn( + result.addAll(getPartitionNamesNoTxn(table.getCatName(), table.getDbName(), table.getTableName(), maxParts)); if (defaultPartName == null || defaultPartName.isEmpty()) { defaultPartName = MetastoreConf.getVar(getConf(), ConfVars.DEFAULTPARTITIONNAME); @@ -3058,7 +3256,7 @@ public class ObjectStore implements RawStore, Configurable { short maxParts, boolean isValidatedFilter) throws MetaException { Map<String, Object> params = new HashMap<>(); String jdoFilter = - makeQueryFilterString(table.getDbName(), table, tree, params, isValidatedFilter); + makeQueryFilterString(table.getCatName(), table.getDbName(), table, tree, params, isValidatedFilter); if (jdoFilter == null) { assert !isValidatedFilter; return null; @@ -3085,7 +3283,8 @@ public class ObjectStore implements RawStore, Configurable { private Integer getNumPartitionsViaOrmFilter(Table table, ExpressionTree tree, boolean isValidatedFilter) throws MetaException { Map<String, Object> params = new HashMap<>(); - String jdoFilter = makeQueryFilterString(table.getDbName(), table, tree, params, isValidatedFilter); + String jdoFilter = makeQueryFilterString(table.getCatName(), table.getDbName(), table, tree, + params, isValidatedFilter); if (jdoFilter == null) { assert !isValidatedFilter; return null; @@ -3110,29 +3309,29 @@ public class ObjectStore implements RawStore, Configurable { * @param partNames Partition names to get the objects for. * @return Resulting partitions. */ - private List<Partition> getPartitionsViaOrmFilter( + private List<Partition> getPartitionsViaOrmFilter(String catName, String dbName, String tblName, List<String> partNames) throws MetaException { if (partNames.isEmpty()) { return new ArrayList<>(); } ObjectPair<Query, Map<String, String>> queryWithParams = - getPartQueryWithParams(dbName, tblName, partNames); + getPartQueryWithParams(catName, dbName, tblName, partNames); Query query = queryWithParams.getFirst(); query.setResultClass(MPartition.class); query.setClass(MPartition.class); query.setOrdering("partitionName ascending"); @SuppressWarnings("unchecked") List<MPartition> mparts = (List<MPartition>)query.executeWithMap(queryWithParams.getSecond()); - List<Partition> partitions = convertToParts(dbName, tblName, mparts); + List<Partition> partitions = convertToParts(catName, dbName, tblName, mparts); if (query != null) { query.closeAll(); } return partitions; } - private void dropPartitionsNoTxn(String dbName, String tblName, List<String> partNames) { + private void dropPartitionsNoTxn(String catName, String dbName, String tblName, List<String> partNames) { ObjectPair<Query, Map<String, String>> queryWithParams = - getPartQueryWithParams(dbName, tblName, partNames); + getPartQueryWithParams(catName, dbName, tblName, partNames); Query query = queryWithParams.getFirst(); query.setClass(MPartition.class); long deleted = query.deletePersistentAll(queryWithParams.getSecond()); @@ -3147,9 +3346,9 @@ public class ObjectStore implements RawStore, Configurable { * they are referenced by other SDs. */ private HashSet<MColumnDescriptor> detachCdsFromSdsNoTxn( - String dbName, String tblName, List<String> partNames) { + String catName, String dbName, String tblName, List<String> partNames) { ObjectPair<Query, Map<String, String>> queryWithParams = - getPartQueryWithParams(dbName, tblName, partNames); + getPartQueryWithParams(catName, dbName, tblName, partNames); Query query = queryWithParams.getFirst(); query.setClass(MPartition.class); query.setResult("sd"); @@ -3169,9 +3368,10 @@ public class ObjectStore implements RawStore, Configurable { return candidateCds; } - private ObjectPair<Query, Map<String, String>> getPartQueryWithParams(String dbName, - String tblName, List<String> partNames) { - StringBuilder sb = new StringBuilder("table.tableName == t1 && table.database.name == t2 && ("); + private ObjectPair<Query, Map<String, String>> getPartQueryWithParams( + String catName, String dbName, String tblName, List<String> partNames) { + StringBuilder sb = new StringBuilder("table.tableName == t1 && table.database.name == t2 &&" + + " table.database.catalogName == t3 && ("); int n = 0; Map<String, String> params = new HashMap<>(); for (Iterator<String> itr = partNames.iterator(); itr.hasNext();) { @@ -3189,14 +3389,15 @@ public class ObjectStore implements RawStore, Configurable { LOG.debug(" JDOQL filter is {}", sb); params.put("t1", normalizeIdentifier(tblName)); params.put("t2", normalizeIdentifier(dbName)); + params.put("t3", normalizeIdentifier(catName)); query.declareParameters(makeParameterDeclarationString(params)); return new ObjectPair<>(query, params); } @Override - public List<Partition> getPartitionsByFilter(String dbName, String tblName, + public List<Partition> getPartitionsByFilter(String catName, String dbName, String tblName, String filter, short maxParts) throws MetaException, NoSuchObjectException { - return getPartitionsByFilterInternal(dbName, tblName, filter, maxParts, true, true); + return getPartitionsByFilterInternal(catName, dbName, tblName, filter, maxParts, true, true); } /** Helper class for getting stuff w/transaction, direct SQL, perf logging, etc. */ @@ -3206,14 +3407,15 @@ public class ObjectStore implements RawStore, Configurable { private boolean doUseDirectSql; private long start; private Table table; - protected final String dbName, tblName; + protected final String catName, dbName, tblName; private boolean success = false; protected T results = null; - public GetHelper(String dbName, String tblName, boolean allowSql, boolean allowJdo) - throws MetaException { + public GetHelper(String catalogName, String dbName, String tblName, + boolean allowSql, boolean allowJdo) throws MetaException { assert allowSql || allowJdo; this.allowJdo = allowJdo; + this.catName = normalizeIdentifier(catalogName); this.dbName = normalizeIdentifier(dbName); if (tblName != null){ this.tblName = normalizeIdentifier(tblName); @@ -3284,7 +3486,7 @@ public class ObjectStore implements RawStore, Configurable { start = doTrace ? System.nanoTime() : 0; openTransaction(); if (initTable && (tblName != null)) { - table = ensureGetTable(dbName, tblName); + table = ensureGetTable(catName, dbName, tblName); } doUseDirectSql = doUseDirectSql && canUseDirectSql(this); } @@ -3324,7 +3526,7 @@ public class ObjectStore implements RawStore, Configurable { start = doTrace ? System.nanoTime() : 0; openTransaction(); if (table != null) { - table = ensureGetTable(dbName, tblName); + table = ensureGetTable(catName, dbName, tblName); } } else { start = doTrace ? System.nanoTime() : 0; @@ -3393,9 +3595,9 @@ public class ObjectStore implements RawStore, Configurable { } private abstract class GetListHelper<T> extends GetHelper<List<T>> { - public GetListHelper( - String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException { - super(dbName, tblName, allowSql, allowJdo); + public GetListHelper(String catName, String dbName, String tblName, boolean allowSql, + boolean allowJdo) throws MetaException { + super(catName, dbName, tblName, allowSql, allowJdo); } @Override @@ -3413,9 +3615,9 @@ public class ObjectStore implements RawStore, Configurable { * @param allowJdo Whether or not we allow ORM to perform this query. * @throws MetaException */ - public GetDbHelper( - String dbName,boolean allowSql, boolean allowJdo) throws MetaException { - super(dbName,null,allowSql,allowJdo); + public GetDbHelper(String catalogName, String dbName,boolean allowSql, boolean allowJdo) + throws MetaException { + super(catalogName, dbName,null,allowSql,allowJdo); } @Override @@ -3425,9 +3627,9 @@ public class ObjectStore implements RawStore, Configurable { } private abstract class GetStatHelper extends GetHelper<ColumnStatistics> { - public GetStatHelper( - String dbName, String tblName, boolean allowSql, boolean allowJdo) throws MetaException { - super(dbName, tblName, allowSql, allowJdo); + public GetStatHelper(String catalogName, String dbName, String tblName, boolean allowSql, + boolean allowJdo) throws MetaException { + super(catalogName, dbName, tblName, allowSql, allowJdo); } @Override @@ -3437,12 +3639,12 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public int getNumPartitionsByFilter(String dbName, String tblName, + public int getNumPartitionsByFilter(String catName, String dbName, String tblName, String filter) throws MetaException, NoSuchObjectException { final ExpressionTree exprTree = org.apache.commons.lang
<TRUNCATED>
