This is an automated email from the ASF dual-hosted git repository. ngangam pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new a0059ff HIVE-25517: Move table location logic from translation layer into Warehouse APIs(Sourabh Goyal via Naveen Gangam) a0059ff is described below commit a0059ff4e8b957ab666e7a683210e3e1a61b5730 Author: Sourabh Goyal <soura...@cloudera.com> AuthorDate: Mon Sep 13 23:37:38 2021 -0700 HIVE-25517: Move table location logic from translation layer into Warehouse APIs(Sourabh Goyal via Naveen Gangam) Change-Id: I143c0ca19717e6fab5daffefc921004efa174883 --- .../metastore/TestHiveMetastoreTransformer.java | 40 ++++++++++++++++++---- .../apache/hadoop/hive/metastore/Warehouse.java | 3 +- .../metastore/MetastoreDefaultTransformer.java | 8 +---- 3 files changed, 36 insertions(+), 15 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java index 4eb55e0..f9d19b0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Locale; import org.apache.hadoop.hive.metastore.client.builder.GetTablesRequestBuilder; import org.apache.hadoop.hive.metastore.api.Catalog; @@ -44,6 +45,7 @@ import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_NONE; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.ACCESSTYPE_READONLY; @@ -154,11 +156,12 @@ public class TestHiveMetastoreTransformer { /** * EXTERNAL_TABLE - * 1) Old table with no capabilities - * 2a) New table with capabilities with no client requirements - * 2b) New table with capabilities with no matching client requirements - * 2c) New table with capabilities with partial match requirements - * 2d) New table with capabilities with full match requirements + * 1) Old table (name in upper case) with no capabilities + * 2) Old table with no capabilities + * 3a) New table with capabilities with no client requirements + * 3b) New table with capabilities with no matching client requirements + * 3c) New table with capabilities with partial match requirements + * 3d) New table with capabilities with full match requirements */ @Test public void testTransformerExternalTable() throws Exception { @@ -168,9 +171,10 @@ public class TestHiveMetastoreTransformer { Map<String, Object> tProps = new HashMap<>(); int buckets = 32; - String tblName = basetblName; + // create external table with uppercase + String tblNameUpper = "TAB_EXT1"; tProps.put("DBNAME", dbName); - tProps.put("TBLNAME", tblName); + tProps.put("TBLNAME", tblNameUpper); tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE); tProps.put("BUCKETS", buckets); StringBuilder properties = new StringBuilder(); @@ -178,6 +182,28 @@ public class TestHiveMetastoreTransformer { tProps.put("PROPERTIES", properties.toString()); Table tbl = createTableWithCapabilities(tProps); + Table hiveTbl = client.getTable(dbName, tblNameUpper.toLowerCase(Locale.ROOT)); + Path actualPath = new Path(hiveTbl.getSd().getLocation()); + Database db = client.getDatabase(dbName); + LOG.info("Table=" + tblNameUpper + ",Table Details=" + hiveTbl); + assertEquals("Created and retrieved tables do not match:" + hiveTbl.getTableName() + ":" + + tblNameUpper.toLowerCase(Locale.ROOT), hiveTbl.getTableName(), + tblNameUpper.toLowerCase(Locale.ROOT)); + Path expectedTablePath = new Path(db.getLocationUri(), tblNameUpper.toLowerCase(Locale.ROOT)); + assertEquals(String.format("Table location %s is not a subset of the database location %s", + actualPath.toString(), db.getLocationUri()), expectedTablePath.toString(), actualPath.toString()); + + String tblName = basetblName; + tProps = new HashMap<>(); + tProps.put("DBNAME", dbName); + tProps.put("TBLNAME", tblName); + tProps.put("TBLTYPE", TableType.EXTERNAL_TABLE); + tProps.put("BUCKETS", buckets); + properties = new StringBuilder(); + properties.append("EXTERNAL").append("=").append("TRUE"); + tProps.put("PROPERTIES", properties.toString()); + tbl = createTableWithCapabilities(tProps); + // retrieve the table Table tbl2 = client.getTable(dbName, tblName); LOG.info("Table=" + tblName + ",Access=" + tbl2.getAccessType()); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java index 97df316..46593aa 100755 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -355,7 +355,8 @@ public class Warehouse { Path dbPath = null; if (isExternal) { dbPath = new Path(db.getLocationUri()); - if (FileUtils.isSubdirectory(getWhRoot().toString(), dbPath.toString() + Path.SEPARATOR)) { + if (FileUtils.isSubdirectory(getWhRoot().toString(), dbPath.toString()) || + getWhRoot().equals(dbPath)) { // db metadata incorrect, find new location based on external warehouse root dbPath = getDefaultExternalDatabasePath(db.getName()); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java index 74e69cc..99fc9e4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java @@ -939,13 +939,7 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme return table; } } else { - dbLocation = Path.getPathWithoutSchemeAndAuthority(new Path(db.getLocationUri())); - Path tablePath = null; - if (!FileUtils.isSubdirectory(whRootPath.toString(), dbLocation.toString())) { - tablePath = new Path(db.getLocationUri(), table.getTableName().toLowerCase()); - } else { - tablePath = hmsHandler.getWh().getDefaultTablePath(db, table.getTableName(), true); - } + Path tablePath = hmsHandler.getWh().getDefaultTablePath(db, table.getTableName(), true); table.getSd().setLocation(tablePath.toString()); } }