HIVE-17189 : Fix backwards incompatibility in HiveMetaStoreClient (Vihang Karajgaonkar, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b8bd4594 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b8bd4594 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b8bd4594 Branch: refs/heads/branch-2.3 Commit: b8bd4594bef718b1eeac9fceb437d7df7b480ed1 Parents: 0ff4d5b Author: Vihang Karajgaonkar <vih...@cloudera.com> Authored: Thu Jul 27 16:23:47 2017 -0700 Committer: Sahil Takiar <stak...@cloudera.com> Committed: Tue Nov 7 08:15:47 2017 -0800 ---------------------------------------------------------------------- .../hive/metastore/TestHiveMetaStore.java | 49 ++++++++++++++++++ .../hive/metastore/HiveMetaStoreClient.java | 23 +++++++++ .../hadoop/hive/metastore/IMetaStoreClient.java | 52 +++++++++++++++++++- .../ql/metadata/SessionHiveMetaStoreClient.java | 13 +++++ 4 files changed, 136 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/b8bd4594/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 1b0b537..4b1df8e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -608,6 +608,55 @@ public abstract class TestHiveMetaStore extends TestCase { assertEquals(" should have returned 50 partitions", maxParts, partitions.size()); } + public void testAlterTableCascade() throws Throwable { + // create a table with multiple partitions + String dbName = "compdb"; + String tblName = "comptbl"; + String typeName = "Person"; + + cleanUp(dbName, tblName, typeName); + + List<List<String>> values = new ArrayList<List<String>>(); + values.add(makeVals("2008-07-01 14:13:12", "14")); + values.add(makeVals("2008-07-01 14:13:12", "15")); + values.add(makeVals("2008-07-02 14:13:12", "15")); + values.add(makeVals("2008-07-03 14:13:12", "151")); + + createMultiPartitionTableSchema(dbName, tblName, typeName, values); + Table tbl = client.getTable(dbName, tblName); + List<FieldSchema> cols = tbl.getSd().getCols(); + cols.add(new FieldSchema("new_col", serdeConstants.STRING_TYPE_NAME, "")); + tbl.getSd().setCols(cols); + //add new column with cascade option + client.alter_table(dbName, tblName, tbl, true); + // + Table tbl2 = client.getTable(dbName, tblName); + Assert.assertEquals("Unexpected number of cols", 3, tbl2.getSd().getCols().size()); + Assert.assertEquals("Unexpected column name", "new_col", tbl2.getSd().getCols().get(2).getName()); + //get a partition + List<String> pvalues = new ArrayList<>(2); + pvalues.add("2008-07-01 14:13:12"); + pvalues.add("14"); + Partition partition = client.getPartition(dbName, tblName, pvalues); + Assert.assertEquals("Unexpected number of cols", 3, partition.getSd().getCols().size()); + Assert.assertEquals("Unexpected column name", "new_col", partition.getSd().getCols().get(2).getName()); + + //add another column + cols = tbl.getSd().getCols(); + cols.add(new FieldSchema("new_col2", serdeConstants.STRING_TYPE_NAME, "")); + tbl.getSd().setCols(cols); + //add new column with no cascade option + client.alter_table(dbName, tblName, tbl, false); + tbl2 = client.getTable(dbName, tblName); + Assert.assertEquals("Unexpected number of cols", 4, tbl2.getSd().getCols().size()); + Assert.assertEquals("Unexpected column name", "new_col2", tbl2.getSd().getCols().get(3).getName()); + //get partition, this partition should not have the newly added column since cascade option + //was false + partition = client.getPartition(dbName, tblName, pvalues); + Assert.assertEquals("Unexpected number of cols", 3, partition.getSd().getCols().size()); + } + + public void testListPartitionNames() throws Throwable { // create a table with multiple partitions String dbName = "compdb"; http://git-wip-us.apache.org/repos/asf/hive/blob/b8bd4594/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index c5db77f..b1a9782 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -50,6 +50,7 @@ import java.security.PrivilegedExceptionAction; import javax.security.auth.login.LoginException; import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.auth.HiveAuthUtils; import org.apache.hadoop.hive.common.classification.InterfaceAudience; @@ -359,6 +360,16 @@ public class HiveMetaStoreClient implements IMetaStoreClient { } @Override + public void alter_table(String defaultDatabaseName, String tblName, Table table, + boolean cascade) throws InvalidOperationException, MetaException, TException { + EnvironmentContext environmentContext = new EnvironmentContext(); + if (cascade) { + environmentContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE); + } + alter_table_with_environmentContext(defaultDatabaseName, tblName, table, environmentContext); + } + + @Override public void alter_table_with_environmentContext(String dbname, String tbl_name, Table new_tbl, EnvironmentContext envContext) throws InvalidOperationException, MetaException, TException { client.alter_table_with_environment_context(dbname, tbl_name, new_tbl, envContext); @@ -1493,12 +1504,24 @@ public class HiveMetaStoreClient implements IMetaStoreClient { } @Override + public void alter_partition(String dbName, String tblName, Partition newPart) + throws InvalidOperationException, MetaException, TException { + client.alter_partition_with_environment_context(dbName, tblName, newPart, null); + } + + @Override public void alter_partition(String dbName, String tblName, Partition newPart, EnvironmentContext environmentContext) throws InvalidOperationException, MetaException, TException { client.alter_partition_with_environment_context(dbName, tblName, newPart, environmentContext); } @Override + public void alter_partitions(String dbName, String tblName, List<Partition> newParts) + throws InvalidOperationException, MetaException, TException { + client.alter_partitions_with_environment_context(dbName, tblName, newParts, null); + } + + @Override public void alter_partitions(String dbName, String tblName, List<Partition> newParts, EnvironmentContext environmentContext) throws InvalidOperationException, MetaException, TException { client.alter_partitions_with_environment_context(dbName, tblName, newParts, environmentContext); http://git-wip-us.apache.org/repos/asf/hive/blob/b8bd4594/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index e9df1e1..e7ead6b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public; @@ -707,6 +708,14 @@ public interface IMetaStoreClient { void alter_table(String defaultDatabaseName, String tblName, Table table) throws InvalidOperationException, MetaException, TException; + /** + * Use alter_table_with_environmentContext instead of alter_table with cascade option + * passed in EnvironmentContext using {@code StatsSetupConst.CASCADE} + */ + @Deprecated + void alter_table(String defaultDatabaseName, String tblName, Table table, + boolean cascade) throws InvalidOperationException, MetaException, TException; + //wrapper of alter_table_with_cascade void alter_table_with_environmentContext(String defaultDatabaseName, String tblName, Table table, EnvironmentContext environmentContext) throws InvalidOperationException, MetaException, @@ -780,6 +789,26 @@ public interface IMetaStoreClient { boolean dropPartition(String db_name, String tbl_name, String name, boolean deleteData) throws NoSuchObjectException, MetaException, TException; + + /** + * updates a partition to new partition + * + * @param dbName + * database of the old partition + * @param tblName + * table name of the old partition + * @param newPart + * new partition + * @throws InvalidOperationException + * if the old partition does not exist + * @throws MetaException + * if error in updating metadata + * @throws TException + * if error in communicating with metastore server + */ + void alter_partition(String dbName, String tblName, Partition newPart) + throws InvalidOperationException, MetaException, TException; + /** * updates a partition to new partition * @@ -815,7 +844,28 @@ public interface IMetaStoreClient { * @throws TException * if error in communicating with metastore server */ - void alter_partitions(String dbName, String tblName, List<Partition> newParts, EnvironmentContext environmentContext) + void alter_partitions(String dbName, String tblName, List<Partition> newParts) + throws InvalidOperationException, MetaException, TException; + + /** + * updates a list of partitions + * + * @param dbName + * database of the old partition + * @param tblName + * table name of the old partition + * @param newParts + * list of partitions + * @param environmentContext + * @throws InvalidOperationException + * if the old partition does not exist + * @throws MetaException + * if error in updating metadata + * @throws TException + * if error in communicating with metastore server + */ + void alter_partitions(String dbName, String tblName, List<Partition> newParts, + EnvironmentContext environmentContext) throws InvalidOperationException, MetaException, TException; /** http://git-wip-us.apache.org/repos/asf/hive/blob/b8bd4594/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java index 8eb011e..109bc3a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java @@ -280,6 +280,19 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I return super.getSchema(dbName, tableName); } + @Deprecated + @Override + public void alter_table(String dbname, String tbl_name, org.apache.hadoop.hive.metastore.api.Table new_tbl, + boolean cascade) throws InvalidOperationException, MetaException, TException { + org.apache.hadoop.hive.metastore.api.Table old_tbl = getTempTable(dbname, tbl_name); + if (old_tbl != null) { + //actually temp table does not support partitions, cascade is not applicable here + alterTempTable(dbname, tbl_name, old_tbl, new_tbl, null); + return; + } + super.alter_table(dbname, tbl_name, new_tbl, cascade); + } + @Override public void alter_table(String dbname, String tbl_name, org.apache.hadoop.hive.metastore.api.Table new_tbl) throws InvalidOperationException,
