Repository: hive Updated Branches: refs/heads/branch-3 8584947ef -> a3e535f94
http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index a4df509..4b2f961 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -32,8 +32,6 @@ import org.apache.hadoop.hive.ql.io.BucketCodec; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; @@ -191,8 +189,9 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { checkExpected(rs, expected, "Unexpected row count after ctas from non acid table"); runStatementOnDriver("insert into " + Table.ACIDTBL + makeValuesClause(values)); + //todo: try this with acid default - it seem making table acid in listener is too late runStatementOnDriver("create table myctas2 stored as ORC TBLPROPERTIES ('transactional" + - "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL);//todo: try this with acid default - it seem makeing table acid in listener is too late + "'='true', 'transactional_properties'='default') as select a, b from " + Table.ACIDTBL); rs = runStatementOnDriver("select ROW__ID, a, b, INPUT__FILE__NAME from myctas2 order by ROW__ID"); String expected2[][] = { {"{\"writeid\":1,\"bucketid\":536870912,\"rowid\":0}\t3\t4", "warehouse/myctas2/delta_0000001_0000001_0000/bucket_00000"}, @@ -234,7 +233,7 @@ public class TestTxnNoBuckets extends TxnCommandsBaseForTests { /** * Insert into unbucketed acid table from union all query - * Union All is flattend so nested subdirs are created and acid move drops them since + * Union All is flattened so nested subdirs are created and acid move drops them since * delta dirs have unique names */ @Test @@ -529,11 +528,26 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver CommandProcessorResponse cpr = runStatementOnDriverNegative("create table myctas " + "clustered by (a) into 2 buckets stored as ORC TBLPROPERTIES ('transactional'='true') as " + "select a, b from " + Table.NONACIDORCTBL); - int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode();//this code doesn't propagate + int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode(); //this code doesn't propagate // Assert.assertEquals("Wrong msg", ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode(), cpr.getErrorCode()); Assert.assertTrue(cpr.getErrorMessage().contains("CREATE-TABLE-AS-SELECT does not support")); } /** + * Currently CTAS doesn't support partitioned tables. Correspondingly Acid only supports CTAS for + * un-partitioned tables. This test is here to make sure that if CTAS is made to support + * un-partitioned tables, that it raises a red flag for Acid. + */ + @Test + public void testCtasPartitioned() throws Exception { + runStatementOnDriver("insert into " + Table.NONACIDNONBUCKET + "(a,b) values(1,2),(1,3)"); + CommandProcessorResponse cpr = runStatementOnDriverNegative("create table myctas partitioned " + + "by (b int) stored as " + + "ORC TBLPROPERTIES ('transactional'='true') as select a, b from " + Table.NONACIDORCTBL); + int j = ErrorMsg.CTAS_PARCOL_COEXISTENCE.getErrorCode();//this code doesn't propagate + Assert.assertTrue(cpr.getErrorMessage().contains("CREATE-TABLE-AS-SELECT does not support " + + "partitioning in the target table")); + } + /** * Tests to check that we are able to use vectorized acid reader, * VectorizedOrcAcidRowBatchReader, when reading "original" files, * i.e. those that were written before the table was converted to acid. http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index b0f64db..fcf34f0 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -2033,7 +2033,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { * @param partition * @return */ - private Partition deepCopy(Partition partition) { + protected Partition deepCopy(Partition partition) { Partition copy = null; if (partition != null) { copy = new Partition(partition); http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index c5da7b5..a7acdcb 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -3033,23 +3033,7 @@ public class ObjectStore implements RawStore, Configurable { throw new NoSuchObjectException(getCatalogQualifiedTableName(catName, dbName, tableName) + " table not found"); } - List<FieldSchema> partCols = table.getPartitionKeys(); - int numPartKeys = partCols.size(); - if (part_vals.size() > numPartKeys) { - throw new MetaException("Incorrect number of partition values." - + " numPartKeys=" + numPartKeys + ", part_val=" + part_vals.size()); - } - partCols = partCols.subList(0, part_vals.size()); - // Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... - // where partVal is either the escaped partition value given as input, - // or a regex of the form ".*" - // This works because the "=" and "/" separating key names and partition key/values - // are not escaped. - String partNameMatcher = Warehouse.makePartName(partCols, part_vals, ".*"); - // add ".*" to the regex to match anything else afterwards the partial spec. - if (part_vals.size() < numPartKeys) { - partNameMatcher += ".*"; - } + String partNameMatcher = MetaStoreUtils.makePartNameMatcher(table, part_vals); Query query = queryWrapper.query = pm.newQuery(MPartition.class); StringBuilder queryFilter = new StringBuilder("table.database.name == dbName"); queryFilter.append(" && table.database.catalogName == catName"); http://git-wip-us.apache.org/repos/asf/hive/blob/a3e535f9/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 8ea6051..d022bc0 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -934,6 +934,26 @@ public class MetaStoreUtils { } return pvals; } + public static String makePartNameMatcher(Table table, List<String> partVals) throws MetaException { + List<FieldSchema> partCols = table.getPartitionKeys(); + int numPartKeys = partCols.size(); + if (partVals.size() > numPartKeys) { + throw new MetaException("Incorrect number of partition values." + + " numPartKeys=" + numPartKeys + ", part_val=" + partVals); + } + partCols = partCols.subList(0, partVals.size()); + // Construct a pattern of the form: partKey=partVal/partKey2=partVal2/... + // where partVal is either the escaped partition value given as input, + // or a regex of the form ".*" + // This works because the "=" and "/" separating key names and partition key/values + // are not escaped. + String partNameMatcher = Warehouse.makePartName(partCols, partVals, ".*"); + // add ".*" to the regex to match anything else afterwards the partial spec. + if (partVals.size() < numPartKeys) { + partNameMatcher += ".*"; + } + return partNameMatcher; + } /** * @param schema1: The first schema to be compared
