This is an automated email from the ASF dual-hosted git repository. dkuzmenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new e3751ab5453 HIVE-26374: Query based compaction fails for tables with CDT and columns with Reserved Keywords (Chiran Ravani, reviewed by Denys Kuzmenko, Sai Hemanth Gantasala) e3751ab5453 is described below commit e3751ab545370f9b252d0b4a07bc315037541a95 Author: Chiran Ravani <chiran54...@gmail.com> AuthorDate: Fri Jul 15 06:59:37 2022 -0400 HIVE-26374: Query based compaction fails for tables with CDT and columns with Reserved Keywords (Chiran Ravani, reviewed by Denys Kuzmenko, Sai Hemanth Gantasala) Closes #3419 --- .../ql/txn/compactor/TestCrudCompactorOnTez.java | 65 ++++++++++++++++++++++ .../apache/hadoop/hive/ql/exec/DDLPlanUtils.java | 2 +- .../ql/txn/compactor/CompactionQueryBuilder.java | 15 +++-- 3 files changed, 73 insertions(+), 9 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java index ece4d5290e1..cffb58bc41c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java @@ -2351,4 +2351,69 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest { Assert.assertEquals("base_10000000_v0000009", baseName); } + @Test + public void testCompactionShouldNotFailOnStructField() throws Exception { + conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true); + String dbName = "default"; + String tblName = "compaction_hive_26374"; + + TxnStore txnHandler = TxnUtils.getTxnStore(conf); + TestDataProvider testDP = new TestDataProvider(); + + // Create test table + executeStatementOnDriver("drop table if exists " + tblName, driver); + executeStatementOnDriver("CREATE TABLE " + tblName + "(col1 array<struct<arr_col1:int, `timestamp`:string>>)" + + "STORED AS ORC TBLPROPERTIES('transactional'='true')", driver); + + // Insert test data into test table + executeStatementOnDriver("INSERT INTO TABLE " + tblName + + " SELECT ARRAY(NAMED_STRUCT('arr_col1',1,'timestamp','2022-07-05 21:51:20.371'))",driver); + executeStatementOnDriver("INSERT INTO TABLE " + tblName + + " SELECT ARRAY(NAMED_STRUCT('arr_col1',2,'timestamp','2022-07-05 21:51:20.371'))",driver); + + // Find the location of the table + IMetaStoreClient msClient = new HiveMetaStoreClient(conf); + Table table = msClient.getTable(dbName, tblName); + FileSystem fs = FileSystem.get(conf); + // Verify deltas (delta_0000001_0000001_0000, delta_0000002_0000002_0000) are present + Assert.assertEquals("Delta directories does not match before compaction", + Arrays.asList("delta_0000001_0000001_0000", "delta_0000002_0000002_0000"), + CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null)); + + // Get all data before compaction is run + List<String> expectedData = testDP.getAllData(tblName); + + //Do a compaction directly and wait for it to finish + CompactionRequest rqst = new CompactionRequest(dbName, tblName, CompactionType.MAJOR); + CompactionResponse resp = txnHandler.compact(rqst); + runWorker(conf); + + CompactorTestUtil.runCleaner(conf); + + //Check if the compaction succeed + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + List<ShowCompactResponseElement> compacts = rsp.getCompacts(); + Assert.assertEquals("Expecting 1 rows and found " + compacts.size(), 1, compacts.size()); + Assert.assertEquals("Expecting compaction state 'succeeded' and found:" + compacts.get(0).getState(), + "succeeded", compacts.get(0).getState()); + // Should contain only one base directory now + FileStatus[] status = fs.listStatus(new Path(table.getSd().getLocation())); + int inputFileCount = 0; + for(FileStatus file: status) { + inputFileCount++; + } + Assert.assertEquals("Expecting 1 file and found "+ inputFileCount, 1, inputFileCount); + + // Check bucket file name + List<String> baseDir = CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.baseFileFilter, table, null); + List<String> expectedBucketFiles = Arrays.asList("bucket_00000"); + Assert.assertEquals("Bucket names are not matching after compaction", expectedBucketFiles, + CompactorTestUtil + .getBucketFileNames(fs, table, null, baseDir.get(0))); + + // Verify all contents + List<String> actualData = testDP.getAllData(tblName); + Assert.assertEquals(expectedData, actualData); + } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java index 80561179242..62248ff4f71 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java @@ -837,7 +837,7 @@ public class DDLPlanUtils { /** * Struct fields are identifiers, need to be put between ``. */ - private String formatType(TypeInfo typeInfo) { + public static String formatType(TypeInfo typeInfo) { switch (typeInfo.getCategory()) { case PRIMITIVE: return typeInfo.getTypeName(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java index d9aeeeed8d6..8a297533888 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.util.DirectionUtils; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hive.common.util.HiveStringUtils; import org.slf4j.Logger; @@ -43,6 +44,7 @@ import org.slf4j.LoggerFactory; import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -418,16 +420,13 @@ class CompactionQueryBuilder { + "`currentTransaction` bigint, `row` struct<"); } List<FieldSchema> cols = sourceTab.getSd().getCols(); - boolean isFirst = true; + List<String> columnDescs = new ArrayList<>(); for (FieldSchema col : cols) { - if (!isFirst) { - query.append(", "); - } - isFirst = false; - query.append("`").append(col.getName()).append("` "); - query.append(crud ? ":" : ""); - query.append(col.getType()); + String columnType = DDLPlanUtils.formatType(TypeInfoUtils.getTypeInfoFromTypeString(col.getType())); + String columnDesc = "`" + col.getName() + "` " + (crud ? ":" : "") + columnType; + columnDescs.add(columnDesc); } + query.append(StringUtils.join(',',columnDescs)); query.append(crud ? ">" : ""); query.append(") "); }