HIVE-13115: MetaStore Direct SQL getPartitions call fail when the columns schemas for a partition are null (Ratandeep Ratti reviewed by Carl Steinbach)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/69cfd357 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/69cfd357 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/69cfd357 Branch: refs/heads/llap Commit: 69cfd357eb482c426161aa3c4a00d574ee29416e Parents: 9686209 Author: Carl Steinbach <c...@apache.org> Authored: Sun Mar 27 15:41:38 2016 -0700 Committer: Carl Steinbach <c...@apache.org> Committed: Sun Mar 27 15:41:38 2016 -0700 ---------------------------------------------------------------------- .../hive/metastore/TestHiveMetaStore.java | 20 +++++++++++- .../hive/metastore/MetaStoreDirectSql.java | 32 +++++++++++--------- 2 files changed, 37 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/69cfd357/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 5da4165..83fb15c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -280,6 +280,24 @@ public abstract class TestHiveMetaStore extends TestCase { } assertTrue("Partitions are not same", part.equals(part_get)); + // check null cols schemas for a partition + List<String> vals6 = makeVals("2016-02-22 00:00:00", "16"); + Partition part6 = makePartitionObject(dbName, tblName, vals6, tbl, "/part5"); + part6.getSd().setCols(null); + LOG.info("Creating partition will null field schema"); + client.add_partition(part6); + LOG.info("Listing all partitions for table " + dbName + "." + tblName); + final List<Partition> partitions = client.listPartitions(dbName, tblName, (short) -1); + boolean foundPart = false; + for (Partition p : partitions) { + if (p.getValues().equals(vals6)) { + assertNull(p.getSd().getCols()); + LOG.info("Found partition " + p + " having null field schema"); + foundPart = true; + } + } + assertTrue(foundPart); + String partName = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=14"; String part2Name = "ds=" + FileUtils.escapePathName("2008-07-01 14:13:12") + "/hr=15"; String part3Name = "ds=" + FileUtils.escapePathName("2008-07-02 14:13:12") + "/hr=15"; @@ -313,7 +331,7 @@ public abstract class TestHiveMetaStore extends TestCase { partialVals.clear(); partialVals.add(""); partialNames = client.listPartitionNames(dbName, tblName, partialVals, (short) -1); - assertTrue("Should have returned 4 partition names", partialNames.size() == 4); + assertTrue("Should have returned 5 partition names", partialNames.size() == 5); assertTrue("Not all part names returned", partialNames.containsAll(partNames)); // Test partition listing with a partial spec - hr is specified but ds is not http://git-wip-us.apache.org/repos/asf/hive/blob/69cfd357/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index d51f58d..06e9f78 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -557,14 +557,14 @@ class MetaStoreDirectSql { Long sdId = extractSqlLong(fields[1]); Long colId = extractSqlLong(fields[2]); Long serdeId = extractSqlLong(fields[3]); - // A partition must have either everything set, or nothing set if it's a view. - if (sdId == null || colId == null || serdeId == null) { + // A partition must have at least sdId and serdeId set, or nothing set if it's a view. + if (sdId == null || serdeId == null) { if (isView == null) { isView = isViewTable(dbName, tblName); } if ((sdId != null || colId != null || serdeId != null) || !isView) { - throw new MetaException("Unexpected null for one of the IDs, SD " + sdId + ", column " - + colId + ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view"); + throw new MetaException("Unexpected null for one of the IDs, SD " + sdId + + ", serde " + serdeId + " for a " + (isView ? "" : "non-") + " view"); } } @@ -580,7 +580,7 @@ class MetaStoreDirectSql { partitions.put(partitionId, part); if (sdId == null) continue; // Probably a view. - assert colId != null && serdeId != null; + assert serdeId != null; // We assume each partition has an unique SD. StorageDescriptor sd = new StorageDescriptor(); @@ -605,14 +605,16 @@ class MetaStoreDirectSql { sdSb.append(sdId).append(","); part.setSd(sd); - List<FieldSchema> cols = colss.get(colId); - // We expect that colId will be the same for all (or many) SDs. - if (cols == null) { - cols = new ArrayList<FieldSchema>(); - colss.put(colId, cols); - colsSb.append(colId).append(","); + if (colId != null) { + List<FieldSchema> cols = colss.get(colId); + // We expect that colId will be the same for all (or many) SDs. + if (cols == null) { + cols = new ArrayList<FieldSchema>(); + colss.put(colId, cols); + colsSb.append(colId).append(","); + } + sd.setCols(cols); } - sd.setCols(cols); // We assume each SD has an unique serde. SerDeInfo serde = new SerDeInfo(); @@ -658,8 +660,10 @@ class MetaStoreDirectSql { assert serdeSb.length() == 0 && colsSb.length() == 0; return orderedResult; // No SDs, probably a view. } - String sdIds = trimCommaList(sdSb), serdeIds = trimCommaList(serdeSb), - colIds = trimCommaList(colsSb); + + String sdIds = trimCommaList(sdSb); + String serdeIds = trimCommaList(serdeSb); + String colIds = trimCommaList(colsSb); // Get all the stuff for SD. Don't do empty-list check - we expect partitions do have SDs. queryText = "select \"SD_ID\", \"PARAM_KEY\", \"PARAM_VALUE\" from \"SD_PARAMS\""