Repository: incubator-atlas Updated Branches: refs/heads/master dd66e1580 -> 985465fc7
ATLAS-588 import-hive.sh fails while importing partitions for a non-partitioned table (sumasai via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/985465fc Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/985465fc Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/985465fc Branch: refs/heads/master Commit: 985465fc73be355986e326ee3d491d3b6a077e05 Parents: dd66e15 Author: Shwetha GS <[email protected]> Authored: Thu Mar 24 11:34:16 2016 +0530 Committer: Shwetha GS <[email protected]> Committed: Thu Mar 24 11:34:16 2016 +0530 ---------------------------------------------------------------------- .../atlas/hive/bridge/HiveMetaStoreBridge.java | 7 ++++- .../hive/bridge/HiveMetaStoreBridgeTest.java | 31 ++++++++++++++++++++ release-log.txt | 1 + 3 files changed, 38 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index ea38750..0680e65 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -18,6 +18,7 @@ package org.apache.atlas.hive.bridge; +import com.google.common.base.Joiner; import com.sun.jersey.api.client.ClientResponse; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; @@ -435,7 +436,11 @@ public class HiveMetaStoreBridge { List<Partition> tableParts = hiveClient.getPartitions(table); for (Partition hivePart : tableParts) { - registerPartition(tableReferenceable, sdReferenceable, hivePart); + if (hivePart.getValues() != null && hivePart.getValues().size() > 0) { + registerPartition(tableReferenceable, sdReferenceable, hivePart); + } else { + LOG.info("Skipping partition for table {} since partition values are {}", getTableQualifiedName(clusterName, table.getDbName(), table.getTableName()), StringUtils.join(hivePart.getValues(), ",")); + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java index f8cfb71..c717c0f 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java @@ -35,6 +35,7 @@ import org.codehaus.jettison.json.JSONObject; import org.mockito.ArgumentMatcher; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.testng.Assert; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import scala.actors.threadpool.Arrays; @@ -163,6 +164,34 @@ public class HiveMetaStoreBridgeTest { new Integer(lastAccessTime)))); } + @Test + public void testImportWhenPartitionKeysAreNull() throws Exception { + setupDB(hiveClient, TEST_DB_NAME); + Table hiveTable = setupTable(hiveClient, TEST_DB_NAME, TEST_TABLE_NAME); + + returnExistingDatabase(TEST_DB_NAME, atlasClient, CLUSTER_NAME); + + when(atlasClient.searchByDSL(HiveMetaStoreBridge.getTableDSLQuery(CLUSTER_NAME, TEST_DB_NAME, + TEST_TABLE_NAME, + HiveDataTypes.HIVE_TABLE.getName()))).thenReturn( + getEntityReference("82e06b34-9151-4023-aa9d-b82103a50e77")); + when(atlasClient.getEntity("82e06b34-9151-4023-aa9d-b82103a50e77")).thenReturn(createTableReference()); + + Partition partition = mock(Partition.class); + when(partition.getTable()).thenReturn(hiveTable); + List partitionValues = Arrays.asList(new String[]{}); + when(partition.getValues()).thenReturn(partitionValues); + + when(hiveClient.getPartitions(hiveTable)).thenReturn(Arrays.asList(new Partition[]{partition})); + + HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(CLUSTER_NAME, hiveClient, atlasClient); + try { + bridge.importHiveMetadata(); + } catch (Exception e) { + Assert.fail("Partition with null key caused import to fail with exception ", e); + } + } + private JSONArray getPartitionReference(String id) throws JSONException { JSONObject resultEntry = new JSONObject(); resultEntry.put(HiveMetaStoreBridge.SEARCH_ENTRY_GUID_ATTR, id); @@ -203,4 +232,6 @@ public class HiveMetaStoreBridgeTest { return attrValue.equals(((Referenceable) o).get(attrName)); } } + + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/985465fc/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 02f2528..4569e55 100644 --- a/release-log.txt +++ b/release-log.txt @@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-588 import-hive.sh fails while importing partitions for a non-partitioned table (sumasai via shwethags) ATLAS-575 jetty-maven-plugin fails with ShutdownMonitorThread already started (shwethags) ATLAS-408 UI : Add a close link (x) on the top right when Tag is added (darshankumar89 via shwethags) ATLAS-524 Support alter database (sumasai via shwethags)
