Repository: incubator-atlas Updated Branches: refs/heads/master 44dbfe576 -> ad7604fcf
ATLAS-523 Support alter view ( sumasai via shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ad7604fc Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ad7604fc Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ad7604fc Branch: refs/heads/master Commit: ad7604fcf4bf4f84a66b339b8ea1f4473cd9f84a Parents: 44dbfe5 Author: Suma Shivaprasad <[email protected]> Authored: Mon Mar 14 10:52:28 2016 -0700 Committer: Suma Shivaprasad <[email protected]> Committed: Mon Mar 14 10:53:00 2016 -0700 ---------------------------------------------------------------------- .../org/apache/atlas/hive/hook/HiveHook.java | 7 +- .../org/apache/atlas/hive/hook/HiveHookIT.java | 123 +++++++++++++++---- release-log.txt | 1 + 3 files changed, 99 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ad7604fc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index b5a4ce9..fc2a57a 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -86,7 +86,6 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { private static final long keepAliveTimeDefault = 10; private static final int queueSizeDefault = 10000; - private static boolean typesRegistered = false; private static Configuration atlasProperties; class HiveEvent { @@ -212,6 +211,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { case CREATETABLE_AS_SELECT: case CREATEVIEW: + case ALTERVIEW_AS: case LOAD: case EXPORT: case IMPORT: @@ -229,15 +229,12 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext { case ALTERTABLE_CLUSTER_SORT: case ALTERTABLE_BUCKETNUM: case ALTERTABLE_PROPERTIES: + case ALTERVIEW_PROPERTIES: case ALTERTABLE_SERDEPROPERTIES: case ALTERTABLE_SERIALIZER: alterTable(dgiBridge, event); break; - case ALTERVIEW_AS: - //update inputs/outputs? - break; - case ALTERTABLE_ADDCOLS: case ALTERTABLE_REPLACECOLS: case ALTERTABLE_RENAMECOL: http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ad7604fc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index cb215db..f2084b0 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -18,8 +18,8 @@ package org.apache.atlas.hive.hook; +import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; -import groovy.transform.Immutable; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasClient; import org.apache.atlas.AtlasServiceException; @@ -38,13 +38,6 @@ import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; -import org.apache.hadoop.io.NullWritable; -import org.apache.hadoop.io.Writable; -import org.apache.hadoop.mapreduce.InputFormat; -import org.apache.hadoop.mapreduce.InputSplit; -import org.apache.hadoop.mapreduce.JobContext; -import org.apache.hadoop.mapreduce.RecordReader; -import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; @@ -52,14 +45,7 @@ import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import java.io.DataInput; -import java.io.DataOutput; import java.io.File; -import java.io.IOException; -import java.net.URLClassLoader; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -221,6 +207,55 @@ public class HiveHookIT { } @Test + public void testAlterViewAsSelect() throws Exception { + + //Create the view from table1 + String table1Name = createTable(); + String viewName = tableName(); + String query = "create view " + viewName + " as select * from " + table1Name; + runCommand(query); + + String table1Id = assertTableIsRegistered(DEFAULT_DB, table1Name); + assertProcessIsRegistered(query); + String viewId = assertTableIsRegistered(DEFAULT_DB, viewName); + + //Check lineage which includes table1 + String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName); + JSONObject response = dgiCLient.getInputGraph(datasetName); + JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices"); + Assert.assertTrue(vertices.has(viewId)); + Assert.assertTrue(vertices.has(table1Id)); + + //Alter the view from table2 + String table2Name = createTable(); + query = "alter view " + viewName + " as select * from " + table2Name; + runCommand(query); + + //Check if alter view process is reqistered + assertProcessIsRegistered(query); + String table2Id = assertTableIsRegistered(DEFAULT_DB, table2Name); + Assert.assertEquals(assertTableIsRegistered(DEFAULT_DB, viewName), viewId); + + //Check lineage which includes both table1 and table2 + datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName); + response = dgiCLient.getInputGraph(datasetName); + vertices = response.getJSONObject("values").getJSONObject("vertices"); + Assert.assertTrue(vertices.has(viewId)); + + //This is through the alter view process + Assert.assertTrue(vertices.has(table2Id)); + + //This is through the Create view process + Assert.assertTrue(vertices.has(table1Id)); + + //Outputs dont exist + response = dgiCLient.getOutputGraph(datasetName); + vertices = response.getJSONObject("values").getJSONObject("vertices"); + Assert.assertEquals(vertices.length(), 0); + } + + + @Test public void testLoadData() throws Exception { String tableName = createTable(false); @@ -520,7 +555,7 @@ public class HiveHookIT { } private String getSerializedProps(Map<String, String> expectedProps) { - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); for(String expectedPropKey : expectedProps.keySet()) { if(sb.length() > 0) { sb.append(","); @@ -535,34 +570,68 @@ public class HiveHookIT { @Test public void testAlterTableProperties() throws Exception { String tableName = createTable(); + final String fmtQuery = "alter table %s %s TBLPROPERTIES (%s)"; + testAlterProperties(tableName, fmtQuery); + } + + private void testAlterProperties(String tableName, String fmtQuery) throws Exception { + + final String SET_OP = "set"; + final String UNSET_OP = "unset"; + final Map<String, String> expectedProps = new HashMap<String, String>() {{ put("testPropKey1", "testPropValue1"); put("comment", "test comment"); }}; - final String fmtQuery = "alter table %s set TBLPROPERTIES (%s)"; - String query = String.format(fmtQuery, tableName, getSerializedProps(expectedProps)); + String query = String.format(fmtQuery, tableName, SET_OP, getSerializedProps(expectedProps)); runCommand(query); - verifyTableProperties(tableName, expectedProps); - + verifyTableProperties(tableName, expectedProps, false); expectedProps.put("testPropKey2", "testPropValue2"); //Add another property - query = String.format(fmtQuery, tableName, getSerializedProps(expectedProps)); + query = String.format(fmtQuery, tableName, SET_OP, getSerializedProps(expectedProps)); + runCommand(query); + + verifyTableProperties(tableName, expectedProps, false); + + //Unset all the props + StringBuilder sb = new StringBuilder("'"); + query = String.format(fmtQuery, tableName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb, expectedProps.keySet()).append('\'')); + runCommand(query); + + verifyTableProperties(tableName, expectedProps, true); + } + + @Test + public void testAlterViewProperties() throws Exception { + String tableName = createTable(); + String viewName = tableName(); + String query = "create view " + viewName + " as select * from " + tableName; runCommand(query); - verifyTableProperties(tableName, expectedProps); + final String fmtQuery = "alter view %s %s TBLPROPERTIES (%s)"; + testAlterProperties(viewName, fmtQuery); } - private void verifyTableProperties(String tableName, Map<String, String> expectedProps) throws Exception { + private void verifyTableProperties(String tableName, Map<String, String> expectedProps, boolean checkIfNotExists) throws Exception { String tableId = assertTableIsRegistered(DEFAULT_DB, tableName); Referenceable tableRef = dgiCLient.getEntity(tableId); Map<String, String> parameters = (Map<String, String>) tableRef.get(HiveDataModelGenerator.PARAMETERS); - Assert.assertNotNull(parameters); - //Comment should exist since SET TBLPOPERTIES only adds properties. Doe not remove existing ones - for (String propKey : expectedProps.keySet()) { - Assert.assertEquals(parameters.get(propKey), expectedProps.get(propKey)); + if (checkIfNotExists == false) { + //Check if properties exist + Assert.assertNotNull(parameters); + for (String propKey : expectedProps.keySet()) { + Assert.assertEquals(parameters.get(propKey), expectedProps.get(propKey)); + } + } else { + //Check if properties dont exist + if (expectedProps != null && parameters != null) { + for (String propKey : expectedProps.keySet()) { + Assert.assertFalse(parameters.containsKey(propKey)); + } + } } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ad7604fc/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index a4cd781..9c73da2 100644 --- a/release-log.txt +++ b/release-log.txt @@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags) ALL CHANGES: +ATLAS-523 Support alter view (sumasai via shwethags) ATLAS-555 Tag creation from UI fails due to missing description attribute (guptaneeru via shwethags) ATLAS-522 Support Alter table commands (sumasai via shwethags) ATLAS-512 Decouple currently integrating components from availability of Atlas service for raising metadata events ( yhemanth via sumasai)
