Repository: incubator-atlas Updated Branches: refs/heads/master b273520e0 -> 864cc5bc3
ATLAS-54 Rename configs in hive hook (shwethags) Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/864cc5bc Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/864cc5bc Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/864cc5bc Branch: refs/heads/master Commit: 864cc5bc30a03eaef6ffc4f3f93e40c42b30af3d Parents: b273520 Author: Shwetha GS <[email protected]> Authored: Wed Jul 8 12:23:33 2015 +0530 Committer: Shwetha GS <[email protected]> Committed: Wed Jul 8 12:23:33 2015 +0530 ---------------------------------------------------------------------- .../atlas/hive/bridge/HiveMetaStoreBridge.java | 6 +++--- .../java/org/apache/atlas/hive/hook/HiveHook.java | 11 +++++++---- addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki | 16 ++++++++-------- .../atlas/hive/hook/BaseSSLAndKerberosTest.java | 10 +--------- .../java/org/apache/atlas/hive/hook/HiveHookIT.java | 16 ++++++++++------ .../org/apache/atlas/hive/hook/SSLHiveHookIT.java | 10 +--------- release-log.txt | 1 + 7 files changed, 31 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java index a8f6a67..4d1af4e 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java @@ -56,11 +56,11 @@ import java.util.Set; */ public class HiveMetaStoreBridge { private static final String DEFAULT_DGI_URL = "http://localhost:21000/";; - public static final String HIVE_CLUSTER_NAME = "hive.cluster.name"; + public static final String HIVE_CLUSTER_NAME = "atlas.cluster.name"; public static final String DEFAULT_CLUSTER_NAME = "primary"; private final String clusterName; - public static final String DGI_URL_PROPERTY = "hive.hook.dgi.url"; + public static final String ATLAS_ENDPOINT = "atlas.rest.address"; private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class); @@ -78,7 +78,7 @@ public class HiveMetaStoreBridge { public HiveMetaStoreBridge(HiveConf hiveConf, String doAsUser, UserGroupInformation ugi) throws Exception { clusterName = hiveConf.get(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME); hiveClient = Hive.get(hiveConf); - atlasClient = new AtlasClient(hiveConf.get(DGI_URL_PROPERTY, DEFAULT_DGI_URL), ugi, doAsUser); + atlasClient = new AtlasClient(hiveConf.get(ATLAS_ENDPOINT, DEFAULT_DGI_URL), ugi, doAsUser); } public AtlasClient getAtlasClient() { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java index 3d55e85..fe26446 100755 --- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java +++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java @@ -60,9 +60,12 @@ public class HiveHook implements ExecuteWithHookContext { private static final int WAIT_TIME = 3; private static ExecutorService executor; - private static final String MIN_THREADS = "hive.hook.dgi.minThreads"; - private static final String MAX_THREADS = "hive.hook.dgi.maxThreads"; - private static final String KEEP_ALIVE_TIME = "hive.hook.dgi.keepAliveTime"; + public static final String CONF_PREFIX = "atlas.hook.hive."; + + private static final String MIN_THREADS = CONF_PREFIX + "minThreads"; + private static final String MAX_THREADS = CONF_PREFIX + "maxThreads"; + private static final String KEEP_ALIVE_TIME = CONF_PREFIX + "keepAliveTime"; + public static final String CONF_SYNC = CONF_PREFIX + "synchronous"; private static final int minThreadsDefault = 5; private static final int maxThreadsDefault = 5; @@ -131,7 +134,7 @@ public class HiveHook implements ExecuteWithHookContext { // clone to avoid concurrent access final HiveEvent event = new HiveEvent(); final HiveConf conf = new HiveConf(hookContext.getConf()); - boolean debug = conf.get("hive.hook.dgi.synchronous", "false").equals("true"); + boolean debug = conf.get(CONF_SYNC, "false").equals("true"); event.conf = conf; event.inputs = hookContext.getInputs(); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki b/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki index e5646b9..647af27 100644 --- a/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki +++ b/addons/hive-bridge/src/site/twiki/Bridge-Hive.twiki @@ -25,11 +25,11 @@ hive conf directory: * Atlas endpoint - Add the following property with the Atlas endpoint for your set-up <verbatim> <property> - <name>hive.hook.dgi.url</name> + <name>atlas.rest.address</name> <value>http://localhost:21000/</value> </property> <property> - <name>hive.cluster.name</name> + <name>atlas.cluster.name</name> <value>primary</value> </property> </verbatim> @@ -50,21 +50,21 @@ The hook submits the request to a thread pool executor to avoid blocking the com * Add the following properties in hive-ste.xml with the Atlas endpoint for your set-up <verbatim> <property> - <name>hive.hook.dgi.url</name> + <name>atlas.rest.address</name> <value>http://localhost:21000/</value> </property> <property> - <name>hive.cluster.name</name> + <name>atlas.cluster.name</name> <value>primary</value> </property> </verbatim> * Add 'export HIVE_AUX_JARS_PATH=<dgi package>/hook/hive' in hive-env.sh The following properties in hive-site.xml control the thread pool details: - * hive.hook.dgi.minThreads - core number of threads. default 5 - * hive.hook.dgi.maxThreads - maximum number of threads. default 5 - * hive.hook.dgi.keepAliveTime - keep alive time in msecs. default 10 - * hive.hook.dgi.synchronous - boolean, true to run the hook synchronously. default false + * atlas.hook.hive.minThreads - core number of threads. default 5 + * atlas.hook.hive.maxThreads - maximum number of threads. default 5 + * atlas.hook.hive.keepAliveTime - keep alive time in msecs. default 10 + * atlas.hook.hive.synchronous - boolean, true to run the hook synchronously. default false ---++ Limitations * Since database name, table name and column names are case insensitive in hive, the corresponding names in entities are lowercase. So, any search APIs should use lowercase while querying on the entity names http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java index 4669577..5ae6fe2 100644 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/BaseSSLAndKerberosTest.java @@ -123,14 +123,6 @@ public class BaseSSLAndKerberosTest extends BaseSecurityTest { } protected HiveConf getHiveConf() { - HiveConf hiveConf = new HiveConf(this.getClass()); - hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, ""); - hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName()); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/atlas"); - hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL); - hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true"); - hiveConf.set("hive.hook.dgi.synchronous", "true"); - return hiveConf; + return HiveHookIT.createHiveConf(DGI_URL); } } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java index 95f4eeb..2bf3aba 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java @@ -54,7 +54,7 @@ public class HiveHookIT { @BeforeClass public void setUp() throws Exception { //Set-up hive session - HiveConf conf = getHiveConf(); + HiveConf conf = createHiveConf(); driver = new Driver(conf); ss = new SessionState(conf, System.getProperty("user.name")); ss = SessionState.start(ss); @@ -63,15 +63,19 @@ public class HiveHookIT { dgiCLient = new AtlasClient(DGI_URL); } - private HiveConf getHiveConf() { - HiveConf hiveConf = new HiveConf(this.getClass()); + public static HiveConf createHiveConf() { + return createHiveConf(DGI_URL); + } + + public static HiveConf createHiveConf(String atlasEndpoint) { + HiveConf hiveConf = new HiveConf(HiveHookIT.class); hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, ""); hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName()); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/metastore"); - hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL); - hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true"); - hiveConf.set("hive.hook.dgi.synchronous", "true"); + hiveConf.set(HiveMetaStoreBridge.ATLAS_ENDPOINT, atlasEndpoint); + hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:./target/metastore_db;create=true"); + hiveConf.set(HiveHook.CONF_SYNC, "true"); hiveConf.set(HiveMetaStoreBridge.HIVE_CLUSTER_NAME, CLUSTER_NAME); hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODE, true); //to not use hdfs hiveConf.setVar(HiveConf.ConfVars.HIVETESTMODEPREFIX, ""); http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java ---------------------------------------------------------------------- diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java index 5c17b64..54bfaf6 100755 --- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java +++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/SSLHiveHookIT.java @@ -187,15 +187,7 @@ public class SSLHiveHookIT { } private HiveConf getHiveConf() { - HiveConf hiveConf = new HiveConf(this.getClass()); - hiveConf.setVar(HiveConf.ConfVars.PREEXECHOOKS, ""); - hiveConf.setVar(HiveConf.ConfVars.POSTEXECHOOKS, HiveHook.class.getName()); - hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, System.getProperty("user.dir") + "/target/atlas"); - hiveConf.set(HiveMetaStoreBridge.DGI_URL_PROPERTY, DGI_URL); - hiveConf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:./target/metastore_db;create=true"); - hiveConf.set("hive.hook.dgi.synchronous", "true"); - return hiveConf; + return HiveHookIT.createHiveConf(DGI_URL); } private void runCommand(String cmd) throws Exception { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/864cc5bc/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index f527845..39ff8c2 100644 --- a/release-log.txt +++ b/release-log.txt @@ -4,6 +4,7 @@ Apache Atlas Release Notes --trunk - unreleased INCOMPATIBLE CHANGES: +ATLAS-54 Rename configs in hive hook (shwethags) ATLAS-3 Mixed Index creation fails with Date types (suma.shivaprasad via shwethags) ALL CHANGES:
