svn commit: r1553563 - /hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
Author: brock Date: Thu Dec 26 19:44:45 2013 New Revision: 1553563 URL: http://svn.apache.org/r1553563 Log: HIVE-5524 - Unwanted delay in getting Hive metastore connection with METASTORE_CLIENT_CONNECT_RETRY_DELAY (Rajesh Balamohan via Brock Noland) Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1553563&r1=1553562&r2=1553563&view=diff == --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Thu Dec 26 19:44:45 2013 @@ -322,7 +322,7 @@ public class HiveMetaStoreClient impleme } } // Wait before launching the next round of connection retries. - if (retryDelaySeconds > 0) { + if (!isConnected && retryDelaySeconds > 0) { try { LOG.info("Waiting " + retryDelaySeconds + " seconds before next connection attempt."); Thread.sleep(retryDelaySeconds * 1000);
svn commit: r1553607 - /hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
Author: daijy Date: Fri Dec 27 00:07:52 2013 New Revision: 1553607 URL: http://svn.apache.org/r1553607 Log: HIVE-5702: add logging on WebHCat startup to print all env var and Configuration (Eugene Koifman via Daniel Dai) Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1553607&r1=1553606&r2=1553607&view=diff == --- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original) +++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Fri Dec 27 00:07:52 2013 @@ -19,7 +19,13 @@ package org.apache.hive.hcatalog.templeton; import java.io.File; +import java.io.StringBufferInputStream; import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -141,16 +147,52 @@ public class AppConfig extends Configura set("env." + e.getKey(), e.getValue()); String templetonDir = getTempletonDir(); -for (String fname : TEMPLETON_CONF_FILENAMES) +for (String fname : TEMPLETON_CONF_FILENAMES) { + logConfigLoadAttempt(templetonDir + File.separator + fname); if (! loadOneClasspathConfig(fname)) loadOneFileConfig(templetonDir, fname); - +} String hadoopConfDir = getHadoopConfDir(); -for (String fname : HADOOP_CONF_FILENAMES) +for (String fname : HADOOP_CONF_FILENAMES) { + logConfigLoadAttempt(hadoopConfDir + File.separator + fname); loadOneFileConfig(hadoopConfDir, fname); +} ProxyUserSupport.processProxyuserConfig(this); +LOG.info(dumpEnvironent()); + } + private static void logConfigLoadAttempt(String path) { +LOG.info("Attempting to load config file: " + path); } + /** + * Dumps all env and config state. Should be called once on WebHCat start up to facilitate + * support/debugging. Later it may be worth adding a REST call which will return this data. + */ + private String dumpEnvironent() { +StringBuilder sb = new StringBuilder("WebHCat environment:\n"); +Map env = System.getenv(); +List propKeys = new ArrayList(env.keySet()); +Collections.sort(propKeys); +for(String propKey : propKeys) { + sb.append(propKey).append('=').append(env.get(propKey)).append('\n'); +} +sb.append("Configration properties: \n"); +Iterator> configIter = this.iterator(); +List> configVals = new ArrayList>(); +while(configIter.hasNext()) { + configVals.add(configIter.next()); +} +Collections.sort(configVals, new Comparator> () { + @Override + public int compare(Map.Entry ent, Map.Entry ent2) { +return ent.getKey().compareTo(ent2.getKey()); + } +}); +for(Map.Entry entry : configVals) { + sb.append(entry.getKey()).append('=').append(entry.getValue()).append('\n'); +} +return sb.toString(); + } public void startCleanup() { JobState.getStorageInstance(this).startCleanup(this); } @@ -182,7 +224,7 @@ public class AppConfig extends Configura URL x = getResource(fname); if (x != null) { addResource(x); - LOG.debug("loaded config from classpath " + x); + LOG.info("loaded config from classpath " + x); return true; }
svn commit: r1553627 - /hive/branches/tez/ql/pom.xml
Author: gunther Date: Fri Dec 27 06:05:37 2013 New Revision: 1553627 URL: http://svn.apache.org/r1553627 Log: HIVE-6101 (part 4): Classpath is incorrect for hadoop-1 tests on tez (Gunther Hagleitner) Modified: hive/branches/tez/ql/pom.xml Modified: hive/branches/tez/ql/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/pom.xml?rev=1553627&r1=1553626&r2=1553627&view=diff == --- hive/branches/tez/ql/pom.xml (original) +++ hive/branches/tez/ql/pom.xml Fri Dec 27 06:05:37 2013 @@ -231,6 +231,14 @@ org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + org.apache.hadoop hadoop-hdfs @@ -255,6 +263,14 @@ org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + org.apache.hadoop hadoop-hdfs @@ -279,6 +295,14 @@ org.apache.hadoop + hadoop-mapreduce-client-jobclient + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + org.apache.hadoop hadoop-hdfs
svn commit: r1553628 - in /hive/branches/tez: ./ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreC
Author: gunther Date: Fri Dec 27 06:06:32 2013 New Revision: 1553628 URL: http://svn.apache.org/r1553628 Log: Merge latest trunk into branch. (Gunther Hagleitner) Modified: hive/branches/tez/ (props changed) hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Propchange: hive/branches/tez/ -- Merged /hive/trunk:r1553448-1553627 Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1553628&r1=1553627&r2=1553628&view=diff == --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Fri Dec 27 06:06:32 2013 @@ -19,7 +19,13 @@ package org.apache.hive.hcatalog.templeton; import java.io.File; +import java.io.StringBufferInputStream; import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -141,16 +147,52 @@ public class AppConfig extends Configura set("env." + e.getKey(), e.getValue()); String templetonDir = getTempletonDir(); -for (String fname : TEMPLETON_CONF_FILENAMES) +for (String fname : TEMPLETON_CONF_FILENAMES) { + logConfigLoadAttempt(templetonDir + File.separator + fname); if (! loadOneClasspathConfig(fname)) loadOneFileConfig(templetonDir, fname); - +} String hadoopConfDir = getHadoopConfDir(); -for (String fname : HADOOP_CONF_FILENAMES) +for (String fname : HADOOP_CONF_FILENAMES) { + logConfigLoadAttempt(hadoopConfDir + File.separator + fname); loadOneFileConfig(hadoopConfDir, fname); +} ProxyUserSupport.processProxyuserConfig(this); +LOG.info(dumpEnvironent()); + } + private static void logConfigLoadAttempt(String path) { +LOG.info("Attempting to load config file: " + path); } + /** + * Dumps all env and config state. Should be called once on WebHCat start up to facilitate + * support/debugging. Later it may be worth adding a REST call which will return this data. + */ + private String dumpEnvironent() { +StringBuilder sb = new StringBuilder("WebHCat environment:\n"); +Map env = System.getenv(); +List propKeys = new ArrayList(env.keySet()); +Collections.sort(propKeys); +for(String propKey : propKeys) { + sb.append(propKey).append('=').append(env.get(propKey)).append('\n'); +} +sb.append("Configration properties: \n"); +Iterator> configIter = this.iterator(); +List> configVals = new ArrayList>(); +while(configIter.hasNext()) { + configVals.add(configIter.next()); +} +Collections.sort(configVals, new Comparator> () { + @Override + public int compare(Map.Entry ent, Map.Entry ent2) { +return ent.getKey().compareTo(ent2.getKey()); + } +}); +for(Map.Entry entry : configVals) { + sb.append(entry.getKey()).append('=').append(entry.getValue()).append('\n'); +} +return sb.toString(); + } public void startCleanup() { JobState.getStorageInstance(this).startCleanup(this); } @@ -182,7 +224,7 @@ public class AppConfig extends Configura URL x = getResource(fname); if (x != null) { addResource(x); - LOG.debug("loaded config from classpath " + x); + LOG.info("loaded config from classpath " + x); return true; } Modified: hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1553628&r1=1553627&r2=1553628&view=diff == --- hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original) +++ hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Fri Dec 27 06:06:32 2013 @@ -322,7 +322,7 @@ public class HiveMetaStoreClient impleme } } // Wait before launching the next round of connection retries. - if (retryDelaySeconds > 0) { + if (!isConnected && retryDelaySeconds > 0) { try { LOG.info("Waiting " + retryDelaySeconds + " seconds before next connection attempt."); Thread.sleep(retryDelaySeconds * 1000);