Author: ekoifman
Date: Fri Nov 21 19:24:54 2014
New Revision: 1640981
URL: http://svn.apache.org/r1640981
Log:
HIVE-8877 improve context logging during job submission via WebHCat (reviewed
by Thejas Nair)
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1640981&r1=1640980&r2=1640981&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java
Fri Nov 21 19:24:54 2014
@@ -28,12 +28,14 @@ import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.StringTokenizer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.SystemVariables;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hive.hcatalog.templeton.tool.JobState;
@@ -224,14 +226,8 @@ public class AppConfig extends Configura
* support/debugging. Later it may be worth adding a REST call which will
return this data.
*/
private String dumpEnvironent() {
- StringBuilder sb = new StringBuilder("WebHCat environment:\n");
- Map<String, String> env = System.getenv();
- List<String> propKeys = new ArrayList<String>(env.keySet());
- Collections.sort(propKeys);
- for(String propKey : propKeys) {
- sb.append(propKey).append('=').append(env.get(propKey)).append('\n');
- }
- sb.append("Configration properties: \n");
+ StringBuilder sb = TempletonUtils.dumpPropMap("========WebHCat
System.getenv()========", System.getenv());
+ sb.append("START========WebHCat AppConfig.iterator()========: \n");
Iterator<Map.Entry<String, String>> configIter = this.iterator();
List<Map.Entry<String, String>> configVals = new
ArrayList<Map.Entry<String, String>>();
while(configIter.hasNext()) {
@@ -245,8 +241,19 @@ public class AppConfig extends Configura
});
for(Map.Entry<String, String> entry : configVals) {
//use get() to make sure variable substitution works
-
sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n');
+ if(entry.getKey().toLowerCase().contains("path")) {
+ StringTokenizer st = new StringTokenizer(get(entry.getKey()),
File.pathSeparator);
+ sb.append(entry.getKey()).append("=\n");
+ while(st.hasMoreTokens()) {
+ sb.append("
").append(st.nextToken()).append(File.pathSeparator).append('\n');
+ }
+ }
+ else {
+
sb.append(entry.getKey()).append('=').append(get(entry.getKey())).append('\n');
+ }
}
+ sb.append("END========WebHCat AppConfig.iterator()========: \n");
+ sb.append(TempletonUtils.dumpPropMap("========WebHCat
System.getProperties()========", System.getProperties()));
return sb.toString();
}
public void startCleanup() {
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java?rev=1640981&r1=1640980&r2=1640981&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java
Fri Nov 21 19:24:54 2014
@@ -18,6 +18,7 @@
*/
package org.apache.hive.hcatalog.templeton.tool;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -34,6 +35,8 @@ import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
+import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -408,4 +411,34 @@ public class TempletonUtils {
}
return null;
}
+ public static StringBuilder dumpPropMap(String header, Properties props) {
+ Map<String, String> map = new HashMap<String, String>();
+ for(Map.Entry<Object, Object> ent : props.entrySet()) {
+ map.put(ent.getKey().toString(), ent.getValue() == null ? null :
ent.getValue().toString());
+ }
+ return dumpPropMap(header, map);
+ }
+ public static StringBuilder dumpPropMap(String header, Map<String, String>
map) {
+ StringBuilder sb = new StringBuilder("START").append(header).append(":\n");
+ List<String> propKeys = new ArrayList<String>(map.keySet());
+ Collections.sort(propKeys);
+ for(String propKey : propKeys) {
+ if(propKey.toLowerCase().contains("path")) {
+ StringTokenizer st = new StringTokenizer(map.get(propKey),
File.pathSeparator);
+ if(st.countTokens() > 1) {
+ sb.append(propKey).append("=\n");
+ while (st.hasMoreTokens()) {
+ sb.append("
").append(st.nextToken()).append(File.pathSeparator).append('\n');
+ }
+ }
+ else {
+ sb.append(propKey).append('=').append(map.get(propKey)).append('\n');
+ }
+ }
+ else {
+ sb.append(propKey).append('=').append(map.get(propKey)).append('\n');
+ }
+ }
+ return sb.append("END").append(header).append('\n');
+ }
}
Modified:
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
URL:
http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java?rev=1640981&r1=1640980&r2=1640981&view=diff
==============================================================================
---
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
(original)
+++
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TrivialExecService.java
Fri Nov 21 19:24:54 2014
@@ -20,7 +20,6 @@ package org.apache.hive.hcatalog.templet
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
import java.io.File;
import java.io.IOException;
@@ -28,6 +27,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.StringTokenizer;
/**
@@ -61,16 +61,47 @@ final class TrivialExecService {
pb.environment().remove(key);
}
pb.environment().putAll(environmentVariables);
- logDebugInfo("Starting process with env:", pb.environment());
+ logDebugInfo("========Starting process with env:========",
pb.environment());
+ printContentsOfDir(".");
return pb.start();
}
private static void logDebugInfo(String msg, Map<String, String> props) {
- LOG.info(msg);
- List<String> keys = new ArrayList<String>();
- keys.addAll(props.keySet());
- Collections.sort(keys);
- for(String key : keys) {
- LOG.info(key + "=" + props.get(key));
- }
+ StringBuilder sb = TempletonUtils.dumpPropMap(msg, props);
+ LOG.info(sb.toString());
+ String sqoopHome = props.get("SQOOP_HOME");
+ if(TempletonUtils.isset(sqoopHome)) {
+ //this is helpful when Sqoop is installed on each node in the cluster to
make sure
+ //relevant jars (JDBC in particular) are present on the node running the
command
+ printContentsOfDir(sqoopHome + File.separator+ "lib");
+ }
+ }
+ /**
+ * Print files and directories in current directory. Will list files in the
sub-directory (only 1 level deep)
+ * time honored tradition in WebHCat of borrowing from Oozie
+ */
+ private static void printContentsOfDir(String dir) {
+ File folder = new File(dir);
+ StringBuilder sb = new StringBuilder("Files in '").append(dir).append("'
dir:").append(folder.getAbsolutePath()).append('\n');
+
+ File[] listOfFiles = folder.listFiles();
+ for (File fileName : listOfFiles) {
+ if (fileName.isFile()) {
+ sb.append("File: ").append(fileName.getName()).append('\n');
+ }
+ else if (fileName.isDirectory()) {
+ sb.append("Dir: ").append(fileName.getName()).append('\n');
+ File subDir = new File(fileName.getName());
+ File[] moreFiles = subDir.listFiles();
+ for (File subFileName : moreFiles) {
+ if (subFileName.isFile()) {
+ sb.append("--File: ").append(subFileName.getName()).append('\n');
+ }
+ else if (subFileName.isDirectory()) {
+ sb.append("--Dir: ").append(subFileName.getName()).append('\n');
+ }
+ }
+ }
+ }
+ LOG.info(sb.toString());
}
}
