OOZIE-2799 Setting log location for spark sql on hive (satishsaley)

Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/77817e73
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/77817e73
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/77817e73

Branch: refs/heads/oya
Commit: 77817e73588f665cbe43b0971cd93465d40711fb
Parents: 4483f6d
Author: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Authored: Wed Feb 15 09:18:47 2017 -0800
Committer: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Committed: Wed Feb 15 09:18:47 2017 -0800

----------------------------------------------------------------------
 .../action/hadoop/SparkActionExecutor.java      |  6 +++
 release-log.txt                                 |  1 +
 .../apache/oozie/action/hadoop/SparkMain.java   | 47 ++++++++++++++++++++
 3 files changed, 54 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git 
a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java 
b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 8e8d7d3..e2e023e 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -160,4 +160,10 @@ public class SparkActionExecutor extends 
JavaActionExecutor {
     protected String getLauncherMain(Configuration launcherConf, Element 
actionXml) {
         return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, 
SPARK_MAIN_CLASS_NAME);
     }
+
+    @Override
+    public String[] getShareLibFilesForActionConf() {
+        return new String[] { "hive-site.xml" };
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 4562524..df586c9 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
 -- Oozie 4.4.0 release (trunk - unreleased)
 
+OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
 OOZIE-2792 Hive2 action is not parsing Spark application ID from log file 
properly when Hive is on Spark (zhengxb2005 via rkanter)
 OOZIE-2788 Fix jobs API servlet mapping for EmbeddedOozieServer (abhishekbafna 
via rkanter)
 OOZIE-2778 Copy only jetty.version related server dependencies to distro 
(asasvari via abhishekbafna)

http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git 
a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java 
b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 3cc6ccb..db1e197 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.lang.reflect.Field;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -41,6 +42,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.filecache.DistributedCache;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.log4j.PropertyConfigurator;
 import org.apache.spark.deploy.SparkSubmit;
 
@@ -76,6 +78,8 @@ public class SparkMain extends LauncherMain {
     private static final Pattern SPARK_VERSION_1 = Pattern.compile("^1.*");
     private static final String SPARK_YARN_JAR = "spark.yarn.jar";
     private static final String SPARK_YARN_JARS = "spark.yarn.jars";
+    public static final String HIVE_SITE_CONF = "hive-site.xml";
+
     public static void main(String[] args) throws Exception {
         run(SparkMain.class, args);
     }
@@ -89,6 +93,7 @@ public class SparkMain extends LauncherMain {
         setYarnTag(actionConf);
         LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
         String logFile = setUpSparkLog4J(actionConf);
+        setHiveSite(actionConf);
         List<String> sparkArgs = new ArrayList<String>();
 
         sparkArgs.add(MASTER_OPTION);
@@ -225,6 +230,7 @@ public class SparkMain extends LauncherMain {
             jarfilter.filter();
             jarPath = jarfilter.getApplicationJar();
             fixedUris.add(new Path(SPARK_LOG4J_PROPS).toUri());
+            fixedUris.add(new Path(HIVE_SITE_CONF).toUri());
             String cachedFiles = StringUtils.join(fixedUris, ",");
             if (cachedFiles != null && !cachedFiles.isEmpty()) {
                 sparkArgs.add("--files");
@@ -519,6 +525,47 @@ public class SparkMain extends LauncherMain {
     }
 
     /**
+     * Sets up hive-site.xml
+     *
+     * @param hiveConf
+     * @throws IOException
+     */
+    private void setHiveSite(Configuration hiveConf) throws IOException {
+        // See https://issues.apache.org/jira/browse/HIVE-1411
+        hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");
+
+        // To ensure that the logs go into container attempt tmp directory
+        // When unset, default is
+        // System.getProperty("java.io.tmpdir") + File.separator +
+        // System.getProperty("user.name")
+        hiveConf.unset("hive.querylog.location");
+        hiveConf.unset("hive.exec.local.scratchdir");
+
+        // Write the action configuration out to hive-site.xml
+        OutputStream os = null;
+        try {
+            os = new FileOutputStream(HIVE_SITE_CONF);
+            hiveConf.writeXml(os);
+        }
+        finally {
+            if (os != null) {
+                os.close();
+            }
+        }
+        // Reset the hiveSiteURL static variable as we just created
+        // hive-site.xml.
+        // If prepare block had a drop partition it would have been initialized
+        // to null.
+        try {
+            Field declaredField = 
HiveConf.class.getDeclaredField("hiveSiteURL");
+            declaredField.setAccessible(true);
+            declaredField.set(null, 
HiveConf.class.getClassLoader().getResource("hive-site.xml"));
+        }
+        catch (Throwable ignore) {
+        }
+    }
+
+    /**
      * This class is used for filtering out unwanted jars.
      */
     static class JarFilter {

Reply via email to