OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/77817e73 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/77817e73 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/77817e73 Branch: refs/heads/oya Commit: 77817e73588f665cbe43b0971cd93465d40711fb Parents: 4483f6d Author: Satish Subhashrao Saley <sa...@yahoo-inc.com> Authored: Wed Feb 15 09:18:47 2017 -0800 Committer: Satish Subhashrao Saley <sa...@yahoo-inc.com> Committed: Wed Feb 15 09:18:47 2017 -0800 ---------------------------------------------------------------------- .../action/hadoop/SparkActionExecutor.java | 6 +++ release-log.txt | 1 + .../apache/oozie/action/hadoop/SparkMain.java | 47 ++++++++++++++++++++ 3 files changed, 54 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java index 8e8d7d3..e2e023e 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java @@ -160,4 +160,10 @@ public class SparkActionExecutor extends JavaActionExecutor { protected String getLauncherMain(Configuration launcherConf, Element actionXml) { return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME); } + + @Override + public String[] getShareLibFilesForActionConf() { + return new String[] { "hive-site.xml" }; + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 4562524..df586c9 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2799 Setting log location for spark sql on hive (satishsaley) OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter) OOZIE-2788 Fix jobs API servlet mapping for EmbeddedOozieServer (abhishekbafna via rkanter) OOZIE-2778 Copy only jetty.version related server dependencies to distro (asasvari via abhishekbafna) http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 3cc6ccb..db1e197 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -41,6 +42,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.log4j.PropertyConfigurator; import org.apache.spark.deploy.SparkSubmit; @@ -76,6 +78,8 @@ public class SparkMain extends LauncherMain { private static final Pattern SPARK_VERSION_1 = Pattern.compile("^1.*"); private static final String SPARK_YARN_JAR = "spark.yarn.jar"; private static final String SPARK_YARN_JARS = "spark.yarn.jars"; + public static final String HIVE_SITE_CONF = "hive-site.xml"; + public static void main(String[] args) throws Exception { run(SparkMain.class, args); } @@ -89,6 +93,7 @@ public class SparkMain extends LauncherMain { setYarnTag(actionConf); LauncherMainHadoopUtils.killChildYarnJobs(actionConf); String logFile = setUpSparkLog4J(actionConf); + setHiveSite(actionConf); List<String> sparkArgs = new ArrayList<String>(); sparkArgs.add(MASTER_OPTION); @@ -225,6 +230,7 @@ public class SparkMain extends LauncherMain { jarfilter.filter(); jarPath = jarfilter.getApplicationJar(); fixedUris.add(new Path(SPARK_LOG4J_PROPS).toUri()); + fixedUris.add(new Path(HIVE_SITE_CONF).toUri()); String cachedFiles = StringUtils.join(fixedUris, ","); if (cachedFiles != null && !cachedFiles.isEmpty()) { sparkArgs.add("--files"); @@ -519,6 +525,47 @@ public class SparkMain extends LauncherMain { } /** + * Sets up hive-site.xml + * + * @param hiveConf + * @throws IOException + */ + private void setHiveSite(Configuration hiveConf) throws IOException { + // See https://issues.apache.org/jira/browse/HIVE-1411 + hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG"); + + // To ensure that the logs go into container attempt tmp directory + // When unset, default is + // System.getProperty("java.io.tmpdir") + File.separator + + // System.getProperty("user.name") + hiveConf.unset("hive.querylog.location"); + hiveConf.unset("hive.exec.local.scratchdir"); + + // Write the action configuration out to hive-site.xml + OutputStream os = null; + try { + os = new FileOutputStream(HIVE_SITE_CONF); + hiveConf.writeXml(os); + } + finally { + if (os != null) { + os.close(); + } + } + // Reset the hiveSiteURL static variable as we just created + // hive-site.xml. + // If prepare block had a drop partition it would have been initialized + // to null. + try { + Field declaredField = HiveConf.class.getDeclaredField("hiveSiteURL"); + declaredField.setAccessible(true); + declaredField.set(null, HiveConf.class.getClassLoader().getResource("hive-site.xml")); + } + catch (Throwable ignore) { + } + } + + /** * This class is used for filtering out unwanted jars. */ static class JarFilter {
