OOZIE-2709 Log the substituted pig script by expanding macros (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c58f5456 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c58f5456 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c58f5456 Branch: refs/heads/oya Commit: c58f5456ddfdccce0c25a0ad80d86dba5b4ce944 Parents: 2a1ba19 Author: Satish Subhashrao Saley <sa...@yahoo-inc.com> Authored: Thu Jan 19 10:12:39 2017 -0800 Committer: Satish Subhashrao Saley <sa...@yahoo-inc.com> Committed: Thu Jan 19 10:12:39 2017 -0800 ---------------------------------------------------------------------- core/src/main/resources/oozie-default.xml | 8 ++ release-log.txt | 1 + sharelib/pig/pom.xml | 6 +- .../org/apache/oozie/action/hadoop/PigMain.java | 82 +++++++++++++++++--- 4 files changed, 84 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/c58f5456/core/src/main/resources/oozie-default.xml ---------------------------------------------------------------------- diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 943f9bc..ad10386 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -1882,6 +1882,14 @@ will be the requeue interval for the actions which are waiting for a long time w </property> <property> + <name>oozie.action.pig.log.expandedscript</name> + <value>true</value> + <description> + Log the expanded pig script in launcher stdout log + </description> + </property> + + <property> <name>oozie.action.rootlogger.log.level</name> <value>INFO</value> <description> http://git-wip-us.apache.org/repos/asf/oozie/blob/c58f5456/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 346a5ef..8a4aad0 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2709 Log the substituted pig script by expanding macros (satishsaley) OOZIE-2732 Remove login server example (rkanter via abhishekbafna) OOZIE-2756 Extend HTTPS configuration settings for embedded Jetty (asasvari via abhishekbafna) OOZIE-2727 OozieDBCLI creates temporary directories and do not delete them (gezapeti via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/c58f5456/sharelib/pig/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/pig/pom.xml b/sharelib/pig/pom.xml index 233873c..99148d7 100644 --- a/sharelib/pig/pom.xml +++ b/sharelib/pig/pom.xml @@ -140,7 +140,11 @@ </exclusion> </exclusions> </dependency> - + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <scope>compile</scope> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/c58f5456/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index 87c4cc4..7de2e45 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -20,8 +20,10 @@ package org.apache.oozie.action.hadoop; import org.apache.pig.Main; import org.apache.pig.PigRunner; +import org.apache.pig.scripting.ScriptEngine; import org.apache.pig.tools.pigstats.JobStats; import org.apache.pig.tools.pigstats.PigStats; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -34,7 +36,9 @@ import java.io.FileOutputStream; import java.io.BufferedReader; import java.io.FileReader; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStreamReader; import java.util.Arrays; import java.util.HashSet; import java.util.Map; @@ -48,6 +52,7 @@ import java.util.regex.Pattern; public class PigMain extends LauncherMain { private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>(); public static final int STRING_BUFFER_SIZE = 100; + public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherMapper.ACTION_PREFIX + "pig.log.expandedscript"; private static final Pattern[] PIG_JOB_IDS_PATTERNS = { Pattern.compile("HadoopJobId: (job_\\S*)"), @@ -136,17 +141,7 @@ public class PigMain extends LauncherMain { throw new RuntimeException("Error: Pig script file [" + script + "] does not exist"); } - System.out.println("Pig script [" + script + "] content: "); - System.out.println("------------------------"); - BufferedReader br = new BufferedReader(new FileReader(script)); - String line = br.readLine(); - while (line != null) { - System.out.println(line); - line = br.readLine(); - } - br.close(); - System.out.println("------------------------"); - System.out.println(); + printScript(script, ""); arguments.add("-file"); arguments.add(script); @@ -212,6 +207,12 @@ public class PigMain extends LauncherMain { arguments.add(pigArg); } + if (actionConf.getBoolean(LOG_EXPANDED_PIG_SCRIPT, true) + // To avoid Pig running the embedded scripts on dryrun + && ScriptEngine.getSupportedScriptLang(script) == null) { + logExpandedScript(script, arguments); + } + System.out.println("Pig command arguments :"); for (String arg : arguments) { System.out.println(" " + arg); @@ -244,8 +245,65 @@ public class PigMain extends LauncherMain { } } + /** + * Logs the expanded + * + * @param script + * @param arguments + */ + private void logExpandedScript(String script, List<String> arguments) { + List<String> dryrunArgs = new ArrayList<String>(); + dryrunArgs.addAll(arguments); + dryrunArgs.add("-dryrun"); + try { + PigRunner.run(dryrunArgs.toArray(new String[dryrunArgs.size()]), null); + printScript(script + ".expanded", "Expanded"); + } + catch (Exception e) { + System.out.println("Failure while expanding pig script"); + e.printStackTrace(System.out); + } + } - + private void printScript(String name, String type) { + FileInputStream fin = null; + InputStreamReader inReader = null; + BufferedReader bufReader = null; + try { + File script = new File(name); + if (!script.exists()) { + return; + } + System.out.println("-----------------------------------------------------------"); + System.out.println(type + " Pig script [" + name + "] content: "); + System.out.println("-----------------------------------------------------------"); + fin = new FileInputStream(script); + inReader = new InputStreamReader(fin, "UTF-8"); + bufReader = new BufferedReader(inReader); + String line = bufReader.readLine(); + while (line != null) { + System.out.println(line); + line = bufReader.readLine(); + } + bufReader.close(); + } + catch (RuntimeException e) { + throw e; + } + catch (Exception e) { + System.out.println("Unable to read " + name); + e.printStackTrace(System.out); + } + finally { + IOUtils.closeQuietly(fin); + IOUtils.closeQuietly(inReader); + IOUtils.closeQuietly(bufReader); + } + System.out.println(); + System.out.flush(); + System.out.println("-----------------------------------------------------------"); + System.out.println(); + } private void handleError(String pigLog) throws Exception { System.err.println(); System.err.println("Pig logfile dump:");
