Author: zjffdu Date: Wed May 12 16:39:29 2010 New Revision: 943570 URL: http://svn.apache.org/viewvc?rev=943570&view=rev Log: Pig-1406: Allow to run shell commands from grunt
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=943570&r1=943569&r2=943570&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Wed May 12 16:39:29 2010 @@ -23,6 +23,9 @@ Trunk (unreleased changes) INCOMPATIBLE CHANGES IMPROVEMENTS + +PIG-1406: Allow to run shell commands from grunt (zjffdu) + PIG-1398: Marking Pig interfaces for org.apache.pig.data package (gates) PIG-1396: eclipse-files target in build.xml fails to generate necessary classes in src-gen Modified: hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=943570&r1=943569&r2=943570&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Wed May 12 16:39:29 2010 @@ -44,6 +44,7 @@ import jline.ConsoleReaderInputStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FsShell; + import org.apache.hadoop.mapred.JobClient; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobID; @@ -867,6 +868,65 @@ public class GruntParser extends PigScri } } + @Override + protected void processShCommand(String[] cmdTokens) throws IOException{ + StringBuilder builder = new StringBuilder(); + for (String token:cmdTokens){ + builder.append(token + " "); + } + try { + Process executor = Runtime.getRuntime().exec(builder.toString()); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err); + + outPrinter.start(); + errPrinter.start(); + + int ret = executor.waitFor(); + if (ret != 0) { + log.warn("Command failed with exit code = " + ret); + } + } catch (Exception e) { + log.warn("Exception raised from Shell command " + e.getLocalizedMessage()); + } + } + + /** + * StreamPrinter. + * + */ + public static class StreamPrinter extends Thread { + InputStream is; + String type; + PrintStream os; + + public StreamPrinter(InputStream is, String type, PrintStream os) { + this.is = is; + this.type = type; + this.os = os; + } + + @Override + public void run() { + try { + InputStreamReader isr = new InputStreamReader(is); + BufferedReader br = new BufferedReader(isr); + String line = null; + if (type != null) { + while ((line = br.readLine()) != null) { + os.println(type + ">" + line); + } + } else { + while ((line = br.readLine()) != null) { + os.println(line); + } + } + } catch (IOException ioe) { + ioe.printStackTrace(); + } + } + } + private static class ExplainState { public long mTime; public int mCount; Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=943570&r1=943569&r2=943570&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original) +++ hadoop/pig/trunk/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Wed May 12 16:39:29 2010 @@ -66,6 +66,8 @@ public abstract class PigScriptParser abstract protected void printAliases() throws IOException; abstract protected void processFsCommand(String[] cmdTokens) throws IOException; + + abstract protected void processShCommand(String[] cmdTokens) throws IOException; abstract protected void processDescribe(String alias) throws IOException; @@ -133,6 +135,7 @@ SKIP : { // commands TOKEN: {<CAT: "cat">} TOKEN: {<FS: "fs">} +TOKEN: {<SH:"sh">} TOKEN: {<CD: "cd">} TOKEN: {<COPY: "cp">} TOKEN: {<COPYFROMLOCAL: "copyFromLocal">} @@ -396,6 +399,23 @@ void parse() throws IOException: } )+ | + <SH> + ( + t1 = GetPath() + { + cmdTokens.add(t1.image); + while(true){ + try{ + t1=GetPath(); + cmdTokens.add(t1.image); + }catch(ParseException e){ + break; + } + } + processShCommand(cmdTokens.toArray(new String[cmdTokens.size()])); + } + )+ + | <CAT> ( t1 = GetPath() Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java?rev=943570&r1=943569&r2=943570&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestGrunt.java Wed May 12 16:39:29 2010 @@ -849,6 +849,36 @@ public class TestGrunt extends TestCase } @Test + public void testShellCommand(){ + + try { + PigServer server = new PigServer(ExecType.MAPREDUCE,cluster.getProperties()); + PigContext context = server.getPigContext(); + + String strCmd = "sh mkdir test_shell_tmp;"; + + ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes()); + InputStreamReader reader = new InputStreamReader(cmd); + Grunt grunt = new Grunt(new BufferedReader(reader), context); + grunt.exec(); + assertTrue(new File("test_shell_tmp").exists()); + + strCmd = "sh rmdir test_shell_tmp;"; + cmd = new ByteArrayInputStream(strCmd.getBytes()); + reader = new InputStreamReader(cmd); + grunt = new Grunt(new BufferedReader(reader), context); + grunt.exec(); + assertFalse(new File("test_shell_tmp").exists()); + } catch (ExecException e) { + e.printStackTrace(); + fail(); + } catch (Throwable e) { + e.printStackTrace(); + fail(); + } + } + + @Test public void testSetPriority() throws Throwable { PigServer server = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); PigContext context = server.getPigContext();