Repository: knox Updated Branches: refs/heads/master a738b878b -> 47edcbc16
KNOX-712 - submitPig does not allow multiple argument via arg() Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/47edcbc1 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/47edcbc1 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/47edcbc1 Branch: refs/heads/master Commit: 47edcbc16c040f6a68a9e15354439188a79fe3b0 Parents: a738b87 Author: Larry McCay <lmc...@hortonworks.com> Authored: Fri May 20 18:37:03 2016 -0400 Committer: Larry McCay <lmc...@hortonworks.com> Committed: Fri May 20 18:37:03 2016 -0400 ---------------------------------------------------------------------- .../home/samples/ExampleWebHCatPig.groovy | 84 ++++++++++++++++++++ .../apache/hadoop/gateway/shell/job/Pig.java | 8 +- 2 files changed, 87 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/47edcbc1/gateway-release/home/samples/ExampleWebHCatPig.groovy ---------------------------------------------------------------------- diff --git a/gateway-release/home/samples/ExampleWebHCatPig.groovy b/gateway-release/home/samples/ExampleWebHCatPig.groovy new file mode 100644 index 0000000..8699608 --- /dev/null +++ b/gateway-release/home/samples/ExampleWebHCatPig.groovy @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.jayway.jsonpath.JsonPath +import groovy.json.JsonSlurper +import org.apache.hadoop.gateway.shell.Hadoop +import org.apache.hadoop.gateway.shell.hdfs.Hdfs +import org.apache.hadoop.gateway.shell.job.Job + +import static java.util.concurrent.TimeUnit.SECONDS +import org.apache.hadoop.gateway.shell.Credentials + +gateway = "https://localhost:8443/gateway/sandbox"; + +credentials = new Credentials() +credentials.add("ClearInput", "Enter username: ", "user") + .add("HiddenInput", "Enter pas" + "sword: ", "pass") +credentials.collect() + +username = credentials.get("user").string() +pass = credentials.get("pass").string() + +jobDir = "/user/" + username + "/test" + +session = Hadoop.login( gateway, username, pass ) + +println "Delete " + jobDir + ": " + Hdfs.rm( session ).file( jobDir ).recursive().now().statusCode +println "Create " + jobDir + ": " + Hdfs.mkdir( session ).dir( jobDir ).now().statusCode + +id_pig = ''' +A = load 'test/input/$filename' using PigStorage(':'); +B = foreach A generate $0 as id; +dump B; +''' + +fake_passwd = '''ctdean:Chris Dean:secret +pauls:Paul Stolorz:good +carmas:Carlos Armas:evil +dra:Deirdre McClure:marvelous +''' + +Hdfs.put(session).text( id_pig ).to( jobDir + "/input/id.pig" ).now() +Hdfs.put(session).text( fake_passwd ).to( jobDir + "/input/passwd" ).now() + +jobId = Job.submitPig(session) \ + .file("${jobDir}/input/id.pig") \ + .arg("-v") \ + .arg("-p").arg("filename=passwd") \ + .statusDir("${jobDir}/output") \ + .now().jobId + +println "Submitted job: " + jobId + +println "Polling up to 60s for job completion..." +done = false +count = 0 +while( !done && count++ < 60 ) { + sleep( 1000 ) + json = Job.queryStatus(session).jobId(jobId).now().string + done = JsonPath.read( json, "\$.status.jobComplete" ) + print "."; System.out.flush(); +} +println "" +println "Job status: " + done + +text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string +json = (new JsonSlurper()).parseText( text ) +println json.FileStatuses.FileStatus.pathSuffix + +println "Session closed: " + session.shutdown( 10, SECONDS ) http://git-wip-us.apache.org/repos/asf/knox/blob/47edcbc1/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java ---------------------------------------------------------------------- diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java index 67efba6..01d43e6 100644 --- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java +++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Pig.java @@ -38,8 +38,8 @@ public class Pig { private String group; private String file; - private String arg; private String statusDir; + List<NameValuePair> params = new ArrayList<NameValuePair>(); public Request( Hadoop session ) { super( session ); @@ -55,8 +55,8 @@ public class Pig { return this; } - public Request arg( String arg ) { - this.arg = arg; + public Request arg( String value ) { + addParam( params, "arg", value ); return this; } @@ -70,10 +70,8 @@ public class Pig { @Override public Response call() throws Exception { URIBuilder uri = uri( Job.SERVICE_PATH, "/pig" ); - List<NameValuePair> params = new ArrayList<NameValuePair>(); addParam( params, "group", group ); addParam( params, "file", file ); - addParam( params, "arg", arg ); addParam( params, "statusdir", statusDir ); UrlEncodedFormEntity form = new UrlEncodedFormEntity( params ); HttpPost request = new HttpPost( uri.build() );
