Repository: knox Updated Branches: refs/heads/master c6caebd4b -> 43d07683b
KNOX-743 - Add submitSqoop via knox shell api Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/43d07683 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/43d07683 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/43d07683 Branch: refs/heads/master Commit: 43d07683b687ffffe9302e390669bfeccdcf98fc Parents: c6caebd Author: Larry McCay <lmc...@hortonworks.com> Authored: Mon Oct 31 15:59:47 2016 -0400 Committer: Larry McCay <lmc...@hortonworks.com> Committed: Mon Oct 31 15:59:47 2016 -0400 ---------------------------------------------------------------------- .../home/samples/ExampleWebHCatSqoop.groovy | 80 ++++++++++++++++ .../apache/hadoop/gateway/shell/job/Job.java | 4 + .../apache/hadoop/gateway/shell/job/Sqoop.java | 99 ++++++++++++++++++++ 3 files changed, 183 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/43d07683/gateway-release/home/samples/ExampleWebHCatSqoop.groovy ---------------------------------------------------------------------- diff --git a/gateway-release/home/samples/ExampleWebHCatSqoop.groovy b/gateway-release/home/samples/ExampleWebHCatSqoop.groovy new file mode 100644 index 0000000..a5629e3 --- /dev/null +++ b/gateway-release/home/samples/ExampleWebHCatSqoop.groovy @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import com.jayway.jsonpath.JsonPath +import groovy.json.JsonSlurper +import org.apache.hadoop.gateway.shell.Hadoop +import org.apache.hadoop.gateway.shell.hdfs.Hdfs +import org.apache.hadoop.gateway.shell.job.Job + +import static java.util.concurrent.TimeUnit.SECONDS +import org.apache.hadoop.gateway.shell.Credentials + +gateway = "https://localhost:8443/gateway/sandbox" + +credentials = new Credentials() +credentials.add("ClearInput", "Enter username: ", "user") + .add("HiddenInput", "Enter pas" + "sword: ", "pass") +credentials.collect() + +username = credentials.get("user").string() +pass = credentials.get("pass").string() + +jobDir = "/user/" + username + "/test" + +session = Hadoop.login( gateway, username, pass ) + +println "Delete " + jobDir + ": " + Hdfs.rm( session ).file( jobDir ).recursive().now().statusCode +println "Create " + jobDir + ": " + Hdfs.mkdir( session ).dir( jobDir ).now().statusCode + +// Define sqoop options value for sqoop command. +// This use publicly available Genome mysql database. +// If the database is unavailable, setup an alternate database and update the +// db information below. +db = [ driver:"com.mysql.jdbc.Driver", url:"jdbc:mysql://genome-mysql.cse.ucsc.edu/hg38", user:"genome", password:"", name:"hg38", table:"scBlastTab", split:"query" ] + +targetdir = jobDir + "/" + db.table + +sqoop_command = "import --driver ${db.driver} --connect ${db.url} --username ${db.user} --password ${db.password} --table ${db.table} --split-by ${db.split} --target-dir ${targetdir}" + +jobId = Job.submitSqoop(session) \ + .command(sqoop_command) \ + .statusDir("${jobDir}/output") \ + .now().jobId + +println "Submitted job: " + jobId + +println "Polling up to 60s for job completion..." +done = false +count = 0 +while( !done && count++ < 180 ) { + sleep( 1000 ) + json = Job.queryStatus(session).jobId(jobId).now().string + done = JsonPath.read( json, "\$.status.jobComplete" ) + print "."; System.out.flush(); +} +println "" +println "Job status: " + done + +text = Hdfs.ls( session ).dir( jobDir + "/output" ).now().string +json = (new JsonSlurper()).parseText( text ) +println json.FileStatuses.FileStatus.pathSuffix + +println "Content of stderr:" +println Hdfs.get( session ).from( jobDir + "/output/stderr" ).now().string + +println "Session closed: " + session.shutdown( 10, SECONDS ) http://git-wip-us.apache.org/repos/asf/knox/blob/43d07683/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java ---------------------------------------------------------------------- diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java index f374244..ec49fe1 100644 --- a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java +++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Job.java @@ -27,6 +27,10 @@ public class Job { return new Java.Request( session ); } + public static Sqoop.Request submitSqoop( Hadoop session ) { + return new Sqoop.Request( session ); + } + public static Pig.Request submitPig( Hadoop session ) { return new Pig.Request( session ); } http://git-wip-us.apache.org/repos/asf/knox/blob/43d07683/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Sqoop.java ---------------------------------------------------------------------- diff --git a/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Sqoop.java b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Sqoop.java new file mode 100644 index 0000000..124a00f --- /dev/null +++ b/gateway-shell/src/main/java/org/apache/hadoop/gateway/shell/job/Sqoop.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.gateway.shell.job; + +import com.jayway.jsonpath.JsonPath; +import org.apache.hadoop.gateway.shell.AbstractRequest; +import org.apache.hadoop.gateway.shell.BasicResponse; +import org.apache.hadoop.gateway.shell.Hadoop; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; + +public class Sqoop { + + static class Request extends AbstractRequest<Response> { + + private String statusDir; + List<NameValuePair> params = new ArrayList<NameValuePair>(); + + public Request( Hadoop session ) { + super( session ); + } + + public Request command( String command ) { + addParam( params, "command", command ); + return this; + } + + public Request libdir( String libdir ) { + addParam( params, "libdir", libdir ); + return this; + } + + public Request files( String files ) { + addParam( params, "files", files ); + return this; + } + + public Request optionsfile( String optionsFile ) { + addParam( params, "optionsfile", optionsFile ); + return this; + } + + public Request statusDir( String dir ) { + this.statusDir = dir; + return this; + } + + protected Callable<Response> callable() { + return new Callable<Response>() { + @Override + public Response call() throws Exception { + URIBuilder uri = uri( Job.SERVICE_PATH, "/sqoop" ); + addParam( params, "statusdir", statusDir ); + UrlEncodedFormEntity form = new UrlEncodedFormEntity( params ); + HttpPost request = new HttpPost( uri.build() ); + request.setEntity( form ); + return new Response( execute( request ) ); + } + }; + } + + } + + static class Response extends BasicResponse { + + protected Response( HttpResponse response ) { + super( response ); + } + + public String getJobId() throws IOException { + return JsonPath.read( getString(), "$.id" ); + } + + } + +}