AMBARI-9751. Ambari Views: Kerberos support in Pig View (alexantonenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/57645a81 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/57645a81 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/57645a81 Branch: refs/heads/trunk Commit: 57645a814fea816eeea09e37d392d37988823f79 Parents: 3716c55 Author: Alex Antonenko <hiv...@gmail.com> Authored: Mon Feb 23 19:01:49 2015 +0200 Committer: Alex Antonenko <hiv...@gmail.com> Committed: Mon Feb 23 19:40:49 2015 +0200 ---------------------------------------------------------------------- .../pig/resources/jobs/JobResourceManager.java | 2 +- .../view/pig/templeton/client/JSONRequest.java | 278 +++++++++++++++++++ .../pig/templeton/client/RequestWrapper.java | 271 ------------------ .../view/pig/templeton/client/TempletonApi.java | 31 +-- .../pig/templeton/client/TempletonRequest.java | 99 ------- .../apache/ambari/view/pig/utils/HdfsApi.java | 56 +++- contrib/views/pig/src/main/resources/view.xml | 7 + .../org/apache/ambari/view/pig/BasePigTest.java | 1 + .../org/apache/ambari/view/pig/HDFSTest.java | 9 + .../view/pig/test/ScriptTestHDFSUnmanaged.java | 5 +- 10 files changed, 361 insertions(+), 398 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java index 0230625..19cf69f 100644 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/resources/jobs/JobResourceManager.java @@ -332,7 +332,7 @@ public class JobResourceManager extends PersonalCRUDResourceManager<PigJob> { throw new MisconfigurationFormattedException("webhcat.url"); } return new TempletonApi(context.getProperties().get("webhcat.url"), - getTempletonUser(context), getTempletonUser(context), context); + getTempletonUser(context), context); } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/JSONRequest.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/JSONRequest.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/JSONRequest.java new file mode 100644 index 0000000..39a595b --- /dev/null +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/JSONRequest.java @@ -0,0 +1,278 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.view.pig.templeton.client; + +import com.google.gson.Gson; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.core.util.MultivaluedMapImpl; +import org.apache.ambari.view.ViewContext; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.core.UriBuilder; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * Request handler, supports GET, POST, PUT, DELETE methods + * @param <RESPONSE> data type to deserialize response from JSON + */ +public class JSONRequest<RESPONSE> { + protected final Class<RESPONSE> responseClass; + protected final ViewContext context; + protected final WebResource resource; + private String username; + private String doAs; + + protected final Gson gson = new Gson(); + + protected final static Logger LOG = + LoggerFactory.getLogger(JSONRequest.class); + + /** + * Constructor + * @param resource object that represents resource + * @param responseClass model class + * @param username will be used to auth as proxyuser + * @param doAs run query from this user + * @param context View Context instance + */ + public JSONRequest(WebResource resource, Class<RESPONSE> responseClass, String username, String doAs, ViewContext context) { + this.resource = resource; + this.responseClass = responseClass; + this.username = username; + this.context = context; + this.doAs = doAs; + } + + /** + * Main implementation of GET request + * @param resource resource + * @return unmarshalled response data + */ + public RESPONSE get(WebResource resource) throws IOException { + LOG.debug("GET " + resource.toString()); + + InputStream inputStream = readFrom(resource, "GET", null, new HashMap<String, String>()); + + recordLastCurlCommand(String.format("curl \"" + resource.toString() + "\"")); + String responseJson = IOUtils.toString(inputStream); + LOG.debug(String.format("RESPONSE => %s", responseJson)); + return gson.fromJson(responseJson, responseClass); + } + + /** + * Make GET request + * @see #get(WebResource) + */ + public RESPONSE get() throws IOException { + return get(this.resource); + } + + /** + * Make GET request + * @see #get(WebResource) + */ + public RESPONSE get(MultivaluedMapImpl params) throws IOException { + return get(this.resource.queryParams(params)); + } + + /** + * Main implementation of POST request + * @param resource resource + * @param data post body + * @return unmarshalled response data + */ + public RESPONSE post(WebResource resource, MultivaluedMapImpl data) throws IOException { + LOG.debug("POST " + resource.toString()); + LOG.debug("data: " + data.toString()); + + StringBuilder curlBuilder = new StringBuilder(); + + UriBuilder builder = getUriBuilder(data, curlBuilder); + + Map<String, String> headers = new HashMap<String, String>(); + headers.put("Content-Type", "application/x-www-form-urlencoded"); + + recordLastCurlCommand(String.format("curl " + curlBuilder.toString() + " \"" + resource.toString() + "\"")); + InputStream inputStream = readFrom(resource, "POST", builder.build().getRawQuery(), headers); + String responseJson = IOUtils.toString(inputStream); + + LOG.debug(String.format("RESPONSE => %s", responseJson)); + return gson.fromJson(responseJson, responseClass); + } + + /** + * @see #post(WebResource, MultivaluedMapImpl) + */ + public RESPONSE post(MultivaluedMapImpl data) throws IOException { + return post(resource, data); + } + + /** + * @see #post(WebResource, MultivaluedMapImpl) + */ + public RESPONSE post() throws IOException { + return post(resource, new MultivaluedMapImpl()); + } + + /** + * @see #post(WebResource, MultivaluedMapImpl) + */ + public RESPONSE post(MultivaluedMapImpl params, MultivaluedMapImpl data) throws IOException { + return post(resource.queryParams(params), data); + } + + /** + * Main implementation of PUT request + * @param resource resource + * @param data put body + * @return unmarshalled response data + */ + public RESPONSE put(WebResource resource, MultivaluedMapImpl data) throws IOException { + LOG.debug("PUT " + resource.toString()); + + StringBuilder curlBuilder = new StringBuilder(); + + UriBuilder builder = getUriBuilder(data, curlBuilder); + + Map<String, String> headers = new HashMap<String, String>(); + headers.put("Content-Type", "application/x-www-form-urlencoded"); + + recordLastCurlCommand(String.format("curl -X PUT " + curlBuilder.toString() + " \"" + resource.toString() + "\"")); + + InputStream inputStream = readFrom(resource, "PUT", builder.build().getRawQuery(), headers); + String responseJson = IOUtils.toString(inputStream); + + LOG.debug(String.format("RESPONSE => %s", responseJson)); + return gson.fromJson(responseJson, responseClass); + } + + public UriBuilder getUriBuilder(MultivaluedMapImpl data, StringBuilder curlBuilder) { + MultivaluedMapImpl effectiveData; + if (data == null) + effectiveData = new MultivaluedMapImpl(); + else + effectiveData = new MultivaluedMapImpl(data); + + effectiveData.putSingle("user.name", username); + effectiveData.putSingle("doAs", doAs); + + UriBuilder builder = UriBuilder.fromPath("host/"); + for(String key : effectiveData.keySet()) { + for(String value : effectiveData.get(key)) { + builder.queryParam(key, value); + curlBuilder.append(String.format("-d %s=\"%s\" ", key, value.replace("\"", "\\\""))); + } + } + + if (data != null) + LOG.debug("... data: " + builder.build().getRawQuery()); + return builder; + } + + /** + * @see #put(WebResource, MultivaluedMapImpl) + */ + public RESPONSE put(MultivaluedMapImpl data) throws IOException { + return put(resource, data); + } + + /** + * @see #put(WebResource, MultivaluedMapImpl) + */ + public RESPONSE put() throws IOException { + return put(resource, new MultivaluedMapImpl()); + } + + /** + * @see #put(WebResource, MultivaluedMapImpl) + */ + public RESPONSE put(MultivaluedMapImpl params, MultivaluedMapImpl data) throws IOException { + return put(resource.queryParams(params), data); + } + + /** + * Main implementation of DELETE request + * @param resource resource + * @param data delete body + * @return unmarshalled response data + */ + public RESPONSE delete(WebResource resource, MultivaluedMapImpl data) throws IOException { + LOG.debug("DELETE " + resource.toString()); + + StringBuilder curlBuilder = new StringBuilder(); + + UriBuilder builder = getUriBuilder(data, curlBuilder); + + Map<String, String> headers = new HashMap<String, String>(); + headers.put("Content-Type", "application/x-www-form-urlencoded"); + + recordLastCurlCommand(String.format("curl -X DELETE " + curlBuilder.toString() + " \"" + resource.toString() + "\"")); + + InputStream inputStream = readFrom(resource, "DELETE", builder.build().getRawQuery(), headers); + String responseJson = IOUtils.toString(inputStream); + + LOG.debug(String.format("RESPONSE => %s", responseJson)); + return gson.fromJson(responseJson, responseClass); + } + + /** + * @see #delete(WebResource, MultivaluedMapImpl) + */ + public RESPONSE delete(MultivaluedMapImpl data) throws IOException { + return delete(resource, data); + } + + /** + * @see #delete(WebResource, MultivaluedMapImpl) + */ + public RESPONSE delete() throws IOException { + return delete(resource, new MultivaluedMapImpl()); + } + + /** + * @see #delete(WebResource, MultivaluedMapImpl) + */ + public RESPONSE delete(MultivaluedMapImpl params, MultivaluedMapImpl data) throws IOException { + return delete(resource.queryParams(params), data); + } + + private static void recordLastCurlCommand(String curl) { + LOG.info(curl); + } + + public InputStream readFrom(WebResource resource, String method, String body, Map<String, String> headers) throws IOException { + InputStream inputStream; + resource = resource.queryParam("user.name", username) + .queryParam("doAs", doAs); + + if (doAs == null) { + inputStream = context.getURLStreamProvider().readFrom(resource.toString(), + method, body, headers); + } else { + inputStream = context.getURLStreamProvider().readAs(resource.toString(), + method, body, headers, doAs); + } + return inputStream; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/RequestWrapper.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/RequestWrapper.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/RequestWrapper.java deleted file mode 100644 index efd2074..0000000 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/RequestWrapper.java +++ /dev/null @@ -1,271 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.view.pig.templeton.client; - -import com.google.gson.Gson; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.core.util.MultivaluedMapImpl; -import org.apache.ambari.view.ViewContext; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.core.UriBuilder; -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; - -/** - * Request handler, supports GET, POST, PUT, DELETE methods - * @param <RESPONSE> data type to deserialize response from JSON - */ -public class RequestWrapper<RESPONSE> { - protected final Class<RESPONSE> responseClass; - protected final ViewContext context; - protected final WebResource resource; - - protected final Gson gson = new Gson(); - - protected final static Logger LOG = - LoggerFactory.getLogger(RequestWrapper.class); - - /** - * Constructor - * @param resource object that represents resource - * @param responseClass model class - * @param context View Context instance - */ - public RequestWrapper(WebResource resource, Class<RESPONSE> responseClass, ViewContext context) { - this.resource = resource; - this.responseClass = responseClass; - this.context = context; - } - - /** - * Main implementation of GET request - * @param resource resource - * @return unmarshalled response data - */ - public RESPONSE get(WebResource resource) throws IOException { - LOG.debug("GET " + resource.toString()); - - InputStream inputStream = context.getURLStreamProvider().readFrom(resource.toString(), "GET", - null, new HashMap<String, String>()); - - recordLastCurlCommand(String.format("curl \"" + resource.toString() + "\"")); - String responseJson = IOUtils.toString(inputStream); - LOG.debug(String.format("RESPONSE => %s", responseJson)); - return gson.fromJson(responseJson, responseClass); - } - - /** - * Make GET request - * @see #get(WebResource) - */ - public RESPONSE get() throws IOException { - return get(this.resource); - } - - /** - * Make GET request - * @see #get(WebResource) - */ - public RESPONSE get(MultivaluedMapImpl params) throws IOException { - return get(this.resource.queryParams(params)); - } - - /** - * Main implementation of POST request - * @param resource resource - * @param data post body - * @return unmarshalled response data - */ - public RESPONSE post(WebResource resource, MultivaluedMapImpl data) throws IOException { - LOG.debug("POST " + resource.toString()); - LOG.debug("data: " + data.toString()); - - StringBuilder curlBuilder = new StringBuilder(); - - UriBuilder builder = UriBuilder.fromPath("host/"); - for(String key : data.keySet()) { - for(String value : data.get(key)) { - builder.queryParam(key, value); - curlBuilder.append(String.format("-d %s=\"%s\"", key, value.replace("\"", "\\\""))); - } - } - - if (data != null) - LOG.debug("... data: " + builder.build().getRawQuery()); - - Map<String, String> headers = new HashMap<String, String>(); - headers.put("Content-Type", "application/x-www-form-urlencoded"); - - recordLastCurlCommand(String.format("curl " + curlBuilder.toString() + " \"" + resource.toString() + "\"")); - InputStream inputStream = context.getURLStreamProvider().readFrom(resource.toString(), - "POST", builder.build().getRawQuery(), headers); - String responseJson = IOUtils.toString(inputStream); - - LOG.debug(String.format("RESPONSE => %s", responseJson)); - return gson.fromJson(responseJson, responseClass); - } - - /** - * @see #post(WebResource, MultivaluedMapImpl) - */ - public RESPONSE post(MultivaluedMapImpl data) throws IOException { - return post(resource, data); - } - - /** - * @see #post(WebResource, MultivaluedMapImpl) - */ - public RESPONSE post() throws IOException { - return post(resource, new MultivaluedMapImpl()); - } - - /** - * @see #post(WebResource, MultivaluedMapImpl) - */ - public RESPONSE post(MultivaluedMapImpl params, MultivaluedMapImpl data) throws IOException { - return post(resource.queryParams(params), data); - } - - /** - * Main implementation of PUT request - * @param resource resource - * @param data put body - * @return unmarshalled response data - */ - public RESPONSE put(WebResource resource, MultivaluedMapImpl data) throws IOException { - LOG.debug("PUT " + resource.toString()); - - StringBuilder curlBuilder = new StringBuilder(); - - UriBuilder builder = UriBuilder.fromPath("host/"); - for(String key : data.keySet()) { - for(String value : data.get(key)) { - builder.queryParam(key, value); - curlBuilder.append(String.format("-d %s=\"%s\"", key, value.replace("\"", "\\\""))); - } - } - - if (data != null) - LOG.debug("... data: " + builder.build().getRawQuery()); - - Map<String, String> headers = new HashMap<String, String>(); - headers.put("Content-Type", "application/x-www-form-urlencoded"); - - recordLastCurlCommand(String.format("curl -X PUT " + curlBuilder.toString() + " \"" + resource.toString() + "\"")); - - InputStream inputStream = context.getURLStreamProvider().readFrom(resource.toString(), - "PUT", builder.build().getRawQuery(), headers); - String responseJson = IOUtils.toString(inputStream); - - LOG.debug(String.format("RESPONSE => %s", responseJson)); - return gson.fromJson(responseJson, responseClass); - } - - /** - * @see #put(WebResource, MultivaluedMapImpl) - */ - public RESPONSE put(MultivaluedMapImpl data) throws IOException { - return put(resource, data); - } - - /** - * @see #put(WebResource, MultivaluedMapImpl) - */ - public RESPONSE put() throws IOException { - return put(resource, new MultivaluedMapImpl()); - } - - /** - * @see #put(WebResource, MultivaluedMapImpl) - */ - public RESPONSE put(MultivaluedMapImpl params, MultivaluedMapImpl data) throws IOException { - return put(resource.queryParams(params), data); - } - - /** - * Main implementation of DELETE request - * @param resource resource - * @param data delete body - * @return unmarshalled response data - */ - public RESPONSE delete(WebResource resource, MultivaluedMapImpl data) throws IOException { - LOG.debug("DELETE " + resource.toString()); - - StringBuilder curlBuilder = new StringBuilder(); - - UriBuilder builder = UriBuilder.fromPath("host/"); - for(String key : data.keySet()) { - for(String value : data.get(key)) { - builder.queryParam(key, value); - curlBuilder.append(String.format("-d %s=\"%s\"", key, value.replace("\"", "\\\""))); - } - } - - if (data != null) - LOG.debug("... data: " + builder.build().getRawQuery()); - - Map<String, String> headers = new HashMap<String, String>(); - headers.put("Content-Type", "application/x-www-form-urlencoded"); - - recordLastCurlCommand(String.format("curl -X DELETE " + curlBuilder.toString() + " \"" + resource.toString() + "\"")); - - InputStream inputStream = context.getURLStreamProvider().readFrom(resource.toString(), - "DELETE", builder.build().getRawQuery(), headers); - String responseJson = IOUtils.toString(inputStream); - - LOG.debug(String.format("RESPONSE => %s", responseJson)); - return gson.fromJson(responseJson, responseClass); - } - - /** - * @see #delete(WebResource, MultivaluedMapImpl) - */ - public RESPONSE delete(MultivaluedMapImpl data) throws IOException { - return delete(resource, data); - } - - /** - * @see #delete(WebResource, MultivaluedMapImpl) - */ - public RESPONSE delete() throws IOException { - return delete(resource, new MultivaluedMapImpl()); - } - - /** - * @see #delete(WebResource, MultivaluedMapImpl) - */ - public RESPONSE delete(MultivaluedMapImpl params, MultivaluedMapImpl data) throws IOException { - return delete(resource.queryParams(params), data); - } - - private static String lastCurlCommand = null; - private static void recordLastCurlCommand(String curl) { - LOG.info(curl); - lastCurlCommand = curl; - } - - public static String getLastCurlCommand() { - return lastCurlCommand; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java index d45b55c..497ec82 100644 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonApi.java @@ -20,7 +20,6 @@ package org.apache.ambari.view.pig.templeton.client; import com.google.gson.Gson; import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; @@ -30,7 +29,6 @@ import org.apache.ambari.view.ViewContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.core.MediaType; import java.io.File; import java.io.IOException; import java.util.Map; @@ -46,19 +44,16 @@ public class TempletonApi { LoggerFactory.getLogger(TempletonApi.class); protected WebResource service; - private String username; private String doAs; private ViewContext context; /** * TempletonApi constructor * @param api webhcat.url - * @param username templeton username * @param doAs doAs argument * @param context context with URLStreamProvider */ - public TempletonApi(String api, String username, String doAs, ViewContext context) { - this.username = username; + public TempletonApi(String api, String doAs, ViewContext context) { this.doAs = doAs; this.context = context; ClientConfig config = new DefaultClientConfig(); @@ -68,10 +63,10 @@ public class TempletonApi { } /** - * @see #TempletonApi(String,String,String,ViewContext) + * @see #TempletonApi(String,String,ViewContext) */ - public TempletonApi(String api, String username, ViewContext context) { - this(api, username, username, context); + public TempletonApi(String api, ViewContext context) { + this(api, null, context); } /** @@ -98,8 +93,8 @@ public class TempletonApi { } } - TempletonRequest<JobData> request = - new TempletonRequest<JobData>(service.path("pig"), JobData.class, username, doAs, context); + JSONRequest<JobData> request = + new JSONRequest<JobData>(service.path("pig"), JobData.class, doAs, doAs, context); //FIXME: configurable proxyuser return request.post(data); } @@ -132,8 +127,8 @@ public class TempletonApi { * @throws IOException */ public JobInfo checkJob(String jobId) throws IOException { - TempletonRequest<JobInfo> request = - new TempletonRequest<JobInfo>(service.path("jobs").path(jobId), JobInfo.class, username, context); + JSONRequest<JobInfo> request = + new JSONRequest<JobInfo>(service.path("jobs").path(jobId), JobInfo.class, doAs, doAs, context); return request.get(); } @@ -144,8 +139,8 @@ public class TempletonApi { * @throws IOException */ public void killJob(String jobId) throws IOException { - TempletonRequest<JobInfo> request = - new TempletonRequest<JobInfo>(service.path("jobs").path(jobId), JobInfo.class, username, context); + JSONRequest<JobInfo> request = + new JSONRequest<JobInfo>(service.path("jobs").path(jobId), JobInfo.class, doAs, doAs, context); try { request.delete(); @@ -161,9 +156,9 @@ public class TempletonApi { * @throws IOException */ public Status status() throws IOException { - TempletonRequest<Status> request = - new TempletonRequest<Status>(service.path("status"), Status.class, - username, doAs, context); + JSONRequest<Status> request = + new JSONRequest<Status>(service.path("status"), Status.class, + doAs, doAs, context); return request.get(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonRequest.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonRequest.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonRequest.java deleted file mode 100644 index 10cfdd5..0000000 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/templeton/client/TempletonRequest.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ambari.view.pig.templeton.client; - -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.core.util.MultivaluedMapImpl; -import org.apache.ambari.view.ViewContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -/** - * Request handler that adds user.name and doAs - * GET parameters to every request - * @param <RESPONSE> data type to deserialize response from JSON - */ -public class TempletonRequest<RESPONSE> extends RequestWrapper<RESPONSE> { - private String username; - private String doAs; - - protected final static Logger LOG = - LoggerFactory.getLogger(TempletonRequest.class); - - /** - * Constructor - * @param resource object that represents resource - * @param responseClass model class - * @param context View Context instance - * @param username user.name of templeton. user.name will be equal to doAs value - */ - public TempletonRequest(WebResource resource, Class<RESPONSE> responseClass, - String username, ViewContext context) { - this(resource, responseClass, username, username, context); - } - - /** - * Constructor - * @param resource object that represents resource - * @param responseClass model class - * @param context View Context instance - * @param username user.name of templeton - * @param doAs doAs user for templeton - */ - public TempletonRequest(WebResource resource, Class<RESPONSE> responseClass, - String username, String doAs, ViewContext context) { - super(resource, responseClass, context); - this.username = username; - this.doAs = doAs; - } - - @Override - public RESPONSE get(WebResource resource) throws IOException { - MultivaluedMapImpl params = new MultivaluedMapImpl(); - params.add("user.name", username); - params.add("doAs", doAs); - return super.get(resource.queryParams(params)); - } - - @Override - public RESPONSE put(WebResource resource, MultivaluedMapImpl data) throws IOException { - MultivaluedMapImpl params = new MultivaluedMapImpl(); - params.add("user.name", username); - params.add("doAs", doAs); - return super.put(resource.queryParams(params), data); - } - - @Override - public RESPONSE delete(WebResource resource, MultivaluedMapImpl data) throws IOException { - MultivaluedMapImpl params = new MultivaluedMapImpl(); - params.add("user.name", username); - params.add("doAs", doAs); - return super.delete(resource.queryParams(params), data); - } - - @Override - public RESPONSE post(WebResource resource, MultivaluedMapImpl data) throws IOException { - MultivaluedMapImpl params = new MultivaluedMapImpl(); - params.add("user.name", username); - params.add("doAs", doAs); - return super.post(resource.queryParams(params), data); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java index f9dfb35..7af37ad 100644 --- a/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java +++ b/contrib/views/pig/src/main/java/org/apache/ambari/view/pig/utils/HdfsApi.java @@ -51,23 +51,52 @@ public class HdfsApi { private final static Logger LOG = LoggerFactory.getLogger(HdfsApi.class); + private Map<String, String> params; /** * Constructor * @param defaultFs hdfs uri * @param username user.name + * @param params map of parameters * @throws IOException * @throws InterruptedException */ - public HdfsApi(String defaultFs, String username) throws IOException, + public HdfsApi(final String defaultFs, String username, Map<String, String> params) throws IOException, InterruptedException { + this.params = params; + Thread.currentThread().setContextClassLoader(null); conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName()); conf.set("fs.webhdfs.impl", WebHdfsFileSystem.class.getName()); conf.set("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); - fs = FileSystem.get(URI.create(defaultFs), conf); - ugi = UserGroupInformation.createProxyUser(username, - UserGroupInformation.getLoginUser()); + ugi = UserGroupInformation.createProxyUser(username, getProxyUser()); + fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() { + public FileSystem run() throws IOException { + return FileSystem.get(URI.create(defaultFs), conf); + } + }); + } + + private UserGroupInformation getProxyUser() throws IOException { + UserGroupInformation proxyuser; + if (params.containsKey("proxyuser")) { + proxyuser = UserGroupInformation.createRemoteUser(params.get("proxyuser")); + } else { + proxyuser = UserGroupInformation.getCurrentUser(); + } + + proxyuser.setAuthenticationMethod(getAuthenticationMethod()); + return proxyuser; + } + + private UserGroupInformation.AuthenticationMethod getAuthenticationMethod() { + UserGroupInformation.AuthenticationMethod authMethod; + if (params.containsKey("auth")) { + authMethod = UserGroupInformation.AuthenticationMethod.valueOf(params.get("auth")); + } else { + authMethod = UserGroupInformation.AuthenticationMethod.SIMPLE; + } + return authMethod; } /** @@ -320,7 +349,7 @@ public class HdfsApi { } try { - api = new HdfsApi(defaultFS, getHdfsUsername(context)); + api = new HdfsApi(defaultFS, getHdfsUsername(context), getHdfsAuthParams(context)); LOG.info("HdfsApi connected OK"); } catch (IOException e) { String message = "HdfsApi IO error: " + e.getMessage(); @@ -334,6 +363,23 @@ public class HdfsApi { return api; } + private static Map<String, String> getHdfsAuthParams(ViewContext context) { + String auth = context.getProperties().get("webhdfs.auth"); + Map<String, String> params = new HashMap<String, String>(); + if (auth == null || auth.isEmpty()) { + auth = "auth=SIMPLE"; + } + for(String param : auth.split(";")) { + String[] keyvalue = param.split("="); + if (keyvalue.length != 2) { + LOG.error("Can not parse authentication param " + param + " in " + auth); + continue; + } + params.put(keyvalue[0], keyvalue[1]); + } + return params; + } + public static String getHdfsUsername(ViewContext context) { String userName = context.getProperties().get("webhdfs.username"); if (userName == null || userName.compareTo("null") == 0 || userName.compareTo("") == 0) http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/main/resources/view.xml ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/main/resources/view.xml b/contrib/views/pig/src/main/resources/view.xml index 91b604e..6128dc4 100644 --- a/contrib/views/pig/src/main/resources/view.xml +++ b/contrib/views/pig/src/main/resources/view.xml @@ -36,6 +36,13 @@ <required>false</required> </parameter> + <parameter> + <name>webhdfs.auth</name> + <description>Semicolon-separated authentication configs. Default: auth=SIMPLE</description> + <label>WebHDFS Authentication</label> + <required>false</required> + </parameter> + <!-- WebHCat Configs --> <parameter> <name>webhcat.url</name> http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java index ff34247..cd9acad 100644 --- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java +++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/BasePigTest.java @@ -24,6 +24,7 @@ import com.google.inject.Injector; import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.ViewResourceHandler; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.*; import java.io.File; http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/HDFSTest.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/HDFSTest.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/HDFSTest.java index 37b316f..851b413 100644 --- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/HDFSTest.java +++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/HDFSTest.java @@ -20,7 +20,11 @@ package org.apache.ambari.view.pig; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.UserGroupInformation; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -40,10 +44,14 @@ public abstract class HDFSTest extends BasePigTest { Configuration conf = new Configuration(); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath()); + conf.set("hadoop.proxyuser." + System.getProperty("user.name") + ".groups", "*"); + conf.set("hadoop.proxyuser." + System.getProperty("user.name") + ".hosts", "*"); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); hdfsCluster = builder.build(); hdfsURI = hdfsCluster.getURI().toString(); + hdfsCluster.getFileSystem().mkdir(new Path("/tmp"), FsPermission.getDefault()); + hdfsCluster.getFileSystem().setPermission(new Path("/tmp"), new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); } @AfterClass @@ -57,5 +65,6 @@ public abstract class HDFSTest extends BasePigTest { protected void setupProperties(Map<String, String> properties, File baseDir) throws Exception { super.setupProperties(properties, baseDir); properties.put("webhdfs.url", hdfsURI); + properties.put("webhdfs.username", System.getProperty("user.name")); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/57645a81/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java ---------------------------------------------------------------------- diff --git a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java index 2db4f18..32f2763 100644 --- a/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java +++ b/contrib/views/pig/src/test/java/org/apache/ambari/view/pig/test/ScriptTestHDFSUnmanaged.java @@ -23,16 +23,12 @@ import org.apache.ambari.view.ViewContext; import org.apache.ambari.view.ViewResourceHandler; import org.apache.ambari.view.pig.HDFSTest; import org.apache.ambari.view.pig.persistence.DataStoreStorage; -import org.apache.ambari.view.pig.persistence.InstanceKeyValueStorage; import org.apache.ambari.view.pig.persistence.Storage; -import org.apache.ambari.view.pig.resources.files.FileService; import org.apache.ambari.view.pig.resources.scripts.ScriptService; import org.apache.ambari.view.pig.persistence.utils.StorageUtil; import org.apache.ambari.view.pig.resources.scripts.models.PigScript; -import org.apache.ambari.view.pig.services.BaseService; import org.apache.ambari.view.pig.utils.HdfsApi; import org.apache.ambari.view.pig.utils.MisconfigurationFormattedException; -import org.apache.ambari.view.pig.utils.NotFoundFormattedException; import org.json.simple.JSONObject; import org.junit.*; import org.junit.rules.ExpectedException; @@ -64,6 +60,7 @@ public class ScriptTestHDFSUnmanaged extends HDFSTest { @Override @Before public void setUp() throws Exception { + super.setUp(); handler = createNiceMock(ViewResourceHandler.class); context = createNiceMock(ViewContext.class); HdfsApi.dropAllConnections();
