Repository: ranger Updated Branches: refs/heads/master 4603dfa3d -> 9fea06dbb
RANGER-1271 Ranger-Atlas testConnection feature Signed-off-by: Velmurugan Periasamy <v...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/306075df Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/306075df Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/306075df Branch: refs/heads/master Commit: 306075df553e295041ef5b6e5b01a8a6799c9654 Parents: 4603dfa Author: zhangqiang2 <zhangqia...@zte.com.cn> Authored: Thu Jan 5 23:15:12 2017 +0800 Committer: Velmurugan Periasamy <v...@apache.org> Committed: Thu Feb 2 23:29:53 2017 -0500 ---------------------------------------------------------------------- .../services/atlas/RangerServiceAtlas.java | 50 +++- .../services/atlas/client/AtlasClient.java | 292 +++++++++++++++++++ .../atlas/client/AtlasConnectionMgr.java | 30 ++ .../services/atlas/client/AtlasResourceMgr.java | 88 ++++++ src/main/assembly/admin-web.xml | 16 +- 5 files changed, 464 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java ---------------------------------------------------------------------- diff --git a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java index cb5a7eb..ba4d263 100644 --- a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java +++ b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java @@ -21,14 +21,15 @@ package org.apache.ranger.services.atlas; import java.util.ArrayList; import java.util.HashMap; import java.util.List; - +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ranger.plugin.client.BaseClient; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.service.RangerBaseService; import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.services.atlas.client.AtlasResourceMgr; + public class RangerServiceAtlas extends RangerBaseService { @@ -44,19 +45,46 @@ public class RangerServiceAtlas extends RangerBaseService { } @Override - public HashMap<String, Object> validateConfig() throws Exception { - if (LOG.isDebugEnabled()) { - LOG.debug("This method will be implemented"); - } + public HashMap<String,Object> validateConfig() throws Exception { HashMap<String, Object> responseMap = new HashMap<String, Object>(); - String msg = "This feature is not available currently"; - BaseClient.generateResponseDataMap(true, msg, msg, null, null, responseMap); + String serviceName = getServiceName(); + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceAtlas.validateConfig Service: (" + serviceName + " )"); + } + if ( configs != null) { + try { + responseMap = AtlasResourceMgr.validateConfig(serviceName, configs); + } catch (Exception e) { + LOG.error("<== RangerServiceAtlas.validateConfig Error:" + e); + throw e; + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceAtlas.validateConfig Response : (" + responseMap + " )"); + } return responseMap; } + @Override public List<String> lookupResource(ResourceLookupContext context) throws Exception { - //This feature is not available currently - return new ArrayList<String>(); - } + List<String> ret = new ArrayList<String>(); + String serviceName = getServiceName(); + Map<String,String> configs = getConfigs(); + if(LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceAtlas.lookupResource Context: (" + context + ")"); + } + if (context != null) { + try { + ret = AtlasResourceMgr.getAtlasTermResources(serviceName,configs,context); + } catch (Exception e) { + LOG.error( "<==RangerServiceAtlas.lookupResource Error : " + e); + throw e; + } + } + if(LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceAtlas.lookupResource Response: (" + ret + ")"); + } + return ret; + } } http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasClient.java ---------------------------------------------------------------------- diff --git a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasClient.java b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasClient.java new file mode 100644 index 0000000..4f90469 --- /dev/null +++ b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasClient.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.atlas.client; + +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; + +import javax.security.auth.Subject; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.NewCookie; + +import org.apache.log4j.Logger; +import org.apache.ranger.plugin.client.BaseClient; +import org.apache.ranger.plugin.client.HadoopException; +import org.apache.ranger.plugin.util.PasswordUtils; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.core.util.MultivaluedMapImpl; + +public class AtlasClient extends BaseClient { + + private static final Logger LOG = Logger.getLogger(AtlasClient.class); + private static final String EXPECTED_MIME_TYPE = "application/json"; + private static final String ATLAS_STATUS_API_ENDPOINT = "/j_spring_security_check"; + private static final String ATLAS_LIST_TERM_API_ENDPOINT = "/api/atlas/types"; + private static final String errMessage = " You can still save the repository and start creating " + + "policies, but you would not be able to use autocomplete for " + + "resource names. Check ranger_admin.log for more info."; + + private String atlasUrl; + private String userName; + private String password; + + public AtlasClient(String serviceName, Map<String, String> configs) { + + super(serviceName,configs,"atlas-client"); + + this.atlasUrl = configs.get("atlas.rest.address"); + this.userName = configs.get("username"); + this.password = configs.get("password"); + if (this.atlasUrl == null || this.atlasUrl.isEmpty()) { + LOG.error("No value found for configuration 'atlas.rest.address'. Atlas resource lookup will fail"); + } + if (this.userName == null || this.userName.isEmpty()) { + LOG.error("No value found for configuration 'usename'. Atlas resource lookup will fail"); + } + if (this.password == null || this.password.isEmpty()) { + LOG.error("No value found for configuration 'password'. Atlas resource lookup will fail"); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Atlas Client is build with url [" + this.atlasUrl + "] user: [" + this.userName + "], password: [" + "*********" + "]"); + } + } + + public List<String> getTermList( String termNameMatching, List<String> existingTermList) { + if (LOG.isDebugEnabled()) { + LOG.debug("Getting Atlas Terms list for termNameMatching : " + termNameMatching); + } + final String errMsg = errMessage; + List<String> ret = null; + + Callable<List<String>> callableAtlasTermListGetter = new Callable<List<String>>() { + + @Override + public List<String> call() { + List<String> atlasTermListGetter = null; + + Subject subj = getLoginSubject(); + + if (subj != null) { + atlasTermListGetter = Subject.doAs(subj, new PrivilegedAction<List<String>>() { + + @Override + public List<String> run() { + + List<String> lret = new ArrayList<String>(); + + String statusUrl = atlasUrl + ATLAS_STATUS_API_ENDPOINT; + String resultUrl = atlasUrl + ATLAS_LIST_TERM_API_ENDPOINT; + + Client client = null; + ClientResponse statusResponse = null; + ClientResponse resultResponse = null; + + try { + client = Client.create(); + WebResource webResource = client.resource(statusUrl); + MultivaluedMap<String, String> formData = new MultivaluedMapImpl(); + formData.add("j_username", userName); + formData.add("j_password",PasswordUtils.decryptPassword(password)); + try { + statusResponse = webResource.type("application/x-www-form-urlencoded").post( + ClientResponse.class, formData); + } catch (Exception e) { + String msgDesc = "Unable to get a valid statusResponse for " + + "expected mime type : [" + EXPECTED_MIME_TYPE + + "] URL : " + statusUrl + " - got null response."; + LOG.error(msgDesc); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("getTermList():calling " + statusUrl); + } + + if (statusResponse != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("getTermList():response.getStatus()= " + statusResponse.getStatus()); + } + if (statusResponse.getStatus() == 200) { + WebResource webResource2 = client + .resource(resultUrl); + WebResource.Builder builder = webResource2.getRequestBuilder(); + for (NewCookie cook : statusResponse.getCookies()) { builder = builder.cookie(cook); + } + resultResponse = builder.get(ClientResponse.class); + lret.add(resultResponse.getEntity(String.class)); + } else{ + LOG.info("getTermList():response.getStatus()= " + statusResponse.getStatus() + " for URL " + statusUrl + ", so returning null list"); + LOG.info(statusResponse.getEntity(String.class)); + lret = null; + } + } + } catch (Throwable t) { + lret = null; + String msgDesc = "Exception while getting Atlas Term List." + + " URL : " + statusUrl; + HadoopException hdpException = new HadoopException(msgDesc, + t); + LOG.error(msgDesc, t); + hdpException.generateResponseDataMap(false, + BaseClient.getMessage(t), msgDesc + errMsg, null, + null); + throw hdpException; + + } finally { + if (statusResponse != null) { + statusResponse.close(); + } + if (resultResponse != null) { + resultResponse.close(); + } + + if (client != null) { + client.destroy(); + } + } + return lret; + } + } ); + } + return atlasTermListGetter; + } + }; + try { + ret = timedTask(callableAtlasTermListGetter, 5, TimeUnit.SECONDS); + } catch ( Throwable t) { + LOG.error("Unable to get Atlas Terms list from [" + atlasUrl + "]", t); + String msgDesc = "Unable to get a valid response for " + + "expected mime type : [" + EXPECTED_MIME_TYPE + + "] URL : " + atlasUrl; + HadoopException hdpException = new HadoopException(msgDesc, + t); + LOG.error(msgDesc, t); + + hdpException.generateResponseDataMap(false, + BaseClient.getMessage(t), msgDesc + errMsg, null, + null); + throw hdpException; + } + return ret; + } + + public static HashMap<String, Object> connectionTest(String serviceName, + Map<String, String> configs) { + + String errMsg = errMessage; + boolean connectivityStatus = false; + HashMap<String, Object> responseData = new HashMap<String, Object>(); + + AtlasClient AtlasClient = getAtlasClient(serviceName, + configs); + List<String> strList = getAtlasTermResource(AtlasClient, "",null); + + if (strList != null && strList.size() > 0 ) { + if (LOG.isDebugEnabled()) { + LOG.debug("TESTING Term list size" + strList.size() + " Atlas Terms"); + } + connectivityStatus = true; + } + + if (connectivityStatus) { + String successMsg = "ConnectionTest Successful"; + BaseClient.generateResponseDataMap(connectivityStatus, successMsg, + successMsg, null, null, responseData); + } else { + String failureMsg = "Unable to retrieve any Atlas Terms using given parameters."; + BaseClient.generateResponseDataMap(connectivityStatus, failureMsg, + failureMsg + errMsg, null, null, responseData); + } + + return responseData; + } + + public static AtlasClient getAtlasClient(String serviceName, + Map<String, String> configs) { + AtlasClient AtlasClient = null; + if (LOG.isDebugEnabled()) { + LOG.debug("Getting AtlasClient for datasource: " + serviceName); + } + String errMsg = errMessage; + if (configs == null || configs.isEmpty()) { + String msgDesc = "Could not connect as Connection ConfigMap is empty."; + LOG.error(msgDesc); + HadoopException hdpException = new HadoopException(msgDesc); + hdpException.generateResponseDataMap(false, msgDesc, msgDesc + + errMsg, null, null); + throw hdpException; + } else { + AtlasClient = new AtlasClient (serviceName, configs); + } + return AtlasClient; + } + + public static List<String> getAtlasTermResource (final AtlasClient atlasClient, + String atlasTermName, List<String> existingAtlasTermName) { + + List<String> resultList = new ArrayList<String>(); + String errMsg = errMessage; + + try { + if (atlasClient == null) { + String msgDesc = "Unable to get Atlas Terms : AtlasClient is null."; + LOG.error(msgDesc); + HadoopException hdpException = new HadoopException(msgDesc); + hdpException.generateResponseDataMap(false, msgDesc, msgDesc + + errMsg, null, null); + throw hdpException; + } + + if (atlasTermName != null) { + String finalAtlasTermName = atlasTermName.trim(); + resultList = atlasClient + .getTermList(finalAtlasTermName,existingAtlasTermName); + if (resultList != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Returning list of " + resultList.size() + " Atlas Terms"); + } + } + } + }catch (Throwable t) { + String msgDesc = "getAtlasResource: Unable to get Atlas resources."; + LOG.error(msgDesc, t); + HadoopException hdpException = new HadoopException(msgDesc); + + hdpException.generateResponseDataMap(false, + BaseClient.getMessage(t), msgDesc + errMsg, null, null); + throw hdpException; + } + + return resultList; + } + + public static <T> T timedTask(Callable<T> callableObj, long timeout, + TimeUnit timeUnit) throws Exception { + return callableObj.call(); + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasConnectionMgr.java ---------------------------------------------------------------------- diff --git a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasConnectionMgr.java b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasConnectionMgr.java new file mode 100644 index 0000000..84e24c5 --- /dev/null +++ b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasConnectionMgr.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.atlas.client; + +import java.util.Map; + +public class AtlasConnectionMgr { + + public static AtlasClient getAtlasClient(String serviceName, Map<String, String> configs) { + return new AtlasClient(serviceName, configs); + } + +} http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasResourceMgr.java ---------------------------------------------------------------------- diff --git a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasResourceMgr.java b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasResourceMgr.java new file mode 100644 index 0000000..f738601 --- /dev/null +++ b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasResourceMgr.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.atlas.client; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.apache.ranger.plugin.service.ResourceLookupContext; + + +public class AtlasResourceMgr { + public static final Logger LOG = Logger.getLogger(AtlasResourceMgr.class); + private static final String ATLASTERM = "terms"; + + public static HashMap<String, Object> validateConfig(String serviceName, Map<String, String> configs) throws Exception { + + HashMap<String, Object> ret = null; + + if(LOG.isDebugEnabled()) { + LOG.debug("==> AtlasResourceMgr.validateConfig ServiceName: "+ serviceName + "Configs" + configs ); + } + + try { + ret = AtlasClient.connectionTest(serviceName, configs); + } catch (Exception e) { + LOG.error("<== AtlasResourceMgr.validateConfig Error: " + e); + throw e; + } + + if(LOG.isDebugEnabled()) { + LOG.debug("<== AtlasResourceMgr.validateConfig Result : "+ ret ); + } + return ret; + } + + public static List<String> getAtlasTermResources(String serviceName, Map<String, String> configs,ResourceLookupContext context) { + String userInput = context.getUserInput(); + Map<String, List<String>> resourceMap = context.getResources(); + List<String> resultList = null; + List<String> atlasTermList = null; + String atlasTermName = null; + + if ( resourceMap != null && !resourceMap.isEmpty() && + resourceMap.get(ATLASTERM) != null ) { + atlasTermName = userInput; + atlasTermList = resourceMap.get(ATLASTERM); + } else { + atlasTermName = userInput; + } + + if (configs == null || configs.isEmpty()) { + LOG.error("Connection Config is empty"); + } else { + resultList = getAtlasTermResource(serviceName, configs, atlasTermName,atlasTermList); + } + return resultList; + } + + public static List<String> getAtlasTermResource(String serviceName, Map<String, String> configs, String atlasTermName, List<String> atlasTermList) { + final AtlasClient atlasClient = AtlasConnectionMgr.getAtlasClient(serviceName, configs); + List<String> termList = null; + if (atlasClient != null) { + synchronized(atlasClient) { + termList = atlasClient.getTermList(atlasTermName, atlasTermList); + } + } + return termList; + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/src/main/assembly/admin-web.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml index 4779071..966033f 100644 --- a/src/main/assembly/admin-web.xml +++ b/src/main/assembly/admin-web.xml @@ -310,7 +310,21 @@ <include>org.apache.ranger:ranger-nifi-plugin</include> </includes> </moduleSet> - </moduleSets> + + <moduleSet> + <binaries> + <includeDependencies>true</includeDependencies> + <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/atlas</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + </binaries> + <includes> + <include>org.apache.ranger:ranger-atlas-plugin</include> + </includes> + </moduleSet> + + </moduleSets> <fileSets> <fileSet>
