This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new f510319fb RANGER-3855: added RangerMultiSourceUserStoreRetriever
implementation
f510319fb is described below
commit f510319fb23bc23c71e08780e0b59d502b9590d3
Author: Eckman, Barbara <barbara_eck...@cable.comcast.com>
AuthorDate: Thu Nov 17 16:11:45 2022 -0500
RANGER-3855: added RangerMultiSourceUserStoreRetriever implementation
Signed-off-by: Madhan Neethiraj <mad...@apache.org>
---
.../externalretrievers/GetFromDataFile.java | 75 +++++
.../externalretrievers/GetFromURL.java | 224 +++++++++++++
.../contextenricher/externalretrievers/LICENSE | 202 ++++++++++++
.../contextenricher/externalretrievers/NOTICE | 18 +
.../contextenricher/externalretrievers/README.md | 137 ++++++++
.../RangerMultiSourceUserStoreRetriever.java | 365 +++++++++++++++++++++
.../ranger/plugin/util/RangerRolesProvider.java | 2 +-
.../apache/ranger/plugin/util/RangerRolesUtil.java | 2 +-
dev-support/spotbugsIncludeFile.xml | 1 +
9 files changed, 1024 insertions(+), 2 deletions(-)
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/GetFromDataFile.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/GetFromDataFile.java
new file mode 100644
index 000000000..93cf38aac
--- /dev/null
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/GetFromDataFile.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.contextenricher.externalretrievers;
+
+import org.apache.ranger.plugin.contextenricher.RangerAbstractContextEnricher;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+public class GetFromDataFile {
+ private static final Logger LOG =
LoggerFactory.getLogger(GetFromDataFile.class);
+
+ public Map<String, Map<String, String>> getFromDataFile(String dataFile,
String attrName) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> getFromDataFile(dataFile={}, attrName={})",
dataFile, attrName);
+ }
+
+ Map<String, Map<String, String>> ret = new HashMap<>();
+
+ // create an instance so that readProperties() can be used!
+ RangerAbstractContextEnricher ce = new RangerAbstractContextEnricher()
{
+ @Override
+ public void enrich(RangerAccessRequest rangerAccessRequest) {
+ }
+ };
+
+ Properties prop = ce.readProperties(dataFile);
+
+ if (prop == null) {
+ LOG.warn("getFromDataFile({}, {}): failed to read file", dataFile,
attrName);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("read from datafile {}: {}", dataFile, prop);
+ }
+
+ // reformat UserAttrsProp into UserStore format:
+ // format of UserAttrsProp: Map<String, String>
+ // format of UserStore: Map<String, Map<String, String>>
+ for (String user : prop.stringPropertyNames()) {
+ Map<String, String> userAttrs = new HashMap<>();
+
+ userAttrs.put(attrName, prop.getProperty(user));
+
+ ret.put(user, userAttrs);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getFromDataFile(dataFile={}, attrName={}): ret={}",
dataFile, attrName, ret);
+ }
+
+ return ret;
+ }
+}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/GetFromURL.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/GetFromURL.java
new file mode 100644
index 000000000..f9eae3574
--- /dev/null
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/GetFromURL.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.contextenricher.externalretrievers;
+
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.client.methods.RequestBuilder;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class GetFromURL {
+ private static final Logger LOG =
LoggerFactory.getLogger(GetFromURL.class);
+
+ private final Gson gson = new Gson();
+
+ public Map<String, Map<String, String>> getFromURL(String url, String
configFile) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> getFromURL(url={}, configFile={})", url,
configFile);
+ }
+
+ String token = getBearerToken(configFile);
+ HttpUriRequest request = RequestBuilder.get().setUri(url)
+
.setHeader(HttpHeaders.AUTHORIZATION, token)
+
.setHeader(HttpHeaders.CONTENT_TYPE, "text/plain")
+ .build();
+ Map<String, Map<String, String>> ret;
+
+ try (CloseableHttpClient httpClient = HttpClients.createDefault();
+ CloseableHttpResponse response = httpClient.execute(request)) {
+ if (response == null) {
+ throw new IOException("getFromURL(" + url + ") failed: null
response");
+ }
+
+ int statusCode = response.getStatusLine().getStatusCode();
+
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new IOException("getFromURL(" + url + ") failed: http
status=" + response.getStatusLine());
+ }
+
+ HttpEntity httpEntity =
response.getEntity();
+ String stringResult =
EntityUtils.toString(httpEntity);
+ Map resultMap =
gson.fromJson(stringResult, Map.class);
+ Map<String, Map<String, List<String>>> userAttrValues =
(Map<String, Map<String, List<String>>>) resultMap.get("body");
+
+ ret = toUserAttributes(userAttrValues);
+
+ // and ensure response body is fully consumed
+ EntityUtils.consume(httpEntity);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getFromURL(url={}, configFile={}): ret={}", url,
configFile, ret);
+ }
+
+ return ret;
+ }
+
+ private String getBearerToken(String configFile) throws Exception {
+ String secrets = getSecretsFromFile(configFile);
+ JsonObject jsonObject = gson.fromJson(secrets,
JsonObject.class);
+ String tokenURL =
jsonObject.get("tokenUrl").getAsString(); // retrieve tokenURL and create a new
HttpPost object with it:
+ List<Map<String, String>> headers =
gson.fromJson(jsonObject.getAsJsonArray("headers"), List.class);
+ List<Map<String, String>> params =
gson.fromJson(jsonObject.getAsJsonArray("params"), List.class);
+ List<NameValuePair> nvPairs = new ArrayList<>();
+ HttpPost httpPost = new HttpPost(tokenURL);
+
+ // add headers to httpPost object:
+ for (Map<String, String> header : headers) {
+ for (Map.Entry<String, String> e : header.entrySet()) {
+ httpPost.setHeader(e.getKey(), e.getValue());
+ }
+ }
+
+ // add params to httpPost entity:
+ for (Map<String, String> param : params) {
+ for (Map.Entry<String, String> e : param.entrySet()) {
+ nvPairs.add(new BasicNameValuePair(e.getKey(), e.getValue()));
+ }
+ }
+
+ httpPost.setEntity(new UrlEncodedFormEntity(nvPairs,
StandardCharsets.UTF_8));
+
+ String ret;
+
+ // execute httpPost:
+ try (CloseableHttpClient httpClient = HttpClients.createDefault();
+ CloseableHttpResponse response = httpClient.execute(httpPost)) {
+ if (response == null) {
+ throw new IOException("getBearerToken(" + configFile + ")
failed: null response");
+ }
+
+ int statusCode = response.getStatusLine().getStatusCode();
+
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new IOException("getBearerToken(" + configFile + ")
failed: http status=" + response.getStatusLine());
+ }
+
+ HttpEntity httpEntity = response.getEntity();
+ String stringResult =
EntityUtils.toString(httpEntity);
+ Map<String, Object> resultMap = gson.fromJson(stringResult,
Map.class);
+ String token =
resultMap.get("access_token").toString();
+
+ ret = "Bearer " + token;
+
+ // and ensure response body is fully consumed
+ EntityUtils.consume(httpEntity);
+ }
+
+ return ret;
+ }
+
+ private Map<String, Map<String, String>> toUserAttributes(Map<String,
Map<String, List<String>>> userAttrValues){
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> toUserAttributes(userAttrValues={})",
userAttrValues);
+ }
+
+ Map<String, Map<String, String>> ret = new HashMap<>();
+
+ for (Map.Entry<String, Map<String, List<String>>> userEntry :
userAttrValues.entrySet()) {
+ String user = userEntry.getKey();
+ Map<String, List<String>> attrValues = userEntry.getValue();
+ Map<String, String> userAttrs = new HashMap<>();
+
+ for (Map.Entry<String, List<String>> attrEntry :
attrValues.entrySet()) {
+ String attrName = attrEntry.getKey();
+ List<String> values = attrEntry.getValue();
+
+ userAttrs.put(attrName, String.join(",", values));
+ }
+
+ ret.put(user, userAttrs);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== toUserAttributes(userAttrValues={}): ret={}",
userAttrValues, ret);
+ }
+
+ return ret;
+ }
+
+ private String getSecretsFromFile(String configFile) throws IOException {
+ String ret = decodeSecrets(new
String(Files.readAllBytes(Paths.get(configFile))));
+
+ verifyToken(ret);
+
+ return ret;
+ }
+
+ private String decodeSecrets(String encodedSecrets) {
+ return new String(Base64.getDecoder().decode(encodedSecrets));
+ }
+
+ private void verifyToken(String secrets) throws IOException {
+ String errorMessage = "";
+ JsonObject jsonObject = gson.fromJson(secrets, JsonObject.class);
+
+ // verify all necessary items are there
+ if (jsonObject.get("tokenUrl") == null) {
+ errorMessage += "tokenUrl must be specified in the config file; ";
+ }
+
+ if (jsonObject.get("headers") == null) {
+ errorMessage += "headers must be specified in the config file; ";
+ } else { // verify that Content-type, if included, is
application/x-www-form-urlencoded
+ List<Map<String, String>> headers =
gson.fromJson(jsonObject.getAsJsonArray("headers"), List.class);
+
+ for (Map<String, String> header : headers) {
+ if (header.containsKey("Content-Type") &&
!StringUtils.equalsIgnoreCase(header.get("Content-Type"),
"application/x-www-form-urlencoded")) {
+ errorMessage += "Content-Type, if specified, must be
\"application/x-www-form-urlencoded\"; ";
+ }
+ }
+ }
+
+ if (jsonObject.get("params") == null) {
+ errorMessage += "params must be specified in the config file; ";
+ }
+
+ if (!errorMessage.equals("")) {
+ throw new IOException(errorMessage);
+ }
+ }
+}
+
+
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/LICENSE
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/LICENSE
new file mode 100644
index 000000000..d64569567
--- /dev/null
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/NOTICE
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/NOTICE
new file mode 100644
index 000000000..b5c81eb4d
--- /dev/null
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/NOTICE
@@ -0,0 +1,18 @@
+Apache Ranger External User Store Retriever and Apache Ranger Role User Store
Retriever
+
+Copyright 2022 Comcast Cable Communications Management, LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an AS IS BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+PDX-License-Identifier: Apache-2.0
+
+This product includes software developed at Comcast (http://www.comcast.com/).
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/README.md
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/README.md
new file mode 100644
index 000000000..c874419eb
--- /dev/null
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/README.md
@@ -0,0 +1,137 @@
+
+````text
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+````
+
+# Ranger External User Store Retrievers
+
+A library to retrieve userStore entries from sources external to the Ranger
Admin User Store. The top level class is called
RangerMultiSourceUserStoreRetriever.
+
+## Business Value
+
+A counterpart of RangerAdminUserStoreRetriever, instead of retrieving items
from the internal Ranger userStore,
+RangerMultiSourceUserStoreRetriever retrieves userStore entries from other
sources. The userStore entries will persist
+while the plugin is up, and will be refreshed at configured intervals.
+
+This enables ABAC (Attribute-based Access Control) based on the user's
attributes that are retrieved.
+For example, the API could return the set of business partners that the user
has been granted access to.
+Thus, a row filter policy can be built using a condition like this:
+````text
+${{USER.partner}}.includes(partner)
+````
+where partner is the name of a column in a hive table. This enables
+row filter policies in which a user might match multiple conditions, which is
not possible with out-of-the-box Ranger.
+
+## Currently Supported Sources
+
+### Arbitrary API calls (source name: "api")
+
+This code enables additions to the UserStore to be retrieved simply by
creating an API which
+returns data in userStore format, and including attrName, userStoreURL, and
optionally
+configFile and dataFile as contextEnricher options in the host plugin's
service definition.
+
+#### Configuration Items
+
+Configurations are specified in the host plugin's service definition, as
enricherOptions in the contextEnricher
+definition. Configurations specific to this source type:
+
+**attrName** is the attribute whose values are mapped to the user, i.e., the
key to be used in the userStore:
+user -> **attrName** -> attrValues.
+In Ranger policies it appears in ${{USER.**attrName**}} syntax, eg
${{USER.partner}}
+
+**userStoreURL** is the URL from which to retrieve the user-to-attribute
mapping.
+
+**dataFile** is a local java properties file from which the user-to-attribute
mapping can be retrieved. It is optional,
+and intended to be used primarily in development.
+
+**configFile** is the name of the file containing the Base64-encoded secrets
needed as inputs to retrieve
+the Bearer Token needed for access to the userStoreURL. It is optional, for
security reasons. If configFile doesn't
+appear in the EnricherOptions, a default value is constructed as
"/var/ranger/security/"+attrName+".conf".
+
+The config file is a required JSON file which must contain:
+- **tokenUrl** : the name of the url to call to retrieve the Bearer Token
+- **headers**: list of key-value pairs representing names and values of http
headers for the call to the tokenUrl.
+ Note: Content-Type is assumed to be "application/x-www-form-urlencoded".
Inclusion of a different content-type header in the config file will cause a
400 error.
+- **params**: name-value pairs to be added as parameters to the URI's query
portion
+
+Here are the contents of an **example configFile**:
+```json
+{
+ "tokenUrl": "https://security.mycompany.com/token.oauth2";,
+ "headers": [
+ { "Content-Type": "application/x-www-form-urlencoded" } ,
+ { "Accept": "application/json" }
+ ],
+ "params": [
+ { "client_id": "my_user_name" },
+ { "client_secret": "***************" },
+ { "grant_type": "client_credentials" },
+ { "scope": "my_project" }
+ ]
+}
+```
+
+### RangerRoles (source name: "role")
+
+In this case, attributes are retrieved internally from Ranger, based on the
+roles of which the user is a member. No additional coding is needed.
+
+#### Configuration Items
+
+Configurations are specified in the host plugin's service definition, as
enricherOptions in the contextEnricher
+definition. Instead of external storage configurations (eg URL, datafile),
configurations
+specify how to retrieve the roles of interest:
+
+**attrName** is the attribute whose values are mapped to the user, i.e., the
key to be used in the userStore:
+user -> **attrName** -> attrValues. It is also the string used to identify
the role of interest. By convention,
+role names are assumed to have this structure: _attrName.attrValue_, e.g.,
salesRegion.northeast.
+
+## Service Definition Configurations
+In order to ensure that all new userStore entries are retained, there must be
a single userStoreRetrieverClass
+and a single userStore for all retrievers.
+
+**Options at the Context Enricher Level:**
+
+**userStoreRetrieverClassName** is the name of the context enricher that calls
all subsequent retriever methods.
+**userStoreRefresherPollingInterval** defines the interval at which the
userStoreRefresher polls its source, seeking data changes since it was last
refreshed.
+
+Within the options for this enricher, configurations for the individual
retrievers are given in a special format.
+The option key is "retrieverX_*sourceType*", where X is a sequential integer
and sourceType is (currently)
+either "api" or "role". The option value is a string containing
configurations for the individual retrievers, as outlined above,
+specified in a comma-separated Java Property-like format.
+
+Here is the relevant section of a **sample host plugin's service definition**.
Two api retrievers and two role retrievers
+are specified.
+
+```json
+{
+ "contextEnrichers": [
+ {
+ "itemId": 1,
+ "name": "RangerMultiSourceUserStoreRetriever",
+ "enricher":
"org.apache.ranger.plugin.contextenricher.RangerUserStoreEnricher",
+ "enricherOptions": {
+ "userStoreRetrieverClassName":
"org.apache.ranger.plugin.contextenricher.externalretrievers.RangerMultiSourceUserStoreRetriever",
+ "userStoreRefresherPollingInterval": "60000",
+ "retriever0_api":
"attrName=partner,userStoreURL=http://localhost:8000/security/getPartnersByUser";,
+ "retriever1_api":
"attrName=ownedResources,dataFile=/var/ranger/data/userOwnerResource.txt",
+ "retriever2_role": "attrName=salesRegion",
+ "retriever3_role": "attrName=sensitivityLevel"
+ }
+ }
+ ]
+}
+```
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/RangerMultiSourceUserStoreRetriever.java
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/RangerMultiSourceUserStoreRetriever.java
new file mode 100644
index 000000000..7e2462814
--- /dev/null
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/externalretrievers/RangerMultiSourceUserStoreRetriever.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.plugin.contextenricher.externalretrievers;
+
+import org.apache.ranger.admin.client.RangerAdminClient;
+import org.apache.ranger.plugin.contextenricher.RangerUserStoreRetriever;
+import org.apache.ranger.plugin.util.RangerRoles;
+import org.apache.ranger.plugin.util.RangerRolesUtil;
+import org.apache.ranger.plugin.util.RangerUserStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+// Options examples for individual retrievers:
+// "retriever0_api":
"attrName=partner,userStoreURL=http://localhost:8000/security/getPartnersByUser";,
+// "retriever1_role": "attrName=employee"
+
+public class RangerMultiSourceUserStoreRetriever extends
RangerUserStoreRetriever {
+ private static final Logger LOG =
LoggerFactory.getLogger(RangerMultiSourceUserStoreRetriever.class);
+
+ private static final Pattern PATTERN_ROLE_RETRIEVER_NAME =
Pattern.compile("\\d+_role");
+
+ private Map<String, Map<String,String>> retrieverOptions =
Collections.emptyMap();
+ private RangerAdminClient adminClient = null;
+ private RangerUserStore userStore = null;
+ private RangerRolesUtil rolesUtil = new
RangerRolesUtil(new RangerRoles());
+
+ // options come from service-def
+ @Override
+ public void init(Map<String, String> options) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> init(options={})", options);
+ }
+
+ try {
+ retrieverOptions = toRetrieverOptions(options);
+
+ if (hasAnyRoleRetriever()) {
+ adminClient = pluginContext.createAdminClient(pluginConfig);
+ }
+ } catch (Exception e) {
+ LOG.error("init() failed", e);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== init(options={})", options);
+ }
+ }
+
+ @Override
+ public RangerUserStore retrieveUserStoreInfo(long lastKnownVersion, long
lastActivationTimeInMillis) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("=> retrieveUserStoreInfo(lastKnownVersion={},
lastActivationTimeInMillis={})", lastKnownVersion, lastActivationTimeInMillis);
+ }
+
+ // if there are any role type retrievers, get rangerRoles; otherwise
don't bother
+ if (adminClient != null) {
+ try {
+ RangerRoles roles = rolesUtil.getRoles();
+ long rolesVersion = roles.getRoleVersion() != null ?
roles.getRoleVersion() : -1;
+ RangerRoles updatedRoles =
adminClient.getRolesIfUpdated(rolesVersion, lastActivationTimeInMillis);
+
+ if (updatedRoles != null) {
+ rolesUtil = new RangerRolesUtil(updatedRoles);
+ }
+ } catch (Exception e) {
+ LOG.error("retrieveUserStoreInfo(lastKnownVersion={}) failed
to retrieve roles", lastKnownVersion, e);
+ }
+ }
+
+ Map<String, Map<String, String>> userAttrs = null;
+
+ try {
+ userAttrs = retrieveAll();
+ } catch (Exception e) {
+ LOG.error("retrieveUserStoreInfo(lastKnownVersion={}) failed",
lastKnownVersion, e);
+ }
+
+ if (userAttrs != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("retrieveUserStoreInfo(lastKnownVersion={}):
user-attributes={}", lastKnownVersion, userAttrs);
+ }
+
+ userStore = new RangerUserStore();
+
+ userStore.setUserStoreVersion(System.currentTimeMillis());
+ userStore.setUserAttrMapping(userAttrs);
+ } else {
+ LOG.error("retrieveUserStoreInfo(lastKnownVersion={}): failed to
retrieve user-attributes", lastKnownVersion);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== retrieveUserStoreInfo(lastKnownVersion={},
lastActivationTimeInMillis={}): ret={}", lastKnownVersion,
lastActivationTimeInMillis, userStore);
+ }
+
+ return userStore;
+ }
+
+ private Map<String, Map<String,String>> toRetrieverOptions(Map<String,
String> enricherOptions) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> toRetrieverOptions({})", enricherOptions);
+ }
+
+ Map<String, Map<String, String>> ret = new HashMap<>();
+
+ for (Map.Entry<String, String> entry : enricherOptions.entrySet()) {
+ String retrieverName = entry.getKey();
+
+ if (retrieverName.startsWith("retriever")) {
+ String retrieverOptions = entry.getValue();
+
+ ret.put(retrieverName, toRetrieverOptions(retrieverName,
retrieverOptions));
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== toRetrieverOptions({}): ret={}", enricherOptions,
ret);
+ }
+
+ return ret;
+ }
+
+ // Managing options for various retrievals
+ private Map<String, String> toRetrieverOptions(String name, String
options) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> toRetrieverOptions(name={}, options={})", name,
options);
+ }
+
+ Properties prop = new Properties();
+
+ options = options.replaceAll("\\s", "");
+ options = options.replaceAll(",", "\n");
+
+ try {
+ prop.load(new StringReader(options));
+ } catch (Exception e) {
+ LOG.error("toRetrieverOptions(name={}, options={}): failed to
parse retriever options", name, options, e);
+
+ throw new Exception(name + ": failed to parse retriever options: "
+ options, e);
+ }
+
+ Map<String, String> ret = new HashMap<>();
+
+ for (String key : prop.stringPropertyNames()) {
+ ret.put(key, prop.getProperty(key));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== toRetrieverOptions(name={}, options={}): ret={}",
name, options, ret);
+ }
+
+ return ret;
+ }
+
+ private boolean hasAnyRoleRetriever() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> hasAnyRoleRetriever()");
+ }
+
+ boolean ret = false;
+
+ for (String retrieverName : retrieverOptions.keySet()) {
+ Matcher matcher =
PATTERN_ROLE_RETRIEVER_NAME.matcher(retrieverName);
+
+ if (matcher.find()) {
+ ret = true;
+
+ break;
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== hasAnyRoleRetriever(): ret={}", ret);
+ }
+
+ return ret;
+ }
+
+ // top-level retrieval management
+ private Map<String, Map<String, String>> retrieveAll() throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> retrieveAll()");
+ }
+
+ Map<String, Map<String, String>> ret = new HashMap<>();
+
+ for (Map.Entry<String, Map<String, String>> entry :
retrieverOptions.entrySet()) {
+ String name = entry.getKey();
+ Map<String, String> options = entry.getValue();
+ String source =
name.replaceAll("\\w+_","");
+ Map<String, Map<String, String>> userAttrs;
+
+ switch (source) {
+ case "api":
+ userAttrs = retrieveUserAttributes(name, options);
+ break;
+
+ case "role":
+ userAttrs = retrieveUserAttrFromRoles(name, options);
+ break;
+
+ default:
+ throw new Exception("unrecognized retriever source '" +
source + "'. Valid values: api, role");
+ }
+
+ mergeUserAttributes(userAttrs, ret);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== retrieveAll(): ret={}", ret);
+ }
+
+ return ret;
+ }
+
+ // external retrieval
+ private Map<String, Map<String, String>> retrieveUserAttributes(String
retrieverName, Map<String, String> options) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> retrieveUserAttributes(name={}, options={})",
retrieverName, options);
+ }
+
+ String attrName = options.get("attrName");
+ String url = options.get("userStoreURL");
+ String dataFile = options.get("dataFile");
+
+ if (attrName == null) {
+ throw new Exception(retrieverName + ": attrName must be specified
in retriever options");
+ }
+
+ if (url == null && dataFile == null) {
+ throw new Exception(retrieverName + ": url or dataFile must be
specified in retriever options");
+ }
+
+ Map<String, Map<String, String>> ret;
+
+ if (url != null) {
+ GetFromURL gu = new GetFromURL();
+
+ String configFile = options.getOrDefault("configFile",
"/var/ranger/security/" + attrName + ".conf");
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}: configFile={}", retrieverName, configFile);
+ }
+
+ ret = gu.getFromURL(url, configFile); // get user-Attrs mapping
in UserStore format from an API call
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loaded attribute {} from URL {}: {}", attrName,
url, ret);
+ }
+ } else {
+ GetFromDataFile gf = new GetFromDataFile();
+
+ ret = gf.getFromDataFile(dataFile, attrName);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("loaded attribute {} from file {}: {}", attrName,
dataFile, ret);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== retrieveUserAttributes(name={}, options={}):
ret={}", retrieverName, options, ret);
+ }
+
+ return ret;
+ }
+
+ // role-based retrieval
+ /** retrieveSingleRoleUserAttrMapping:
+ *
+ * @param options includes the attribute name of interest, from which to
create the UserStore attribute name
+ * and to identify the role of interest.
+ * @return In UserStore format, maps from user to attrName to attribute
values
+ *
+ * rangerRoles: one object for each role; contains set of users who are
members. The important feature here
+ * * is that it maps roles to users.
rolesUtil.getUserRoleMapping() returns the reverse:
+ * * maps users to roles that they are members of. This is closer
to the UserStore format.
+ */
+ public Map<String, Map<String, String>> retrieveUserAttrFromRoles(String
retrieverName, Map<String, String> options) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> retrieveUserAttrFromRoles(name={}, options={})",
retrieverName, options);
+ }
+
+ Map<String, Map<String, String>> ret = new HashMap<>();
+ Map<String, Set<String>> userToRoles =
rolesUtil.getUserRoleMapping();
+ String attrName = options.get("attrName");
+ String rolePrefix = attrName + ".";
+ Pattern pattern = Pattern.compile("^.*" +
rolePrefix + ".*$");
+
+ for (Map.Entry<String, Set<String>> entry : userToRoles.entrySet()) {
+ String user = entry.getKey();
+ Set<String> roles = entry.getValue();
+ List<String> attrValues = new ArrayList<>();
+
+ for (String role : roles) {
+ Matcher matcher = pattern.matcher(role);
+
+ if (matcher.find()) {
+ String value = matcher.group().replace(rolePrefix, "");
+
+ attrValues.add(value);
+ }
+ }
+
+ if (!attrValues.isEmpty()) {
+ Map<String, String> userAttrs = new HashMap<>();
+
+ userAttrs.put(attrName, String.join(",", attrValues));
+
+ ret.put(user, userAttrs);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== retrieveUserAttrFromRoles(name={}, options={}):
ret={}", retrieverName, options, ret);
+ }
+
+ return ret;
+ }
+
+ private void mergeUserAttributes(Map<String, Map<String, String>> source,
Map<String, Map<String, String>> dest) {
+ if (dest.size() == 0) {
+ dest.putAll(source);
+ } else {
+ for (Map.Entry<String, Map<String, String>> e : source.entrySet())
{
+ String userName = e.getKey();
+ Map<String, String> userAttrs = e.getValue();
+
+ if (dest.containsKey(userName)) {
+ Map<String, String> existingAttrs = dest.get(userName);
+
+ existingAttrs.putAll(userAttrs);
+ } else {
+ dest.put(userName, userAttrs);
+ }
+ }
+ }
+ }
+}
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesProvider.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesProvider.java
index 6efd13f80..bea82f5b8 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesProvider.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesProvider.java
@@ -139,7 +139,7 @@ public class RangerRolesProvider {
plugIn.setRoles(roles);
rangerUserGroupRolesSetInPlugin = true;
setLastActivationTimeInMillis(System.currentTimeMillis());
- lastKnownRoleVersion = roles.getRoleVersion();
+ lastKnownRoleVersion = roles.getRoleVersion()
!= null ? roles.getRoleVersion() : -1;;
} else {
if (!rangerUserGroupRolesSetInPlugin &&
!serviceDefSetInPlugin) {
plugIn.setRoles(null);
diff --git
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesUtil.java
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesUtil.java
index f785e1186..40b2652a9 100644
---
a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesUtil.java
+++
b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRolesUtil.java
@@ -44,7 +44,7 @@ public class RangerRolesUtil {
public RangerRolesUtil(RangerRoles roles) {
if (roles != null) {
this.roles = roles;
- roleVersion = roles.getRoleVersion();
+ roleVersion = roles.getRoleVersion() != null ?
roles.getRoleVersion() : -1;
if (CollectionUtils.isNotEmpty(roles.getRangerRoles())) {
for (RangerRole role : roles.getRangerRoles()) {
diff --git a/dev-support/spotbugsIncludeFile.xml
b/dev-support/spotbugsIncludeFile.xml
index 3621e8c08..9a0a9261a 100644
--- a/dev-support/spotbugsIncludeFile.xml
+++ b/dev-support/spotbugsIncludeFile.xml
@@ -45,6 +45,7 @@
<Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS" />
<Bug pattern="IL_INFINITE_RECURSIVE_LOOP" />
<Bug pattern="DMI_RANDOM_USED_ONLY_ONCE" />
+ <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
</Or>
</Not>
</Match>
