rameeshm commented on code in PR #1262: URL: https://github.com/apache/ranger/pull/1262#discussion_r4136387850
########## security-admin/src/main/java/org/apache/ranger/opensearch/OpenSearchAccessAuditsService.java: ########## @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.opensearch; + +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.ranger.common.MessageEnums; +import org.apache.ranger.common.PropertiesUtil; +import org.apache.ranger.common.RESTErrorUtil; +import org.apache.ranger.common.SearchCriteria; +import org.apache.ranger.db.XXServiceDefDao; +import org.apache.ranger.entity.XXService; +import org.apache.ranger.entity.XXServiceDef; +import org.apache.ranger.opensearch.OpenSearchUtil.OpenSearchSearchResult; +import org.apache.ranger.view.VXAccessAudit; +import org.apache.ranger.view.VXAccessAuditList; +import org.apache.ranger.view.VXLong; +import org.elasticsearch.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Service +@Scope("singleton") +public class OpenSearchAccessAuditsService extends org.apache.ranger.AccessAuditsService { Review Comment: org.apache.ranger.AccessAuditsService can we use import instead as standard practice? ########## security-admin/src/main/java/org/apache/ranger/opensearch/OpenSearchUtil.java: ########## @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.opensearch; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.entity.ContentType; +import org.apache.http.nio.entity.NStringEntity; +import org.apache.http.util.EntityUtils; +import org.apache.ranger.common.PropertiesUtil; +import org.apache.ranger.common.SearchCriteria; +import org.apache.ranger.common.SearchField; +import org.apache.ranger.common.SortField; +import org.apache.ranger.common.StringUtil; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.stream.Collectors; + +@Component +public class OpenSearchUtil { + private static final Logger LOG = LoggerFactory.getLogger(OpenSearchUtil.class); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String LUCENE_SPECIAL_CHARS = "+-=&|><!(){}[]^\"~*?:\\/"; + private static final String DATE_FORMAT_STR = "yyyy-MM-dd'T'HH:mm:ss'Z'"; + + private static final ThreadLocal<DateFormat> DATE_FORMAT = ThreadLocal.withInitial(() -> { + SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT_STR); + String timeZone = PropertiesUtil.getProperty("xa.elasticSearch.timezone"); Review Comment: xa.elasticSearch.timezone . Can we opensearch specific property name like ranger.audit.opensearch.timezone ########## embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/OpenSearchIndexBootStrapper.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.server.tomcat; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.nio.entity.NStringEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.logging.Logger; + +public class OpenSearchIndexBootStrapper extends Thread { + private static final Logger LOG = Logger.getLogger(OpenSearchIndexBootStrapper.class.getName()); + + private static final String CONFIG_PREFIX = "ranger.audit.opensearch"; + private static final String CONFIG_URLS = CONFIG_PREFIX + ".urls"; + private static final String CONFIG_PORT = CONFIG_PREFIX + ".port"; + private static final String CONFIG_PROTOCOL = CONFIG_PREFIX + ".protocol"; + private static final String CONFIG_USER = CONFIG_PREFIX + ".user"; + private static final String CONFIG_PASSWORD = CONFIG_PREFIX + ".password"; + private static final String CONFIG_INDEX = CONFIG_PREFIX + ".index"; + private static final String CONFIG_INTERVAL = CONFIG_PREFIX + ".time.interval"; + private static final String CONFIG_SHARDS = CONFIG_PREFIX + ".no.shards"; + private static final String CONFIG_REPLICAS = CONFIG_PREFIX + ".no.replica"; + private static final String CONFIG_MAX_RETRY = CONFIG_PREFIX + ".max.retry"; + private static final String SCHEMA_FILE_NAME = "ranger_opensearch_schema.json"; + + private volatile RestClient client; + private String index; + private String urls; + private String protocol; + private String user; + private String password; + private int port; + private int noOfShards; + private int noOfReplicas; + private int maxRetry; + private long timeInterval; + + @Override + public void run() { + LOG.info("OpenSearchIndexBootStrapper: starting..."); + + readConfig(); + + if (StringUtils.isBlank(urls) || "NONE".equalsIgnoreCase(urls)) { + LOG.severe("OpenSearch URL is not configured. Aborting bootstrap."); + + return; + } + + int retryCounter = 0; + + while (maxRetry == -1 || retryCounter < maxRetry) { + retryCounter++; + + try { + LOG.info("Trying to acquire OpenSearch connection"); + + connect(); + + if (client == null) { + logErrorAndWait(retryCounter, "Failed to create OpenSearch client"); + + continue; + } + + LOG.info("Connection to OpenSearch established successfully"); + + if (indexExists()) { + LOG.info("Index '" + index + "' already exists. Bootstrap complete."); + + return; + } + + createIndex(); + + LOG.info("Index '" + index + "' created successfully."); + + return; + } catch (Exception e) { + logErrorAndWait(retryCounter, e.getMessage()); + } + } + + LOG.severe("OpenSearch index bootstrap failed after " + retryCounter + " attempts."); + } + + private void readConfig() { + urls = EmbeddedServerUtil.getConfig(CONFIG_URLS, ""); + protocol = EmbeddedServerUtil.getConfig(CONFIG_PROTOCOL, "http"); + user = EmbeddedServerUtil.getConfig(CONFIG_USER, ""); + password = EmbeddedServerUtil.getConfig(CONFIG_PASSWORD, ""); + port = Integer.parseInt(EmbeddedServerUtil.getConfig(CONFIG_PORT, "9200")); Review Comment: CONFIG_PORT if this empty string this may result in NumberFormat Exception. Please use EmbeddedServerUtil.getIntConfig. check all the occurance ########## embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/OpenSearchIndexBootStrapper.java: ########## @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.server.tomcat; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.entity.ContentType; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.nio.entity.NStringEntity; +import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.logging.Logger; + +public class OpenSearchIndexBootStrapper extends Thread { + private static final Logger LOG = Logger.getLogger(OpenSearchIndexBootStrapper.class.getName()); + + private static final String CONFIG_PREFIX = "ranger.audit.opensearch"; + private static final String CONFIG_URLS = CONFIG_PREFIX + ".urls"; + private static final String CONFIG_PORT = CONFIG_PREFIX + ".port"; + private static final String CONFIG_PROTOCOL = CONFIG_PREFIX + ".protocol"; + private static final String CONFIG_USER = CONFIG_PREFIX + ".user"; + private static final String CONFIG_PASSWORD = CONFIG_PREFIX + ".password"; + private static final String CONFIG_INDEX = CONFIG_PREFIX + ".index"; + private static final String CONFIG_INTERVAL = CONFIG_PREFIX + ".time.interval"; + private static final String CONFIG_SHARDS = CONFIG_PREFIX + ".no.shards"; + private static final String CONFIG_REPLICAS = CONFIG_PREFIX + ".no.replica"; + private static final String CONFIG_MAX_RETRY = CONFIG_PREFIX + ".max.retry"; + private static final String SCHEMA_FILE_NAME = "ranger_opensearch_schema.json"; + + private volatile RestClient client; + private String index; + private String urls; + private String protocol; + private String user; + private String password; + private int port; + private int noOfShards; + private int noOfReplicas; + private int maxRetry; + private long timeInterval; + + @Override + public void run() { + LOG.info("OpenSearchIndexBootStrapper: starting..."); + + readConfig(); + + if (StringUtils.isBlank(urls) || "NONE".equalsIgnoreCase(urls)) { + LOG.severe("OpenSearch URL is not configured. Aborting bootstrap."); + + return; + } + + int retryCounter = 0; + + while (maxRetry == -1 || retryCounter < maxRetry) { + retryCounter++; + + try { + LOG.info("Trying to acquire OpenSearch connection"); + + connect(); + + if (client == null) { + logErrorAndWait(retryCounter, "Failed to create OpenSearch client"); + + continue; + } + + LOG.info("Connection to OpenSearch established successfully"); + + if (indexExists()) { + LOG.info("Index '" + index + "' already exists. Bootstrap complete."); + + return; + } + + createIndex(); + + LOG.info("Index '" + index + "' created successfully."); + + return; + } catch (Exception e) { + logErrorAndWait(retryCounter, e.getMessage()); + } + } + + LOG.severe("OpenSearch index bootstrap failed after " + retryCounter + " attempts."); + } + + private void readConfig() { + urls = EmbeddedServerUtil.getConfig(CONFIG_URLS, ""); + protocol = EmbeddedServerUtil.getConfig(CONFIG_PROTOCOL, "http"); + user = EmbeddedServerUtil.getConfig(CONFIG_USER, ""); + password = EmbeddedServerUtil.getConfig(CONFIG_PASSWORD, ""); + port = Integer.parseInt(EmbeddedServerUtil.getConfig(CONFIG_PORT, "9200")); + index = EmbeddedServerUtil.getConfig(CONFIG_INDEX, "ranger_audits"); + noOfShards = Integer.parseInt(EmbeddedServerUtil.getConfig(CONFIG_SHARDS, "1")); + noOfReplicas = Integer.parseInt(EmbeddedServerUtil.getConfig(CONFIG_REPLICAS, "1")); + maxRetry = Integer.parseInt(EmbeddedServerUtil.getConfig(CONFIG_MAX_RETRY, "30")); + timeInterval = Long.parseLong(EmbeddedServerUtil.getConfig(CONFIG_INTERVAL, "60000")); + } + + private void connect() { + if (client != null) { + return; + } + + HttpHost[] hosts = Arrays.stream(urls.split(",")).map(String::trim).filter(h -> !h.isEmpty()).map(h -> new HttpHost(h, port, protocol)).toArray(HttpHost[]::new); + + RestClientBuilder builder = RestClient.builder(hosts); + + if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && !"NONE".equalsIgnoreCase(user) && !"NONE".equalsIgnoreCase(password)) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password)); + builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + } + + client = builder.build(); + } + + private boolean indexExists() throws IOException { + Request request = new Request("HEAD", "/" + index); + request.addParameter("ignore", "404"); + + Response response = client.performRequest(request); + + return response.getStatusLine().getStatusCode() == 200; + } + + private void createIndex() throws IOException { + String schemaJson = loadSchemaFile(); + String body = buildCreateIndexBody(schemaJson); + + Request request = new Request("PUT", "/" + index); + + request.setEntity(new NStringEntity(body, ContentType.APPLICATION_JSON)); + + Response response = client.performRequest(request); + String result = EntityUtils.toString(response.getEntity()); + + if (response.getStatusLine().getStatusCode() >= 400) { + throw new IOException("Failed to create index '" + index + "': " + result); + } + + LOG.info("Create index response: " + result); + } + + private String buildCreateIndexBody(String mappingsJson) { + return "{\"settings\":{\"number_of_shards\":" + noOfShards + + ",\"number_of_replicas\":" + noOfReplicas + + "},\"mappings\":" + mappingsJson + "}"; + } + + private String loadSchemaFile() throws IOException { + String jarLocation; + + try { + jarLocation = this.getClass().getProtectionDomain().getCodeSource().getLocation().toURI().getPath(); + } catch (Exception e) { + throw new IOException("Cannot determine Ranger home directory", e); + } + + String rangerHomeDir = new File(jarLocation).getParentFile().getParentFile().getParentFile().getPath(); + String schemaPath = Paths.get(rangerHomeDir, "contrib", "opensearch_for_audit_setup", "conf", SCHEMA_FILE_NAME).toString(); + + File schemaFile = new File(schemaPath); + + if (!schemaFile.exists()) { + throw new IOException("OpenSearch schema file not found: " + schemaPath); + } + + return new String(Files.readAllBytes(Paths.get(schemaPath))); Review Comment: can be return new String(Files.readAllBytes(Paths.get(schemaPath)), StandardCharsets.UTF_8); ########## security-admin/src/main/java/org/apache/ranger/opensearch/OpenSearchMgr.java: ########## @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.opensearch; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.config.Lookup; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.ranger.audit.destination.OpenSearchAuditDestination; +import org.apache.ranger.authorization.credutils.CredentialsProviderUtil; +import org.apache.ranger.authorization.credutils.kerberos.KerberosCredentialsProvider; +import org.apache.ranger.common.PropertiesUtil; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosTicket; + +import java.security.PrivilegedActionException; +import java.util.Arrays; +import java.util.Date; +import java.util.Locale; + +@Component +public class OpenSearchMgr { + private static final Logger LOG = LoggerFactory.getLogger(OpenSearchMgr.class); + + private static final String CONFIG_PREFIX = OpenSearchAuditDestination.CONFIG_PREFIX; + private static final String CONFIG_URLS = OpenSearchAuditDestination.CONFIG_URLS; + private static final String CONFIG_PORT = OpenSearchAuditDestination.CONFIG_PORT; + private static final String CONFIG_PROTOCOL = OpenSearchAuditDestination.CONFIG_PROTOCOL; + private static final String CONFIG_USER = OpenSearchAuditDestination.CONFIG_USER; + private static final String CONFIG_PASSWORD = OpenSearchAuditDestination.CONFIG_PASSWORD; + private static final String CONFIG_INDEX = OpenSearchAuditDestination.CONFIG_INDEX; + private static final String CONFIG_AUTH_TYPE = OpenSearchAuditDestination.CONFIG_AUTH_TYPE; + private static final String CONFIG_KERBEROS_PRINCIPAL = OpenSearchAuditDestination.CONFIG_KERBEROS_PRINCIPAL; + private static final String CONFIG_KERBEROS_KEYTAB = OpenSearchAuditDestination.CONFIG_KERBEROS_KEYTAB; + + private volatile RestClient client; + private Subject subject; + private String user; + private String password; + private String index; + private String loginPrincipal; + private String loginKeytab; + + public RestClient getClient() { + RestClient me = client; + + if (me != null && subject != null) { + KerberosTicket ticket = CredentialsProviderUtil.getTGT(subject); + + try { + if (new Date().getTime() > ticket.getEndTime().getTime()) { + client = null; + CredentialsProviderUtil.ticketExpireTime80 = 0; + + me = connect(); + } else if (CredentialsProviderUtil.ticketWillExpire(ticket)) { + subject = CredentialsProviderUtil.login(loginPrincipal, loginKeytab); + } + } catch (PrivilegedActionException e) { + LOG.error("PrivilegedActionException:", e); + + throw new RuntimeException(e); + } + + return me; + } else { + me = connect(); + } + + return me; + } + + String getIndex() { + return index; + } + + void setIndex(String index) { + this.index = index; + } + + synchronized RestClient connect() { + RestClient me = client; + + if (me == null) { + synchronized (OpenSearchMgr.class) { Review Comment: Do we need synchronized again here as connect() is synchronized -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
