This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ambari-infra.git
The following commit(s) were added to refs/heads/master by this push: new c4c350c AMBARI-24918 - Infra Manager: ssl support (#17) c4c350c is described below commit c4c350c98edbb0c830153eb7fe43e944e6dc0c23 Author: kasakrisz <33458261+kasakr...@users.noreply.github.com> AuthorDate: Wed Nov 21 12:07:05 2018 +0100 AMBARI-24918 - Infra Manager: ssl support (#17) --- .../test/java/org/apache/ambari/infra/Solr.java | 2 +- .../ambari/infra/solr/metrics/MetricsIT.java | 2 +- ambari-infra-manager/docker/docker-compose.yml | 6 +++ .../conf/InfraManagerWebServerCustomizer.java | 37 ++++++++++++++++- ...sitePasswordStore.java => CompositeSecret.java} | 14 +++---- ...tyEnvironment.java => EnvironmentalSecret.java} | 13 ++++-- ...itePasswordStore.java => HadoopCredential.java} | 22 ++++++----- .../infra/conf/security/HadoopCredentialStore.java | 24 +++++------ .../conf/security/InfraManagerSecurityConfig.java | 31 +++++++++++++-- .../{SecurityEnvironment.java => S3Secrets.java} | 20 +++++++--- .../security/{PasswordStore.java => Secret.java} | 4 +- .../{SecurityEnvironment.java => SslSecrets.java} | 19 ++++++--- .../archive/DocumentArchivingConfiguration.java | 7 ++-- .../ambari/infra/job/archive/S3AccessCsv.java | 43 ++++++++++---------- .../ambari/infra/job/archive/S3Uploader.java | 27 ++++++++----- ...wordStoreTest.java => CompositeSecretTest.java} | 16 ++++---- .../ambari/infra/job/archive/S3AccessCsvTest.java | 46 ++++++++++------------ .../docker/infra-solr-docker-compose.sh | 4 +- 18 files changed, 215 insertions(+), 122 deletions(-) diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java index 7bc952a..f149cd8 100644 --- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java +++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java @@ -86,7 +86,7 @@ public class Solr { public void createSolrCollection(String collectionName) { logger.info("Creating collection"); - runCommand(new String[]{"docker", "exec", "docker_solr_1", "solr", "create_collection", "-force", "-c", collectionName, "-d", Paths.get(configSetPath, "configsets", collectionName, "conf").toString(), "-n", collectionName + "_conf"}); + runCommand(new String[]{"docker", "exec", "solr", "solr", "create_collection", "-force", "-c", collectionName, "-d", Paths.get(configSetPath, "configsets", collectionName, "conf").toString(), "-n", collectionName + "_conf"}); } public QueryResponse query(SolrQuery query) { diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java index c400aee..6f17442 100644 --- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java +++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java @@ -52,7 +52,7 @@ public class MetricsIT { logger.info("Creating new docker containers for testing Ambari Infra Solr Metrics plugin ..."); runCommand(new String[]{shellScriptLocation, "start"}); - Solr solr = new Solr("/usr/lib/ambari-infra-solr/server/solr"); + Solr solr = new Solr(); solr.waitUntilSolrIsUp(); solr.createSolrCollection(HADOOP_LOGS_COLLECTION); diff --git a/ambari-infra-manager/docker/docker-compose.yml b/ambari-infra-manager/docker/docker-compose.yml index 3fa21b2..5051820 100644 --- a/ambari-infra-manager/docker/docker-compose.yml +++ b/ambari-infra-manager/docker/docker-compose.yml @@ -15,6 +15,7 @@ version: '3.3' services: zookeeper: + container_name: zookeeper image: zookeeper:${ZOOKEEPER_VERSION:-3.4.10} restart: always hostname: zookeeper @@ -27,6 +28,7 @@ services: ZOO_SERVERS: server.1=zookeeper:2888:3888 solr: # TODO: use infra-solr + container_name: solr image: solr:${SOLR_VERSION:-7.5.0} restart: always hostname: solr @@ -47,6 +49,7 @@ services: volumes: - $AMBARI_INFRA_LOCATION/ambari-infra-manager/docker/configsets:/opt/solr/configsets fakes3: + container_name: fakes3 image: localstack/localstack hostname: fakes3 ports: @@ -61,6 +64,7 @@ services: env_file: - Profile namenode: + container_name: hdfs_namenode image: flokkr/hadoop-hdfs-namenode:${HADOOP_VERSION:-3.0.0} hostname: namenode ports: @@ -73,6 +77,7 @@ services: networks: - infra-network datanode: + container_name: hdfs_datanode image: flokkr/hadoop-hdfs-datanode:${HADOOP_VERSION:-3.0.0} links: - namenode @@ -81,6 +86,7 @@ services: networks: - infra-network inframanager: + container_name: inframanager image: ambari-infra-manager:v1.0 restart: always hostname: infra-manager.apache.org diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java index 06174a0..e560ae9 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java @@ -18,13 +18,17 @@ */ package org.apache.ambari.infra.conf; +import static org.apache.commons.lang3.StringUtils.isNotBlank; + import java.time.Duration; import javax.inject.Inject; +import org.apache.ambari.infra.conf.security.SslSecrets; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.web.ServerProperties; import org.springframework.boot.web.embedded.jetty.JettyServletWebServerFactory; +import org.springframework.boot.web.server.Ssl; import org.springframework.boot.web.server.WebServerFactoryCustomizer; import org.springframework.stereotype.Component; @@ -34,18 +38,47 @@ public class InfraManagerWebServerCustomizer implements WebServerFactoryCustomiz @Value("${infra-manager.server.port:61890}") private int port; + @Value("${infra-manager.server.ssl.enabled:false}") + private boolean sslEnabled; + @Inject private ServerProperties serverProperties; + @Inject + private SslSecrets sslSecrets; + private static final Integer SESSION_TIMEOUT = 60 * 30; - private static final String INFRA_MANAGER_SESSIONID = "INFRAMANAGER_SESSIONID"; + private static final String INFRA_MANAGER_SESSION_ID = "INFRAMANAGER_SESSIONID"; private static final String INFRA_MANAGER_APPLICATION_NAME = "infra-manager"; @Override public void customize(JettyServletWebServerFactory factory) { factory.setPort(port); factory.setDisplayName(INFRA_MANAGER_APPLICATION_NAME); - factory.getSession().getCookie().setName(INFRA_MANAGER_SESSIONID); + factory.getSession().getCookie().setName(INFRA_MANAGER_SESSION_ID); factory.getSession().setTimeout(Duration.ofSeconds(SESSION_TIMEOUT)); + + Ssl ssl = new Ssl(); + String keyStore = System.getProperty("javax.net.ssl.keyStore"); + if (isNotBlank(keyStore)) { + ssl.setKeyStore(keyStore); + ssl.setKeyStoreType(System.getProperty("javax.net.ssl.keyStoreType")); + String keyStorePassword = sslSecrets.getKeyStorePassword().get().orElseThrow(() -> new IllegalStateException("Password for keystore is not set!")); + ssl.setKeyStorePassword(keyStorePassword); + System.setProperty("javax.net.ssl.keyStorePassword", keyStorePassword); + } + + String trustStore = System.getProperty("javax.net.ssl.trustStore"); + if (isNotBlank(trustStore)) { + ssl.setTrustStore(trustStore); + ssl.setTrustStoreType(System.getProperty("javax.net.ssl.trustStoreType")); + String trustStorePassword = sslSecrets.getTrustStorePassword().get().orElseThrow(() -> new IllegalStateException("Password for truststore is not set!")); + ssl.setKeyStorePassword(trustStorePassword); + System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword); + } + + ssl.setEnabled(sslEnabled); + + factory.setSsl(ssl); } } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java similarity index 70% copy from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java copy to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java index 6d32963..e8ab52e 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java @@ -20,17 +20,17 @@ package org.apache.ambari.infra.conf.security; import java.util.Optional; -public class CompositePasswordStore implements PasswordStore { - private PasswordStore[] passwordStores; +public class CompositeSecret implements Secret { + private Secret[] secrets; - public CompositePasswordStore(PasswordStore... passwordStores) { - this.passwordStores = passwordStores; + public CompositeSecret(Secret... secrets) { + this.secrets = secrets; } @Override - public Optional<String> getPassword(String propertyName) { - for (PasswordStore passwordStore : passwordStores) { - Optional<String> optionalPassword = passwordStore.getPassword(propertyName); + public Optional<String> get() { + for (Secret secret : secrets) { + Optional<String> optionalPassword = secret.get(); if (optionalPassword.isPresent()) return optionalPassword; } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java similarity index 72% copy from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java copy to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java index 8e3387b..887767b 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java @@ -20,9 +20,16 @@ package org.apache.ambari.infra.conf.security; import java.util.Optional; -public class SecurityEnvironment implements PasswordStore { +public class EnvironmentalSecret implements Secret { + + private final String environmentalVariableName; + + public EnvironmentalSecret(String environmentalVariableName) { + this.environmentalVariableName = environmentalVariableName; + } + @Override - public Optional<String> getPassword(String propertyName) { - return Optional.ofNullable(System.getenv(propertyName)); + public Optional<String> get() { + return Optional.ofNullable(System.getenv(environmentalVariableName)); } } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java similarity index 63% rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java index 6d32963..8fba08a 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java @@ -20,20 +20,22 @@ package org.apache.ambari.infra.conf.security; import java.util.Optional; -public class CompositePasswordStore implements PasswordStore { - private PasswordStore[] passwordStores; +public class HadoopCredential implements Secret { - public CompositePasswordStore(PasswordStore... passwordStores) { - this.passwordStores = passwordStores; + private final HadoopCredentialStore hadoopCredentialStore; + private final String propertyName; + + public HadoopCredential(HadoopCredentialStore hadoopCredentialStore, String propertyName) { + this.propertyName = propertyName; + this.hadoopCredentialStore = hadoopCredentialStore; } @Override - public Optional<String> getPassword(String propertyName) { - for (PasswordStore passwordStore : passwordStores) { - Optional<String> optionalPassword = passwordStore.getPassword(propertyName); - if (optionalPassword.isPresent()) - return optionalPassword; + public Optional<String> get() { + if (hadoopCredentialStore == null) { + return Optional.empty(); } - return Optional.empty(); + + return hadoopCredentialStore.get(propertyName).map(String::new); } } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java index 957a45d..08a8804 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java @@ -21,13 +21,11 @@ package org.apache.ambari.infra.conf.security; import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.commons.lang3.ArrayUtils.isNotEmpty; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Optional; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class HadoopCredentialStore implements PasswordStore { - private static final Logger logger = LogManager.getLogger(InfraManagerSecurityConfig.class); +public class HadoopCredentialStore { public static final String CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY = "hadoop.security.credential.provider.path"; private final String credentialStoreProviderPath; @@ -36,8 +34,7 @@ public class HadoopCredentialStore implements PasswordStore { this.credentialStoreProviderPath = credentialStoreProviderPath; } - @Override - public Optional<String> getPassword(String propertyName) { + public Optional<char[]> get(String key) { try { if (isBlank(credentialStoreProviderPath)) { return Optional.empty(); @@ -45,11 +42,14 @@ public class HadoopCredentialStore implements PasswordStore { org.apache.hadoop.conf.Configuration config = new org.apache.hadoop.conf.Configuration(); config.set(CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY, credentialStoreProviderPath); - char[] passwordChars = config.getPassword(propertyName); - return (isNotEmpty(passwordChars)) ? Optional.of(new String(passwordChars)) : Optional.empty(); - } catch (Exception e) { - logger.warn("Could not load password {} from credential store.", propertyName); - return Optional.empty(); + char[] passwordChars = config.getPassword(key); + return (isNotEmpty(passwordChars)) ? Optional.of(passwordChars) : Optional.empty(); + } catch (IOException e) { + throw new UncheckedIOException(String.format("Could not load password %s from credential store.", key), e); } } + + public Secret getSecret(String key) { + return new HadoopCredential(this, key); + } } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java index 45b79b3..0e5196d 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java @@ -18,21 +18,44 @@ */ package org.apache.ambari.infra.conf.security; +import static org.apache.ambari.infra.conf.security.HadoopCredentialStore.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY; + import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import static org.apache.ambari.infra.conf.security.HadoopCredentialStore.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY; - @Configuration public class InfraManagerSecurityConfig { @Value("${"+ CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY + ":}") private String credentialStoreProviderPath; + @Bean + public HadoopCredentialStore hadoopCredentialStore() { + return new HadoopCredentialStore(credentialStoreProviderPath); + } + + @Bean + public S3Secrets s3SecretStore(HadoopCredentialStore hadoopCredentialStore) { + return new S3Secrets(s3AccessKeyId(hadoopCredentialStore), s3SecretKeyId(hadoopCredentialStore)); + } + + private Secret s3AccessKeyId(HadoopCredentialStore hadoopCredentialStore) { + return new CompositeSecret( + hadoopCredentialStore.getSecret( "AWS_ACCESS_KEY_ID"), + new EnvironmentalSecret("AWS_ACCESS_KEY_ID")); + } + + private Secret s3SecretKeyId(HadoopCredentialStore hadoopCredentialStore) { + return new CompositeSecret( + hadoopCredentialStore.getSecret( "AWS_SECRET_ACCESS_KEY"), + new EnvironmentalSecret("AWS_SECRET_ACCESS_KEY")); + } @Bean - public PasswordStore passwords() { - return new CompositePasswordStore(new HadoopCredentialStore(credentialStoreProviderPath), new SecurityEnvironment()); + public SslSecrets sslSecrets(HadoopCredentialStore hadoopCredentialStore) { + return new SslSecrets( + hadoopCredentialStore.getSecret("infra_manager_keystore_password"), + hadoopCredentialStore.getSecret("infra_manager_truststore_password")); } } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java similarity index 67% copy from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java copy to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java index 8e3387b..30a1ca9 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java @@ -18,11 +18,21 @@ */ package org.apache.ambari.infra.conf.security; -import java.util.Optional; +public class S3Secrets { + private final Secret s3AccessKeyId; + private final Secret s3SecretAccessKey; -public class SecurityEnvironment implements PasswordStore { - @Override - public Optional<String> getPassword(String propertyName) { - return Optional.ofNullable(System.getenv(propertyName)); + public S3Secrets(Secret s3AccessKeyId, Secret s3SecretAccessKey) { + this.s3AccessKeyId = s3AccessKeyId; + this.s3SecretAccessKey = s3SecretAccessKey; + } + + + public Secret getS3AccessKeyId() { + return s3AccessKeyId; + } + + public Secret getS3SecretAccessKey() { + return s3SecretAccessKey; } } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java similarity index 91% rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java index 19848fe..e4f54a3 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java @@ -20,6 +20,6 @@ package org.apache.ambari.infra.conf.security; import java.util.Optional; -public interface PasswordStore { - Optional<String> getPassword(String propertyName); +public interface Secret { + Optional<String> get(); } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java similarity index 66% rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java index 8e3387b..6323e95 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java @@ -18,11 +18,20 @@ */ package org.apache.ambari.infra.conf.security; -import java.util.Optional; +public class SslSecrets { + private final Secret keyStorePassword; + private final Secret trustStorePassword; -public class SecurityEnvironment implements PasswordStore { - @Override - public Optional<String> getPassword(String propertyName) { - return Optional.ofNullable(System.getenv(propertyName)); + public SslSecrets(Secret keyStorePassword, Secret trustStorePassword) { + this.keyStorePassword = keyStorePassword; + this.trustStorePassword = trustStorePassword; + } + + public Secret getKeyStorePassword() { + return keyStorePassword; + } + + public Secret getTrustStorePassword() { + return trustStorePassword; } } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java index 319cc5b..b35c1d2 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java @@ -27,12 +27,11 @@ import java.io.File; import javax.inject.Inject; import org.apache.ambari.infra.conf.InfraManagerDataConfig; -import org.apache.ambari.infra.conf.security.PasswordStore; +import org.apache.ambari.infra.conf.security.S3Secrets; import org.apache.ambari.infra.job.AbstractJobsConfiguration; import org.apache.ambari.infra.job.JobContextRepository; import org.apache.ambari.infra.job.JobScheduler; import org.apache.ambari.infra.job.ObjectSource; -import org.apache.hadoop.fs.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.batch.core.Job; @@ -91,7 +90,7 @@ public class DocumentArchivingConfiguration extends AbstractJobsConfiguration<Ar @Value("#{jobParameters[end]}") String intervalEnd, DocumentWiper documentWiper, JobContextRepository jobContextRepository, - PasswordStore passwordStore) { + S3Secrets s3Secrets) { File baseDir = new File(infraManagerDataConfig.getDataFolder(), "exporting"); CompositeFileAction fileAction = new CompositeFileAction(new BZip2Compressor()); @@ -99,7 +98,7 @@ public class DocumentArchivingConfiguration extends AbstractJobsConfiguration<Ar case S3: fileAction.add(new S3Uploader( parameters.s3Properties().orElseThrow(() -> new IllegalStateException("S3 properties are not provided!")), - passwordStore)); + s3Secrets)); break; case HDFS: org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration(); diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java index 7c4de52..3f541b8 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java @@ -25,31 +25,40 @@ import java.io.FileReader; import java.io.IOException; import java.io.Reader; import java.io.UncheckedIOException; -import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Optional; -import org.apache.ambari.infra.conf.security.PasswordStore; +import org.apache.ambari.infra.conf.security.Secret; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -public class S3AccessCsv implements PasswordStore { +public class S3AccessCsv implements Secret { private static final Logger logger = LogManager.getLogger(S3AccessCsv.class); + public static final String ACCESS_KEY_ID = "Access key ID"; + public static final String SECRET_ACCESS_KEY = "Secret access key"; - public static S3AccessCsv file(String path) { + + public static S3AccessCsv file(String path, String propertyName) { try { - return new S3AccessCsv(new FileReader(path)); + return new S3AccessCsv(new FileReader(path), propertyName); } catch (FileNotFoundException e) { throw new UncheckedIOException(e); } } - private Map<String, String> passwordMap = new HashMap<>(); + private final Reader reader; + private final String propertyName; + + S3AccessCsv(Reader reader, String propertyName) { + this.reader = reader; + this.propertyName = propertyName; + } - public S3AccessCsv(Reader reader) { + @Override + public Optional<String> get() { try (CSVParser csvParser = CSVParser.parse(reader, DEFAULT.withHeader( S3AccessKeyNames.AccessKeyId.getCsvName(), S3AccessKeyNames.SecretAccessKey.getCsvName()))) { Iterator<CSVRecord> iterator = csvParser.iterator(); @@ -62,8 +71,8 @@ public class S3AccessCsv implements PasswordStore { throw new S3AccessCsvFormatException("Csv file contains less than 2 columns!"); } - checkColumnExists(record, S3AccessKeyNames.AccessKeyId); - checkColumnExists(record, S3AccessKeyNames.SecretAccessKey); + checkColumnExists(record, ACCESS_KEY_ID); + checkColumnExists(record, SECRET_ACCESS_KEY); if (!iterator.hasNext()) { throw new S3AccessCsvFormatException("Csv file contains header only!"); @@ -72,23 +81,15 @@ public class S3AccessCsv implements PasswordStore { record = iterator.next(); Map<String, Integer> header = csvParser.getHeaderMap(); - for (S3AccessKeyNames keyNames : S3AccessKeyNames.values()) - passwordMap.put(keyNames.getEnvVariableName(), record.get(header.get(keyNames.getCsvName()))); + return Optional.ofNullable(record.get(header.get(propertyName))); } catch (IOException e) { throw new UncheckedIOException(e); - } catch (S3AccessCsvFormatException e) { - logger.warn("Unable to parse csv file: {}", e.getMessage()); } } - private void checkColumnExists(CSVRecord record, S3AccessKeyNames s3AccessKeyName) { - if (!s3AccessKeyName.getCsvName().equals(record.get(s3AccessKeyName.getCsvName()))) { - throw new S3AccessCsvFormatException(String.format("Csv file does not contain the required column: '%s'", s3AccessKeyName.getCsvName())); + private void checkColumnExists(CSVRecord record, String s3AccessKeyName) { + if (!s3AccessKeyName.equals(record.get(s3AccessKeyName))) { + throw new S3AccessCsvFormatException(String.format("Csv file does not contain the required column: '%s'", s3AccessKeyName)); } } - - @Override - public Optional<String> getPassword(String propertyName) { - return Optional.ofNullable(passwordMap.get(propertyName)); - } } diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java index 3e1310a..f3c92b7 100644 --- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java +++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java @@ -9,8 +9,9 @@ import java.io.UncheckedIOException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; -import org.apache.ambari.infra.conf.security.CompositePasswordStore; -import org.apache.ambari.infra.conf.security.PasswordStore; +import org.apache.ambari.infra.conf.security.CompositeSecret; +import org.apache.ambari.infra.conf.security.S3Secrets; +import org.apache.ambari.infra.conf.security.Secret; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.xmlpull.v1.XmlPullParserException; @@ -51,22 +52,28 @@ public class S3Uploader extends AbstractFileAction { private final MinioClient client; private final String keyPrefix; private final String bucketName; + private final Secret s3AccessKey; + private final Secret s3SecretKey; - public S3Uploader(S3Properties s3Properties, PasswordStore passwordStore) { + public S3Uploader(S3Properties s3Properties, S3Secrets s3Secrets) { logger.info("Initializing S3 client with " + s3Properties); this.keyPrefix = s3Properties.getS3KeyPrefix(); this.bucketName = s3Properties.getS3BucketName(); - PasswordStore compositePasswordStore = passwordStore; - if (isNotBlank((s3Properties.getS3AccessFile()))) - compositePasswordStore = new CompositePasswordStore(passwordStore, S3AccessCsv.file(s3Properties.getS3AccessFile())); + if (isNotBlank(s3Properties.getS3AccessFile())) { + this.s3AccessKey = new CompositeSecret(s3Secrets.getS3AccessKeyId(), S3AccessCsv.file(s3Properties.getS3AccessFile(), "Access key ID")); + this.s3SecretKey = new CompositeSecret(s3Secrets.getS3SecretAccessKey(), S3AccessCsv.file(s3Properties.getS3AccessFile(), "Secret access key")); + } + else { + this.s3AccessKey = s3Secrets.getS3AccessKeyId(); + this.s3SecretKey = s3Secrets.getS3SecretAccessKey(); + } try { - client = new MinioClient(s3Properties.getS3EndPoint(), compositePasswordStore.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()) - .orElseThrow(() -> new IllegalArgumentException("Access key Id is not present!")), - compositePasswordStore.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()) - .orElseThrow(() -> new IllegalArgumentException("Secret Access Key is not present!"))); + client = new MinioClient(s3Properties.getS3EndPoint(), + s3AccessKey.get().orElseThrow(() -> new IllegalArgumentException("Access key Id is not present!")), + s3SecretKey.get().orElseThrow(() -> new IllegalArgumentException("Secret Access Key is not present!"))); if (!client.bucketExists(bucketName)) client.makeBucket(bucketName); diff --git a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java similarity index 70% rename from ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java rename to ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java index 26a6953..78b0269 100644 --- a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java +++ b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java @@ -1,11 +1,11 @@ package org.apache.ambari.infra.conf.security; -import org.junit.Test; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; import java.util.Optional; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import org.junit.Test; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -25,24 +25,24 @@ import static org.junit.Assert.assertThat; * specific language governing permissions and limitations * under the License. */ -public class CompositePasswordStoreTest { +public class CompositeSecretTest { @Test public void testGetPasswordReturnNullIfNoPasswordStoresWereAdded() { - assertThat(new CompositePasswordStore().getPassword("any").isPresent(), is(false)); + assertThat(new CompositeSecret().get().isPresent(), is(false)); } @Test public void testGetPasswordReturnNullIfPasswordNotFoundInAnyStore() { - assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.empty()).getPassword("any").isPresent(), is(false)); + assertThat(new CompositeSecret(Optional::empty, Optional::empty).get().isPresent(), is(false)); } @Test public void testGetPasswordReturnPasswordFromFirstStoreIfExists() { - assertThat(new CompositePasswordStore((prop) -> Optional.of("Pass"), (prop) -> Optional.empty()).getPassword("any").get(), is("Pass")); + assertThat(new CompositeSecret(() -> Optional.of("Pass"), Optional::empty).get().get(), is("Pass")); } @Test public void testGetPasswordReturnPasswordFromSecondStoreIfNotExistsInFirst() { - assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.of("Pass")).getPassword("any").get(), is("Pass")); + assertThat(new CompositeSecret(Optional::empty, () -> Optional.of("Pass")).get().get(), is("Pass")); } } \ No newline at end of file diff --git a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java index e34a222..eb4b011 100644 --- a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java +++ b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java @@ -1,11 +1,12 @@ package org.apache.ambari.infra.job.archive; -import org.junit.Test; +import static org.apache.ambari.infra.job.archive.S3AccessCsv.ACCESS_KEY_ID; +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; import java.io.StringReader; -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; +import org.junit.Test; /* * Licensed to the Apache Software Foundation (ASF) under one @@ -33,38 +34,33 @@ public class S3AccessCsvTest { private static final String ANY_CSV_FILE = "Column1,Column2\n" + "Foo,Bar\n"; - @Test + @Test(expected = S3AccessCsvFormatException.class) public void testGetPasswordReturnsNullIfInputIsEmpty() { - S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("")); - assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false)); - assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false)); + S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(""), ACCESS_KEY_ID); + assertThat(accessCsv.get().isPresent(), is(false)); } @Test public void testGetPasswordReturnsAccessAndSecretKeyIfInputIsAValidS3AccessFile() { - S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(VALID_ACCESS_FILE)); - assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).get(), is("someKey")); - assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).get(), is("someSecret")); + S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(VALID_ACCESS_FILE), ACCESS_KEY_ID); + assertThat(accessCsv.get().get(), is("someKey")); } - @Test - public void testGetPasswordReturnsNullIfNotAValidS3AccessFileProvided() { - S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(ANY_CSV_FILE)); - assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false)); - assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false)); + @Test(expected = S3AccessCsvFormatException.class) + public void testGetPasswordThrowsExceptionIfNotAValidS3AccessFileProvided() { + S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(ANY_CSV_FILE), ACCESS_KEY_ID); + assertThat(accessCsv.get().isPresent(), is(false)); } - @Test - public void testGetPasswordReturnsNullIfAHeaderOnlyS3AccessFileProvided() { - S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Secret access key\n")); - assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false)); - assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false)); + @Test(expected = S3AccessCsvFormatException.class) + public void testGetPasswordThrowsExceptionIfAHeaderOnlyS3AccessFileProvided() { + S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Secret access key\n"), ACCESS_KEY_ID); + assertThat(accessCsv.get().isPresent(), is(false)); } - @Test - public void testGetPasswordReturnsNullIfOnlyOneValidColumnProvided() { - S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Column\n")); - assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false)); - assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false)); + @Test(expected = S3AccessCsvFormatException.class) + public void testGetPasswordThrowsExceptionIfOnlyOneValidColumnProvided() { + S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Column\n"), ACCESS_KEY_ID); + assertThat(accessCsv.get().isPresent(), is(false)); } } \ No newline at end of file diff --git a/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh b/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh index 7ddb757..a3df897 100755 --- a/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh +++ b/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh @@ -101,8 +101,8 @@ function create_collection() { pushd $sdir/../ local AMBARI_SOLR_MANAGER_LOCATION=$(pwd) cd $AMBARI_SOLR_MANAGER_LOCATION/docker - docker exec docker_solr_1 solr create_collection -force -c hadoop_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/hadoop_logs/conf -n hadoop_logs_conf - docker exec docker_solr_1 solr create_collection -force -c audit_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/audit_logs/conf -n audit_logs_conf + docker exec solr solr create_collection -force -c hadoop_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/hadoop_logs/conf -n hadoop_logs_conf + docker exec solr solr create_collection -force -c audit_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/audit_logs/conf -n audit_logs_conf popd }