Repository: ambari Updated Branches: refs/heads/trunk 7b0ee28ef -> 9c952c300
AMBARI-19645. Log Search: support credential store api - part 1 (oleewere) Change-Id: I00e5229da73b78dd0da998f947c208cbc631b81b Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9c952c30 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9c952c30 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9c952c30 Branch: refs/heads/trunk Commit: 9c952c300881623de5911dab06fa24f2a934b1a7 Parents: 7b0ee28 Author: oleewere <oleew...@gmail.com> Authored: Tue Jan 24 00:15:09 2017 +0100 Committer: oleewere <oleew...@gmail.com> Committed: Tue Jan 24 00:30:25 2017 +0100 ---------------------------------------------------------------------- .../apache/ambari/logfeeder/util/SSLUtil.java | 52 +++++++++++-- .../src/main/scripts/run.sh | 78 ++++++++++---------- .../apache/ambari/logsearch/util/SSLUtil.java | 65 ++++++++++++---- 3 files changed, 135 insertions(+), 60 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9c952c30/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java index ea9f45d..80b34e0 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SSLUtil.java @@ -21,19 +21,27 @@ package org.apache.ambari.logfeeder.util; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; import java.io.File; public class SSLUtil { + private static final Logger LOG = Logger.getLogger(SSLUtil.class); + private static final String KEYSTORE_LOCATION_ARG = "javax.net.ssl.keyStore"; private static final String TRUSTSTORE_LOCATION_ARG = "javax.net.ssl.trustStore"; private static final String KEYSTORE_TYPE_ARG = "javax.net.ssl.keyStoreType"; private static final String TRUSTSTORE_TYPE_ARG = "javax.net.ssl.trustStoreType"; private static final String KEYSTORE_PASSWORD_ARG = "javax.net.ssl.keyStorePassword"; private static final String TRUSTSTORE_PASSWORD_ARG = "javax.net.ssl.trustStorePassword"; + private static final String KEYSTORE_PASSWORD_PROPERTY_NAME = "logfeeder_keystore_password"; + private static final String TRUSTSTORE_PASSWORD_PROPERTY_NAME = "logfeeder_truststore_password"; private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt"; private static final String TRUSTSTORE_PASSWORD_FILE = "ts_pass.txt"; - + + private static final String CREDENTIAL_STORE_PROVIDER_PATH = "hadoop.security.credential.provider.path"; private static final String LOGFEEDER_CERT_DEFAULT_FOLDER = "/etc/ambari-logsearch-portal/conf/keys"; private static final String LOGFEEDER_STORE_DEFAULT_PASSWORD = "bigdata"; @@ -66,17 +74,48 @@ public class SSLUtil { } public static void ensureStorePasswords() { - ensureStorePassword(KEYSTORE_LOCATION_ARG, KEYSTORE_PASSWORD_ARG, KEYSTORE_PASSWORD_FILE); - ensureStorePassword(TRUSTSTORE_LOCATION_ARG, TRUSTSTORE_PASSWORD_ARG, TRUSTSTORE_PASSWORD_FILE); + ensureStorePassword(KEYSTORE_LOCATION_ARG, KEYSTORE_PASSWORD_ARG, KEYSTORE_PASSWORD_PROPERTY_NAME, KEYSTORE_PASSWORD_FILE); + ensureStorePassword(TRUSTSTORE_LOCATION_ARG, TRUSTSTORE_PASSWORD_ARG, TRUSTSTORE_PASSWORD_PROPERTY_NAME, TRUSTSTORE_PASSWORD_FILE); } - private static void ensureStorePassword(String locationArg, String pwdArg, String pwdFile) { + private static void ensureStorePassword(String locationArg, String pwdArg, String propertyName, String fileName) { if (StringUtils.isNotEmpty(System.getProperty(locationArg)) && StringUtils.isEmpty(System.getProperty(pwdArg))) { - String password = getPasswordFromFile(pwdFile); + String password = getPassword(propertyName, fileName); System.setProperty(pwdArg, password); } } + private static String getPassword(String propertyName, String fileName) { + String credentialStorePassword = getPasswordFromCredentialStore(propertyName); + if (credentialStorePassword != null) { + return credentialStorePassword; + } + + String filePassword = getPasswordFromFile(fileName); + if (filePassword != null) { + return filePassword; + } + + return LOGFEEDER_STORE_DEFAULT_PASSWORD; + } + + private static String getPasswordFromCredentialStore(String propertyName) { + try { + String providerPath = LogFeederUtil.getStringProperty(CREDENTIAL_STORE_PROVIDER_PATH); + if (providerPath == null) { + return null; + } + + Configuration config = new Configuration(); + config.set(CREDENTIAL_STORE_PROVIDER_PATH, providerPath); + char[] passwordChars = config.getPassword(propertyName); + return (ArrayUtils.isNotEmpty(passwordChars)) ? new String(passwordChars) : null; + } catch (Exception e) { + LOG.warn(String.format("Could not load password %s from credential store, using default password", propertyName)); + return null; + } + } + private static String getPasswordFromFile(String fileName) { try { File pwdFile = new File(LOGFEEDER_CERT_DEFAULT_FOLDER, fileName); @@ -87,7 +126,8 @@ public class SSLUtil { return FileUtils.readFileToString(pwdFile); } } catch (Exception e) { - throw new RuntimeException("Exception occurred during read/write password file for keystore/truststore.", e); + LOG.warn("Exception occurred during read/write password file for keystore/truststore.", e); + return null; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/9c952c30/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh index 645c5f0..53cd17f 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh @@ -19,49 +19,48 @@ cd `dirname $0`; script_dir=`pwd`; cd $curr_dir foreground=0 if [ "$1" = "-foreground" ]; then - foreground=1 - shift + foreground=1 + shift fi if [ ! -z "$LOGFEEDER_INCLUDE" ]; then - source $LOGFEEDER_INCLUDE + source $LOGFEEDER_INCLUDE fi if [ ! -z "$LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE" ]; then - source $LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE + source $LOGSEARCH_SOLR_CLIENT_SSL_INCLUDE fi JAVA=java if [ -x $JAVA_HOME/bin/java ]; then - JAVA=$JAVA_HOME/bin/java + JAVA=$JAVA_HOME/bin/java fi if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then - LOGFEEDER_JAVA_MEM="-Xmx512m" + LOGFEEDER_JAVA_MEM="-Xmx512m" fi if [ "$LOGFILE" = "" ]; then - LOGFILE="/var/log/logfeeder/logfeeder.out" + LOGFILE="/var/log/logfeeder/logfeeder.out" fi if [ "$PID_FILE" = "" ]; then - LOGFEEDER_PID_DIR=$HOME - PID_FILE=$LOGFEEDER_PID_DIR/logsearch-logfeeder-$USER.pid + LOGFEEDER_PID_DIR=$HOME + PID_FILE=$LOGFEEDER_PID_DIR/logsearch-logfeeder-$USER.pid fi if [ "$LOGFEEDER_CONF_DIR" = "" ]; then - LOGFEEDER_CONF_DIR="/etc/logfeeder/conf" - if [ ! -d $LOGFEEDER_CONF_DIR ]; then - if [ -d $script_dir/classes ]; then - LOGFEEDER_CONF_DIR=$script_dir/classes - fi + LOGFEEDER_CONF_DIR="/etc/logfeeder/conf" + if [ ! -d $LOGFEEDER_CONF_DIR ]; then + if [ -d $script_dir/classes ]; then + LOGFEEDER_CONF_DIR=$script_dir/classes + fi fi - fi LOGFEEDER_DEBUG_SUSPEND=${LOGFEEDER_DEBUG_SUSPEND:-n} if [ "$LOGFEEDER_DEBUG" = "true" ] && [ ! -z "$LOGFEEDER_DEBUG_PORT" ]; then - LOGFEEDER_JAVA_OPTS="$LOGSEARCH_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND " + LOGFEEDER_JAVA_OPTS="$LOGFEEDER_JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=$LOGFEEDER_DEBUG_PORT,server=y,suspend=$LOGFEEDER_DEBUG_SUSPEND " fi LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log @@ -74,32 +73,31 @@ if [ "$LOGFEEDER_SSL" = "true" ]; then fi if [ $foreground -eq 0 ]; then - if [ -f ${PID_FILE} ]; then - PID=`cat ${PID_FILE}` - if kill -0 $PID 2>/dev/null; then - echo "logfeeder already running (${PID}) killing..." - kill $PID 2>/dev/null - sleep 5 - if kill -0 $PID 2>/dev/null; then - echo "logfeeder still running. Will kill process forcefully in another 10 seconds..." - sleep 10 - kill -9 $PID 2>/dev/null - sleep 2 - fi - fi - - if kill -0 $PID 2>/dev/null; then - echo "ERROR: Even after all efforts to stop logfeeder, it is still running. pid=$PID. Please manually kill the service and try again." - exit 1 - fi + if [ -f ${PID_FILE} ]; then + PID=`cat ${PID_FILE}` + if kill -0 $PID 2>/dev/null; then + echo "logfeeder already running (${PID}) killing..." + kill $PID 2>/dev/null + sleep 5 + if kill -0 $PID 2>/dev/null; then + echo "logfeeder still running. Will kill process forcefully in another 10 seconds..." + sleep 10 + kill -9 $PID 2>/dev/null + sleep 2 + fi fi - echo "Starting logfeeder. Output file=$LOGFILE pid_file=$PID_FILE" - #LOGFEEDER_CLI_CLASSPATH= - #set -x - nohup $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $* > $LOGFILE 2>&1 & - echo $! > $PID_FILE + if kill -0 $PID 2>/dev/null; then + echo "ERROR: Even after all efforts to stop logfeeder, it is still running. pid=$PID. Please manually kill the service and try again." + exit 1 + fi + fi + + echo "Starting logfeeder. Output file=$LOGFILE pid_file=$PID_FILE" + #LOGFEEDER_CLI_CLASSPATH=set -x + nohup $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGFEEDER_GC_OPTS $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $* > $LOGFILE 2>&1 & + echo $! > $PID_FILE else - $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $* + $JAVA -cp "$LOGFEEDER_CLI_CLASSPATH:$LOGFEEDER_CONF_DIR:$script_dir/libs/*:$script_dir/classes" $LOGFEEDER_JAVA_MEM $LOGFEEDER_JAVA_OPTS $JMX org.apache.ambari.logfeeder.LogFeeder $* fi http://git-wip-us.apache.org/repos/asf/ambari/blob/9c952c30/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java index 2fb4ff3..e0111e7 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SSLUtil.java @@ -21,8 +21,12 @@ package org.apache.ambari.logsearch.util; import javax.net.ssl.SSLContext; +import org.apache.ambari.logsearch.common.PropertiesHelper; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.bouncycastle.jce.X509Principal; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.x509.X509V3CertificateGenerator; @@ -64,9 +68,12 @@ public class SSLUtil { private static final String TRUSTSTORE_PASSWORD_ARG = "javax.net.ssl.trustStorePassword"; private static final String TRUSTSTORE_TYPE_ARG = "javax.net.ssl.trustStoreType"; private static final String DEFAULT_TRUSTSTORE_TYPE = "JKS"; + private static final String KEYSTORE_PASSWORD_PROPERTY_NAME = "logsearch_keystore_password"; + private static final String TRUSTSTORE_PASSWORD_PROPERTY_NAME = "logsearch_truststore_password"; private static final String KEYSTORE_PASSWORD_FILE = "ks_pass.txt"; private static final String TRUSTSTORE_PASSWORD_FILE = "ts_pass.txt"; - + private static final String CREDENTIAL_STORE_PROVIDER_PATH = "hadoop.security.credential.provider.path"; + private SSLUtil() { throw new UnsupportedOperationException(); } @@ -104,8 +111,8 @@ public class SSLUtil { } public static SslContextFactory getSslContextFactory() { - setPasswordIfSysPropIsEmpty(KEYSTORE_PASSWORD_ARG, KEYSTORE_PASSWORD_FILE); - setPasswordIfSysPropIsEmpty(TRUSTSTORE_PASSWORD_ARG, TRUSTSTORE_PASSWORD_FILE); + setPasswordIfSysPropIsEmpty(KEYSTORE_PASSWORD_ARG, KEYSTORE_PASSWORD_PROPERTY_NAME, KEYSTORE_PASSWORD_FILE); + setPasswordIfSysPropIsEmpty(TRUSTSTORE_PASSWORD_ARG, TRUSTSTORE_PASSWORD_PROPERTY_NAME, TRUSTSTORE_PASSWORD_FILE); SslContextFactory sslContextFactory = new SslContextFactory(); sslContextFactory.setKeyStorePath(getKeyStoreLocation()); sslContextFactory.setKeyStorePassword(getKeyStorePassword()); @@ -137,20 +144,50 @@ public class SSLUtil { } } - private static String getPasswordFromFile(String certFolder, String fileName, String defaultPassword) { + private static String getPasswordFromFile(String fileName) { try { - String pwdFileName = String.format("%s/%s", certFolder, fileName); - File pwdFile = new File(pwdFileName); + File pwdFile = new File(LOGSEARCH_CERT_DEFAULT_FOLDER, fileName); if (!pwdFile.exists()) { - FileUtils.writeStringToFile(pwdFile, defaultPassword); - return defaultPassword; + FileUtils.writeStringToFile(pwdFile, LOGSEARCH_KEYSTORE_DEFAULT_PASSWORD); + return LOGSEARCH_KEYSTORE_DEFAULT_PASSWORD; } else { return FileUtils.readFileToString(pwdFile); } } catch (Exception e) { - String errMsg = "Exception occurred during read/write password file for keystore."; - throw new RuntimeException(errMsg, e); + LOG.warn("Exception occurred during read/write password file for keystore/truststore.", e); + return null; + } + } + + private static String getPasswordFromCredentialStore(String propertyName) { + try { + String providerPath = PropertiesHelper.getProperty(CREDENTIAL_STORE_PROVIDER_PATH); + if (providerPath == null) { + return null; + } + + Configuration config = new Configuration(); + config.set(CREDENTIAL_STORE_PROVIDER_PATH, providerPath); + char[] passwordChars = config.getPassword(propertyName); + return (ArrayUtils.isNotEmpty(passwordChars)) ? new String(passwordChars) : null; + } catch (Exception e) { + LOG.warn(String.format("Could not load password %s from credential store, using default password", propertyName)); + return null; + } + } + + private static String getPassword(String propertyName, String fileName) { + String credentialStorePassword = getPasswordFromCredentialStore(propertyName); + if (credentialStorePassword != null) { + return credentialStorePassword; + } + + String filePassword = getPasswordFromFile(fileName); + if (filePassword != null) { + return filePassword; } + + return LOGSEARCH_KEYSTORE_DEFAULT_PASSWORD; } /** @@ -200,10 +237,10 @@ public class SSLUtil { } } - private static void setPasswordIfSysPropIsEmpty(String prop, String pwdFile) { - if (StringUtils.isEmpty(System.getProperty(prop))) { - String password = getPasswordFromFile(LOGSEARCH_CERT_DEFAULT_FOLDER, pwdFile, LOGSEARCH_KEYSTORE_DEFAULT_PASSWORD); - System.setProperty(prop, password); + private static void setPasswordIfSysPropIsEmpty(String pwdArg, String propertyName, String fileName) { + if (StringUtils.isEmpty(System.getProperty(pwdArg))) { + String password = getPassword(propertyName, fileName); + System.setProperty(pwdArg, password); } }