AMBARI-19806: After setting up hadoop credential, cannot start Hive Metastore
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d340fe93 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d340fe93 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d340fe93 Branch: refs/heads/branch-dev-patch-upgrade Commit: d340fe93b51b82dc380ad7783cd63b7e76b72224 Parents: f36af3d Author: Nahappan Somasundaram <[email protected]> Authored: Tue Jan 31 15:34:38 2017 -0800 Committer: Nahappan Somasundaram <[email protected]> Committed: Wed Feb 1 15:09:44 2017 -0800 ---------------------------------------------------------------------- .../ambari_agent/CustomServiceOrchestrator.py | 4 + ambari-server/pom.xml | 14 + .../server/credentialapi/CredentialUtil.java | 580 +++++++++++++++++ .../0.12.0.2.0/package/scripts/params_linux.py | 54 +- .../credentialapi/CredentialUtilTest.java | 644 +++++++++++++++++++ 5 files changed, 1292 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py index 8f1848c..9baaf08 100644 --- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py +++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py @@ -75,6 +75,9 @@ class CustomServiceOrchestrator(): # The property name used by the hadoop credential provider CREDENTIAL_PROVIDER_PROPERTY_NAME = 'hadoop.security.credential.provider.path' + # Property name for credential store class path + CREDENTIAL_STORE_CLASS_PATH_NAME = 'credentialStoreClassPath' + def __init__(self, config, controller): self.config = config self.tmp_dir = config.get('agent', 'prefix') @@ -286,6 +289,7 @@ class CustomServiceOrchestrator(): os.chmod(file_path, 0644) # group and others should have read access so that the service user can read # Add JCEKS provider path instead config[self.CREDENTIAL_PROVIDER_PROPERTY_NAME] = provider_path + config[self.CREDENTIAL_STORE_CLASS_PATH_NAME] = cs_lib_path return cmd_result http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index 0441867..89b734e 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -134,6 +134,15 @@ value="org.apache.ambari.server.DBConnectionVerification" /> </manifest> </jar> + <jar destfile="target/CredentialUtil.jar"> + <fileset dir="${basedir}/target/classes/"> + <include name="**/CredentialUtil*.class" /> + </fileset> + <manifest> + <attribute name="Main-Class" + value="org.apache.ambari.server.credentialapi.CredentialUtil" /> + </manifest> + </jar> </tasks> </configuration> <goals> @@ -1469,6 +1478,11 @@ <version>${hadoop.version}</version> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + </dependency> + <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> <version>${jetty.version}</version> http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-server/src/main/java/org/apache/ambari/server/credentialapi/CredentialUtil.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/credentialapi/CredentialUtil.java b/ambari-server/src/main/java/org/apache/ambari/server/credentialapi/CredentialUtil.java new file mode 100644 index 0000000..e6d7a37 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/credentialapi/CredentialUtil.java @@ -0,0 +1,580 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.server.credentialapi; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.CredentialShell; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; + +/** + * Command line utility that wraps over CredentialShell. Extends the + * create command to overwrite a credential if it exists. Also + * provides the ability to get the decrypted password. + * + * The CLI structure and code is the same as CredentialShell. + */ +public class CredentialUtil extends Configured implements Tool { + /** + * List of supported commands. + */ + final static private String COMMANDS = + " [--help]\n" + + " [" + CreateCommand.USAGE + "]\n" + + " [" + DeleteCommand.USAGE + "]\n" + + " [" + ListCommand.USAGE + "]\n" + + " [" + GetCommand.USAGE + "]\n"; + + /** + * JCEKS provider prefix. + */ + public static final String jceksPrefix = JavaKeyStoreProvider.SCHEME_NAME + "://file"; + + /** + * Local JCEKS provider prefix. + */ + public static final String localJceksPrefix = "localjceks://file"; + + /** + * Password alias + */ + private String alias = null; + + /** + * Password specified using the -value option + */ + private String value = null; + + /** + * Provider specified using the -provider option + */ + protected CredentialProvider provider; + + /** + * When creating a credential, overwrite the credential if it exists. + * If -n option is specified, this will be set to false. + */ + private boolean overwrite = true; + + /** + * Prompt for user confirmation before deleting/overwriting a credential. + * If -f option is specified, it will be set to false. In the case + * of a create command, it will be set to false if -n or -f is specified. + */ + private boolean interactive = true; + + /** + * One of the supported credential commands. + */ + private Command command = null; + + /** + * Main program. + * + * @param args Command line arguments + * @throws Exception + */ + public static void main(String[] args) throws Exception { + int res = ToolRunner.run(new Configuration(), new CredentialUtil(), args); + System.exit(res); + } + + /** + * Called by ToolRunner.run(). This is the entry point to the tool. + * Parses the command line arguments and executes the appropriate command. + * + * @param args - Arguments supplied by the user. + * @return - 0 if successful. 1 in case of a failure. + * @throws Exception - If something goes wrong during command execution. + */ + @Override + public int run(String[] args) throws Exception { + int exitCode = 1; + + for (int i = 0; i < args.length; ++i) { + if (args[i].equals("create")) { + if (i == args.length - 1) { + return 1; + } + command = new CreateCommand(); + alias = args[++i]; + if (alias.equals("-h") || alias.equals("-help")) { + printUsage(); + return 0; + } + } else if (args[i].equals("get")) { + if (i == args.length - 1) { + return 1; + } + command = new GetCommand(); + alias = args[++i]; + if (alias.equals("-h") || alias.equals("-help")) { + printUsage(); + return 0; + } + } else if (args[i].equals("delete")) { + if (i == args.length - 1) { + printUsage(); + return 1; + } + command = new DeleteCommand(); + alias = args[++i]; + if (alias.equals("-help")) { + printUsage(); + return 0; + } + } else if (args[i].equals("list")) { + if (i < args.length - 1) { + alias = args[i + 1]; + } + command = new ListCommand(); + if (alias.equals("-h") || alias.equals("-help")) { + printUsage(); + return 0; + } + alias = "not required"; + } else if (args[i].equals("-provider")) { + if (i == args.length - 1) { + return 1; + } + String providerPath = getNormalizedPath(args[++i]); + getConf().set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, providerPath); + provider = getCredentialProvider(); + } else if (args[i].equals("-f") || args[i].equals("-force")) { + interactive = false; + overwrite = true; + } else if (args[i].equals("-n")) { + interactive = false; + overwrite = false; + } else if (args[i].equals("-v") || args[i].equals("-value")) { + value = args[++i]; + } else if (args[i].equals("-h") || args[i].equals("-help")) { + printUsage(); + return 0; + } else { + printUsage(); + ToolRunner.printGenericCommandUsage(System.err); + return 1; + } + } + + if (command == null) { + printUsage(); + } + else if (command.validate()) { + exitCode = command.execute(); + } + + return exitCode; + } + + /** + * Prints a command specific usage or overall tool usage. + */ + protected void printUsage() { + System.out.println(getUsagePrefix() + COMMANDS); + if (command != null) { + System.out.println(command.getUsage()); + } + else { + System.out.println("=========================================================" + + "======"); + System.out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC); + System.out.println("=========================================================" + + "======"); + System.out.println(DeleteCommand.USAGE + ":\n\n" + DeleteCommand.DESC); + System.out.println("=========================================================" + + "======"); + System.out.println(ListCommand.USAGE + ":\n\n" + ListCommand.DESC); + System.out.println("=========================================================" + + "======"); + System.out.println(GetCommand.USAGE + ":\n\n" + GetCommand.DESC); + } + } + + /** + * Overridden by the command line driver to provide name of the tool. + * + * @return - CLI specific information, like tool name, year, copyright, etc. + */ + protected String getUsagePrefix() { + return "Usage: "; + } + + /* + * Normalize the providerPath to jceks://file/<file_path> or localjceks://file/<file_path> + */ + private static String getNormalizedPath(String providerPath) { + if (providerPath != null) { + String jceksPath; + + if (providerPath.startsWith("/")) { + providerPath = providerPath.substring(1); + } + + jceksPath = StringUtils.lowerCase(providerPath.trim()); + + if (!jceksPath.startsWith(StringUtils.lowerCase(jceksPrefix)) && + !jceksPath.startsWith(localJceksPrefix)) { + providerPath = jceksPrefix + "/" + providerPath; + } + } + + return providerPath; + } + + /** + * Gets the provider object for the user specified provider. + * + * @return - A credential provider. + */ + private CredentialProvider getCredentialProvider() { + CredentialProvider provider = null; + + List<CredentialProvider> providers; + try { + providers = CredentialProviderFactory.getProviders(getConf()); + provider = providers.get(0); + } catch (IOException e) { + e.printStackTrace(System.err); + } + + return provider; + } + + /** + * CredentialCommand base class + */ + private abstract class Command { + /** + * Validates the user input. + * + * @return - True if inputs are valid. False otherwise. + */ + public boolean validate() { + boolean rc = true; + + if (alias == null || alias.isEmpty()) { + System.out.println("There is no alias specified. Please provide the" + + "mandatory <alias>. See the usage description with -help."); + rc = false; + } + + if (provider == null) { + System.out.println("There are no valid CredentialProviders configured." + + "\nCredential will not be created.\n" + + "Consider using the -provider option to indicate the provider" + + " to use."); + rc = false; + } + + return rc; + } + + /** + * Gets command usage and description. + * + * @return + */ + public abstract String getUsage(); + + /** + * Called by run(). Implemented by the concrete command classes. + * + * @return - 0 if successful. 1 on failure. + * @throws Exception - If something goes wrong. + */ + public abstract int execute() throws Exception; + } + + /** + * Gets the credential for the specified alias from the + * specified provider. + */ + private class GetCommand extends Command { + public static final String USAGE = "get <alias> [-provider provider-path]"; + public static final String DESC = + "The get subcommand gets the credential for the specified alias\n" + + "from the provider specified through the -provider argument.\n"; + + /** + * Executes the get command. Prints the clear text password on the command line. + * + * @return - 0 on success; 1 on failure. + * @throws IOException + */ + @Override + public int execute() throws IOException { + int exitCode = 0; + + try { + String credential = getCredential(); + if (credential == null) { + exitCode = 1; + } else { + System.out.println(credential); + } + } catch (IOException ex) { + System.out.println("Cannot get the credential for the specified alias." + + ": " + ex.getMessage()); + throw ex; + } + + return exitCode; + } + + /** + * Gets the clear text password from the credential provider. + * + * @return - Decrypted password for the specified alias. + * @throws IOException + */ + private String getCredential() throws IOException { + String credential = null; + CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias); + + if (credEntry != null) { + char[] password = credEntry.getCredential(); + if (password != null) { + credential = String.valueOf(password); + } + } + + return credential; + } + + /** + * Usage and description. + * + * @return + */ + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } + + /** + * Creates a new credential for the alias specified or overwrites an + * existing credential + */ + private class CreateCommand extends Command { + /** + * Usage summary + */ + public static final String USAGE = + "create <alias> [-value credential] [-provider provider-path] [-f | -n]"; + + /** + * Command description + */ + public static final String DESC = + "The create subcommand creates a new credential or overwrites\n" + + "an existing credential for the name specified\n" + + "as the <alias> argument within the provider indicated through\n" + + "the -provider argument. The command asks for confirmation to\n" + + "overwrite the existing credential unless the -f option is specified.\n" + + "Specify -n to not overwrite if the credential exists.\nThe option specified last wins."; + + /** + * Creates or updates the specified credential. + * + * @return - 0 on success; 1 on failure. + * @throws Exception + */ + @Override + public int execute() throws Exception { + int exitCode = 0; + CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias); + + if (credEntry != null) { + /* + * If credential already exists, overwrite if -f flag was specified. + * overwrite is true if -f was specified. + * overwrite is false if -n was specified. + * if neither options were specified, prompt the user. + */ + if (interactive) { + // prompt the user to confirm or reject the overwrite + overwrite = ToolRunner + .confirmPrompt("You are about to OVERWRITE the credential " + + alias + " from CredentialProvider " + provider.toString() + + ". Continue? "); + } + + if (overwrite) { + // delete the existing credential + DeleteCommand deleteCommand = new DeleteCommand(); + exitCode = deleteCommand.execute(); + } else { + // nothing to do + return 0; + } + } + + // create new or overwrite existing credential if delete succeeded + if (exitCode == 0) { + exitCode = createCredential(); + } + + return exitCode; + } + + /** + * Usage and description. + * @return + */ + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + + /** + * Creates the specified credential. A credential with the same alias + * should not exist. It must be deleted before this method is called. + * + * @return - 0 on success; 1 on failure. + * @throws Exception - If the alias already exists. + */ + private int createCredential() throws Exception { + int exitCode; + List<String> args = new ArrayList<>(); + + args.add("create"); + args.add(alias); + if (value != null) { + args.add("-value"); + args.add(value); + } + + String[] toolArgs = args.toArray(new String[args.size()]); + + exitCode = ToolRunner.run(getConf(), new CredentialShell(), toolArgs); + + return exitCode; + } + } + + /** + * Deletes the credential specified by the alias from the + * specified provider. + */ + private class DeleteCommand extends Command { + public static final String USAGE = + "delete <alias> [-f] [-provider provider-path]"; + public static final String DESC = + "The delete subcommand deletes the credential specified\n" + + "as the <alias> argument from within the provider indicated\n" + + "through the -provider argument. The command asks for\n" + + "confirmation unless the -f option is specified."; + + /** + * Deletes the specified alias. Prompts for user confirmation + * if -f option is not specified. + * @return + * @throws Exception + */ + @Override + public int execute() throws Exception { + int exitCode; + List<String> args = new ArrayList<>(); + + args.add("delete"); + args.add(alias); + if (!interactive) { + args.add("-f"); + } + + String[] toolArgs = args.toArray(new String[args.size()]); + + exitCode = ToolRunner.run(getConf(), new CredentialShell(), toolArgs); + + return exitCode; + } + + /** + * Usage and description. + * + * @return + */ + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } + + + /** + * Lists all the aliases contained in the specified provider. + */ + private class ListCommand extends Command { + /** + * Command usage + */ + public static final String USAGE = "list [-provider provider-path]"; + + /** + * Command description + */ + public static final String DESC = + "The list subcommand displays the aliases contained within \n" + + "a particular provider - as configured in core-site.xml or\n " + + "indicated through the -provider argument."; + + /** + * Executes the list command. + * + * @return - 0 if successful; 1 otherwise. + * @throws Exception - If something goes wrong. + */ + @Override + public int execute() throws Exception { + int exitCode; + List<String> args = new ArrayList<>(); + + args.add("list"); + + String[] toolArgs = args.toArray(new String[args.size()]); + + exitCode = ToolRunner.run(getConf(), new CredentialShell(), toolArgs); + + return exitCode; + } + + /** + * Usage and description. + * + * @return + */ + @Override + public String getUsage() { + return USAGE + ":\n\n" + DESC; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 8451de1..211fe0a 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -46,6 +46,11 @@ from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_f from resource_management.libraries.functions.setup_ranger_plugin_xml import get_audit_configs from resource_management.libraries.functions.get_architecture import get_architecture +from resource_management.core.utils import PasswordString +from resource_management.core.shell import checked_call +from resource_management.core.logger import Logger +from ambari_commons.inet_utils import download_file + # Default log4j version; put config files under /etc/hive/conf log4j_version = '1' @@ -55,6 +60,10 @@ tmp_dir = Script.get_tmp_dir() architecture = get_architecture() sudo = AMBARI_SUDO_BINARY +credential_store_enabled = False +if 'credentialStoreEnabled' in config: + credential_store_enabled = config['credentialStoreEnabled'] + stack_root = status_params.stack_root stack_name = status_params.stack_name stack_name_uppercase = stack_name.upper() @@ -218,7 +227,46 @@ execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_ hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL'] -hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] +jdk_location = config['hostLevelParams']['jdk_location'] + +credential_util_cmd = 'org.apache.ambari.server.credentialapi.CredentialUtil' +credential_util_jar = 'CredentialUtil.jar' + +# Gets the hive metastore password from its JCEKS provider, if available. +def getHiveMetastorePassword(): + passwd = '' + if 'hadoop.security.credential.provider.path' in config['configurations']['hive-site']: + # Try to download CredentialUtil.jar from ambari-server resources + cs_lib_path = config['configurations']['hive-site']['credentialStoreClassPath'] + credential_util_dir = cs_lib_path.split('*')[0] # Remove the trailing '*' + credential_util_path = os.path.join(credential_util_dir, credential_util_jar) + credential_util_url = jdk_location + credential_util_jar + try: + download_file(credential_util_url, credential_util_path) + except Exception, e: + message = 'Error downloading {0} from Ambari Server resources. {1}'.format(credential_util_url, str(e)) + Logger.error(message) + raise + + # Execute a get command on the CredentialUtil CLI to get the password for the specified alias + java_home = config['hostLevelParams']['java_home'] + java_bin = '{java_home}/bin/java'.format(java_home=java_home) + alias = 'javax.jdo.option.ConnectionPassword' + provider_path = config['configurations']['hive-site']['hadoop.security.credential.provider.path'] + cmd = (java_bin, '-cp', cs_lib_path, credential_util_cmd, 'get', alias, '-provider', provider_path) + cmd_result, std_out_msg = checked_call(cmd) + if cmd_result != 0: + message = 'The following error occurred while executing {0}: {1}'.format(' '.join(cmd), std_out_msg) + Logger.error(message) + raise + std_out_lines = std_out_msg.split('\n') + passwd = std_out_lines[-1] # Get the last line of the output, to skip warnings if any. + return passwd + +if credential_store_enabled: + hive_metastore_user_passwd = PasswordString(getHiveMetastorePassword()) +else: + hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type'] @@ -237,8 +285,6 @@ if 'roleCommand' in config and 'CUSTOM_COMMAND' == config['roleCommand']: #JDBC driver jar name hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] -jdk_location = config['hostLevelParams']['jdk_location'] - java_share_dir = '/usr/share/java' hive_database_name = config['configurations']['hive-env']['hive_database_name'] hive_database = config['configurations']['hive-env']['hive_database'] @@ -796,4 +842,4 @@ if enable_ranger_hive: if has_ranger_admin and stack_supports_ranger_audit_db and xa_audit_db_flavor.lower() == 'sqla': xa_audit_db_is_enabled = False -# ranger hive plugin section end +# ranger hive plugin section end \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d340fe93/ambari-server/src/test/java/org/apache/ambari/server/credentialapi/CredentialUtilTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/credentialapi/CredentialUtilTest.java b/ambari-server/src/test/java/org/apache/ambari/server/credentialapi/CredentialUtilTest.java new file mode 100644 index 0000000..abc7cfe --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/credentialapi/CredentialUtilTest.java @@ -0,0 +1,644 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.credentialapi; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.List; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ToolRunner; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + + +/** + * Tests CredentialUtilTest. + */ +public class CredentialUtilTest { + /** + * Cleans up itself after a test method is run. + */ + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + /** + * Redirect System.out to a stream. CredentialShell() writes to System.out. + * We want to capture that. + */ + private final ByteArrayOutputStream out = new ByteArrayOutputStream(); + + /** + * Redirect System.err to a stream. + */ + private final ByteArrayOutputStream err = new ByteArrayOutputStream(); + + /** + * CRUD command verbs + */ + private static final String CREATE_VERB = "create"; + private static final String DELETE_VERB = "delete"; + private static final String LIST_VERB = "list"; + private static final String GET_VERB = "get"; + + + /** + * Gets a temporary folder name. + * + * @return + */ + private String getTempFolderName() { + return temporaryFolder.getRoot().getAbsolutePath(); + } + + /** + * Constructs a valid JCEKS provider path from the specified file name. + * + * @param fileName + * @return + */ + private String getProviderPath(String fileName) { + return CredentialUtil.jceksPrefix + getTempFolderName() + "/" + fileName; + } + + /** + * Constructs the create arguments to create a new credential. + * + * @param alias + * @param credential + * @param providerPath + * @return + */ + private String[] getCreateArgs(String alias, String credential, String providerPath) { + List<String> args = new ArrayList<String>(); + + args.add(CREATE_VERB); + args.add(alias); + args.add("-value"); + args.add(credential); + args.add("-provider"); + args.add(providerPath); + + return args.toArray(new String[args.size()]); + } + + /** + * Constructs the create arguments with no overwrite existing option. + * + * @param alias + * @param credential + * @param providerPath + * @return + */ + private String[] getSafeCreateArgs(String alias, String credential, String providerPath) { + List<String> args = new ArrayList<String>(); + + args.add(CREATE_VERB); + args.add(alias); + args.add("-value"); + args.add(credential); + args.add("-provider"); + args.add(providerPath); + args.add("-n"); /* do not overwrite if credential exists */ + + return args.toArray(new String[args.size()]); + } + + /** + * Constructs the create arguments with overwrite. + * + * @param alias + * @param credential + * @param providerPath + * @return + */ + private String[] getUpdateArgs(String alias, String credential, String providerPath) { + List<String> args = new ArrayList<String>(); + + args.add(CREATE_VERB); + args.add(alias); + args.add("-value"); + args.add(credential); + args.add("-provider"); + args.add(providerPath); + args.add("-f"); /* overwrite */ + + return args.toArray(new String[args.size()]); + } + + /** + * Constructs the delete arguments. + * + * @param alias + * @param providerPath + * @return + */ + private String[] getDeleteArgs(String alias, String providerPath) { + List<String> args = new ArrayList<String>(); + + args.add(DELETE_VERB); + args.add(alias); + args.add("-provider"); + args.add(providerPath); + args.add("-f"); /* suppress prompt to confirm */ + + return args.toArray(new String[args.size()]); + } + + /** + * List arguments. + * + * @param providerPath + * @return + */ + private String[] getListArgs(String providerPath) { + List<String> args = new ArrayList<String>(); + + args.add(LIST_VERB); + args.add("-provider"); + args.add(providerPath); + + return args.toArray(new String[args.size()]); + } + + /** + * Get arguments. + * + * @param alias + * @param providerPath + * @return + */ + private String[] getGetArgs(String alias, String providerPath) { + List<String> args = new ArrayList<String>(); + + args.add(GET_VERB); + args.add(alias); + args.add("-provider"); + args.add(providerPath); + + return args.toArray(new String[args.size()]); + } + + /** + * Executes the given arguments. + * + * @param args + * @return + * @throws Exception + */ + private int executeCommand(String[] args) throws Exception { + return ToolRunner.run(new Configuration(), new CredentialUtil(), args); + } + + /** + * Set up the streams before each test. + */ + @Before + public void setupStreams() { + System.setOut(new PrintStream(out)); + System.setErr(new PrintStream(err)); + } + + /** + * Clean up the stream after each test. + */ + @After + public void teardownStreams() { + System.setOut(null); + System.setErr(null); + } + + /** + * Creates a non-existing credential + * + * @throws Exception + */ + @Test + public void testCreateCommand() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args = getCreateArgs(alias, credential, providerPath); + + int exitCode = executeCommand(args); + + Assert.assertEquals(exitCode, 0); + } + + /** + * Overwrites an existing credential. + * + * @throws Exception + */ + @Test + public void testCreateCommandOverwriteExisting() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Create a new credential + */ + args = getCreateArgs(alias, credential, providerPath); + exitCode = ToolRunner.run(new Configuration(), new CredentialUtil(), args); + Assert.assertEquals(exitCode, 0); + + /* + * Update the created credential + */ + credential = "MyUpdatedTopSecretPassword"; + args = getUpdateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + } + + /** + * Updates a non-existing credential. Should create it. + * + * @throws Exception + */ + @Test + public void testCreateCommandOverwriteNonExisting() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Update a non-existing credential. Should create it. + */ + args = getUpdateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + } + + /** + * Safely creates a credential. If credential already exists, nothing is done. + * + * @throws Exception + */ + @Test + public void testSafeCreateCommandExisting() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Create a new credential + */ + args = getCreateArgs(alias, credential, providerPath); + exitCode = ToolRunner.run(new Configuration(), new CredentialUtil(), args); + Assert.assertEquals(exitCode, 0); + + /* + * Safely update the previously created credential. Nothing is done. + */ + credential = "AnotherTopSecretPassword"; + args = getSafeCreateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + } + + /** + * Safely creates a credential. If it does not exist, it will be created. + * + * @throws Exception + */ + @Test + public void testSafeCreateCommandNotExisting() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Safely update a non-existing credential. Should create the credential. + */ + credential = "AnotherTopSecretPassword"; + args = getSafeCreateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + } + + /** + * Delete an existing credential + * + * @throws Exception + */ + @Test + public void testDeleteCommandExisting() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Create a new credential + */ + args = getCreateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + + /* + * Delete the above credential + */ + args = getDeleteArgs(alias, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + } + + /** + * Delete a non-existing credential + * + * @throws Exception + */ + @Test + public void testDeleteCommandNonExisting() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Delete a non-existing credential. Should fail with exit code 1. + */ + args = getDeleteArgs(alias, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 1); + } + + /** + * Retrieve an existing credential. + * + * @throws Exception + */ + @Test + public void testGetCommandExisting() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Create a new credential + */ + args = getCreateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + out.reset(); + + /* + * Get the existing credential. + */ + args = getGetArgs(alias, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + + String retrievedCredential = out.toString().trim(); + Assert.assertEquals(credential, retrievedCredential); + } + + /** + * Retrieve a non-existing credential. + * + * @throws Exception + */ + @Test + public void testGetCommandNonExisting() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Get a non-existing credential. Should fail with exit code 1. + */ + args = getGetArgs(alias, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 1); + } + + /** + * Create, delete and attempt to get the alias. + * + * @throws Exception + */ + @Test + public void testGetCommandAfterDeletion() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Create a new credential + */ + args = getCreateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + + /* + * Delete the above credential + */ + args = getDeleteArgs(alias, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + + /* + * Get the existing credential. Should not be there. + */ + args = getGetArgs(alias, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 1); + } + + /** + * Execute get on an invalid provider path. + * + * @throws Exception + */ + @Test + public void testGetCommandWithNonExistingProvider() throws Exception { + String alias = "javax.jdo.option.ConnectionPassword"; + String credential = "MyTopSecretPassword"; + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + + /* + * Create a new credential + */ + args = getCreateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + + /* + * Get a credential. Should not be there. + */ + args = getGetArgs(alias, "BadProvider.jceks"); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 1); + } + + /** + * List all aliases. + * + * @throws Exception + */ + @Test + public void testListCommand() throws Exception { + String providerPath = getProviderPath("CreateCommandTest.jceks"); + String[] args; + int exitCode; + final int numEntries = 5; + Properties properties = new Properties(); + + /* + * Create some alias password entries. + */ + for (int i = 0; i < numEntries; ++i) { + String alias = String.format("alias_%d", i + 1); + String credential = String.format("credential_%d", i + 1); + properties.setProperty(alias, credential); + args = getCreateArgs(alias, credential, providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + } + + out.reset(); + + /* + * List all aliases. + */ + args = getListArgs(providerPath); + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + List<String> aliases = Arrays.asList(out.toString().split(System.getProperty("line.separator"))); + Enumeration enumeration = properties.propertyNames(); + while (enumeration.hasMoreElements()) { + String alias = (String)enumeration.nextElement(); + Assert.assertTrue(aliases.contains(alias)); + } + } + + /** + * Prints the tool usage. + * + * @throws Exception + */ + @Test + public void testToolUsage() throws Exception { + String[] args = new String[1]; + int exitCode; + + /* + * Invoke tool help + */ + args[0] = "-help"; + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + Assert.assertTrue(!out.toString().isEmpty()); + } + + /** + * Invoke Create command help + * + * @throws Exception + */ + @Test + public void testCreateCommandUsage() throws Exception { + String[] args = new String[2]; + int exitCode; + + args[0] = CREATE_VERB; + args[1] = "-help"; + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + Assert.assertTrue(!out.toString().isEmpty()); + } + + /* + * Invoke Delete command help + * + * @throws Exception + */ + @Test + public void testDeleteCommandUsage() throws Exception { + String[] args = new String[2]; + int exitCode; + + args[0] = DELETE_VERB; + args[1] = "-help"; + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + Assert.assertTrue(!out.toString().isEmpty()); + } + + /* + * Invoke List command help + * + * @throws Exception + */ + @Test + public void testListCommandUsage() throws Exception { + String[] args = new String[2]; + int exitCode; + + args[0] = LIST_VERB; + args[1] = "-help"; + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + Assert.assertTrue(!out.toString().isEmpty()); + } + + /* + * Invoke Get command help + * + * @throws Exception + */ + @Test + public void testGetCommandUsage() throws Exception { + String[] args = new String[2]; + int exitCode; + + args[0] = GET_VERB; + args[1] = "-help"; + exitCode = executeCommand(args); + Assert.assertEquals(exitCode, 0); + Assert.assertTrue(!out.toString().isEmpty()); + } +} \ No newline at end of file
