[ https://issues.apache.org/jira/browse/NIFI-1831?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15433310#comment-15433310 ]
ASF GitHub Bot commented on NIFI-1831: -------------------------------------- Github user alopresto commented on a diff in the pull request: https://github.com/apache/nifi/pull/834#discussion_r75919751 --- Diff: nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy --- @@ -0,0 +1,540 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.properties + +import groovy.io.GroovyPrintWriter +import org.apache.commons.cli.CommandLine +import org.apache.commons.cli.CommandLineParser +import org.apache.commons.cli.DefaultParser +import org.apache.commons.cli.HelpFormatter +import org.apache.commons.cli.Options +import org.apache.commons.cli.ParseException +import org.apache.commons.codec.binary.Hex +import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException +import org.apache.nifi.toolkit.tls.commandLine.ExitCode +import org.apache.nifi.util.NiFiProperties +import org.apache.nifi.util.console.TextDevice +import org.apache.nifi.util.console.TextDevices +import org.bouncycastle.crypto.generators.SCrypt +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.crypto.Cipher +import java.nio.charset.StandardCharsets +import java.security.KeyException +import java.security.Security + +class ConfigEncryptionTool { + private static final Logger logger = LoggerFactory.getLogger(ConfigEncryptionTool.class) + + public String bootstrapConfPath + public String niFiPropertiesPath + public String outputNiFiPropertiesPath + public String loginIdentityProvidersPath + + private String keyHex + private String password + private NiFiProperties niFiProperties + + private boolean usingPassword = true + + private static final String HELP_ARG = "help" + private static final String BOOTSTRAP_CONF_ARG = "bootstrapConf" + private static final String NIFI_PROPERTIES_ARG = "niFiProperties" + private static final String OUTPUT_NIFI_PROPERTIES_ARG = "outputNiFiProperties" + private static final String KEY_ARG = "key" + private static final String PASSWORD_ARG = "password" + private static final String USE_KEY_ARG = "useRawKey" + + private static final int MIN_PASSWORD_LENGTH = 12 + + // Strong parameters as of 12 Aug 2016 + private static final int SCRYPT_N = 2**16 + private static final int SCRYPT_R = 8 + private static final int SCRYPT_P = 1 + + private static + final String BOOTSTRAP_KEY_COMMENT = "# Master key in hexadecimal format for encrypted sensitive configuration values" + private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key=" + private static final String JAVA_HOME = "JAVA_HOME" + private static final String NIFI_TOOLKIT_HOME = "NIFI_TOOLKIT_HOME" + private static final String SEP = System.lineSeparator() + + private static final String FOOTER = buildFooter() + + private static + final String DEFAULT_DESCRIPTION = "This tool reads from a nifi.properties file with plain sensitive configuration values, prompts the user for a master key, and encrypts each value. It will replace the plain value with the protected value in the same file (or write to a new nifi.properties file if specified)." + + private static String buildHeader(String description = DEFAULT_DESCRIPTION) { + "${SEP}${description}${SEP * 2}" + } + + private static String buildFooter() { + "${SEP}Java home: ${System.getenv(JAVA_HOME)}${SEP}NiFi Toolkit home: ${System.getenv(NIFI_TOOLKIT_HOME)}" + } + + private final Options options; + private final String header; + + + public ConfigEncryptionTool() { + this(DEFAULT_DESCRIPTION) + } + + public ConfigEncryptionTool(String description) { + this.header = buildHeader(description) + this.options = new Options() + options.addOption("h", HELP_ARG, false, "Prints this usage message") + options.addOption("n", NIFI_PROPERTIES_ARG, true, "The nifi.properties file containing unprotected config values (will be overwritten)") --- End diff -- No, you were right. I added a test to capture it and resolved it. > Allow encrypted passwords in configuration files > ------------------------------------------------ > > Key: NIFI-1831 > URL: https://issues.apache.org/jira/browse/NIFI-1831 > Project: Apache NiFi > Issue Type: New Feature > Components: Configuration, Core Framework > Affects Versions: 0.6.1 > Reporter: Andy LoPresto > Assignee: Andy LoPresto > Priority: Critical > Labels: configuration, encryption, password, security > Fix For: 1.0.0 > > Original Estimate: 504h > Remaining Estimate: 504h > > Storing passwords in plaintext in configuration files is not a security best > practice. While file access can be restricted through OS permissions, these > configuration files can be accidentally checked into source control, shared > or deployed to multiple instances, etc. > NiFi should allow a deployer to provide an encrypted password in the > configuration file to minimize exposure of the passwords. On application > start-up, NiFi should decrypt the passwords in memory. NiFi should also > include a utility to encrypt the raw passwords (and optionally populate the > configuration files and provide additional metadata in the configuration > files). > I am aware this simply shifts the responsibility/delegation of trust from the > passwords in the properties file to a new location on the same system, but > mitigating the visibility of the raw passwords in the properties file can be > one step in a defense in depth approach and is often mandated by security > policies within organizations using NiFi. > The key used for encryption should not be hard-coded into the application > source code, nor should it be universally consistent. The key could be > determined by reading static information from the deployed system and feeding > it to a key derivation function based on a cryptographically-secure hash > function, such as PBKDF2, bcrypt, or scrypt. However, this does introduce > upgrade, system migration, and portability issues. These challenges will have > to be kept in consideration when determining the key derivation process. > Manual key entry is a possibility, and then the master key would only be > present in memory, but this prevents automatic reboot on loss of power or > other recovery scenario. > This must be backward-compatible to allow systems with plaintext passwords to > continue operating. Options for achieving this are to only attempt to decrypt > passwords when a sibling property is present, or to match a specific format. > For these examples, I have used the following default values: > {code} > password: thisIsABadPassword > key: 0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 > iv: 0123456789ABCDEFFEDCBA9876543210 > algorithm: AES/CBC 256-bit > {code} > **Note: These values should not be used in production systems -- the key and > IV are common test values, and an AEAD cipher is preferable to provide cipher > text integrity assurances, however OpenSSL does not support the use of AEAD > ciphers for command-line encryption at this time** > Example 1: *here the sibling property indicates the password is encrypted and > with which implementation; the absence of the property would default to a raw > password* > {code} > hw12203:/Users/alopresto/Workspace/scratch/encrypted-passwords (master) > alopresto > 🔓 0s @ 16:25:56 $ echo "thisIsABadPassword" > password.txt > hw12203:/Users/alopresto/Workspace/scratch/encrypted-passwords (master) > alopresto > 🔓 0s @ 16:26:47 $ ossl aes-256-cbc -e -nosalt -p -K > 0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 -iv > 0123456789ABCDEFFEDCBA9876543210 -a -in password.txt -out password.enc > key=0123456789ABCDEFFEDCBA98765432100123456789ABCDEFFEDCBA9876543210 > iv =0123456789ABCDEFFEDCBA9876543210 > hw12203:/Users/alopresto/Workspace/scratch/encrypted-passwords (master) > alopresto > 🔓 0s @ 16:27:09 $ xxd password.enc > 0000000: 5643 5856 6146 6250 4158 364f 5743 7646 VCXVaFbPAX6OWCvF > 0000010: 6963 6b76 4a63 7744 3854 6b67 3731 4c76 ickvJcwD8Tkg71Lv > 0000020: 4d38 6d32 7952 4776 5739 413d 0a M8m2yRGvW9A=. > hw12203:/Users/alopresto/Workspace/scratch/encrypted-passwords (master) > alopresto > 🔓 0s @ 16:27:16 $ more password.enc > VCXVaFbPAX6OWCvFickvJcwD8Tkg71LvM8m2yRGvW9A= > hw12203:/Users/alopresto/Workspace/scratch/encrypted-passwords (master) > alopresto > 🔓 0s @ 16:27:55 $ > {code} > In {{nifi.properties}}: > {code} > nifi.security.keystorePasswd=VCXVaFbPAX6OWCvFickvJcwD8Tkg71LvM8m2yRGvW9A= > nifi.security.keystorePasswd.encrypted=AES-CBC-256 > {code} > Example 2: *here the encrypted password has a header tag indicating both that > it is encrypted and the algorithm used* > {code:java} > @Test > public void testShouldDecryptPassword() throws Exception { > // Arrange > KeyedCipherProvider cipherProvider = new AESKeyedCipherProvider() > final String PLAINTEXT = "thisIsABadPassword" > logger.info("Expected: ${Hex.encodeHexString(PLAINTEXT.bytes)}") > final byte[] IV = Hex.decodeHex("0123456789ABCDEFFEDCBA9876543210" as > char[]) > final byte[] LOCAL_KEY = > Hex.decodeHex("0123456789ABCDEFFEDCBA9876543210" * 2 as char[]) > // Generated via openssl enc -a > final String CIPHER_TEXT = > "VCXVaFbPAX6OWCvFickvJcwD8Tkg71LvM8m2yRGvW9A=" > byte[] cipherBytes = Base64.decoder.decode(CIPHER_TEXT) > SecretKey localKey = new SecretKeySpec(LOCAL_KEY, "AES") > EncryptionMethod encryptionMethod = EncryptionMethod.AES_CBC > logger.info("Using algorithm: ${encryptionMethod.getAlgorithm()}") > logger.info("Cipher text: \$nifipw\$${CIPHER_TEXT} > ${cipherBytes.length + 8}") > // Act > Cipher cipher = cipherProvider.getCipher(encryptionMethod, localKey, > IV, false) > byte[] recoveredBytes = cipher.doFinal(cipherBytes) > > // OpenSSL adds a newline character during encryption > String recovered = new String(recoveredBytes, "UTF-8").trim() > logger.info("Recovered: ${recovered} > ${Hex.encodeHexString(recoveredBytes)}") > // Assert > assert PLAINTEXT.equals(recovered) > } > {code} > In {{nifi.properties}}: > {code} > nifi.security.keystorePasswd=$nifipw$VCXVaFbPAX6OWCvFickvJcwD8Tkg71LvM8m2yRGvW9A= > {code} > Ideally, NiFi would use a pluggable implementation architecture to allow > users to integrate with a variety of secret management services. There are > both commercial and open source solutions, including CyberArk Enterprise > Password Vault [1], Hashicorp Vault [2], and Square Keywhiz [3]. In the > future, this could also be extended to Hardware Security Modules (HSM) like > SafeNet Luna [4] and Amazon CloudHSM [5]. > [1] > http://www.cyberark.com/products/privileged-account-security-solution/enterprise-password-vault/ > [2] https://www.vaultproject.io/ > [3] https://square.github.io/keywhiz/ > [4] > http://www.safenet-inc.com/data-encryption/hardware-security-modules-hsms/luna-hsms-key-management/luna-sa-network-hsm/ > [5] https://aws.amazon.com/cloudhsm/ -- This message was sent by Atlassian JIRA (v6.3.4#6332)