Repository: nifi Updated Branches: refs/heads/master 66783c18b -> 46ce7aaa3
NIFI-5341 Enabled groovy tests in nifi-runtime Fixed tests in NiFiGroovyTest in the nifi-runtime module Updated NiFi.createBootstrapClassLoader to log a warning if lib/bootstrap does not exist rather than throwing a FileNotFoundException, since it already catches MalformedUrlException if there's an issue adding one of the bootstrap JARs to the bootstrap classpath Explicitly handling InvocationTargetException in NiFi.initializeProperties to unwrap the cause and rewrap as an IllegalArgumentException to propogate the real cause of the underlying exception thrown by NiFiPropertiesLoader This closes #2821. Signed-off-by: Andy LoPresto <alopre...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/46ce7aaa Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/46ce7aaa Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/46ce7aaa Branch: refs/heads/master Commit: 46ce7aaa3298fa32d7f0c6beca9d45c5d43d3180 Parents: 66783c1 Author: Jeff Storck <jtsw...@gmail.com> Authored: Tue Jun 26 22:33:19 2018 -0400 Committer: Andy LoPresto <alopre...@apache.org> Committed: Wed Jul 18 11:35:55 2018 -0700 ---------------------------------------------------------------------- .../nifi-framework/nifi-runtime/pom.xml | 35 +++++++++++++ .../src/main/java/org/apache/nifi/NiFi.java | 27 ++++++---- .../org/apache/nifi/NiFiGroovyTest.groovy | 52 ++++++++++---------- ...rties_protected_aes_different_key.properties | 2 +- 4 files changed, 78 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/46ce7aaa/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml index 97a9c73..1649fd4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml @@ -53,4 +53,39 @@ <scope>test</scope> </dependency> </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.5</version> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>src/main/groovy</source> + </sources> + </configuration> + </execution> + <execution> + <id>add-test-source</id> + <phase>generate-test-sources</phase> + <goals> + <goal>add-test-source</goal> + </goals> + <configuration> + <sources> + <source>src/test/groovy</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/46ce7aaa/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java index 68ee8c7..13b60e8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java @@ -191,16 +191,20 @@ public class NiFi { SLF4JBridgeHandler.install(); } - private static ClassLoader createBootstrapClassLoader() throws IOException { + private static ClassLoader createBootstrapClassLoader() { //Get list of files in bootstrap folder final List<URL> urls = new ArrayList<>(); - Files.list(Paths.get("lib/bootstrap")).forEach(p -> { - try { - urls.add(p.toUri().toURL()); - } catch (final MalformedURLException mef) { - LOGGER.warn("Unable to load " + p.getFileName() + " due to " + mef, mef); - } - }); + try { + Files.list(Paths.get("lib/bootstrap")).forEach(p -> { + try { + urls.add(p.toUri().toURL()); + } catch (final MalformedURLException mef) { + LOGGER.warn("Unable to load " + p.getFileName() + " due to " + mef, mef); + } + }); + } catch (IOException ioe) { + LOGGER.warn("Unable to access lib/bootstrap to create bootstrap classloader", ioe); + } //Create the bootstrap classloader return new URLClassLoader(urls.toArray(new URL[0]), Thread.currentThread().getContextClassLoader()); } @@ -295,7 +299,7 @@ public class NiFi { } } - protected static NiFiProperties convertArgumentsToValidatedNiFiProperties(String[] args) throws IOException { + protected static NiFiProperties convertArgumentsToValidatedNiFiProperties(String[] args) { final ClassLoader bootstrap = createBootstrapClassLoader(); NiFiProperties properties = initializeProperties(args, bootstrap); properties.validate(); @@ -327,7 +331,10 @@ public class NiFi { final NiFiProperties properties = (NiFiProperties) getMethod.invoke(loaderInstance); LOGGER.info("Loaded {} properties", properties.size()); return properties; - } catch (final IllegalAccessException | InvocationTargetException | NoSuchMethodException | ClassNotFoundException reex) { + } catch (InvocationTargetException wrappedException) { + final String msg = "There was an issue decrypting protected properties"; + throw new IllegalArgumentException(msg, wrappedException.getCause() == null ? wrappedException : wrappedException.getCause()); + } catch (final IllegalAccessException | NoSuchMethodException | ClassNotFoundException reex) { final String msg = "Unable to access properties loader in the expected manner - apparent classpath or build issue"; throw new IllegalArgumentException(msg, reex); } catch (final RuntimeException e) { http://git-wip-us.apache.org/repos/asf/nifi/blob/46ce7aaa/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy index f192d79..2c4900a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy @@ -25,7 +25,6 @@ import org.apache.nifi.util.NiFiProperties import org.bouncycastle.jce.provider.BouncyCastleProvider import org.junit.After import org.junit.AfterClass -import org.junit.Before import org.junit.BeforeClass import org.junit.Test import org.junit.runner.RunWith @@ -34,6 +33,7 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.slf4j.bridge.SLF4JBridgeHandler +import java.nio.file.Paths import java.security.Security @RunWith(JUnit4.class) @@ -43,7 +43,6 @@ class NiFiGroovyTest extends GroovyTestCase { private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) private static final String TEST_RES_PATH = NiFiGroovyTest.getClassLoader().getResource(".").toURI().getPath() - private static final File workDir = new File("./target/work/jetty/") @BeforeClass public static void setUpOnce() throws Exception { @@ -58,13 +57,6 @@ class NiFiGroovyTest extends GroovyTestCase { logger.info("Identified test resources path as ${TEST_RES_PATH}") } - @Before - public void setUp() throws Exception { - if (!workDir.exists()) { - workDir.mkdirs() - } - } - @After public void tearDown() throws Exception { NiFiPropertiesLoader.@sensitivePropertyProviderFactory = null @@ -88,7 +80,7 @@ class NiFiGroovyTest extends GroovyTestCase { System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, plainPropertiesPath) // Act - NiFiProperties loadedProperties = NiFi.initializeProperties(args) + NiFiProperties loadedProperties = NiFi.initializeProperties(args, NiFiGroovyTest.class.classLoader) // Assert assert loadedProperties.size() > 0 @@ -105,14 +97,13 @@ class NiFiGroovyTest extends GroovyTestCase { NiFi.main(args) // Assert - assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: There was an issue decrypting protected properties" } @Test public void testParseArgsShouldSplitCombinedArgs() throws Exception { // Arrange - final String DIFFERENT_KEY = "0" * 64 - def args = ["-k ${DIFFERENT_KEY}"] as String[] + def args = ["-K filename"] as String[] // Act def parsedArgs = NiFi.parseArgs(args) @@ -125,7 +116,7 @@ class NiFiGroovyTest extends GroovyTestCase { @Test public void testMainShouldHandleBadArgs() throws Exception { // Arrange - def args = ["-k"] as String[] + def args = ["-K"] as String[] System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties") @@ -135,14 +126,16 @@ class NiFiGroovyTest extends GroovyTestCase { // Assert assert TestAppender.events.collect { it.getFormattedMessage() - }.contains("The bootstrap process passed the -k flag without a key") - assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + }.contains("The bootstrap process passed the -K flag without a filename") + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key" } @Test - public void testMainShouldHandleMalformedBootstrapKey() throws Exception { + public void testMainShouldHandleMalformedBootstrapKeyFromFile() throws Exception { // Arrange - def args = ["-k", "BAD KEY"] as String[] + def passwordFile = Paths.get(TEST_RES_PATH, "NiFiProperties", "password-testMainShouldHandleMalformedBootstrapKeyFromFile.txt").toFile() + passwordFile.text = "BAD KEY" + def args = ["-K", passwordFile.absolutePath] as String[] System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties") @@ -150,32 +143,36 @@ class NiFiGroovyTest extends GroovyTestCase { NiFi.main(args) // Assert - assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key" } @Test - public void testInitializePropertiesShouldSetBootstrapKeyFromArgs() throws Exception { + public void testInitializePropertiesShouldSetBootstrapKeyFromFile() throws Exception { // Arrange final String DIFFERENT_KEY = "0" * 64 - def args = ["-k", DIFFERENT_KEY] as String[] + def passwordFile = Paths.get(TEST_RES_PATH, "NiFiProperties", "password-testInitializePropertiesShouldSetBootstrapKeyFromFile.txt").toFile() + passwordFile.text = DIFFERENT_KEY + def args = ["-K", passwordFile.absolutePath] as String[] String testPropertiesPath = "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties" System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, testPropertiesPath) - NiFiProperties unprocessedProperties = new NiFiPropertiesLoader().loadRaw(new File(testPropertiesPath)) + def protectedNiFiProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(new File(testPropertiesPath)) + NiFiProperties unprocessedProperties = protectedNiFiProperties.internalNiFiProperties def protectedKeys = getProtectedKeys(unprocessedProperties) logger.info("Reading from raw properties file gives protected properties: ${protectedKeys}") // Act - NiFiProperties properties = NiFi.initializeProperties(args) + NiFiProperties properties = NiFi.initializeProperties(args, NiFiGroovyTest.class.classLoader) // Assert // Ensure that there were protected properties, they were encrypted using AES/GCM (128/256 bit key), and they were decrypted (raw value != retrieved value) assert !hasProtectedKeys(properties) - def unprotectedProperties = decrypt(unprocessedProperties, DIFFERENT_KEY) - getProtectedPropertyKeys(unprocessedProperties).every { k, v -> - String rawValue = unprocessedProperties.getProperty(k) + def unprotectedProperties = decrypt(protectedNiFiProperties, DIFFERENT_KEY) + def protectedPropertyKeys = getProtectedPropertyKeys(unprocessedProperties) + protectedPropertyKeys.every { k, v -> + String rawValue = protectedNiFiProperties.getProperty(k) logger.raw("${k} -> ${rawValue}") String retrievedValue = properties.getProperty(k) logger.decrypted("${k} -> ${retrievedValue}") @@ -188,6 +185,7 @@ class NiFiGroovyTest extends GroovyTestCase { String decryptedProperty = unprotectedProperties.getProperty(k) logger.assert("${retrievedValue} == ${decryptedProperty}") assert retrievedValue == decryptedProperty + true } } @@ -202,7 +200,7 @@ class NiFiGroovyTest extends GroovyTestCase { } private static Set<String> getProtectedKeys(NiFiProperties properties) { - properties.getPropertyKeys().findAll { it.endsWith(".protected") } + properties.getPropertyKeys().findAll { it.endsWith(".protected") }.collect { it - ".protected"} } private static NiFiProperties decrypt(NiFiProperties encryptedProperties, String keyHex) { http://git-wip-us.apache.org/repos/asf/nifi/blob/46ce7aaa/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties index 9c52fe6..c11541c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties @@ -134,7 +134,7 @@ nifi.sensitive.props.key=dQU402Mz4J+t+e18||6+ictR0Nssq3/rR/d8fq5CFAKmpakr9jCyPIJ nifi.sensitive.props.key.protected=aes/gcm/256 nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL nifi.sensitive.props.provider=BC -nifi.sensitive.props.additional.keys= +nifi.sensitive.props.additional.keys=nifi.ui.banner.text nifi.security.keystore=/path/to/keystore.jks nifi.security.keystoreType=JKS
