This is an automated email from the ASF dual-hosted git repository.
jgresock pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new c968090 NIFI-9350 Add NiFi Registry NarProvider implementation
c968090 is described below
commit c96809012b51156872c0446473953f0cf8548f93
Author: Bryan Bende <[email protected]>
AuthorDate: Thu Oct 28 17:28:23 2021 -0400
NIFI-9350 Add NiFi Registry NarProvider implementation
Signed-off-by: Joe Gresock <[email protected]>
This closes #5497.
---
.../nifi/nar/NarProviderInitializationContext.java | 7 ++
.../src/main/asciidoc/administration-guide.adoc | 31 +++++
.../extension/AbstractExtensionBundleMetadata.java | 82 ++++++++++++++
.../extension/AbstractExtensionRegistry.java | 83 ++++++++++++++
.../NiFiRegistryExtensionBundleMetadata.java | 109 ++++++++++++++++++
.../extension/NiFiRegistryExtensionRegistry.java | 125 +++++++++++++++++++++
.../extension/NiFiRegistryNarProvider.java | 104 +++++++++++++++++
.../NiFiRegistryExtensionBundleMetadataTest.java | 51 +++++++++
.../extension/ExtensionBundleMetadata.java | 30 +++--
.../nifi/registry/extension/ExtensionRegistry.java | 92 +++++++++++++++
.../extension/ExtensionRegistryException.java | 22 ++--
.../services/org.apache.nifi.nar.NarProvider | 15 +++
.../nifi-framework-nar-loading-utils/pom.xml | 4 +
.../java/org/apache/nifi/nar/NarAutoLoader.java | 3 +-
...pertyBasedNarProviderInitializationContext.java | 21 +++-
...pertyBasedNarProviderInitializationContext.java | 9 +-
.../src/main/resources/conf/nifi.properties | 23 +++-
17 files changed, 784 insertions(+), 27 deletions(-)
diff --git
a/nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
b/nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
index 26d6b7f..e5efdc1 100644
---
a/nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
+++
b/nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.nar;
+import javax.net.ssl.SSLContext;
import java.util.Map;
/**
@@ -27,4 +28,10 @@ public interface NarProviderInitializationContext {
* @return Returns with the available properties.
*/
Map<String, String> getProperties();
+
+ /**
+ * @return Returns an SSLContext created from NiFi's keystore/truststore
+ */
+ SSLContext getNiFiSSLContext();
+
}
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc
b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 5fc55a2..5508b90 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -4083,6 +4083,37 @@ To configure custom properties for use with NiFi’s
Expression Language:
Custom properties can also be configured in the NiFi UI. See the
<<user-guide.adoc#Variables_Window,Variables Window>> section in the User Guide
for more information.
+[[nar_provider_properties]]
+=== NAR Provider Properties
+
+These properties allow configuring one or more NAR providers. A NAR provider
retrieves NARs from an external source and copies them to the directory
specified by `nifi.nar.library.autoload.directory`.
+
+Each NAR provider property follows the format
`nifi.nar.library.provider.<identifier>.<property-name>` and each provider must
have at least one property named `implementation`.
+
+==== HDFS NAR Provider ====
+
+The HDFS NAR provider retrieves NARs using the Hadoop FileSystem API. This can
be used with a traditional HDFS instance or with cloud storage, such as `s3a`
or `abfs`. In order to use cloud storage, the Hadoop Libraries NAR must be
re-built with the cloud storage profiles enabled.
+
+|====
+|*Property*|*Description*
+|`nifi.nar.library.provider.hdfs.implementation`| The fully qualified class
name of the implementation class which is
`org.apache.nifi.nar.hadoop.HDFSNarProvider`.
+|`nifi.nar.library.provider.hdfs.resources`| The comma separated list of
configuration resources, such as `core-site.xml`.
+|`nifi.nar.library.provider.hdfs.storage.location`| The optional storage
location, such as `hdfs://hdfs-location`. If not specified, the `defaultFs`
from `core-site.xml` will be used.
+|`nifi.nar.library.provider.hdfs.source.directory`| The directory within the
storage location where NARs are located.
+|`nifi.nar.library.provider.hdfs.kerberos.principal`| An optional Kerberos
principal for authentication. If specified, one of keytab or password must also
be specified.
+|`nifi.nar.library.provider.hdfs.kerberos.keytab`| An optional Kerberos keytab
for authentication.
+|`nifi.nar.library.provider.hdfs.kerberos.password`| An optional Kerberos
password for authentication.
+|====
+
+==== NiFi Registry NAR Provider ====
+
+The NiFi Registry NAR provider retrieves NARs from a NiFi Registry instance.
In a secure installation, this provider will retrieve NARs from all buckets
that the NiFi server is authorized to read from.
+
+|====
+|*Property*|*Description*
+|`nifi.nar.library.provider.nifi-registry.implementation`| The fully qualified
class name of the implementation class which is
`org.apache.nifi.registry.extension.NiFiRegistryNarProvider`.
+|`nifi.nar.library.provider.nifi-registry.url`| The URL of the NiFi Registry
instance, such as `http://localhost:18080`. If the URL begins with `https`,
then the NiFi keystore and truststore will be used to make the TLS connection.
+|====
[[upgrading_nifi]]
== Upgrading NiFi
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/AbstractExtensionBundleMetadata.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/AbstractExtensionBundleMetadata.java
new file mode 100644
index 0000000..a1d93eb
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/AbstractExtensionBundleMetadata.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.extension;
+
+import org.apache.commons.lang3.Validate;
+
+import java.util.Objects;
+
+/**
+ * Base class for implementations of ExtensionBundleMetadata.
+ */
+public class AbstractExtensionBundleMetadata implements
ExtensionBundleMetadata {
+
+ private final String registryIdentifier;
+ private final String group;
+ private final String artifact;
+ private final String version;
+
+ public AbstractExtensionBundleMetadata(final String registryIdentifier,
final String group, final String artifact, final String version) {
+ this.registryIdentifier = Validate.notBlank(registryIdentifier);
+ this.group = Validate.notBlank(group);
+ this.artifact = Validate.notBlank(artifact);
+ this.version = Validate.notBlank(version);
+ }
+
+ @Override
+ public String getRegistryIdentifier() {
+ return registryIdentifier;
+ }
+
+ @Override
+ public String getGroup() {
+ return group;
+ }
+
+ @Override
+ public String getArtifact() {
+ return artifact;
+ }
+
+ @Override
+ public String getVersion() {
+ return version;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(group, artifact, version, registryIdentifier);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ final AbstractExtensionBundleMetadata otherMetadata =
(AbstractExtensionBundleMetadata) o;
+ return Objects.equals(group, otherMetadata.group)
+ && Objects.equals(artifact, otherMetadata.artifact)
+ && Objects.equals(version, otherMetadata.version)
+ && Objects.equals(registryIdentifier,
otherMetadata.registryIdentifier);
+ }
+
+}
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/AbstractExtensionRegistry.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/AbstractExtensionRegistry.java
new file mode 100644
index 0000000..69f9f90
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/AbstractExtensionRegistry.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.extension;
+
+import org.apache.commons.lang3.Validate;
+
+import javax.net.ssl.SSLContext;
+
+/**
+ * Base class for implementations of ExtensionRegistry.
+ *
+ * @param <T> the type of bundle metadata
+ */
+public abstract class AbstractExtensionRegistry<T extends
ExtensionBundleMetadata> implements ExtensionRegistry<T> {
+
+ private final String identifier;
+ private final SSLContext sslContext;
+
+ private volatile String url;
+ private volatile String name;
+ private volatile String description;
+
+ public AbstractExtensionRegistry(final String identifier, final String
url, final String name, final SSLContext sslContext) {
+ this.identifier = Validate.notBlank(identifier);
+ this.url = url;
+ this.name = name;
+ this.sslContext = sslContext;
+ }
+
+ @Override
+ public String getIdentifier() {
+ return identifier;
+ }
+
+ @Override
+ public String getDescription() {
+ return description;
+ }
+
+ @Override
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ @Override
+ public String getURL() {
+ return url;
+ }
+
+ @Override
+ public void setURL(String url) {
+ this.url = url;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public void setName(final String name) {
+ this.name = name;
+ }
+
+ protected SSLContext getSSLContext() {
+ return this.sslContext;
+ }
+
+}
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryExtensionBundleMetadata.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryExtensionBundleMetadata.java
new file mode 100644
index 0000000..828b464
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryExtensionBundleMetadata.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.extension;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
+
+/**
+ * NiFi Registry implementation of ExtensionBundleMetadata which adds
bundleIdentifier to the fields.
+ */
+public class NiFiRegistryExtensionBundleMetadata extends
AbstractExtensionBundleMetadata {
+
+ private static final String SEPARATOR = "::";
+ private static final String LOCATION_FORMAT = String.join(SEPARATOR, "%s",
"%s", "%s", "%s.nar");
+
+ private final String bundleIdentifier;
+
+ private NiFiRegistryExtensionBundleMetadata(final Builder builder) {
+ super(builder.registryIdentifier, builder.group, builder.artifact,
builder.version);
+ this.bundleIdentifier = Validate.notBlank(builder.bundleIdentifier);
+ }
+
+ public String getBundleIdentifier() {
+ return bundleIdentifier;
+ }
+
+ /**
+ * @return a location string that will be returned from a NarProvider and
passed back to the fetch method,
+ * also serves as the filename used by the NarProvider
+ */
+ public String toLocationString() {
+ return String.format(LOCATION_FORMAT, getGroup(), getArtifact(),
getVersion(), getBundleIdentifier());
+ }
+
+ /**
+ * Creates a new Builder from parsing a location string.
+ *
+ * @param location the location string
+ * @return a builder populated from the location string
+ */
+ public static Builder fromLocationString(final String location) {
+ if (StringUtils.isBlank(location)) {
+ throw new IllegalArgumentException("Location is null or blank");
+ }
+
+ final String[] locationParts = location.split(SEPARATOR);
+ if (locationParts.length != 4) {
+ throw new IllegalArgumentException("Invalid location: " +
location);
+ }
+
+ return new Builder()
+ .group(locationParts[0])
+ .artifact(locationParts[1])
+ .version(locationParts[2])
+ .bundleIdentifier(locationParts[3].replace(".nar", ""));
+ }
+
+ public static class Builder {
+
+ private String registryIdentifier;
+ private String group;
+ private String artifact;
+ private String version;
+ private String bundleIdentifier;
+
+ public Builder registryIdentifier(final String registryIdentifier) {
+ this.registryIdentifier = registryIdentifier;
+ return this;
+ }
+
+ public Builder group(final String group) {
+ this.group = group;
+ return this;
+ }
+
+ public Builder artifact(final String artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ public Builder version(final String version) {
+ this.version = version;
+ return this;
+ }
+
+ public Builder bundleIdentifier(final String bundleIdentifier) {
+ this.bundleIdentifier = bundleIdentifier;
+ return this;
+ }
+
+ public NiFiRegistryExtensionBundleMetadata build() {
+ return new NiFiRegistryExtensionBundleMetadata(this);
+ }
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryExtensionRegistry.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryExtensionRegistry.java
new file mode 100644
index 0000000..fae0714
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryExtensionRegistry.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.extension;
+
+import org.apache.nifi.authorization.user.NiFiUser;
+import org.apache.nifi.registry.client.BundleVersionClient;
+import org.apache.nifi.registry.client.NiFiRegistryClient;
+import org.apache.nifi.registry.client.NiFiRegistryClientConfig;
+import org.apache.nifi.registry.client.NiFiRegistryException;
+import org.apache.nifi.registry.client.RequestConfig;
+import org.apache.nifi.registry.client.impl.JerseyNiFiRegistryClient;
+import org.apache.nifi.registry.client.impl.request.ProxiedEntityRequestConfig;
+import org.apache.nifi.registry.extension.bundle.BundleVersionFilterParams;
+import org.apache.nifi.registry.extension.bundle.BundleVersionMetadata;
+
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * NiFi Registry implementation of ExtensionRegistry.
+ */
+public class NiFiRegistryExtensionRegistry extends
AbstractExtensionRegistry<NiFiRegistryExtensionBundleMetadata> {
+
+ private NiFiRegistryClient registryClient;
+
+ public NiFiRegistryExtensionRegistry(final String identifier, final String
url, final String name, final SSLContext sslContext) {
+ super(identifier, url, name, sslContext);
+ }
+
+ private synchronized NiFiRegistryClient getRegistryClient() {
+ if (registryClient != null) {
+ return registryClient;
+ }
+
+ final NiFiRegistryClientConfig config = new
NiFiRegistryClientConfig.Builder()
+ .connectTimeout(30000)
+ .readTimeout(30000)
+ .sslContext(getSSLContext())
+ .baseUrl(getURL())
+ .build();
+
+ registryClient = new JerseyNiFiRegistryClient.Builder()
+ .config(config)
+ .build();
+
+ return registryClient;
+ }
+
+ private synchronized void invalidateClient() {
+ this.registryClient = null;
+ }
+
+ @Override
+ public void setURL(final String url) {
+ super.setURL(url);
+ invalidateClient();
+ }
+
+ @Override
+ public Set<NiFiRegistryExtensionBundleMetadata>
getExtensionBundleMetadata(final NiFiUser user)
+ throws IOException, ExtensionRegistryException {
+ final RequestConfig requestConfig = getRequestConfig(user);
+ final NiFiRegistryClient registryClient = getRegistryClient();
+ final BundleVersionClient bundleVersionClient =
registryClient.getBundleVersionClient(requestConfig);
+
+ try {
+ final List<BundleVersionMetadata> bundleVersions =
bundleVersionClient.getBundleVersions(BundleVersionFilterParams.empty());
+ return bundleVersions.stream().map(bv ->
map(bv)).collect(Collectors.toSet());
+ } catch (final NiFiRegistryException nre) {
+ throw new ExtensionRegistryException(nre.getMessage(), nre);
+ }
+ }
+
+ @Override
+ public InputStream getExtensionBundleContent(final NiFiUser user, final
NiFiRegistryExtensionBundleMetadata bundleMetadata)
+ throws IOException, ExtensionRegistryException {
+ final RequestConfig requestConfig = getRequestConfig(user);
+ final NiFiRegistryClient registryClient = getRegistryClient();
+ final BundleVersionClient bundleVersionClient =
registryClient.getBundleVersionClient(requestConfig);
+
+ try {
+ return
bundleVersionClient.getBundleVersionContent(bundleMetadata.getBundleIdentifier(),
bundleMetadata.getVersion());
+ } catch (NiFiRegistryException nre) {
+ throw new ExtensionRegistryException(nre.getMessage(), nre);
+ }
+ }
+
+ private RequestConfig getRequestConfig(final NiFiUser user) {
+ final String identity = getIdentity(user);
+ return identity == null ? null : new
ProxiedEntityRequestConfig(identity);
+ }
+
+ private String getIdentity(final NiFiUser user) {
+ return (user == null || user.isAnonymous()) ? null :
user.getIdentity();
+ }
+
+ private NiFiRegistryExtensionBundleMetadata map(final
BundleVersionMetadata bundleVersionMetadata) {
+ return new NiFiRegistryExtensionBundleMetadata.Builder()
+ .group(bundleVersionMetadata.getGroupId())
+ .artifact(bundleVersionMetadata.getArtifactId())
+ .version(bundleVersionMetadata.getVersion())
+ .bundleIdentifier(bundleVersionMetadata.getBundleId())
+ .registryIdentifier(getIdentifier())
+ .build();
+ }
+
+}
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryNarProvider.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryNarProvider.java
new file mode 100644
index 0000000..6689d31
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryNarProvider.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.extension;
+
+import org.apache.nifi.nar.NarProvider;
+import org.apache.nifi.nar.NarProviderInitializationContext;
+import org.apache.nifi.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * NarProvider implementation that retrieves NARs from NiFi Registry. The
current implementation will retrieve bundles
+ * from all buckets that the NiFi server is authorized to read from (generally
will be all buckets).
+ *
+ * Example configuration for nifi.properties:
+ *
nifi.nar.library.provider.nifi-registry.implementation=org.apache.nifi.registry.extension.NiFiRegistryNarProvider
+ * nifi.nar.library.provider.nifi-registry.url=http://localhost:18080
+ *
+ */
+public class NiFiRegistryNarProvider implements NarProvider {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(NiFiRegistryNarProvider.class);
+
+ private static final String NIFI_REGISTRY_CLIENT_ID =
"nifi-registry-nar-provider";
+ private static final String NIFI_REGISTRY_CLIENT_NAME = "NiFi Registry NAR
Provider";
+
+ static final String URL_PROPERTY = "url";
+
+ private volatile NiFiRegistryExtensionRegistry extensionRegistry;
+ private volatile boolean initialized = false;
+
+ @Override
+ public void initialize(final NarProviderInitializationContext
initializationContext) {
+ final String url =
initializationContext.getProperties().get(URL_PROPERTY);
+ if (StringUtils.isBlank(url)) {
+ throw new IllegalArgumentException("NiFiRegistryNarProvider
requires a `url` property");
+ }
+
+ final SSLContext sslContext =
initializationContext.getNiFiSSLContext();
+ if (url.startsWith("https") && sslContext == null) {
+ throw new IllegalStateException("NiFi TLS properties must be
specified in order to connect to NiFi Registry via https");
+ }
+
+ extensionRegistry = new
NiFiRegistryExtensionRegistry(NIFI_REGISTRY_CLIENT_ID, url,
NIFI_REGISTRY_CLIENT_NAME, sslContext);
+ initialized = true;
+ }
+
+ @Override
+ public Collection<String> listNars() throws IOException {
+ if (!initialized) {
+ LOGGER.error("Provider is not initialized");
+ }
+
+ try {
+ final Set<NiFiRegistryExtensionBundleMetadata> bundleMetadata =
extensionRegistry.getExtensionBundleMetadata(null);
+ return bundleMetadata.stream().map(bm ->
bm.toLocationString()).collect(Collectors.toSet());
+ } catch (final ExtensionRegistryException ere) {
+ LOGGER.error("Unable to retrieve listing of NARs from NiFi
Registry at []", extensionRegistry.getURL(), ere);
+ return Collections.emptySet();
+ }
+ }
+
+ @Override
+ public InputStream fetchNarContents(final String location) throws
IOException {
+ if (!initialized) {
+ LOGGER.error("Provider is not initialized");
+ }
+
+ final NiFiRegistryExtensionBundleMetadata bundleMetadata =
NiFiRegistryExtensionBundleMetadata.fromLocationString(location)
+ .registryIdentifier(extensionRegistry.getIdentifier())
+ .build();
+
+ LOGGER.debug("Fetching NAR contents for bundleIdentifier [{}] and
version [{}]",
+ bundleMetadata.getBundleIdentifier(),
bundleMetadata.getVersion());
+
+ try {
+ return extensionRegistry.getExtensionBundleContent(null,
bundleMetadata);
+ } catch (ExtensionRegistryException e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
+}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/registry/NiFiRegistryExtensionBundleMetadataTest.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/registry/NiFiRegistryExtensionBundleMetadataTest.java
new file mode 100644
index 0000000..74cba5c
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/registry/NiFiRegistryExtensionBundleMetadataTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry;
+
+import org.apache.nifi.registry.extension.NiFiRegistryExtensionBundleMetadata;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class NiFiRegistryExtensionBundleMetadataTest {
+
+ @Test
+ public void testValidLocation() {
+ final NiFiRegistryExtensionBundleMetadata bundleMetadata = new
NiFiRegistryExtensionBundleMetadata.Builder()
+ .registryIdentifier("registry1")
+ .group("group1")
+ .artifact("artifact1")
+ .version("2")
+ .bundleIdentifier("123")
+ .build();
+
+ final String location = bundleMetadata.toLocationString();
+ assertNotNull(location);
+
+ final NiFiRegistryExtensionBundleMetadata parsedBundleMetadata =
+
NiFiRegistryExtensionBundleMetadata.fromLocationString(location)
+ .registryIdentifier("registry1")
+ .build();
+
+ assertEquals(bundleMetadata.getRegistryIdentifier(),
parsedBundleMetadata.getRegistryIdentifier());
+ assertEquals(bundleMetadata.getGroup(),
parsedBundleMetadata.getGroup());
+ assertEquals(bundleMetadata.getArtifact(),
parsedBundleMetadata.getArtifact());
+ assertEquals(bundleMetadata.getVersion(),
parsedBundleMetadata.getVersion());
+ assertEquals(bundleMetadata.getBundleIdentifier(),
parsedBundleMetadata.getBundleIdentifier());
+ }
+}
diff --git
a/nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionBundleMetadata.java
similarity index 60%
copy from
nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
copy to
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionBundleMetadata.java
index 26d6b7f..856b769 100644
---
a/nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionBundleMetadata.java
@@ -14,17 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.nar;
-
-import java.util.Map;
+package org.apache.nifi.registry.extension;
/**
- * Contains necessary information for extensions of NAR auto loader
functionality.
+ * The metadata for an extension bundle that exists in an extension registry.
*/
-public interface NarProviderInitializationContext {
+public interface ExtensionBundleMetadata {
+
+ /**
+ * @return the identifier of the extension registry that the bundle
belongs to
+ */
+ String getRegistryIdentifier();
/**
- * @return Returns with the available properties.
+ * @return the group id of the bundle
*/
- Map<String, String> getProperties();
-}
+ String getGroup();
+
+ /**
+ * @return the artifact id of the bundle
+ */
+ String getArtifact();
+
+ /**
+ * @return the version of the bundle
+ */
+ String getVersion();
+
+}
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionRegistry.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionRegistry.java
new file mode 100644
index 0000000..7a3ee88
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionRegistry.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.registry.extension;
+
+import org.apache.nifi.authorization.user.NiFiUser;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Set;
+
+/**
+ * Represents an extension registry that can be used to list/retrieve
available bundles.
+ *
+ * @param <T> the type of bundle metadata returned from listing bundles
+ */
+public interface ExtensionRegistry<T extends ExtensionBundleMetadata> {
+
+ /**
+ * @return the identifier of the registry
+ */
+ String getIdentifier();
+
+ /**
+ * @return the description of the registry
+ */
+ String getDescription();
+
+ /**
+ * @param description the description of the registry
+ */
+ void setDescription(String description);
+
+ /**
+ * @return the url of the registry
+ */
+ String getURL();
+
+ /**
+ * @param url the url of the registry
+ */
+ void setURL(String url);
+
+ /**
+ * @return the name of the registry
+ */
+ String getName();
+
+ /**
+ * @param name the name of the registry
+ */
+ void setName(String name);
+
+ /**
+ * Retrieves a listing of all available bundles in the given registry.
+ *
+ * @param user an optional end user making the request
+ * @return the set of bundle metadata for available bundles
+ * @throws IOException if an I/O error occurs
+ * @throws ExtensionRegistryException if a non I/O error occurs
+ */
+ Set<T> getExtensionBundleMetadata(NiFiUser user)
+ throws IOException, ExtensionRegistryException;
+
+ /**
+ * Retrieves the content of a bundle specified by the given bundle
metadata.
+ *
+ * @param user an optional end user making the request
+ * @param bundleMetadata a bundle metadata specifying the bundle to
retrieve
+ * @return an InputStream to the content of the specified bundle
+ * @throws IOException if an I/O error occurs
+ * @throws ExtensionRegistryException if a non I/O error occurs
+ */
+ InputStream getExtensionBundleContent(NiFiUser user, T bundleMetadata)
+ throws IOException, ExtensionRegistryException;
+
+}
+
+
diff --git
a/nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionRegistryException.java
similarity index 68%
copy from
nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
copy to
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionRegistryException.java
index 26d6b7f..770d5f1 100644
---
a/nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionRegistryException.java
@@ -14,17 +14,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.nifi.nar;
-
-import java.util.Map;
+package org.apache.nifi.registry.extension;
/**
- * Contains necessary information for extensions of NAR auto loader
functionality.
+ * Indicates a non-I/O related error from an extension registry.
*/
-public interface NarProviderInitializationContext {
+public class ExtensionRegistryException extends Exception {
+
+ public ExtensionRegistryException(String message) {
+ super(message);
+ }
+
+ public ExtensionRegistryException(String message, Throwable cause) {
+ super(message, cause);
+ }
- /**
- * @return Returns with the available properties.
- */
- Map<String, String> getProperties();
-}
+}
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/META-INF/services/org.apache.nifi.nar.NarProvider
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/META-INF/services/org.apache.nifi.nar.NarProvider
new file mode 100644
index 0000000..018d498
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/META-INF/services/org.apache.nifi.nar.NarProvider
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.nifi.registry.extension.NiFiRegistryNarProvider
\ No newline at end of file
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/pom.xml
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/pom.xml
index 0770f5f..9029630 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/pom.xml
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/pom.xml
@@ -28,6 +28,10 @@
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-security-utils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
<artifactId>nifi-documentation</artifactId>
</dependency>
<dependency>
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/NarAutoLoader.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/NarAutoLoader.java
index b0725b0..ad5b461 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/NarAutoLoader.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/NarAutoLoader.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.nar;
+import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.util.FileUtils;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
@@ -57,7 +58,7 @@ public class NarAutoLoader {
this.extensionManager = Objects.requireNonNull(extensionManager);
}
- public synchronized void start() throws IllegalAccessException,
InstantiationException, ClassNotFoundException, IOException {
+ public synchronized void start() throws IllegalAccessException,
InstantiationException, ClassNotFoundException, IOException, TlsException {
if (started) {
return;
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/PropertyBasedNarProviderInitializationContext.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/PropertyBasedNarProviderInitializationContext.java
index 528b079..4f5edd0 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/PropertyBasedNarProviderInitializationContext.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/PropertyBasedNarProviderInitializationContext.java
@@ -16,8 +16,13 @@
*/
package org.apache.nifi.nar;
+import org.apache.nifi.security.util.SslContextFactory;
+import org.apache.nifi.security.util.StandardTlsConfiguration;
+import org.apache.nifi.security.util.TlsConfiguration;
+import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.util.NiFiProperties;
+import javax.net.ssl.SSLContext;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -32,10 +37,12 @@ public class PropertyBasedNarProviderInitializationContext
implements NarProvide
static final String BASIC_PREFIX = "nifi.nar.library.provider.";
private final Map<String, String> properties;
+ private final SSLContext sslContext;
private final String name;
- public PropertyBasedNarProviderInitializationContext(final NiFiProperties
properties, final String name) {
+ public PropertyBasedNarProviderInitializationContext(final NiFiProperties
properties, final String name) throws TlsException {
this.properties = extractProperties(properties, name);
+ this.sslContext = createSSLContext(properties);
this.name = name;
}
@@ -44,7 +51,12 @@ public class PropertyBasedNarProviderInitializationContext
implements NarProvide
return properties;
}
- public Map<String, String> extractProperties(final NiFiProperties
properties, final String name) {
+ @Override
+ public SSLContext getNiFiSSLContext() {
+ return sslContext;
+ }
+
+ private Map<String, String> extractProperties(final NiFiProperties
properties, final String name) {
final String prefix = BASIC_PREFIX + name + ".";
final Map<String, String> candidates =
properties.getPropertiesWithPrefix(prefix);
final Map<String, String> result = new HashMap<>();
@@ -59,4 +71,9 @@ public class PropertyBasedNarProviderInitializationContext
implements NarProvide
return result;
}
+
+ private SSLContext createSSLContext(final NiFiProperties properties)
throws TlsException {
+ final TlsConfiguration tlsConfiguration =
StandardTlsConfiguration.fromNiFiProperties(properties);
+ return SslContextFactory.createSslContext(tlsConfiguration);
+ }
}
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/java/org/apache/nifi/nar/TestPropertyBasedNarProviderInitializationContext.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/java/org/apache/nifi/nar/TestPropertyBasedNarProviderInitializationContext.java
index d95d699..c0ba0c5 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/java/org/apache/nifi/nar/TestPropertyBasedNarProviderInitializationContext.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/java/org/apache/nifi/nar/TestPropertyBasedNarProviderInitializationContext.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.nar;
+import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.util.NiFiProperties;
import org.junit.Assert;
import org.junit.Test;
@@ -37,7 +38,7 @@ public class
TestPropertyBasedNarProviderInitializationContext {
NiFiProperties properties;
@Test
- public void testEmptyProperties() {
+ public void testEmptyProperties() throws TlsException {
// when
final PropertyBasedNarProviderInitializationContext testSubject = new
PropertyBasedNarProviderInitializationContext(properties, PROVIDER_NAME);
final Map<String, String> result = testSubject.getProperties();
@@ -48,7 +49,7 @@ public class
TestPropertyBasedNarProviderInitializationContext {
}
@Test
- public void testGuardedPropertiesAreNotReturned() {
+ public void testGuardedPropertiesAreNotReturned() throws TlsException {
// given
final Map<String, String> availableProperties = new HashMap<>();
availableProperties.put(PREFIX + "implementation", "value");
@@ -64,7 +65,7 @@ public class
TestPropertyBasedNarProviderInitializationContext {
}
@Test
- public void testPropertiesWouldHaveEmptyKeyAreNotReturned() {
+ public void testPropertiesWouldHaveEmptyKeyAreNotReturned() throws
TlsException {
// given
final Map<String, String> availableProperties = new HashMap<>();
availableProperties.put(PREFIX, "value");
@@ -80,7 +81,7 @@ public class
TestPropertyBasedNarProviderInitializationContext {
}
@Test
- public void testPrefixIsRemoved() {
+ public void testPrefixIsRemoved() throws TlsException {
// given
final Map<String, String> availableProperties = new HashMap<>();
availableProperties.put(PREFIX + "key1", "value1");
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
index fdbe39d..5541943 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
@@ -336,5 +336,24 @@ nifi.diagnostics.on.shutdown.max.filecount=10
# The diagnostics folder's maximum permitted size in bytes. If the limit is
exceeded, the oldest files are deleted.
nifi.diagnostics.on.shutdown.max.directory.size=10 MB
-
-
+# NAR Provider Properties #
+# These properties allow configuring one or more NAR providers. A NAR provider
retrieves NARs from an external source
+# and copies them to the directory specified by
nifi.nar.library.autoload.directory.
+#
+# Each NAR provider property follows the format:
+# nifi.nar.library.provider.<identifier>.<property-name>
+#
+# Each NAR provider must have at least one property named "implementation".
+#
+# Example HDFS NAR Provider:
+#
nifi.nar.library.provider.hdfs.implementation=org.apache.nifi.nar.hadoop.HDFSNarProvider
+#
nifi.nar.library.provider.hdfs.resources=/path/to/core-site.xml,/path/to/hdfs-site.xml
+# nifi.nar.library.provider.hdfs.storage.location=hdfs://hdfs-location
+# nifi.nar.library.provider.hdfs.source.directory=/nars
+# [email protected]
+# nifi.nar.library.provider.hdfs.kerberos.keytab=/path/to/nifi.keytab
+# nifi.nar.library.provider.hdfs.kerberos.password=
+#
+# Example NiFi Registry NAR Provider:
+#
nifi.nar.library.provider.nifi-registry.implementation=org.apache.nifi.registry.extension.NiFiRegistryNarProvider
+# nifi.nar.library.provider.nifi-registry.url=http://localhost:18080
