This is an automated email from the ASF dual-hosted git repository. chriss pushed a commit to branch NIFI-7060 in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 6c1913af56471a8b762f42237860882ba4b2e445 Author: Chris Sampson <chris.sampso...@gmail.com> AuthorDate: Tue Feb 28 18:17:30 2023 +0000 NIFI-7060 update NiFi and NiFi Registry Docker Image start scripts to populate properties from env vars --- nifi-docker/dockerhub/sh/common.sh | 8 +- nifi-docker/dockerhub/sh/nifi_env_from_file.sh | 102 +++++++++ nifi-docker/dockerhub/sh/secure.sh | 56 ++--- nifi-docker/dockerhub/sh/start.sh | 228 ++++++++++++--------- .../sh/update_cluster_state_management.sh | 4 +- nifi-docker/dockerhub/sh/update_oidc_properties.sh | 21 +- nifi-docker/dockermaven/pom.xml | 2 +- .../nifi-registry-docker/dockerhub/sh/common.sh | 14 ++ .../dockerhub/sh/nifi_registry_env_from_file.sh | 98 +++++++++ .../nifi-registry-docker/dockerhub/sh/secure.sh | 37 ++-- .../nifi-registry-docker/dockerhub/sh/start.sh | 62 ++++-- .../dockerhub/sh/update_database.sh | 14 +- .../dockerhub/sh/update_flow_provider.sh | 9 +- .../dockerhub/sh/update_login_providers.sh | 1 + .../dockerhub/sh/update_oidc_properties.sh | 18 +- .../dockermaven/integration-test.sh | 26 +-- .../nifi-registry-docker-maven/dockermaven/pom.xml | 18 +- .../dockermaven/sh/common.sh | 28 --- .../dockermaven/sh/secure.sh | 57 ------ .../dockermaven/sh/start.sh | 63 ------ .../dockermaven/sh/update_bundle_provider.sh | 48 ----- .../dockermaven/sh/update_database.sh | 24 --- .../dockermaven/sh/update_flow_provider.sh | 47 ----- .../dockermaven/sh/update_login_providers.sh | 48 ----- .../dockermaven/sh/update_oidc_properties.sh | 27 --- 25 files changed, 481 insertions(+), 579 deletions(-) diff --git a/nifi-docker/dockerhub/sh/common.sh b/nifi-docker/dockerhub/sh/common.sh index e3ab9d8369..49d1c0c76b 100755 --- a/nifi-docker/dockerhub/sh/common.sh +++ b/nifi-docker/dockerhub/sh/common.sh @@ -20,13 +20,14 @@ prop_replace () { target_file="${3:-${nifi_props_file}}" echo "File [${target_file}] replacing [${1}]" - sed -i -e "s|^$1=.*$|$1=$2|" "${target_file}" + # use case-insensitive match for the property name to support mixed-case properties (e.g. keystoreType) + sed -i -e "s|^($1)=.*$|\1=$2|i" "${target_file}" } uncomment() { target_file="${2}" echo "File [${target_file}] uncommenting [${1}]" - sed -i -e "s|^\#$1|$1|" "${target_file}" + sed -i -e "s|^\#($1)|\1|i" "${target_file}" } # 1 - property key to add or replace @@ -34,7 +35,8 @@ uncomment() { # 3 - file to perform replacement inline prop_add_or_replace () { target_file="${3:-${nifi_props_file}}" - property_found=$(awk -v property="${1}" 'index($0, property) == 1') + # case-insensitive matching of property name (e.g. keystoreType) + property_found=$(awk -v property="${1}" 'index(toLower($0), property) == 1') if [ -z "${property_found}" ]; then echo "File [${target_file}] adding [${1}]" echo "$1=$2" >> "${target_file}" diff --git a/nifi-docker/dockerhub/sh/nifi_env_from_file.sh b/nifi-docker/dockerhub/sh/nifi_env_from_file.sh new file mode 100755 index 0000000000..01d5a713e1 --- /dev/null +++ b/nifi-docker/dockerhub/sh/nifi_env_from_file.sh @@ -0,0 +1,102 @@ +#!/bin/bash -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail + +# Allow environment variables to be set by creating a file with the +# contents, and setting an environment variable with the suffix _FILE to +# point to it. This can be used to provide secrets to a container, without +# the values being specified explicitly when running the container. +# +# Note that only supported environment variables are processed, in order +# to avoid unexpected failures when an environment sets a "*_FILE" variable +# that doesn't contain a filename. +# +# This script is intended to be sourced, not executed, and modifies the +# environment. + +sensitive_files=( + NIFI_SECURITY_KEYSTOREPASSWD_FILE + NIFI_SECURITY_KEYPASSWD_FILE + NIFI_SECURITY_TRUSTSTOREPASSWD_FILE + NIFI_SECURITY_USER_OIDC_CLIENT_SECRET_FILE + NIFI_REPOSITORY_ENCRYPTION_KEY_PROVIDER_KEYSTORE_PASSWORD_FILE + NIFI_ZOOKEEPER_SECURITY_KEYSTOREPASSWD_FILE + NIFI_ZOOKEEPER_SECURITY_TRUSTTOREPASSWD_FILE + NIFI_NAR_LIBRARY_PROVIDER_HDFS_KERBEROS_PASSWORD_FILE + NIFI_SENSITIVE_PROPS_KEY_FILE + SINGLE_USER_CREDENTIALS_PASSWORD_FILE +) + +for VAR_NAME_FILE in "${sensitive_files[@]}"; do + if [[ -n "${!VAR_NAME_FILE}" ]]; then + VAR_NAME="${VAR_NAME_FILE%_FILE}" + + if env | grep "^${VAR_NAME}="; then + echo "ERROR: Both $VAR_NAME_FILE and $VAR_NAME are set. These are mutually exclusive." >&2 + exit 1 + fi + + if [[ ! -e "${!VAR_NAME_FILE}" ]]; then + # Maybe the file doesn't exist, maybe we just can't read it due to file permissions. + # Check permissions on each part of the path + path='' + if ! echo "${!VAR_NAME_FILE}" | grep -q '^/'; then + path='.' + fi + + dirname "${!VAR_NAME_FILE}" | tr '/' '\n' | while read -r part; do + if [[ "$path" == "/" ]]; then + path="${path}${part}" + else + path="$path/$part" + fi + + if ! [[ -x "$path" ]]; then + echo "ERROR: Cannot read ${!VAR_NAME_FILE} from $VAR_NAME_FILE, due to lack of permissions on '$path'" 2>&1 + exit 1 + fi + done + + if ! [[ -r "${!VAR_NAME_FILE}" ]]; then + echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE is not readable." 2>&1 + else + echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE does not exist" >&2 + fi + + exit 1 + fi + + FILE_PERMS="$(stat -L -c '%a' "${!VAR_NAME_FILE}")" + + if [[ "$FILE_PERMS" != "400" && "$FILE_PERMS" != "600" ]]; then + if [[ -L "${!VAR_NAME_FILE}" ]]; then + echo "ERROR: File $(readlink "${!VAR_NAME_FILE}") (target of symlink ${!VAR_NAME_FILE} from $VAR_NAME_FILE) must have file permissions 400 or 600, but actually has: $FILE_PERMS" >&2 + else + echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE must have file permissions 400 or 600, but actually has: $FILE_PERMS" >&2 + fi + exit 1 + fi + + echo "Setting $VAR_NAME from $VAR_NAME_FILE at ${!VAR_NAME_FILE}" >&2 + export "$VAR_NAME"="$(cat "${!VAR_NAME_FILE}")" + + unset VAR_NAME + # Unset the suffixed environment variable + unset "$VAR_NAME_FILE" + fi +done diff --git a/nifi-docker/dockerhub/sh/secure.sh b/nifi-docker/dockerhub/sh/secure.sh index 70622a6905..2a69e1e4d1 100755 --- a/nifi-docker/dockerhub/sh/secure.sh +++ b/nifi-docker/dockerhub/sh/secure.sh @@ -21,57 +21,26 @@ scripts_dir='/opt/nifi/scripts' [ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" # Perform idempotent changes of configuration to support secure environments -echo 'Configuring environment with SSL settings' +echo 'Checking environment TLS settings present' -: "${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."}" -if [ ! -f "${KEYSTORE_PATH}" ]; then - echo "Keystore file specified (${KEYSTORE_PATH}) does not exist." +: "${NIFI_SECURITY_KEYSTORE:?"Must specify an absolute path to the keystore being used."}" +if [ ! -f "${NIFI_SECURITY_KEYSTORE}" ]; then + echo "Keystore file specified (${NIFI_SECURITY_KEYSTORE}) does not exist." exit 1 fi -: "${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."}" -: "${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."}" +: "${NIFI_SECURITY_KEYSTORETYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."}" +: "${NIFI_SECURITY_KEYSTOREPASSWD:?"Must specify the password of the keystore being used."}" -: "${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."}" -if [ ! -f "${TRUSTSTORE_PATH}" ]; then - echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist." +: "${NIFI_SECURITY_TRUSTSTORE:?"Must specify an absolute path to the truststore being used."}" +if [ ! -f "${NIFI_SECURITY_TRUSTSTORE}" ]; then + echo "Keystore file specified (${NIFI_SECURITY_TRUSTSTORE}) does not exist." exit 1 fi -: "${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."}" -: "${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."}" +: "${NIFI_SECURITY_TRUSTSTORETYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."}" +: "${NIFI_SECURITY_TRUSTSTOREPASSWD:?"Must specify the password of the truststore being used."}" -prop_replace 'nifi.security.keystore' "${KEYSTORE_PATH}" -prop_replace 'nifi.security.keystoreType' "${KEYSTORE_TYPE}" -prop_replace 'nifi.security.keystorePasswd' "${KEYSTORE_PASSWORD}" -prop_replace 'nifi.security.keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" -prop_replace 'nifi.security.truststore' "${TRUSTSTORE_PATH}" -prop_replace 'nifi.security.truststoreType' "${TRUSTSTORE_TYPE}" -prop_replace 'nifi.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}" -# shellcheck disable=SC2154 -prop_replace 'keystore' "${KEYSTORE_PATH}" "${nifi_toolkit_props_file}" -prop_replace 'keystoreType' "${KEYSTORE_TYPE}" "${nifi_toolkit_props_file}" -prop_replace 'keystorePasswd' "${KEYSTORE_PASSWORD}" "${nifi_toolkit_props_file}" -prop_replace 'keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" "${nifi_toolkit_props_file}" -prop_replace 'truststore' "${TRUSTSTORE_PATH}" "${nifi_toolkit_props_file}" -prop_replace 'truststoreType' "${TRUSTSTORE_TYPE}" "${nifi_toolkit_props_file}" -# shellcheck disable=SC2086 -prop_replace 'truststorePasswd' "${TRUSTSTORE_PASSWORD}" "${nifi_toolkit_props_file}" - -# Disable HTTP and enable HTTPS -prop_replace 'nifi.web.http.port' '' -prop_replace 'nifi.web.http.host' '' -prop_replace 'nifi.web.https.port' "${NIFI_WEB_HTTPS_PORT:-8443}" -prop_replace 'nifi.web.https.host' "${NIFI_WEB_HTTPS_HOST:-$hostname}" -prop_replace 'nifi.remote.input.secure' 'true' -# Enable the property only for cluster install -prop_replace 'nifi.cluster.protocol.is.secure' "${NIFI_CLUSTER_IS_NODE:-false}" - -# Setup nifi-toolkit -prop_replace 'baseUrl' "https://${NIFI_WEB_HTTPS_HOST:-$hostname}:${NIFI_WEB_HTTPS_PORT:-8443}" "${nifi_toolkit_props_file}" - -# Configure Authorizer and Login Identity Provider -prop_replace 'nifi.security.user.authorizer' "${NIFI_SECURITY_USER_AUTHORIZER:-managed-authorizer}" -prop_replace 'nifi.security.user.login.identity.provider' "${NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER}" +export NIFI_SECURITY_USER_AUTHORIZER="${NIFI_SECURITY_USER_AUTHORIZER:-managed-authorizer}" # Establish initial user and an associated admin identity sed -i -e 's|<property name="Initial User Identity 1"></property>|<property name="Initial User Identity 1">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' "${NIFI_HOME}/conf/authorizers.xml" @@ -81,4 +50,5 @@ if [ -n "${NODE_IDENTITY}" ]; then sed -i -e 's|<property name="Node Identity 1"></property>|<property name="Node Identity 1">'"${NODE_IDENTITY}"'</property>|' "${NIFI_HOME}/conf/authorizers.xml" fi +# shellcheck disable=SC2154 prop_replace 'proxiedEntity' "${INITIAL_ADMIN_IDENTITY}" "${nifi_toolkit_props_file}" diff --git a/nifi-docker/dockerhub/sh/start.sh b/nifi-docker/dockerhub/sh/start.sh index 35cf333718..21dcd0119c 100755 --- a/nifi-docker/dockerhub/sh/start.sh +++ b/nifi-docker/dockerhub/sh/start.sh @@ -20,140 +20,168 @@ scripts_dir='/opt/nifi/scripts' # shellcheck source=./common.sh [ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" +# read sensitive vales from files (if present) +. "${scripts_dir}/nifi_env_from_file.sh" + # Override JVM memory settings if [ -n "${NIFI_JVM_HEAP_INIT}" ]; then - # shellcheck disable=SC2154 - prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" "${nifi_bootstrap_file}" + # shellcheck disable=SC2154 + prop_replace 'java.arg.2' "-Xms${NIFI_JVM_HEAP_INIT}" "${nifi_bootstrap_file}" fi if [ -n "${NIFI_JVM_HEAP_MAX}" ]; then - prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" "${nifi_bootstrap_file}" + prop_replace 'java.arg.3' "-Xmx${NIFI_JVM_HEAP_MAX}" "${nifi_bootstrap_file}" fi if [ -n "${NIFI_JVM_DEBUGGER}" ]; then - uncomment "java.arg.debug" "${nifi_bootstrap_file}" + uncomment "java.arg.debug" "${nifi_bootstrap_file}" fi -# Replace NiFi properties with environment variables -NIFI_ENV_VARS=$(printenv | awk -F= '/^NIFI_/ {print $1}') - -for ENV_VAR in $NIFI_ENV_VARS; do - PROP_NAME=$(echo "$ENV_VAR" | tr _ . | tr '[:upper:]' '[:lower:]') - PROP_VALUE=$(printenv "$ENV_VAR") - prop_replace "$PROP_NAME" "$PROP_VALUE" -done - -# Establish baseline properties -prop_replace 'nifi.web.https.port' "${NIFI_WEB_HTTPS_PORT:-8443}" -prop_replace 'nifi.web.https.host' "${NIFI_WEB_HTTPS_HOST:-$hostname}" -prop_replace 'nifi.web.proxy.host' "${NIFI_WEB_PROXY_HOST}" -prop_replace 'nifi.remote.input.host' "${NIFI_REMOTE_INPUT_HOST:-$hostname}" -prop_replace 'nifi.remote.input.socket.port' "${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" -prop_replace 'nifi.remote.input.secure' 'true' -prop_replace 'nifi.cluster.protocol.is.secure' 'true' - -# Set nifi-toolkit properties files and baseUrl -"${scripts_dir}/toolkit.sh" -# shellcheck disable=SC2154 -prop_replace 'baseUrl' "https://${NIFI_WEB_HTTPS_HOST:-$hostname}:${NIFI_WEB_HTTPS_PORT:-8443}" "${nifi_toolkit_props_file}" - -prop_replace 'keystore' "${NIFI_HOME}/conf/keystore.p12" "${nifi_toolkit_props_file}" -prop_replace 'keystoreType' "PKCS12" "${nifi_toolkit_props_file}" -prop_replace 'truststore' "${NIFI_HOME}/conf/truststore.p12" "${nifi_toolkit_props_file}" -prop_replace 'truststoreType' "PKCS12" "${nifi_toolkit_props_file}" - -if [ -n "${NIFI_WEB_HTTP_PORT}" ]; then - prop_replace 'nifi.web.https.port' '' - prop_replace 'nifi.web.https.host' '' - prop_replace 'nifi.web.http.port' "${NIFI_WEB_HTTP_PORT}" - prop_replace 'nifi.web.http.host' "${NIFI_WEB_HTTP_HOST:-$hostname}" - prop_replace 'nifi.remote.input.secure' 'false' - prop_replace 'nifi.cluster.protocol.is.secure' 'false' - prop_replace 'nifi.security.keystore' '' - prop_replace 'nifi.security.keystoreType' '' - prop_replace 'nifi.security.truststore' '' - prop_replace 'nifi.security.truststoreType' '' - prop_replace 'nifi.security.user.login.identity.provider' '' - prop_replace 'keystore' '' "${nifi_toolkit_props_file}" - prop_replace 'keystoreType' '' "${nifi_toolkit_props_file}" - prop_replace 'truststore' '' "${nifi_toolkit_props_file}" - prop_replace 'truststoreType' '' "${nifi_toolkit_props_file}" - prop_replace 'baseUrl' "http://${NIFI_WEB_HTTP_HOST:-$hostname}:${NIFI_WEB_HTTP_PORT}" "${nifi_toolkit_props_file}" - - if [ -n "${NIFI_WEB_PROXY_HOST}" ]; then - echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Unsetting nifi.web.proxy.host.' - prop_replace 'nifi.web.proxy.host' '' - fi +# set default values for some properties if not otherwise specified +export NIFI_REMOTE_INPUT_SOCKET_PORT="${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" +if [ -z "${NIFI_WEB_HTTP_PORT}" ]; then + export NIFI_WEB_HTTPS_PORT="${NIFI_WEB_HTTPS_PORT:-8443}" + export NIFI_WEB_HTTPS_HOST="${NIFI_WEB_HTTPS_HOST:-$hostname}" + export NIFI_WEB_HTTP_HOST= + export BASE_URL="https://${NIFI_WEB_HTTPS_HOST}:${NIFI_WEB_HTTPS_PORT}" + export NIFI_REMOTE_INPUT_HOST="${NIFI_REMOTE_INPUT_HOST:-$hostname}" + export NIFI_REMOTE_INPUT_SECURE=true + export NIFI_CLUSTER_PROTOCOL_IS_SECURE=true + export NIFI_SECURITY_KEYSTORE="${NIFI_SECURITY_KEYSTORE:-${KEYSTORE_PATH:-${NIFI_HOME}/conf/keystore.p12}}" + export NIFI_SECURITY_KEYSTORETYPE="${NIFI_SECURITY_KEYSTORETYPE:-${KEYSTORE_TYPE:-PKCS12}}" + export NIFI_SECURITY_KEYSTOREPASSWD="${NIFI_SECURITY_KEYSTOREPASSWD:-${KEYSTORE_PASSWORD:-}}" + export NIFI_SECURITY_KEYPASSWD="${NIFI_SECURITY_KEYPASSWD:-${KEY_PASSWORD:-${NIFI_SECURITY_KEYSTOREPASSWD:-}}}" + export NIFI_SECURITY_TRUSTSTORE="${NIFI_SECURITY_KEYSTORE:-${TRUSTSTORE_PATH:-${NIFI_HOME}/conf/truststore.p12}}" + export NIFI_SECURITY_TRUSTSTORETYPE=PKCS12 + export NIFI_SECURITY_TRUSTSTOREPASSWD="${NIFI_SECURITY_TRUSTSTOREPASSWD:-${TRUSTSTORE_PASSWORD:-}}" + + if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then + echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode. The NiFi UI may be inaccessible if using port mapping or connecting through a proxy.' + fi else - if [ -z "${NIFI_WEB_PROXY_HOST}" ]; then - echo 'NIFI_WEB_PROXY_HOST was not set but NiFi is configured to run in a secure mode. The NiFi UI may be inaccessible if using port mapping or connecting through a proxy.' - fi + export NIFI_WEB_HTTPS_PORT= + export NIFI_WEB_HTTPS_HOST= + export NIFI_WEB_HTTP_HOST="${NIFI_WEB_HTTP_HOST:-$hostname}" + export BASE_URL="http://${NIFI_WEB_HTTP_HOST}:${NIFI_WEB_HTTP_PORT}" + export NIFI_REMOTE_INPUT_HOST="${NIFI_REMOTE_INPUT_HOST:-$hostname}" + export NIFI_REMOTE_INPUT_SOCKET_PORT="${NIFI_REMOTE_INPUT_SOCKET_PORT:-10000}" + export NIFI_REMOTE_INPUT_SECURE=false + export NIFI_CLUSTER_PROTOCOL_IS_SECURE=false + export NIFI_SECURITY_KEYSTORE= + export NIFI_SECURITY_KEYSTORETYPE= + export NIFI_SECURITY_KEYSTOREPASSWD= + export NIFI_SECURITY_KEYPASSWD= + export NIFI_SECURITY_TRUSTSTORE= + export NIFI_SECURITY_TRUSTSTORETYPE= + export NIFI_SECURITY_TRUSTSTOREPASSWD= + export NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER= + + if [ -n "${NIFI_WEB_PROXY_HOST}" ]; then + echo 'NIFI_WEB_PROXY_HOST was set but NiFi is not configured to run in a secure mode. Unsetting nifi.web.proxy.host.' + fi fi -prop_replace 'nifi.variable.registry.properties' "${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}" -prop_replace 'nifi.cluster.is.node' "${NIFI_CLUSTER_IS_NODE:-false}" -prop_replace 'nifi.cluster.node.address' "${NIFI_CLUSTER_ADDRESS:-$hostname}" -prop_replace 'nifi.cluster.node.protocol.port' "${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}" -prop_replace 'nifi.cluster.node.protocol.max.threads' "${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}" -prop_replace 'nifi.zookeeper.connect.string' "${NIFI_ZK_CONNECT_STRING:-}" -prop_replace 'nifi.zookeeper.root.node' "${NIFI_ZK_ROOT_NODE:-/nifi}" -prop_replace 'nifi.cluster.flow.election.max.wait.time' "${NIFI_ELECTION_MAX_WAIT:-5 mins}" -prop_replace 'nifi.cluster.flow.election.max.candidates' "${NIFI_ELECTION_MAX_CANDIDATES:-}" -prop_replace 'nifi.web.proxy.context.path' "${NIFI_WEB_PROXY_CONTEXT_PATH:-}" +export NIFI_VARIABLE_REGISTRY_PROPERTIES="${NIFI_VARIABLE_REGISTRY_PROPERTIES:-}" + +# setup cluster properties +export NIFI_CLUSTER_IS_NODE="${NIFI_CLUSTER_IS_NODE:-false}" +export NIFI_CLUSTER_NODE_ADDRESS="${NIFI_CLUSTER_NODE_ADDRESS:-${NIFI_CLUSTER_ADDRESS:-$hostname}}" +export NIFI_CLUSTER_NODE_PROTOCOL_PORT="${NIFI_CLUSTER_NODE_PROTOCOL_PORT:-}" +export NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS="${NIFI_CLUSTER_NODE_PROTOCOL_MAX_THREADS:-50}" +export NIFI_ZOOKEEPER_CONNECT_STRING="${NIFI_ZOOKEEPER_CONNECT_STRING:=${NIFI_ZK_CONNECT_STRING:-}}" +export NIFI_ZOOKEEPER_ROOT_NODE="${NIFI_ZOOKEEPER_ROOT_NODE:-${NIFI_ZK_ROOT_NODE:-/nifi}}" +export NIFI_CLUSTER_FLOW_ELECTION_MAX_WAIT_TIME="${NIFI_CLUSTER_FLOW_ELECTION_MAX_WAIT_TIME:-${NIFI_ELECTION_MAX_WAIT:-5 mins}}" +export NIFI_CLUSTER_FLOW_ELECTION_MAX_CANDIDATES="${NIFI_CLUSTER_FLOW_ELECTION_MAX_CANDIDATES:-${NIFI_ELECTION_MAX_CANDIDATES:-}}" +export NIFI_WEB_PROXY_CONTEXT_PATH="${NIFI_WEB_PROXY_CONTEXT_PATH:-}" # Set analytics properties -prop_replace 'nifi.analytics.predict.enabled' "${NIFI_ANALYTICS_PREDICT_ENABLED:-false}" -prop_replace 'nifi.analytics.predict.interval' "${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}" -prop_replace 'nifi.analytics.query.interval' "${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}" -prop_replace 'nifi.analytics.connection.model.implementation' "${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}" -prop_replace 'nifi.analytics.connection.model.score.name' "${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}" -prop_replace 'nifi.analytics.connection.model.score.threshold' "${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}" +export NIFI_ANALYTICS_PREDICT_ENABLED="${NIFI_ANALYTICS_PREDICT_ENABLED:-false}" +export NIFI_ANALYTICS_PREDICT_INTERVAL="${NIFI_ANALYTICS_PREDICT_INTERVAL:-3 mins}" +export NIFI_ANALYTICS_QUERY_INTERVAL="${NIFI_ANALYTICS_QUERY_INTERVAL:-5 mins}" +export NIFI_ANALYTICS_CONNECTION_MODEL_IMPLEMENTATION="${NIFI_ANALYTICS_CONNECTION_MODEL_IMPLEMENTATION:-${NIFI_ANALYTICS_MODEL_IMPLEMENTATION:-org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares}}" +export NIFI_ANALYTICS_CONNECTION_MODEL_SCORE_NAME="${NIFI_ANALYTICS_CONNECTION_MODEL_SCORE_NAME:-${NIFI_ANALYTICS_MODEL_SCORE_NAME:-rSquared}}" +export NIFI_ANALYTICS_CONNECTION_MODEL_SCORE_THRESHOLD="${NIFI_ANALYTICS_CONNECTION_MODEL_SCORE_THRESHOLD:-${NIFI_ANALYTICS_MODEL_SCORE_THRESHOLD:-.90}}" # Add NAR provider properties -# nifi-registry NAR provider -if [ -n "${NIFI_NAR_LIBRARY_PROVIDER_NIFI_REGISTRY_URL}" ]; then - prop_add_or_replace 'nifi.nar.library.provider.nifi-registry.implementation' 'org.apache.nifi.registry.extension.NiFiRegistryExternalResourceProvider' - prop_add_or_replace 'nifi.nar.library.provider.nifi-registry.url' "${NIFI_NAR_LIBRARY_PROVIDER_NIFI_REGISTRY_URL}" +export NIFI_NAR_LIBRARY_PROVIDER_NIFI__REGISTRY_URL="${NIFI_NAR_LIBRARY_PROVIDER_NIFI__REGISTRY_URL:-${NIFI_NAR_LIBRARY_PROVIDER_NIFI_REGISTRY_URL:-}}" +if [ -n "${NIFI_NAR_LIBRARY_PROVIDER_NIFI__REGISTRY_URL}" ]; then + export NIFI_NAR_LIBRARY_PROVIDER_NIFI__REGISTRY_IMPLEMENTATION=org.apache.nifi.registry.extension.NiFiRegistryExternalResourceProvider fi - -if [ -n "${NIFI_SENSITIVE_PROPS_KEY}" ]; then - prop_replace 'nifi.sensitive.props.key' "${NIFI_SENSITIVE_PROPS_KEY}" +export NIFI_NAR_LIBRARY_PROVIDER_LOCAL__FILES_SOURCE_DIR="${NIFI_NAR_LIBRARY_PROVIDER_LOCAL__FILES_SOURCE_DIR:-}" +if [ -n "${NIFI_NAR_LIBRARY_PROVIDER_LOCAL__FILES_SOURCE_DIR}" ]; then + export NIFI_NAR_LIBRARY_PROVIDER_LOCAL__FILES_IMPLEMENTATION=org.apache.nifi.nar.provider.LocalDirectoryNarProvider fi +# setup single user credentials (if provided) if [ -n "${SINGLE_USER_CREDENTIALS_USERNAME}" ] && [ -n "${SINGLE_USER_CREDENTIALS_PASSWORD}" ]; then - "${NIFI_HOME}/bin/nifi.sh" set-single-user-credentials "${SINGLE_USER_CREDENTIALS_USERNAME}" "${SINGLE_USER_CREDENTIALS_PASSWORD}" + "${NIFI_HOME}/bin/nifi.sh" set-single-user-credentials "${SINGLE_USER_CREDENTIALS_USERNAME}" "${SINGLE_USER_CREDENTIALS_PASSWORD}" fi +# Setup cluster state management . "${scripts_dir}/update_cluster_state_management.sh" # Check if we are secured or unsecured case ${AUTH} in - tls) - echo 'Enabling Two-Way SSL user authentication' - . "${scripts_dir}/secure.sh" - ;; - ldap) - echo 'Enabling LDAP user authentication' - # Reference ldap-provider in properties - export NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER="ldap-provider" - - . "${scripts_dir}/secure.sh" - . "${scripts_dir}/update_login_providers.sh" - ;; - oidc) - echo 'Enabling OIDC user authentication' - - . "${scripts_dir}/secure.sh" - . "${scripts_dir}/update_oidc_properties.sh" - ;; +tls) + echo 'Enabling Two-Way TLS user authentication' + # check TLS settings are set + . "${scripts_dir}/secure.sh" + ;; +ldap) + echo 'Enabling LDAP user authentication' + # check TLS settings are set + . "${scripts_dir}/secure.sh" + # Reference ldap-provider in properties + export NIFI_SECURITY_USER_LOGIN_IDENTITY_PROVIDER="ldap-provider" + + . "${scripts_dir}/update_login_providers.sh" + ;; +oidc) + echo 'Enabling OIDC user authentication' + # check TLS settings are set + . "${scripts_dir}/secure.sh" + # check OIDC properties are set + . "${scripts_dir}/update_oidc_properties.sh" + ;; +*) + echo 'Assuming single-user authentication' + # don't set passwords for single-user auth + export NIFI_SECURITY_KEYSTOREPASSWD= + export NIFI_SECURITY_KEYPASSWD= + export NIFI_SECURITY_TRUSTSTOREPASSWD= + ;; esac + +# Set nifi-toolkit properties files and baseUrl +"${scripts_dir}/toolkit.sh" +# shellcheck disable=SC2154 +prop_replace 'baseUrl' "${BASE_URL}" "${nifi_toolkit_props_file}" +prop_replace 'keystore' "${NIFI_SECURITY_KEYSTORE}" "${nifi_toolkit_props_file}" +prop_replace 'keystoreType' "${NIFI_SECURITY_KEYSTORETYPE}" "${nifi_toolkit_props_file}" +[ -n "${NIFI_SECURITY_KEYSTOREPASSWD}" ] && prop_replace 'keystorePasswd' "${NIFI_SECURITY_KEYSTOREPASSWD}" "${nifi_toolkit_props_file}" +[ -n "${NIFI_SECURITY_KEYPASSWD}" ] && prop_replace 'keyPasswd' "${NIFI_SECURITY_KEYPASSWD}" "${nifi_toolkit_props_file}" +prop_replace 'truststore' "${NIFI_SECURITY_TRUSTSTORE}" "${nifi_toolkit_props_file}" +prop_replace 'truststoreType' "${NIFI_SECURITY_TRUSTSTORETYPE}" "${nifi_toolkit_props_file}" +[ -n "${NIFI_SECURITY_TRUSTSTOREPASSWD}" ] && prop_replace 'truststorePasswd' "${NIFI_SECURITY_TRUSTSTOREPASSWD}" "${nifi_toolkit_props_file}" + + +# Replace NiFi properties with environment variables +nifi_env_vars=$(printenv | awk -F= '/^NIFI_/ {print $1}' | grep -vE '^NIFI_JVM_' | grep -vE '_(HOME|DIR)$') + +for nifi_env_var in ${nifi_env_vars}; do + # mixed-case properties will be matched case-insensitively within the prop_add_or_replace/prop_replace functions + prop_name=$(echo "${nifi_env_var}" | sed -e 's/__/-/' | tr _ . | tr '[:upper:]' '[:lower:]') + prop_value=$(printenv "${nifi_env_var}") + prop_add_or_replace "${prop_name}" "${prop_value}" +done + # Continuously provide logs so that 'docker logs' can produce them "${NIFI_HOME}/bin/nifi.sh" run & nifi_pid="$!" tail -F --pid=${nifi_pid} "${NIFI_HOME}/logs/nifi-app.log" & -trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT; +trap 'echo Received trapped signal, beginning shutdown...;./bin/nifi.sh stop;exit 0;' TERM HUP INT trap ":" EXIT echo NiFi running with PID ${nifi_pid}. diff --git a/nifi-docker/dockerhub/sh/update_cluster_state_management.sh b/nifi-docker/dockerhub/sh/update_cluster_state_management.sh index 07b9d61f82..d1e49be99d 100755 --- a/nifi-docker/dockerhub/sh/update_cluster_state_management.sh +++ b/nifi-docker/dockerhub/sh/update_cluster_state_management.sh @@ -27,5 +27,5 @@ edit_property() { fi } -edit_property 'Connect String' "${NIFI_ZK_CONNECT_STRING}" -edit_property "Root Node" "${NIFI_ZK_ROOT_NODE}" +edit_property 'Connect String' "${NIFI_ZOOKEEPER_CONNECT_STRING}" +edit_property "Root Node" "${NIFI_ZOOKEEPER_ROOT_NODE}" diff --git a/nifi-docker/dockerhub/sh/update_oidc_properties.sh b/nifi-docker/dockerhub/sh/update_oidc_properties.sh index 827a40edba..3d0db5aa47 100644 --- a/nifi-docker/dockerhub/sh/update_oidc_properties.sh +++ b/nifi-docker/dockerhub/sh/update_oidc_properties.sh @@ -15,13 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -prop_replace 'nifi.security.user.oidc.discovery.url' "${NIFI_SECURITY_USER_OIDC_DISCOVERY_URL}" -prop_replace 'nifi.security.user.oidc.connect.timeout' "${NIFI_SECURITY_USER_OIDC_CONNECT_TIMEOUT}" -prop_replace 'nifi.security.user.oidc.read.timeout' "${NIFI_SECURITY_USER_OIDC_READ_TIMEOUT}" -prop_replace 'nifi.security.user.oidc.client.id' "${NIFI_SECURITY_USER_OIDC_CLIENT_ID}" -prop_replace 'nifi.security.user.oidc.client.secret' "${NIFI_SECURITY_USER_OIDC_CLIENT_SECRET}" -prop_replace 'nifi.security.user.oidc.preferred.jwsalgorithm' "${NIFI_SECURITY_USER_OIDC_PREFERRED_JWSALGORITHM}" -prop_replace 'nifi.security.user.oidc.additional.scopes' "${NIFI_SECURITY_USER_OIDC_ADDITIONAL_SCOPES}" -prop_replace 'nifi.security.user.oidc.claim.identifying.user' "${NIFI_SECURITY_USER_OIDC_CLAIM_IDENTIFYING_USER}" -prop_replace 'nifi.security.user.oidc.fallback.claims.identifying.user' "${NIFI_SECURITY_USER_OIDC_FALLBACK_CLAIMS_IDENTIFYING_USER}" -prop_replace 'nifi.security.user.oidc.truststore.strategy' "${NIFI_SECURITY_USER_OIDC_TRUSTSTORE_STRATEGY}" +: "${NIFI_SECURITY_USER_OIDC_DISCOVERY_URL:?"Must specify the OIDC Discovery URL."}" +export NIFI_SECURITY_USER_OIDC_CONNECT_TIMEOUT="${NIFI_SECURITY_USER_OIDC_CONNECT_TIMEOUT:-}" +export NIFI_SECURITY_USER_OIDC_READ_TIMEOUT="${NIFI_SECURITY_USER_OIDC_READ_TIMEOUT:-}" +: "${NIFI_SECURITY_USER_OIDC_CLIENT_ID:?"Must specify the OIDC Client ID."}" +: "${NIFI_SECURITY_USER_OIDC_CLIENT_SECRET:?"Must specify the OIDC Client Secret."}" +: "${NIFI_SECURITY_USER_OIDC_PREFERRED_JWSALGORITHM:?"Must specify the OIDC Preferred JWS Algorithm."}" +export NIFI_SECURITY_USER_OIDC_ADDITIONAL_SCOPES="${NIFI_SECURITY_USER_OIDC_ADDITIONAL_SCOPES:-}" +export NIFI_SECURITY_USER_OIDC_CLAIM_IDENTIFYING_USER="${NIFI_SECURITY_USER_OIDC_CLAIM_IDENTIFYING_USER:-}" +export NIFI_SECURITY_USER_OIDC_FALLBACK_CLAIMS_IDENTIFYING_USER="${NIFI_SECURITY_USER_OIDC_FALLBACK_CLAIMS_IDENTIFYING_USER:-}" +export NIFI_SECURITY_USER_OIDC_TRUSTSTORE_STRATEGY="${NIFI_SECURITY_USER_OIDC_TRUSTSTORE_STRATEGY:-}" +export NIFI_SECURITY_USER_OIDC_CLAIM_GROUPS="${NIFI_SECURITY_USER_OIDC_CLAIM_GROUPS:-}" diff --git a/nifi-docker/dockermaven/pom.xml b/nifi-docker/dockermaven/pom.xml index ca24fbfeae..6237e01473 100644 --- a/nifi-docker/dockermaven/pom.xml +++ b/nifi-docker/dockermaven/pom.xml @@ -62,7 +62,7 @@ <!-- Copy generated artifact to nifi-docker --> <plugin> <artifactId>maven-antrun-plugin</artifactId> - <version>3.0.0</version> + <version>3.1.0</version> <executions> <execution> <id>copy-scripts-for-docker</id> diff --git a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/common.sh b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/common.sh index d5b3c27168..11074e97a7 100755 --- a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/common.sh +++ b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/common.sh @@ -23,6 +23,20 @@ prop_replace () { sed -i -e "s|^$1=.*$|$1=$2|" "${target_file}" } +# 1 - property key to add or replace +# 2 - property value to use +# 3 - file to perform replacement inline +prop_add_or_replace () { + target_file="${3:-${nifi_registry_props_file}}" + property_found=$(awk -v property="${1}" 'index($0, property) == 1') + if [ -z "${property_found}" ]; then + echo "File [${target_file}] adding [${1}]" + echo "$1=$2" >> "${target_file}" + else + prop_replace "$1" "$2" "$3" + fi +} + # NIFI_REGISTRY_HOME is defined by an ENV command in the backing Dockerfile export nifi_registry_props_file="${NIFI_REGISTRY_HOME}/conf/nifi-registry.properties" hostname=$(hostname) diff --git a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/nifi_registry_env_from_file.sh b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/nifi_registry_env_from_file.sh new file mode 100755 index 0000000000..5d5ba7c65f --- /dev/null +++ b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/nifi_registry_env_from_file.sh @@ -0,0 +1,98 @@ +#!/bin/bash -e + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o pipefail + +# Allow environment variables to be set by creating a file with the +# contents, and setting an environment variable with the suffix _FILE to +# point to it. This can be used to provide secrets to a container, without +# the values being specified explicitly when running the container. +# +# Note that only supported environment variables are processed, in order +# to avoid unexpected failures when an environment sets a "*_FILE" variable +# that doesn't contain a filename. +# +# This script is intended to be sourced, not executed, and modifies the +# environment. + +sensitive_files=( + NIFI_REGISTRY_SECURITY_KEYSTOREPASSWD_FILE + NIFI_REGISTRY_SECURITY_KEYPASSWD_FILE + NIFI_REGISTRY_SECURITY_TRUSTSTOREPASSWD_FILE + NIFI_REGISTRY_DB_PASSWORD_FILE + NIFI_REGISTRY_SENSITIVE_PROPS_ADDITIONAL_KEYS_FILE + NIFI_REGISTRY_SECURITY_USER_OIDC_CLIENT_SECRET_FILE +) + +for VAR_NAME_FILE in "${sensitive_files[@]}"; do + if [[ -n "${!VAR_NAME_FILE}" ]]; then + VAR_NAME="${VAR_NAME_FILE%_FILE}" + + if env | grep "^${VAR_NAME}="; then + echo "ERROR: Both $VAR_NAME_FILE and $VAR_NAME are set. These are mutually exclusive." >&2 + exit 1 + fi + + if [[ ! -e "${!VAR_NAME_FILE}" ]]; then + # Maybe the file doesn't exist, maybe we just can't read it due to file permissions. + # Check permissions on each part of the path + path='' + if ! echo "${!VAR_NAME_FILE}" | grep -q '^/'; then + path='.' + fi + + dirname "${!VAR_NAME_FILE}" | tr '/' '\n' | while read -r part; do + if [[ "$path" == "/" ]]; then + path="${path}${part}" + else + path="$path/$part" + fi + + if ! [[ -x "$path" ]]; then + echo "ERROR: Cannot read ${!VAR_NAME_FILE} from $VAR_NAME_FILE, due to lack of permissions on '$path'" 2>&1 + exit 1 + fi + done + + if ! [[ -r "${!VAR_NAME_FILE}" ]]; then + echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE is not readable." 2>&1 + else + echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE does not exist" >&2 + fi + + exit 1 + fi + + FILE_PERMS="$(stat -L -c '%a' "${!VAR_NAME_FILE}")" + + if [[ "$FILE_PERMS" != "400" && "$FILE_PERMS" != "600" ]]; then + if [[ -L "${!VAR_NAME_FILE}" ]]; then + echo "ERROR: File $(readlink "${!VAR_NAME_FILE}") (target of symlink ${!VAR_NAME_FILE} from $VAR_NAME_FILE) must have file permissions 400 or 600, but actually has: $FILE_PERMS" >&2 + else + echo "ERROR: File ${!VAR_NAME_FILE} from $VAR_NAME_FILE must have file permissions 400 or 600, but actually has: $FILE_PERMS" >&2 + fi + exit 1 + fi + + echo "Setting $VAR_NAME from $VAR_NAME_FILE at ${!VAR_NAME_FILE}" >&2 + export "$VAR_NAME"="$(cat "${!VAR_NAME_FILE}")" + + unset VAR_NAME + # Unset the suffixed environment variable + unset "$VAR_NAME_FILE" + fi +done diff --git a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/secure.sh b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/secure.sh index e17c7aefdd..a6750a151e 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/secure.sh +++ b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/secure.sh @@ -23,35 +23,28 @@ scripts_dir='/opt/nifi-registry/scripts' # Perform idempotent changes of configuration to support secure environments echo 'Configuring environment with SSL settings' -: "${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."}" -if [ ! -f "${KEYSTORE_PATH}" ]; then - echo "Keystore file specified (${KEYSTORE_PATH}) does not exist." +export NIFI_REGISTRY_SECURITY_KEYSTORE="${NIFI_REGISTRY_SECURITY_KEYSTORE:-${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."}}" +if [ ! -f "${NIFI_REGISTRY_SECURITY_KEYSTORE}" ]; then + echo "Keystore file specified (${NIFI_REGISTRY_SECURITY_KEYSTORE}) does not exist." exit 1 fi -: "${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."}" -: "${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."}" +export NIFI_REGISTRY_SECURITY_KEYSTORETYPE="${NIFI_REGISTRY_SECURITY_KEYSTORETYPE:-${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."}}" +export NIFI_REGISTRY_SECURITY_KEYSTOREPASSWD="${NIFI_REGISTRY_SECURITY_KEYSTOREPASSWD:-${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."}}" +export NIFI_REGISTRY_SECURITY_KEYSPASSWD="${NIFI_REGISTRY_SECURITY_KEYSPASSWD:-${KEY_PASSWORD:-${NIFI_REGISTRY_SECURITY_KEYSTOREPASSWD}}}" -: "${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."}" -if [ ! -f "${TRUSTSTORE_PATH}" ]; then - echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist." +export NIFI_REGISTRY_SECURITY_TRUSTSTORE="${NIFI_REGISTRY_SECURITY_TRUSTSTORE:-${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."}}" +if [ ! -f "${NIFI_REGISTRY_SECURITY_TRUSTSTORE}" ]; then + echo "Keystore file specified (${NIFI_REGISTRY_SECURITY_TRUSTSTORE}) does not exist." exit 1 fi -: "${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."}" -: "${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."}" - -prop_replace 'nifi.registry.security.keystore' "${KEYSTORE_PATH}" -prop_replace 'nifi.registry.security.keystoreType' "${KEYSTORE_TYPE}" -prop_replace 'nifi.registry.security.keystorePasswd' "${KEYSTORE_PASSWORD}" -prop_replace 'nifi.registry.security.keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" -prop_replace 'nifi.registry.security.truststore' "${TRUSTSTORE_PATH}" -prop_replace 'nifi.registry.security.truststoreType' "${TRUSTSTORE_TYPE}" -prop_replace 'nifi.registry.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}" +export NIFI_REGISTRY_SECURITY_TRUSTSTORETYPE="${NIFI_REGISTRY_SECURITY_TRUSTSTORETYPE:-${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."}}" +export NIFI_REGISTRY_SECURITY_TRUSTSTOREPASSWD="${NIFI_REGISTRY_SECURITY_TRUSTSTOREPASSWD:-${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."}}" # Disable HTTP and enable HTTPS -prop_replace 'nifi.registry.web.http.port' '' -prop_replace 'nifi.registry.web.http.host' '' -prop_replace 'nifi.registry.web.https.port' "${NIFI_REGISTRY_WEB_HTTPS_PORT:-18443}" -prop_replace 'nifi.registry.web.https.host' "${NIFI_REGISTRY_WEB_HTTPS_HOST:-$hostname}" +export NIFI_REGISTRY_WEB_HTTP_PORT= +export NIFI_REGISTRY_WEB_HTTP_HOST= +export NIFI_REGISTRY_WEB_HTTPS_PORT="${NIFI_REGISTRY_WEB_HTTPS_PORT:-18443}" +export NIFI_REGISTRY_WEB_HTTPS_HOST="${NIFI_REGISTRY_WEB_HTTPS_HOST:-$hostname}" # Establish initial user and an associated admin identity sed -i -e 's|<property name="Initial User Identity 1">.*</property>|<property name="Initial User Identity 1">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' "${NIFI_REGISTRY_HOME}/conf/authorizers.xml" diff --git a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/start.sh b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/start.sh index 9cdbb26c81..c994fa09c6 100755 --- a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/start.sh +++ b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/start.sh @@ -20,44 +20,62 @@ scripts_dir='/opt/nifi-registry/scripts' # shellcheck source=./common.sh [ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" +# read sensitive vales from files (if present) +. "${scripts_dir}/nifi_registry_env_from_file.sh" + # Establish baseline properties -prop_replace 'nifi.registry.web.http.port' "${NIFI_REGISTRY_WEB_HTTP_PORT:-18080}" -prop_replace 'nifi.registry.web.http.host' "${NIFI_REGISTRY_WEB_HTTP_HOST:-$hostname}" +export NIFI_REGISTRY_WEB_HTTP_PORT="${NIFI_REGISTRY_WEB_HTTP_PORT:-18080}" +export NIFI_REGISTRY_WEB_HTTP_HOST="${NIFI_REGISTRY_WEB_HTTP_HOST:-$hostname}" . "${scripts_dir}/update_database.sh" # Check if we are secured or unsecured case ${AUTH} in - tls) - echo 'Enabling Two-Way SSL user authentication' - . "${scripts_dir}/secure.sh" - ;; - ldap) - echo 'Enabling LDAP user authentication' - # Reference ldap-provider in properties - prop_replace 'nifi.registry.security.identity.provider' 'ldap-identity-provider' - prop_replace 'nifi.registry.security.needClientAuth' 'false' - - . "${scripts_dir}/secure.sh" - . "${scripts_dir}/update_login_providers.sh" - ;; - oidc) - echo 'Enabling OIDC user authentication' - - . "${scripts_dir}/secure.sh" - . "${scripts_dir}/update_oidc_properties.sh" - ;; +tls) + echo 'Enabling Two-Way SSL user authentication' + # check TLS settings are set + . "${scripts_dir}/secure.sh" + ;; +ldap) + echo 'Enabling LDAP user authentication' + # check TLS settings are set + . "${scripts_dir}/secure.sh" + + # Reference ldap-provider in properties + export NIFI_REGISTRY_SECURITY_IDENTITY_PROVIDER=ldap-identity-provider + export NIFI_REGISTRY_SECURITY_NEEDCLIENTAUTH=false + . "${scripts_dir}/update_login_providers.sh" + ;; +oidc) + echo 'Enabling OIDC user authentication' + # check TLS settings are set + . "${scripts_dir}/secure.sh" + # check OIDC properties are set + export NIFI_REGISTRY_SECURITY_NEEDCLIENTAUTH=false + . "${scripts_dir}/update_oidc_properties.sh" + ;; esac . "${scripts_dir}/update_flow_provider.sh" . "${scripts_dir}/update_bundle_provider.sh" + +# Replace NiFi properties with environment variables +nifi_registry_env_vars=$(printenv | awk -F= '/^NIFI_REGISTRY_/ {print $1}' | grep -vE '^NIFI_REGISTRY_S3_' | grep -v '_BINARY_' | grep -vE '_(HOME|DIR)$') + +for nifi_registry_env_var in ${nifi_registry_env_vars}; do + prop_name=$(echo "${nifi_registry_env_var}" | sed -e 's/__/-/' | tr _ . | tr '[:upper:]' '[:lower:]') + prop_value=$(printenv "${nifi_registry_env_var}") + prop_add_or_replace "${prop_name}" "${prop_value}" +done + + # Continuously provide logs so that 'docker logs' can produce them tail -F "${NIFI_REGISTRY_HOME}/logs/nifi-registry-app.log" & "${NIFI_REGISTRY_HOME}/bin/nifi-registry.sh" run & nifi_registry_pid="$!" -trap "echo Received trapped signal, beginning shutdown...;" TERM HUP INT EXIT; +trap "echo Received trapped signal, beginning shutdown...;" TERM HUP INT EXIT echo NiFi-Registry running with PID ${nifi_registry_pid}. wait ${nifi_registry_pid} diff --git a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_database.sh b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_database.sh index 59d94d7b39..a30a7b39e7 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_database.sh +++ b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_database.sh @@ -15,10 +15,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -prop_replace 'nifi.registry.db.url' "${NIFI_REGISTRY_DB_URL:-jdbc:h2:./database/nifi-registry-primary;AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE}" -prop_replace 'nifi.registry.db.driver.class' "${NIFI_REGISTRY_DB_CLASS:-org.h2.Driver}" -prop_replace 'nifi.registry.db.driver.directory' "${NIFI_REGISTRY_DB_DIR:-}" -prop_replace 'nifi.registry.db.username' "${NIFI_REGISTRY_DB_USER:-nifireg}" -prop_replace 'nifi.registry.db.password' "${NIFI_REGISTRY_DB_PASS:-nifireg}" -prop_replace 'nifi.registry.db.maxConnections' "${NIFI_REGISTRY_DB_MAX_CONNS:-5}" -prop_replace 'nifi.registry.db.sql.debug' "${NIFI_REGISTRY_DB_DEBUG_SQL:-false}" +export NIFI_REGISTRY_DB_URL="${NIFI_REGISTRY_DB_URL:-jdbc:h2:./database/nifi-registry-primary;AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE}" +export NIFI_REGISTRY_DB_DRIVER_CLASS="${NIFI_REGISTRY_DB_DRIVER_CLASS:-${NIFI_REGISTRY_DB_CLASS:-org.h2.Driver}}" +export NIFI_REGISTRY_DB_DRIVER_DIRECTORY="${NIFI_REGISTRY_DB_DRIVER_DIRECTORY:-${NIFI_REGISTRY_DB_DIR:-}}" +export NIFI_REGISTRY_DB_USERNAME="${NIFI_REGISTRY_DB_USERNAME:-${NIFI_REGISTRY_DB_USER:-nifireg}}" +export NIFI_REGISTRY_DB_PASSWORD="${NIFI_REGISTRY_DB_PASSWORD:-${NIFI_REGISTRY_DB_PASS:-nifireg}}" +export NIFI_REGISTRY_DB_MAXCONNECTIONS="${NIFI_REGISTRY_DB_MAXCONNECTIONS:-${NIFI_REGISTRY_DB_MAX_CONNS:-5}}" +export NIFI_REGISTRY_DB_SQL_DEBUG="${NIFI_REGISTRY_DB_SQL_DEBUG:-${NIFI_REGISTRY_DB_DEBUG_SQL:-false}}" diff --git a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_flow_provider.sh b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_flow_provider.sh index abcdbee06a..6cdeb1f4e5 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_flow_provider.sh +++ b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_flow_provider.sh @@ -35,14 +35,17 @@ case "${NIFI_REGISTRY_FLOW_PROVIDER}" in file) xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.provider.flow.FileSystemFlowPersistenceProvider" "${providers_file}" ;; + database) + xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.provider.flow.DatabaseFlowPersistenceProvider" "${providers_file}" + ;; git) xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.provider.flow.git.GitFlowPersistenceProvider" "${providers_file}" add_property "Remote To Push" "${NIFI_REGISTRY_GIT_REMOTE:-}" add_property "Remote Access User" "${NIFI_REGISTRY_GIT_USER:-}" add_property "Remote Access Password" "${NIFI_REGISTRY_GIT_PASSWORD:-}" - if [ -n "$NIFI_REGISTRY_GIT_REPO" ]; then - add_property "Remote Clone Repository" "${NIFI_REGISTRY_GIT_REPO:-}" - fi + if [ -n "$NIFI_REGISTRY_GIT_REPO" ]; then + add_property "Remote Clone Repository" "${NIFI_REGISTRY_GIT_REPO:-}" + fi ;; esac diff --git a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_login_providers.sh b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_login_providers.sh index 0036c9922d..944c9ddda4 100755 --- a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_login_providers.sh +++ b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_login_providers.sh @@ -45,3 +45,4 @@ edit_property 'Url' "${LDAP_URL}" edit_property 'User Search Base' "${LDAP_USER_SEARCH_BASE}" edit_property 'User Search Filter' "${LDAP_USER_SEARCH_FILTER}" edit_property 'Identity Strategy' "${LDAP_IDENTITY_STRATEGY}" +edit_property 'Referral Strategy' "${LDAP_REFERRAL_STRATEGY}" diff --git a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_oidc_properties.sh b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_oidc_properties.sh index 827a40edba..6ebca66eac 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_oidc_properties.sh +++ b/nifi-registry/nifi-registry-core/nifi-registry-docker/dockerhub/sh/update_oidc_properties.sh @@ -15,13 +15,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -prop_replace 'nifi.security.user.oidc.discovery.url' "${NIFI_SECURITY_USER_OIDC_DISCOVERY_URL}" -prop_replace 'nifi.security.user.oidc.connect.timeout' "${NIFI_SECURITY_USER_OIDC_CONNECT_TIMEOUT}" -prop_replace 'nifi.security.user.oidc.read.timeout' "${NIFI_SECURITY_USER_OIDC_READ_TIMEOUT}" -prop_replace 'nifi.security.user.oidc.client.id' "${NIFI_SECURITY_USER_OIDC_CLIENT_ID}" -prop_replace 'nifi.security.user.oidc.client.secret' "${NIFI_SECURITY_USER_OIDC_CLIENT_SECRET}" -prop_replace 'nifi.security.user.oidc.preferred.jwsalgorithm' "${NIFI_SECURITY_USER_OIDC_PREFERRED_JWSALGORITHM}" -prop_replace 'nifi.security.user.oidc.additional.scopes' "${NIFI_SECURITY_USER_OIDC_ADDITIONAL_SCOPES}" -prop_replace 'nifi.security.user.oidc.claim.identifying.user' "${NIFI_SECURITY_USER_OIDC_CLAIM_IDENTIFYING_USER}" -prop_replace 'nifi.security.user.oidc.fallback.claims.identifying.user' "${NIFI_SECURITY_USER_OIDC_FALLBACK_CLAIMS_IDENTIFYING_USER}" -prop_replace 'nifi.security.user.oidc.truststore.strategy' "${NIFI_SECURITY_USER_OIDC_TRUSTSTORE_STRATEGY}" +: "${NIFI_REGISTRY_SECURITY_USER_OIDC_DISCOVERY_URL:?"Must specify the OIDC Discovery URL."}" +export NIFI_REGISTRY_SECURITY_USER_OIDC_CONNECT_TIMEOUT="${NIFI_REGISTRY_SECURITY_USER_OIDC_CONNECT_TIMEOUT:?"Must specify the OIDC Connect Timeout."}" +export NIFI_REGISTRY_SECURITY_USER_OIDC_READ_TIMEOUT="${NIFI_REGISTRY_SECURITY_USER_OIDC_READ_TIMEOUT:?"Must specify the OIDC Read Timeout."}" +: "${NIFI_REGISTRY_SECURITY_USER_OIDC_CLIENT_ID:?"Must specify the OIDC Client ID."}" +: "${NIFI_REGISTRY_SECURITY_USER_OIDC_CLIENT_SECRET:?"Must specify the OIDC Client Secret."}" +: "${NIFI_REGISTRY_SECURITY_USER_OIDC_PREFERRED_JWSALGORITHM:?"Must specify the OIDC Preferred JWS Algorithm."}" +export NIFI_REGISTRY_SECURITY_USER_OIDC_ADDITIONAL_SCOPES="${NIFI_REGISTRY_SECURITY_USER_OIDC_ADDITIONAL_SCOPES:-}" +export NIFI_REGISTRY_SECURITY_USER_OIDC_CLAIM_IDENTIFYING_USER="${NIFI_REGISTRY_SECURITY_USER_OIDC_CLAIM_IDENTIFYING_USER:-}" diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/integration-test.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/integration-test.sh index a8d8d66581..6934ea374e 100755 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/integration-test.sh +++ b/nifi-registry/nifi-registry-docker-maven/dockermaven/integration-test.sh @@ -17,34 +17,34 @@ set -exuo pipefail -TAG=$1 -VERSION=$2 +TAG="$1" +VERSION="$2" -container_name=nifi-registry-${TAG}-integration-test +container_name="nifi-registry-${TAG}-integration-test" -trap "{ docker rm -f ${container_name}; }" EXIT +trap '{ docker rm -f ${container_name}; }' EXIT echo "Deleting any existing ${container_name} containers" -docker rm -f ${container_name}; +docker rm -f "${container_name}"; echo "Checking that all files are owned by NiFi" -test -z $(docker run --rm --entrypoint /bin/bash apache/nifi-registry:${TAG} -c "find /opt/nifi-registry ! -user nifi") +test -z "$(docker run --rm --entrypoint /bin/bash apache/nifi-registry:"${TAG}" -c "find /opt/nifi-registry ! -user nifi")" echo "Checking environment variables" -test "/opt/nifi-registry/nifi-registry-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-registry:${TAG} -c 'echo -n $NIFI_REGISTRY_HOME')" -test "/opt/nifi-registry/nifi-registry-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-registry:${TAG} -c "readlink \${NIFI_REGISTRY_BASE_DIR}/nifi-registry-${VERSION}")" +test "/opt/nifi-registry/nifi-registry-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-registry:"${TAG}" -c 'echo -n $NIFI_REGISTRY_HOME')" +test "/opt/nifi-registry/nifi-registry-current" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-registry:"${TAG}" -c "readlink \${NIFI_REGISTRY_BASE_DIR}/nifi-registry-${VERSION}")" -test "/opt/nifi-registry" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-registry:${TAG} -c 'echo -n $NIFI_REGISTRY_BASE_DIR')" +test "/opt/nifi-registry" = "$(docker run --rm --entrypoint /bin/bash apache/nifi-registry:"${TAG}" -c 'echo -n $NIFI_REGISTRY_BASE_DIR')" echo "Starting NiFi Registry container..." -docker run -d --name ${container_name} apache/nifi-registry:${TAG} +docker run -d --name "${container_name}" apache/nifi-registry:"${TAG}" -IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${container_name}) +IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${container_name}") for i in $(seq 1 10) :; do echo "Iteration: ${i}" - if docker exec ${container_name} bash -c " echo Running < /dev/tcp/${IP}/18080"; then + if docker exec "${container_name}" bash -c " echo Running < /dev/tcp/${IP}/18080"; then break fi sleep 10 @@ -54,4 +54,4 @@ echo "Checking NiFi Registry REST API Access" test "200" = "$(docker exec "${container_name}" bash -c "curl -s -o /dev/null -w %{http_code} -k http://${IP}:18080/nifi-registry-api/access")" echo "Stopping NiFi Registry container" -time docker stop ${container_name} +time docker stop "${container_name}" diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/pom.xml b/nifi-registry/nifi-registry-docker-maven/dockermaven/pom.xml index e95cdc0100..8d0135682b 100644 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/pom.xml +++ b/nifi-registry/nifi-registry-docker-maven/dockermaven/pom.xml @@ -46,8 +46,24 @@ <!-- Copy generated artifacts --> <plugin> <artifactId>maven-antrun-plugin</artifactId> - <version>1.8</version> + <version>3.1.0</version> <executions> + <execution> + <id>copy-scripts-for-docker</id> + <phase>process-sources</phase> + <configuration> + <target name="copy docker scripts to nifi-registry-docker for image build"> + <copy todir="${project.basedir}/target/sh" overwrite="true" flatten="true"> + <fileset dir="${project.basedir}/../../nifi-registry-core/nifi-registry-docker/dockerhub/sh" includes="*.sh"> + <include name="*.sh" /> + </fileset> + </copy> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> <execution> <id>copy-for-docker</id> <phase>process-sources</phase> diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/common.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/common.sh deleted file mode 100755 index 0f594d9aed..0000000000 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/common.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -e -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# 1 - value to search for -# 2 - value to replace -# 3 - file to perform replacement inline -prop_replace () { - target_file=${3:-${nifi_registry_props_file}} - echo 'replacing target file ' ${target_file} - sed -i -e "s|^$1=.*$|$1=$2|" ${target_file} -} - -# NIFI_REGISTRY_HOME is defined by an ENV command in the backing Dockerfile -export nifi_registry_props_file=${NIFI_REGISTRY_HOME}/conf/nifi-registry.properties -export hostname=$(hostname) diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/secure.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/secure.sh deleted file mode 100644 index 8a7a5bbed5..0000000000 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/secure.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -scripts_dir='/opt/nifi-registry/scripts' - -[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" - -# Perform idempotent changes of configuration to support secure environments -echo 'Configuring environment with SSL settings' - -: ${KEYSTORE_PATH:?"Must specify an absolute path to the keystore being used."} -if [ ! -f "${KEYSTORE_PATH}" ]; then - echo "Keystore file specified (${KEYSTORE_PATH}) does not exist." - exit 1 -fi -: ${KEYSTORE_TYPE:?"Must specify the type of keystore (JKS, PKCS12, PEM) of the keystore being used."} -: ${KEYSTORE_PASSWORD:?"Must specify the password of the keystore being used."} - -: ${TRUSTSTORE_PATH:?"Must specify an absolute path to the truststore being used."} -if [ ! -f "${TRUSTSTORE_PATH}" ]; then - echo "Keystore file specified (${TRUSTSTORE_PATH}) does not exist." - exit 1 -fi -: ${TRUSTSTORE_TYPE:?"Must specify the type of truststore (JKS, PKCS12, PEM) of the truststore being used."} -: ${TRUSTSTORE_PASSWORD:?"Must specify the password of the truststore being used."} - -prop_replace 'nifi.registry.security.keystore' "${KEYSTORE_PATH}" -prop_replace 'nifi.registry.security.keystoreType' "${KEYSTORE_TYPE}" -prop_replace 'nifi.registry.security.keystorePasswd' "${KEYSTORE_PASSWORD}" -prop_replace 'nifi.registry.security.keyPasswd' "${KEY_PASSWORD:-$KEYSTORE_PASSWORD}" -prop_replace 'nifi.registry.security.truststore' "${TRUSTSTORE_PATH}" -prop_replace 'nifi.registry.security.truststoreType' "${TRUSTSTORE_TYPE}" -prop_replace 'nifi.registry.security.truststorePasswd' "${TRUSTSTORE_PASSWORD}" - -# Disable HTTP and enable HTTPS -prop_replace 'nifi.registry.web.http.port' '' -prop_replace 'nifi.registry.web.http.host' '' -prop_replace 'nifi.registry.web.https.port' "${NIFI_REGISTRY_WEB_HTTPS_PORT:-18443}" -prop_replace 'nifi.registry.web.https.host' "${NIFI_REGISTRY_WEB_HTTPS_HOST:-$HOSTNAME}" - -# Establish initial user and an associated admin identity -sed -i -e 's|<property name="Initial User Identity 1">.*</property>|<property name="Initial User Identity 1">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' ${NIFI_REGISTRY_HOME}/conf/authorizers.xml -sed -i -e 's|<property name="Initial Admin Identity">.*</property>|<property name="Initial Admin Identity">'"${INITIAL_ADMIN_IDENTITY}"'</property>|' ${NIFI_REGISTRY_HOME}/conf/authorizers.xml diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/start.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/start.sh deleted file mode 100755 index 2703395516..0000000000 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/start.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -scripts_dir='/opt/nifi-registry/scripts' - -[ -f "${scripts_dir}/common.sh" ] && . "${scripts_dir}/common.sh" - -# Establish baseline properties -prop_replace 'nifi.registry.web.http.port' "${NIFI_REGISTRY_WEB_HTTP_PORT:-18080}" -prop_replace 'nifi.registry.web.http.host' "${NIFI_REGISTRY_WEB_HTTP_HOST:-$HOSTNAME}" - -. ${scripts_dir}/update_database.sh - -# Check if we are secured or unsecured -case ${AUTH} in - tls) - echo 'Enabling Two-Way SSL user authentication' - . "${scripts_dir}/secure.sh" - ;; - ldap) - echo 'Enabling LDAP user authentication' - # Reference ldap-provider in properties - prop_replace 'nifi.registry.security.identity.provider' 'ldap-identity-provider' - prop_replace 'nifi.registry.security.needClientAuth' 'false' - - . "${scripts_dir}/secure.sh" - . "${scripts_dir}/update_login_providers.sh" - ;; - oidc) - echo 'Enabling OIDC user authentication' - prop_replace 'nifi.registry.security.needClientAuth' 'false' - - . "${scripts_dir}/secure.sh" - . "${scripts_dir}/update_oidc_properties.sh" - ;; -esac - -. "${scripts_dir}/update_flow_provider.sh" -. "${scripts_dir}/update_bundle_provider.sh" - -# Continuously provide logs so that 'docker logs' can produce them -tail -F "${NIFI_REGISTRY_HOME}/logs/nifi-registry-app.log" & -"${NIFI_REGISTRY_HOME}/bin/nifi-registry.sh" run & -nifi_registry_pid="$!" - -trap "echo Received trapped signal, beginning shutdown...;" KILL TERM HUP INT EXIT; - -echo NiFi-Registry running with PID ${nifi_registry_pid}. -wait ${nifi_registry_pid} diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_bundle_provider.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_bundle_provider.sh deleted file mode 100644 index 27d5c940ac..0000000000 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_bundle_provider.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -providers_file=${NIFI_REGISTRY_HOME}/conf/providers.xml -property_xpath='/providers/extensionBundlePersistenceProvider' - -add_property() { - property_name=$1 - property_value=$2 - - if [ -n "${property_value}" ]; then - xmlstarlet ed --inplace --subnode "${property_xpath}" --type elem -n property -v "${property_value}" \ - -i \$prev --type attr -n name -v "${property_name}" \ - "${providers_file}" - fi -} - -xmlstarlet ed --inplace -u "${property_xpath}/property[@name='Extension Bundle Storage Directory']" -v "${NIFI_REGISTRY_BUNDLE_STORAGE_DIR:-./extension_bundles}" "${providers_file}" - -case ${NIFI_REGISTRY_BUNDLE_PROVIDER} in - file) - xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.provider.extension.FileSystemBundlePersistenceProvider" "${providers_file}" - ;; - s3) - xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.aws.S3BundlePersistenceProvider" "${providers_file}" - add_property "Region" "${NIFI_REGISTRY_S3_REGION:-}" - add_property "Bucket Name" "${NIFI_REGISTRY_S3_BUCKET_NAME:-}" - add_property "Key Prefix" "${NIFI_REGISTRY_S3_KEY_PREFIX:-}" - add_property "Credentials Provider" "${NIFI_REGISTRY_S3_CREDENTIALS_PROVIDER:-DEFAULT_CHAIN}" - add_property "Access Key" "${NIFI_REGISTRY_S3_ACCESS_KEY:-}" - add_property "Secret Access Key" "${NIFI_REGISTRY_S3_SECRET_ACCESS_KEY:-}" - add_property "Endpoint URL" "${NIFI_REGISTRY_S3_ENDPOINT_URL:-}" - ;; -esac diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_database.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_database.sh deleted file mode 100644 index 59d94d7b39..0000000000 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_database.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -prop_replace 'nifi.registry.db.url' "${NIFI_REGISTRY_DB_URL:-jdbc:h2:./database/nifi-registry-primary;AUTOCOMMIT=OFF;DB_CLOSE_ON_EXIT=FALSE;LOCK_MODE=3;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE}" -prop_replace 'nifi.registry.db.driver.class' "${NIFI_REGISTRY_DB_CLASS:-org.h2.Driver}" -prop_replace 'nifi.registry.db.driver.directory' "${NIFI_REGISTRY_DB_DIR:-}" -prop_replace 'nifi.registry.db.username' "${NIFI_REGISTRY_DB_USER:-nifireg}" -prop_replace 'nifi.registry.db.password' "${NIFI_REGISTRY_DB_PASS:-nifireg}" -prop_replace 'nifi.registry.db.maxConnections' "${NIFI_REGISTRY_DB_MAX_CONNS:-5}" -prop_replace 'nifi.registry.db.sql.debug' "${NIFI_REGISTRY_DB_DEBUG_SQL:-false}" diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_flow_provider.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_flow_provider.sh deleted file mode 100644 index 95c9099337..0000000000 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_flow_provider.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -providers_file=${NIFI_REGISTRY_HOME}/conf/providers.xml -property_xpath='/providers/flowPersistenceProvider' - -add_property() { - property_name=$1 - property_value=$2 - - if [ -n "${property_value}" ]; then - xmlstarlet ed --inplace --subnode "${property_xpath}" --type elem -n property -v "${property_value}" \ - -i \$prev --type attr -n name -v "${property_name}" \ - "${providers_file}" - fi -} - -xmlstarlet ed --inplace -u "${property_xpath}/property[@name='Flow Storage Directory']" -v "${NIFI_REGISTRY_FLOW_STORAGE_DIR:-./flow_storage}" "${providers_file}" - -case ${NIFI_REGISTRY_FLOW_PROVIDER} in - file) - xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.provider.flow.FileSystemFlowPersistenceProvider" "${providers_file}" - ;; - database) - xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.provider.flow.DatabaseFlowPersistenceProvider" "${providers_file}" - ;; - git) - xmlstarlet ed --inplace -u "${property_xpath}/class" -v "org.apache.nifi.registry.provider.flow.git.GitFlowPersistenceProvider" "${providers_file}" - add_property "Remote To Push" "${NIFI_REGISTRY_GIT_REMOTE:-}" - add_property "Remote Access User" "${NIFI_REGISTRY_GIT_USER:-}" - add_property "Remote Access Password" "${NIFI_REGISTRY_GIT_PASSWORD:-}" - ;; -esac diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_login_providers.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_login_providers.sh deleted file mode 100755 index b6e4650a42..0000000000 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_login_providers.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -login_providers_file=${NIFI_REGISTRY_HOME}/conf/identity-providers.xml -property_xpath='//identityProviders/provider/property' - -# Update a given property in the login-identity-providers file if a value is specified -edit_property() { - property_name=$1 - property_value=$2 - - if [ -n "${property_value}" ]; then - xmlstarlet ed --inplace -u "${property_xpath}[@name='${property_name}']" -v "${property_value}" "${login_providers_file}" - fi -} - -# Remove comments to enable the ldap-provider -sed -i '/To enable the ldap-identity-provider remove/d' "${login_providers_file}" - -edit_property 'Authentication Strategy' "${LDAP_AUTHENTICATION_STRATEGY}" -edit_property 'Manager DN' "${LDAP_MANAGER_DN}" -edit_property 'Manager Password' "${LDAP_MANAGER_PASSWORD}" -edit_property 'TLS - Keystore' "${LDAP_TLS_KEYSTORE}" -edit_property 'TLS - Keystore Password' "${LDAP_TLS_KEYSTORE_PASSWORD}" -edit_property 'TLS - Keystore Type' "${LDAP_TLS_KEYSTORE_TYPE}" -edit_property 'TLS - Truststore' "${LDAP_TLS_TRUSTSTORE}" -edit_property 'TLS - Truststore Password' "${LDAP_TLS_TRUSTSTORE_PASSWORD}" -edit_property 'TLS - Truststore Type' "${LDAP_TLS_TRUSTSTORE_TYPE}" -edit_property 'TLS - Protocol' "${LDAP_TLS_PROTOCOL}" -edit_property 'Url' "${LDAP_URL}" -edit_property 'User Search Base' "${LDAP_USER_SEARCH_BASE}" -edit_property 'User Search Filter' "${LDAP_USER_SEARCH_FILTER}" -edit_property 'Identity Strategy' "${LDAP_IDENTITY_STRATEGY}" -edit_property 'Referral Strategy' "${LDAP_REFERRAL_STRATEGY}" diff --git a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_oidc_properties.sh b/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_oidc_properties.sh deleted file mode 100644 index 77819f69c5..0000000000 --- a/nifi-registry/nifi-registry-docker-maven/dockermaven/sh/update_oidc_properties.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -e - -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -prop_replace 'nifi.registry.security.user.oidc.discovery.url' "${NIFI_REGISTRY_SECURITY_USER_OIDC_DISCOVERY_URL}" -prop_replace 'nifi.registry.security.user.oidc.connect.timeout' "${NIFI_REGISTRY_SECURITY_USER_OIDC_CONNECT_TIMEOUT}" -prop_replace 'nifi.registry.security.user.oidc.read.timeout' "${NIFI_REGISTRY_SECURITY_USER_OIDC_READ_TIMEOUT}" -prop_replace 'nifi.registry.security.user.oidc.client.id' "${NIFI_REGISTRY_SECURITY_USER_OIDC_CLIENT_ID}" -prop_replace 'nifi.registry.security.user.oidc.client.secret' "${NIFI_REGISTRY_SECURITY_USER_OIDC_CLIENT_SECRET}" -prop_replace 'nifi.registry.security.user.oidc.preferred.jwsalgorithm' "${NIFI_REGISTRY_SECURITY_USER_OIDC_PREFERRED_JWSALGORITHM}" -prop_replace 'nifi.registry.security.user.oidc.additional.scopes' "${NIFI_REGISTRY_SECURITY_USER_OIDC_ADDITIONAL_SCOPES}" -prop_replace 'nifi.registry.security.user.oidc.claim.identifying.user' "${NIFI_REGISTRY_SECURITY_USER_OIDC_CLAIM_IDENTIFYING_USER}" -prop_replace 'nifi.registry.security.user.oidc.fallback.claims.identifying.user' "${NIFI_REGISTRY_SECURITY_USER_OIDC_FALLBACK_CLAIMS_IDENTIFYING_USER}" -prop_replace 'nifi.registry.security.user.oidc.truststore.strategy' "${NIFI_REGISTRY_SECURITY_USER_OIDC_TRUSTSTORE_STRATEGY}"