airajena commented on code in PR #102:
URL:
https://github.com/apache/fineract-business-intelligence/pull/102#discussion_r4318219993
##########
compose.yaml:
##########
@@ -165,10 +284,212 @@ services:
retries: 10
start_period: 60s
+ kafka:
+ image: apache/kafka:4.1.1
+ profiles: ["cdc"]
+ hostname: kafka
+ environment:
+ KAFKA_NODE_ID: 1
+ KAFKA_PROCESS_ROLES: broker,controller
+ KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
+ KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:29092
+ KAFKA_ADVERTISED_LISTENERS:
PLAINTEXT://kafka:9092,EXTERNAL://localhost:29092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP:
PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT
+ KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
+ KAFKA_DEFAULT_REPLICATION_FACTOR: ${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}
+ KAFKA_MIN_INSYNC_REPLICAS: ${KAFKA_MIN_INSYNC_REPLICAS:-1}
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:
${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}
+ KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:
${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}
+ KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: ${KAFKA_MIN_INSYNC_REPLICAS:-1}
+ KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+ KAFKA_NUM_NETWORK_THREADS: ${KAFKA_NUM_NETWORK_THREADS:-3}
+ KAFKA_NUM_IO_THREADS: ${KAFKA_NUM_IO_THREADS:-4}
+ KAFKA_LOG_RETENTION_HOURS: ${KAFKA_LOG_RETENTION_HOURS:-72}
+ KAFKA_LOG_SEGMENT_BYTES: ${KAFKA_LOG_SEGMENT_BYTES:-268435456}
+ KAFKA_HEAP_OPTS: "-Xms${KAFKA_HEAP:-512m} -Xmx${KAFKA_HEAP:-512m}"
+ KAFKA_LOG_DIRS: /var/lib/kafka/data
+ ports:
+ - "29092:29092"
+ volumes:
+ - kafka_data:/var/lib/kafka/data
+ mem_limit: ${KAFKA_MEM_LIMIT:-1g}
+ healthcheck:
+ test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh
--bootstrap-server localhost:9092 >/dev/null 2>&1"]
+ interval: 10s
+ timeout: 10s
+ retries: 12
+ start_period: 20s
+
+ kafka-init:
+ build:
+ context: .
+ dockerfile: docker/ingestion/Dockerfile
+ image: fineract-bi-ingestion:latest
+ profiles: ["cdc"]
+ command: ["provision-topics"]
+ environment:
+ SINK_BOOTSTRAP_SERVERS: kafka:9092
+ TOPIC_REPLICATION_FACTOR: ${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}
+ TOPIC_MIN_INSYNC_REPLICAS: ${KAFKA_MIN_INSYNC_REPLICAS:-1}
+ TOPIC_PARTITION_SCALE: ${TOPIC_PARTITION_SCALE:-0.25}
+ TOPIC_MAX_PARTITIONS: ${TOPIC_MAX_PARTITIONS:-6}
+ TOPIC_RETENTION_MS: ${TOPIC_RETENTION_MS:-259200000}
+ TOPIC_ALLOW_PARTITION_INCREASE: ${TOPIC_ALLOW_PARTITION_INCREASE:-false}
+ depends_on:
+ kafka:
+ condition: service_healthy
+ restart: "no"
+
+ connect:
+ image: quay.io/debezium/connect:3.4
+ profiles: ["cdc"]
+ environment:
+ BOOTSTRAP_SERVERS: kafka:9092
+ GROUP_ID: ${CONNECT_GROUP_ID:-fbi-connect-default}
+ CONFIG_STORAGE_TOPIC: fbi_connect_configs
+ OFFSET_STORAGE_TOPIC: fbi_connect_offsets
+ STATUS_STORAGE_TOPIC: fbi_connect_status
+ CONFIG_STORAGE_REPLICATION_FACTOR: ${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}
+ OFFSET_STORAGE_REPLICATION_FACTOR: ${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}
+ STATUS_STORAGE_REPLICATION_FACTOR: ${KAFKA_DEFAULT_REPLICATION_FACTOR:-1}
+ OFFSET_FLUSH_INTERVAL_MS: "5000"
+ HEAP_OPTS: "-Xms${CONNECT_HEAP:-512m} -Xmx${CONNECT_HEAP:-512m}"
+ CONNECT_CONFIG_PROVIDERS: env,file
+ CONNECT_CONFIG_PROVIDERS_ENV_CLASS:
org.apache.kafka.common.config.provider.EnvVarConfigProvider
+ CONNECT_CONFIG_PROVIDERS_FILE_CLASS:
org.apache.kafka.common.config.provider.FileConfigProvider
+ CONNECT_TOPIC_CREATION_ENABLE: "false"
+ CONNECT_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY: All
+ CONNECT_REST_ADVERTISED_HOST_NAME: ${CONNECT_ADVERTISED_HOST:-connect}
+ FINERACT_CDC_PASSWORD: ${FINERACT_CDC_PASSWORD:-}
+ ports:
+ - "${CONNECT_HOST_PORT:-8083}:8083"
Review Comment:
Fixed, with one caveat. Ports are bound to loopback, and the Connect
config-provider allowlists and file path limits are set. Override policy is
None, the Airflow secrets are required with ${VAR:?}, and the k8s NetworkPolicy
is added. Connect REST now also requires Basic auth. The caveat is that anyone
holding those REST credentials could still point a connector at their own host
and receive an FBI_SRC_* secret. In k8s (Strimzi) the NetworkPolicy is the only
control, since it has no REST auth. All of this is in SECURITY.md.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
