This is an automated email from the ASF dual-hosted git repository. Aman-Mittal pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/fineract-business-intelligence.git
commit 5ce6fe31954386103670637d9a7fb508dc85af65 Merge: 93a105e d73fb70 Author: Aman Mittal <[email protected]> AuthorDate: Fri Oct 30 11:14:37 2026 +0530 Merge pull request #102 from airajena/feature/scalable-cdc-platform Scalable ingestion: Kafka CDC engine, control plane, feature flag and Airflow orchestration .env.example | 58 +- .github/workflows/ci.yml | 107 +- .gitignore | 4 + .rat-excludes | 6 + .sqlfluffignore | 2 + README.md | 52 +- SECURITY.md | 30 +- airflow/dags/fbi_control_plane.py | 99 ++ airflow/dags/fbi_ingestion.py | 134 ++ airflow/dags/fbi_source_lifecycle.py | 164 +++ compose.kafka-cluster.yaml | 98 ++ compose.yaml | 367 ++++- contracts/fineract_tables.json | 320 ++++ .sqlfluffignore => controlplane/__init__.py | 5 +- controlplane/cli.py | 411 ++++++ controlplane/connectors.py | 341 +++++ controlplane/contract.py | 134 ++ controlplane/db.py | 133 ++ controlplane/ownership.py | 211 +++ controlplane/ownership_repo.py | 218 +++ controlplane/reconcile.py | 248 ++++ controlplane/reconciler.py | 372 +++++ controlplane/registry.py | 239 +++ controlplane/retire.py | 181 +++ controlplane/secrets.py | 105 ++ controlplane/tasks.py | 303 ++++ dbt/dbt_project.yml | 23 +- dbt/macros/incremental_windows.sql | 160 ++ dbt/macros/metrics.sql | 70 + dbt/models/intermediate/int_change_window.sql | 37 + .../intermediate/int_loan_delinquency_status.sql | 296 ++-- dbt/models/intermediate/int_snapshot_scope.sql | 143 ++ dbt/models/marts/_marts.yml | 26 +- dbt/models/marts/dimensions/dim_client.sql | 1 + dbt/models/marts/dimensions/dim_currency.sql | 3 +- .../marts/dimensions/dim_delinquency_bucket.sql | 8 +- dbt/models/marts/dimensions/dim_office.sql | 1 + dbt/models/marts/dimensions/dim_product.sql | 1 + dbt/models/marts/facts/fact_delinquency_event.sql | 8 +- dbt/models/marts/facts/fact_loan_snapshot.sql | 65 +- .../marts/presentations/mart_delinquency_par.sql | 290 ++-- .../marts/presentations/mart_portfolio_health.sql | 276 ++-- .../presentations/mart_repayment_behavior.sql | 442 +++--- dbt/models/sources.yml | 43 +- dbt/models/staging/_staging.yml | 44 +- ..._m_currency.sql => stg_batch_job_execution.sql} | 35 +- dbt/models/staging/stg_m_client.sql | 34 +- dbt/models/staging/stg_m_currency.sql | 3 + dbt/models/staging/stg_m_delinquency.sql | 40 +- dbt/models/staging/stg_m_loan.sql | 2 + .../staging/stg_m_loan_repayment_schedule.sql | 2 + dbt/models/staging/stg_m_loan_transaction.sql | 2 + dbt/models/staging/stg_m_office.sql | 2 + dbt/models/staging/stg_m_product_loan.sql | 2 + dbt/profiles.yml | 2 +- dbt/seeds/metric_definitions.csv | 13 + ...shot_principal_matches_current_loan_balance.sql | 21 +- .../assert_marts_have_no_orphan_dimension_keys.sql | 22 +- .../assert_metric_catalog_covers_marts.sql} | 28 +- .../assert_metric_consistency_across_marts.sql | 49 + .../assert_no_orphan_sources_in_marts.sql} | 37 +- ...ication_matches_actual_overdue_installments.sql | 21 +- .../assert_repayment_due_matches_schedule.sql | 67 + deploy/k8s/connect-and-sink.yaml | 158 ++ deploy/k8s/kafka.yaml | 104 ++ .../superset_config.py => airflow/Dockerfile} | 31 +- docker/ingestion/Dockerfile | 34 + .../initdb/001_init_warehouse.sql | 11 +- .../initdb/002_create_warehouse_roles.sh | 45 +- docker/superset/asset_specs.py | 4 + docker/superset/init_superset.sh | 5 +- docker/superset/superset_config.py | 17 +- docs/architecture.md | 62 +- docs/cdc-ingestion.md | 828 +++++++++++ docs/testing.md | 34 +- extractor/cli.py | 50 +- extractor/config.py | 115 +- extractor/extractor.py | 1097 +++++++------- extractor/replica_lag_check.py | 4 +- extractor/watermark_manager.py | 58 +- ingestion/ingestion-core/pom.xml | 56 + .../fineract/bi/ingestion/cdc/ChangeEvent.java | 41 + .../bi/ingestion/cdc/DebeziumEnvelopeParser.java | 139 ++ .../bi/ingestion/cdc/PoisonEventException.java | 40 + .../fineract/bi/ingestion/cdc/ValueConverter.java | 90 ++ .../apache/fineract/bi/ingestion/config/Env.java | 87 ++ .../fineract/bi/ingestion/contract/ColumnSpec.java | 43 + .../fineract/bi/ingestion/contract/Contract.java | 79 + .../bi/ingestion/contract/ContractLoader.java | 82 ++ .../bi/ingestion/contract/ContractVerifier.java | 99 ++ .../bi/ingestion/contract/TableContract.java | 62 + .../bi/ingestion/routing/OwnershipCache.java | 102 ++ .../bi/ingestion/routing/OwnershipSnapshot.java | 83 ++ .../bi/ingestion/routing/OwnershipView.java | 28 + .../fineract/bi/ingestion/routing/Route.java | 42 + .../fineract/bi/ingestion/sink/BatchWriter.java | 31 + .../fineract/bi/ingestion/sink/CsvEncoder.java | 49 + .../bi/ingestion/sink/FencedWriteException.java | 32 + .../bi/ingestion/sink/PartitionOffset.java | 22 + .../fineract/bi/ingestion/sink/RoutedEvent.java | 25 + .../fineract/bi/ingestion/sink/SkipCheck.java | 24 + .../fineract/bi/ingestion/sink/SqlFailures.java | 54 + .../bi/ingestion/sink/WarehouseWriter.java | 254 ++++ .../ingestion/cdc/DebeziumEnvelopeParserTest.java | 189 +++ .../bi/ingestion/cdc/ValueConverterTest.java | 61 + .../bi/ingestion/contract/ContractLoaderTest.java | 55 + .../ingestion/routing/OwnershipSnapshotTest.java | 70 + .../bi/ingestion/sink/SqlFailuresTest.java | 52 + .../ingestion/sink/WarehouseWriterEncodeTest.java | 56 + .../sink/WarehouseWriterSkipVerificationTest.java | 162 ++ ingestion/ingestion-loadgen/pom.xml | 88 ++ .../apache/fineract/bi/ingestion/loadgen/Args.java | 57 + .../bi/ingestion/loadgen/EventProducer.java | 242 +++ .../fineract/bi/ingestion/loadgen/GroundTruth.java | 107 ++ .../bi/ingestion/loadgen/LoadGenApplication.java | 52 + .../bi/ingestion/loadgen/SourceWorkload.java | 280 ++++ .../bi/ingestion/loadgen/SyntheticPayloads.java | 134 ++ .../fineract/bi/ingestion/loadgen/Verifier.java | 157 ++ .../src/main/resources/log4j2.xml | 33 + .../ingestion/loadgen/GeneratorRoundTripTest.java | 90 ++ ingestion/ingestion-sink/pom.xml | 110 ++ .../bi/ingestion/app/ConnectionSource.java | 28 + .../bi/ingestion/app/DeadLetterPublisher.java | 111 ++ .../fineract/bi/ingestion/app/DeadLetterSink.java | 27 + .../bi/ingestion/app/ProgressReporter.java | 157 ++ .../fineract/bi/ingestion/app/SinkApplication.java | 147 ++ .../fineract/bi/ingestion/app/SinkConfig.java | 101 ++ .../fineract/bi/ingestion/app/SinkMetrics.java | 183 +++ .../fineract/bi/ingestion/app/SinkWorker.java | 746 ++++++++++ .../bi/ingestion/app/TopicProvisioner.java | 118 ++ .../ingestion-sink/src/main/resources/log4j2.xml | 33 + .../bi/ingestion/app/DeadLetterPublisherTest.java | 90 ++ .../ingestion/app/SinkConfigAndTopicPlanTest.java | 77 + .../bi/ingestion/app/SinkKafkaPostgresTest.java | 400 +++++ .../bi/ingestion/app/SinkWorkerDeliveryTest.java | 486 ++++++ ingestion/pom.xml | 219 +++ loadtest/README.md | 82 ++ loadtest/results/.gitkeep | 0 loadtest/run_stages.sh | 80 + loadtest/verify_incremental_equivalence.sh | 148 ++ scripts/airflow_trigger.py | 89 ++ scripts/bootstrap_cdc_sources.sh | 59 + scripts/bootstrap_source.sh | 7 + scripts/demo_feature_flag.sh | 112 ++ scripts/pipeline_status.sh | 94 ++ scripts/verify_extraction.sh | 2 +- .../datasets/delinquency_par_secure_all_dates.sql | 19 +- .../datasets/delinquency_par_secure_latest.sql | 41 +- .../datasets/portfolio_health_secure_all_dates.sql | 19 +- .../datasets/portfolio_health_secure_latest.sql | 23 +- .../repayment_behavior_secure_all_dates.sql | 19 +- .../datasets/repayment_behavior_secure_latest.sql | 24 +- tests/airflow/test_dag_integrity.py | 85 ++ tests/conftest.py | 70 +- tests/integration/test_control_plane_hardening.py | 807 ++++++++++ tests/integration/test_platform_write_path.py | 404 +++++ tests/integration/test_rls_isolation.py | 115 ++ .../unit/controlplane/__init__.py | 5 - .../controlplane/test_connectors_and_registry.py | 312 ++++ .../controlplane/test_contract_and_reconcile.py | 80 + .../controlplane/test_ownership_state_machine.py | 238 +++ tests/unit/controlplane/test_reconciler.py | 357 +++++ tests/unit/extractor/test_cob_gate.py | 107 -- tests/unit/extractor/test_config.py | 176 --- tests/unit/extractor/test_helpers.py | 123 -- tests/unit/extractor/test_python_engine.py | 350 +++++ tests/unit/extractor/test_reconcile.py | 106 -- tests/unit/extractor/test_source_query.py | 120 -- tests/unit/extractor/test_upsert_rows.py | 149 -- tests/unit/extractor/test_watermark.py | 214 --- tests/unit/superset/test_asset_specs.py | 9 + warehouse/schema/access_control.sql | 90 ++ warehouse/schema/mart_tables.sql | 19 - warehouse/schema/pipeline_state.sql | 55 - warehouse/schema/platform.sql | 1556 ++++++++++++++++++++ warehouse/schema/raw_tables.sql | 276 ---- warehouse/schema/staging_tables.sql | 18 - 177 files changed, 19435 insertions(+), 3024 deletions(-)
