This is an automated email from the ASF dual-hosted git repository.

Aman-Mittal pushed a commit to branch main
in repository 
https://gitbox.apache.org/repos/asf/fineract-business-intelligence.git

commit 5ce6fe31954386103670637d9a7fb508dc85af65
Merge: 93a105e d73fb70
Author: Aman Mittal <[email protected]>
AuthorDate: Fri Oct 30 11:14:37 2026 +0530

    Merge pull request #102 from airajena/feature/scalable-cdc-platform
    
    Scalable ingestion: Kafka CDC engine, control plane, feature flag and 
Airflow orchestration

 .env.example                                       |   58 +-
 .github/workflows/ci.yml                           |  107 +-
 .gitignore                                         |    4 +
 .rat-excludes                                      |    6 +
 .sqlfluffignore                                    |    2 +
 README.md                                          |   52 +-
 SECURITY.md                                        |   30 +-
 airflow/dags/fbi_control_plane.py                  |   99 ++
 airflow/dags/fbi_ingestion.py                      |  134 ++
 airflow/dags/fbi_source_lifecycle.py               |  164 +++
 compose.kafka-cluster.yaml                         |   98 ++
 compose.yaml                                       |  367 ++++-
 contracts/fineract_tables.json                     |  320 ++++
 .sqlfluffignore => controlplane/__init__.py        |    5 +-
 controlplane/cli.py                                |  411 ++++++
 controlplane/connectors.py                         |  341 +++++
 controlplane/contract.py                           |  134 ++
 controlplane/db.py                                 |  133 ++
 controlplane/ownership.py                          |  211 +++
 controlplane/ownership_repo.py                     |  218 +++
 controlplane/reconcile.py                          |  248 ++++
 controlplane/reconciler.py                         |  372 +++++
 controlplane/registry.py                           |  239 +++
 controlplane/retire.py                             |  181 +++
 controlplane/secrets.py                            |  105 ++
 controlplane/tasks.py                              |  303 ++++
 dbt/dbt_project.yml                                |   23 +-
 dbt/macros/incremental_windows.sql                 |  160 ++
 dbt/macros/metrics.sql                             |   70 +
 dbt/models/intermediate/int_change_window.sql      |   37 +
 .../intermediate/int_loan_delinquency_status.sql   |  296 ++--
 dbt/models/intermediate/int_snapshot_scope.sql     |  143 ++
 dbt/models/marts/_marts.yml                        |   26 +-
 dbt/models/marts/dimensions/dim_client.sql         |    1 +
 dbt/models/marts/dimensions/dim_currency.sql       |    3 +-
 .../marts/dimensions/dim_delinquency_bucket.sql    |    8 +-
 dbt/models/marts/dimensions/dim_office.sql         |    1 +
 dbt/models/marts/dimensions/dim_product.sql        |    1 +
 dbt/models/marts/facts/fact_delinquency_event.sql  |    8 +-
 dbt/models/marts/facts/fact_loan_snapshot.sql      |   65 +-
 .../marts/presentations/mart_delinquency_par.sql   |  290 ++--
 .../marts/presentations/mart_portfolio_health.sql  |  276 ++--
 .../presentations/mart_repayment_behavior.sql      |  442 +++---
 dbt/models/sources.yml                             |   43 +-
 dbt/models/staging/_staging.yml                    |   44 +-
 ..._m_currency.sql => stg_batch_job_execution.sql} |   35 +-
 dbt/models/staging/stg_m_client.sql                |   34 +-
 dbt/models/staging/stg_m_currency.sql              |    3 +
 dbt/models/staging/stg_m_delinquency.sql           |   40 +-
 dbt/models/staging/stg_m_loan.sql                  |    2 +
 .../staging/stg_m_loan_repayment_schedule.sql      |    2 +
 dbt/models/staging/stg_m_loan_transaction.sql      |    2 +
 dbt/models/staging/stg_m_office.sql                |    2 +
 dbt/models/staging/stg_m_product_loan.sql          |    2 +
 dbt/profiles.yml                                   |    2 +-
 dbt/seeds/metric_definitions.csv                   |   13 +
 ...shot_principal_matches_current_loan_balance.sql |   21 +-
 .../assert_marts_have_no_orphan_dimension_keys.sql |   22 +-
 .../assert_metric_catalog_covers_marts.sql}        |   28 +-
 .../assert_metric_consistency_across_marts.sql     |   49 +
 .../assert_no_orphan_sources_in_marts.sql}         |   37 +-
 ...ication_matches_actual_overdue_installments.sql |   21 +-
 .../assert_repayment_due_matches_schedule.sql      |   67 +
 deploy/k8s/connect-and-sink.yaml                   |  158 ++
 deploy/k8s/kafka.yaml                              |  104 ++
 .../superset_config.py => airflow/Dockerfile}      |   31 +-
 docker/ingestion/Dockerfile                        |   34 +
 .../initdb/001_init_warehouse.sql                  |   11 +-
 .../initdb/002_create_warehouse_roles.sh           |   45 +-
 docker/superset/asset_specs.py                     |    4 +
 docker/superset/init_superset.sh                   |    5 +-
 docker/superset/superset_config.py                 |   17 +-
 docs/architecture.md                               |   62 +-
 docs/cdc-ingestion.md                              |  828 +++++++++++
 docs/testing.md                                    |   34 +-
 extractor/cli.py                                   |   50 +-
 extractor/config.py                                |  115 +-
 extractor/extractor.py                             | 1097 +++++++-------
 extractor/replica_lag_check.py                     |    4 +-
 extractor/watermark_manager.py                     |   58 +-
 ingestion/ingestion-core/pom.xml                   |   56 +
 .../fineract/bi/ingestion/cdc/ChangeEvent.java     |   41 +
 .../bi/ingestion/cdc/DebeziumEnvelopeParser.java   |  139 ++
 .../bi/ingestion/cdc/PoisonEventException.java     |   40 +
 .../fineract/bi/ingestion/cdc/ValueConverter.java  |   90 ++
 .../apache/fineract/bi/ingestion/config/Env.java   |   87 ++
 .../fineract/bi/ingestion/contract/ColumnSpec.java |   43 +
 .../fineract/bi/ingestion/contract/Contract.java   |   79 +
 .../bi/ingestion/contract/ContractLoader.java      |   82 ++
 .../bi/ingestion/contract/ContractVerifier.java    |   99 ++
 .../bi/ingestion/contract/TableContract.java       |   62 +
 .../bi/ingestion/routing/OwnershipCache.java       |  102 ++
 .../bi/ingestion/routing/OwnershipSnapshot.java    |   83 ++
 .../bi/ingestion/routing/OwnershipView.java        |   28 +
 .../fineract/bi/ingestion/routing/Route.java       |   42 +
 .../fineract/bi/ingestion/sink/BatchWriter.java    |   31 +
 .../fineract/bi/ingestion/sink/CsvEncoder.java     |   49 +
 .../bi/ingestion/sink/FencedWriteException.java    |   32 +
 .../bi/ingestion/sink/PartitionOffset.java         |   22 +
 .../fineract/bi/ingestion/sink/RoutedEvent.java    |   25 +
 .../fineract/bi/ingestion/sink/SkipCheck.java      |   24 +
 .../fineract/bi/ingestion/sink/SqlFailures.java    |   54 +
 .../bi/ingestion/sink/WarehouseWriter.java         |  254 ++++
 .../ingestion/cdc/DebeziumEnvelopeParserTest.java  |  189 +++
 .../bi/ingestion/cdc/ValueConverterTest.java       |   61 +
 .../bi/ingestion/contract/ContractLoaderTest.java  |   55 +
 .../ingestion/routing/OwnershipSnapshotTest.java   |   70 +
 .../bi/ingestion/sink/SqlFailuresTest.java         |   52 +
 .../ingestion/sink/WarehouseWriterEncodeTest.java  |   56 +
 .../sink/WarehouseWriterSkipVerificationTest.java  |  162 ++
 ingestion/ingestion-loadgen/pom.xml                |   88 ++
 .../apache/fineract/bi/ingestion/loadgen/Args.java |   57 +
 .../bi/ingestion/loadgen/EventProducer.java        |  242 +++
 .../fineract/bi/ingestion/loadgen/GroundTruth.java |  107 ++
 .../bi/ingestion/loadgen/LoadGenApplication.java   |   52 +
 .../bi/ingestion/loadgen/SourceWorkload.java       |  280 ++++
 .../bi/ingestion/loadgen/SyntheticPayloads.java    |  134 ++
 .../fineract/bi/ingestion/loadgen/Verifier.java    |  157 ++
 .../src/main/resources/log4j2.xml                  |   33 +
 .../ingestion/loadgen/GeneratorRoundTripTest.java  |   90 ++
 ingestion/ingestion-sink/pom.xml                   |  110 ++
 .../bi/ingestion/app/ConnectionSource.java         |   28 +
 .../bi/ingestion/app/DeadLetterPublisher.java      |  111 ++
 .../fineract/bi/ingestion/app/DeadLetterSink.java  |   27 +
 .../bi/ingestion/app/ProgressReporter.java         |  157 ++
 .../fineract/bi/ingestion/app/SinkApplication.java |  147 ++
 .../fineract/bi/ingestion/app/SinkConfig.java      |  101 ++
 .../fineract/bi/ingestion/app/SinkMetrics.java     |  183 +++
 .../fineract/bi/ingestion/app/SinkWorker.java      |  746 ++++++++++
 .../bi/ingestion/app/TopicProvisioner.java         |  118 ++
 .../ingestion-sink/src/main/resources/log4j2.xml   |   33 +
 .../bi/ingestion/app/DeadLetterPublisherTest.java  |   90 ++
 .../ingestion/app/SinkConfigAndTopicPlanTest.java  |   77 +
 .../bi/ingestion/app/SinkKafkaPostgresTest.java    |  400 +++++
 .../bi/ingestion/app/SinkWorkerDeliveryTest.java   |  486 ++++++
 ingestion/pom.xml                                  |  219 +++
 loadtest/README.md                                 |   82 ++
 loadtest/results/.gitkeep                          |    0
 loadtest/run_stages.sh                             |   80 +
 loadtest/verify_incremental_equivalence.sh         |  148 ++
 scripts/airflow_trigger.py                         |   89 ++
 scripts/bootstrap_cdc_sources.sh                   |   59 +
 scripts/bootstrap_source.sh                        |    7 +
 scripts/demo_feature_flag.sh                       |  112 ++
 scripts/pipeline_status.sh                         |   94 ++
 scripts/verify_extraction.sh                       |    2 +-
 .../datasets/delinquency_par_secure_all_dates.sql  |   19 +-
 .../datasets/delinquency_par_secure_latest.sql     |   41 +-
 .../datasets/portfolio_health_secure_all_dates.sql |   19 +-
 .../datasets/portfolio_health_secure_latest.sql    |   23 +-
 .../repayment_behavior_secure_all_dates.sql        |   19 +-
 .../datasets/repayment_behavior_secure_latest.sql  |   24 +-
 tests/airflow/test_dag_integrity.py                |   85 ++
 tests/conftest.py                                  |   70 +-
 tests/integration/test_control_plane_hardening.py  |  807 ++++++++++
 tests/integration/test_platform_write_path.py      |  404 +++++
 tests/integration/test_rls_isolation.py            |  115 ++
 .../unit/controlplane/__init__.py                  |    5 -
 .../controlplane/test_connectors_and_registry.py   |  312 ++++
 .../controlplane/test_contract_and_reconcile.py    |   80 +
 .../controlplane/test_ownership_state_machine.py   |  238 +++
 tests/unit/controlplane/test_reconciler.py         |  357 +++++
 tests/unit/extractor/test_cob_gate.py              |  107 --
 tests/unit/extractor/test_config.py                |  176 ---
 tests/unit/extractor/test_helpers.py               |  123 --
 tests/unit/extractor/test_python_engine.py         |  350 +++++
 tests/unit/extractor/test_reconcile.py             |  106 --
 tests/unit/extractor/test_source_query.py          |  120 --
 tests/unit/extractor/test_upsert_rows.py           |  149 --
 tests/unit/extractor/test_watermark.py             |  214 ---
 tests/unit/superset/test_asset_specs.py            |    9 +
 warehouse/schema/access_control.sql                |   90 ++
 warehouse/schema/mart_tables.sql                   |   19 -
 warehouse/schema/pipeline_state.sql                |   55 -
 warehouse/schema/platform.sql                      | 1556 ++++++++++++++++++++
 warehouse/schema/raw_tables.sql                    |  276 ----
 warehouse/schema/staging_tables.sql                |   18 -
 177 files changed, 19435 insertions(+), 3024 deletions(-)


Reply via email to