This is an automated email from the ASF dual-hosted git repository.
Aman-Mittal pushed a change to branch main
in repository
https://gitbox.apache.org/repos/asf/fineract-business-intelligence.git
from 93a105e Merge pull request #110 from
apache/dependabot/github_actions/zizmorcore/zizmor-action-0.7.1
add 29089c2 feat: add Kafka CDC and Airflow control plane with
Python/Kafka ingestion feature flag
add 2fe8123 ci: load dbt seeds before running models and tests
add 1d8f6ee test: clone all default mart rows for the second-bank RLS
fixture
add f475ce5 ci: run the extractor backfill once instead of starting its
background pipeline loop
add e0c00e7 docs: add Kafka CDC, control plane and feature flag guide and
update README, architecture and testing
add 18e802d Address review feedback on the CDC platform
add d73fb70 Address second round of review feedback
new 5ce6fe3 Merge pull request #102 from
airajena/feature/scalable-cdc-platform
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.env.example | 58 +-
.github/workflows/ci.yml | 107 +-
.gitignore | 4 +
.rat-excludes | 6 +
.sqlfluffignore | 2 +
README.md | 52 +-
SECURITY.md | 30 +-
airflow/dags/fbi_control_plane.py | 99 ++
airflow/dags/fbi_ingestion.py | 134 ++
airflow/dags/fbi_source_lifecycle.py | 164 +++
compose.kafka-cluster.yaml | 98 ++
compose.yaml | 367 ++++-
contracts/fineract_tables.json | 320 ++++
{tests/unit => controlplane}/__init__.py | 2 +
controlplane/cli.py | 411 ++++++
controlplane/connectors.py | 341 +++++
controlplane/contract.py | 134 ++
controlplane/db.py | 133 ++
controlplane/ownership.py | 211 +++
controlplane/ownership_repo.py | 218 +++
controlplane/reconcile.py | 248 ++++
controlplane/reconciler.py | 372 +++++
controlplane/registry.py | 239 +++
controlplane/retire.py | 181 +++
controlplane/secrets.py | 105 ++
controlplane/tasks.py | 303 ++++
dbt/dbt_project.yml | 23 +-
dbt/macros/incremental_windows.sql | 160 ++
dbt/macros/metrics.sql | 70 +
dbt/models/intermediate/int_change_window.sql | 37 +
.../intermediate/int_loan_delinquency_status.sql | 296 ++--
dbt/models/intermediate/int_snapshot_scope.sql | 143 ++
dbt/models/marts/_marts.yml | 26 +-
dbt/models/marts/dimensions/dim_client.sql | 1 +
dbt/models/marts/dimensions/dim_currency.sql | 3 +-
.../marts/dimensions/dim_delinquency_bucket.sql | 8 +-
dbt/models/marts/dimensions/dim_office.sql | 1 +
dbt/models/marts/dimensions/dim_product.sql | 1 +
dbt/models/marts/facts/fact_delinquency_event.sql | 8 +-
dbt/models/marts/facts/fact_loan_snapshot.sql | 65 +-
.../marts/presentations/mart_delinquency_par.sql | 290 ++--
.../marts/presentations/mart_portfolio_health.sql | 276 ++--
.../presentations/mart_repayment_behavior.sql | 442 +++---
dbt/models/sources.yml | 43 +-
dbt/models/staging/_staging.yml | 44 +-
..._m_currency.sql => stg_batch_job_execution.sql} | 35 +-
dbt/models/staging/stg_m_client.sql | 34 +-
dbt/models/staging/stg_m_currency.sql | 3 +
dbt/models/staging/stg_m_delinquency.sql | 40 +-
dbt/models/staging/stg_m_loan.sql | 2 +
.../staging/stg_m_loan_repayment_schedule.sql | 2 +
dbt/models/staging/stg_m_loan_transaction.sql | 2 +
dbt/models/staging/stg_m_office.sql | 2 +
dbt/models/staging/stg_m_product_loan.sql | 2 +
dbt/profiles.yml | 2 +-
dbt/seeds/metric_definitions.csv | 13 +
...shot_principal_matches_current_loan_balance.sql | 21 +-
.../assert_marts_have_no_orphan_dimension_keys.sql | 22 +-
.../assert_metric_catalog_covers_marts.sql} | 28 +-
.../assert_metric_consistency_across_marts.sql | 49 +
.../assert_no_orphan_sources_in_marts.sql} | 37 +-
...ication_matches_actual_overdue_installments.sql | 21 +-
.../assert_repayment_due_matches_schedule.sql | 67 +
deploy/k8s/connect-and-sink.yaml | 158 ++
deploy/k8s/kafka.yaml | 104 ++
docker/{dbt => airflow}/Dockerfile | 22 +-
docker/ingestion/Dockerfile | 34 +
.../initdb/001_init_warehouse.sql | 11 +-
.../initdb/002_create_warehouse_roles.sh | 45 +-
docker/superset/asset_specs.py | 4 +
docker/superset/init_superset.sh | 5 +-
docker/superset/superset_config.py | 17 +-
docs/architecture.md | 62 +-
docs/cdc-ingestion.md | 828 +++++++++++
docs/testing.md | 34 +-
extractor/cli.py | 50 +-
extractor/config.py | 115 +-
extractor/extractor.py | 1097 +++++++-------
extractor/replica_lag_check.py | 4 +-
extractor/watermark_manager.py | 58 +-
ingestion/ingestion-core/pom.xml | 56 +
.../fineract/bi/ingestion/cdc/ChangeEvent.java | 41 +
.../bi/ingestion/cdc/DebeziumEnvelopeParser.java | 139 ++
.../bi/ingestion/cdc/PoisonEventException.java | 40 +
.../fineract/bi/ingestion/cdc/ValueConverter.java | 90 ++
.../apache/fineract/bi/ingestion/config/Env.java | 87 ++
.../fineract/bi/ingestion/contract/ColumnSpec.java | 43 +
.../fineract/bi/ingestion/contract/Contract.java | 79 +
.../bi/ingestion/contract/ContractLoader.java | 82 ++
.../bi/ingestion/contract/ContractVerifier.java | 99 ++
.../bi/ingestion/contract/TableContract.java | 62 +
.../bi/ingestion/routing/OwnershipCache.java | 102 ++
.../bi/ingestion/routing/OwnershipSnapshot.java | 83 ++
.../bi/ingestion/routing/OwnershipView.java | 28 +
.../fineract/bi/ingestion/routing/Route.java | 42 +
.../fineract/bi/ingestion/sink/BatchWriter.java | 31 +
.../fineract/bi/ingestion/sink/CsvEncoder.java | 49 +
.../bi/ingestion/sink/FencedWriteException.java | 32 +
.../bi/ingestion/sink/PartitionOffset.java | 22 +
.../fineract/bi/ingestion/sink/RoutedEvent.java | 25 +
.../fineract/bi/ingestion/sink/SkipCheck.java | 24 +
.../fineract/bi/ingestion/sink/SqlFailures.java | 54 +
.../bi/ingestion/sink/WarehouseWriter.java | 254 ++++
.../ingestion/cdc/DebeziumEnvelopeParserTest.java | 189 +++
.../bi/ingestion/cdc/ValueConverterTest.java | 61 +
.../bi/ingestion/contract/ContractLoaderTest.java | 55 +
.../ingestion/routing/OwnershipSnapshotTest.java | 70 +
.../bi/ingestion/sink/SqlFailuresTest.java | 52 +
.../ingestion/sink/WarehouseWriterEncodeTest.java | 56 +
.../sink/WarehouseWriterSkipVerificationTest.java | 162 ++
ingestion/ingestion-loadgen/pom.xml | 88 ++
.../apache/fineract/bi/ingestion/loadgen/Args.java | 57 +
.../bi/ingestion/loadgen/EventProducer.java | 242 +++
.../fineract/bi/ingestion/loadgen/GroundTruth.java | 107 ++
.../bi/ingestion/loadgen/LoadGenApplication.java | 52 +
.../bi/ingestion/loadgen/SourceWorkload.java | 280 ++++
.../bi/ingestion/loadgen/SyntheticPayloads.java | 134 ++
.../fineract/bi/ingestion/loadgen/Verifier.java | 157 ++
.../src/main/resources/log4j2.xml | 25 +-
.../ingestion/loadgen/GeneratorRoundTripTest.java | 90 ++
ingestion/ingestion-sink/pom.xml | 110 ++
.../bi/ingestion/app/ConnectionSource.java | 28 +
.../bi/ingestion/app/DeadLetterPublisher.java | 111 ++
.../fineract/bi/ingestion/app/DeadLetterSink.java | 27 +
.../bi/ingestion/app/ProgressReporter.java | 157 ++
.../fineract/bi/ingestion/app/SinkApplication.java | 147 ++
.../fineract/bi/ingestion/app/SinkConfig.java | 101 ++
.../fineract/bi/ingestion/app/SinkMetrics.java | 183 +++
.../fineract/bi/ingestion/app/SinkWorker.java | 746 ++++++++++
.../bi/ingestion/app/TopicProvisioner.java | 118 ++
.../ingestion-sink/src/main/resources/log4j2.xml | 25 +-
.../bi/ingestion/app/DeadLetterPublisherTest.java | 90 ++
.../ingestion/app/SinkConfigAndTopicPlanTest.java | 77 +
.../bi/ingestion/app/SinkKafkaPostgresTest.java | 400 +++++
.../bi/ingestion/app/SinkWorkerDeliveryTest.java | 486 ++++++
ingestion/pom.xml | 219 +++
loadtest/README.md | 82 ++
loadtest/results/.gitkeep | 0
loadtest/run_stages.sh | 80 +
loadtest/verify_incremental_equivalence.sh | 148 ++
scripts/airflow_trigger.py | 89 ++
scripts/bootstrap_cdc_sources.sh | 59 +
scripts/bootstrap_source.sh | 7 +
scripts/demo_feature_flag.sh | 112 ++
scripts/pipeline_status.sh | 94 ++
scripts/verify_extraction.sh | 2 +-
.../datasets/delinquency_par_secure_all_dates.sql | 19 +-
.../datasets/delinquency_par_secure_latest.sql | 41 +-
.../datasets/portfolio_health_secure_all_dates.sql | 19 +-
.../datasets/portfolio_health_secure_latest.sql | 23 +-
.../repayment_behavior_secure_all_dates.sql | 19 +-
.../datasets/repayment_behavior_secure_latest.sql | 24 +-
tests/airflow/test_dag_integrity.py | 85 ++
tests/conftest.py | 70 +-
tests/integration/test_control_plane_hardening.py | 807 ++++++++++
tests/integration/test_platform_write_path.py | 404 +++++
tests/integration/test_rls_isolation.py | 115 ++
tests/{ => unit/controlplane}/__init__.py | 0
.../controlplane/test_connectors_and_registry.py | 312 ++++
.../controlplane/test_contract_and_reconcile.py | 80 +
.../controlplane/test_ownership_state_machine.py | 238 +++
tests/unit/controlplane/test_reconciler.py | 357 +++++
tests/unit/extractor/test_cob_gate.py | 107 --
tests/unit/extractor/test_config.py | 176 ---
tests/unit/extractor/test_helpers.py | 123 --
tests/unit/extractor/test_python_engine.py | 350 +++++
tests/unit/extractor/test_reconcile.py | 106 --
tests/unit/extractor/test_source_query.py | 120 --
tests/unit/extractor/test_upsert_rows.py | 149 --
tests/unit/extractor/test_watermark.py | 214 ---
tests/unit/superset/test_asset_specs.py | 9 +
warehouse/schema/access_control.sql | 90 ++
warehouse/schema/mart_tables.sql | 19 -
warehouse/schema/pipeline_state.sql | 55 -
warehouse/schema/platform.sql | 1556 ++++++++++++++++++++
warehouse/schema/raw_tables.sql | 276 ----
warehouse/schema/staging_tables.sql | 18 -
177 files changed, 19400 insertions(+), 3026 deletions(-)
create mode 100644 airflow/dags/fbi_control_plane.py
create mode 100644 airflow/dags/fbi_ingestion.py
create mode 100644 airflow/dags/fbi_source_lifecycle.py
create mode 100644 compose.kafka-cluster.yaml
create mode 100644 contracts/fineract_tables.json
copy {tests/unit => controlplane}/__init__.py (99%)
create mode 100644 controlplane/cli.py
create mode 100644 controlplane/connectors.py
create mode 100644 controlplane/contract.py
create mode 100644 controlplane/db.py
create mode 100644 controlplane/ownership.py
create mode 100644 controlplane/ownership_repo.py
create mode 100644 controlplane/reconcile.py
create mode 100644 controlplane/reconciler.py
create mode 100644 controlplane/registry.py
create mode 100644 controlplane/retire.py
create mode 100644 controlplane/secrets.py
create mode 100644 controlplane/tasks.py
create mode 100644 dbt/macros/incremental_windows.sql
create mode 100644 dbt/macros/metrics.sql
create mode 100644 dbt/models/intermediate/int_change_window.sql
create mode 100644 dbt/models/intermediate/int_snapshot_scope.sql
copy dbt/models/staging/{stg_m_currency.sql => stg_batch_job_execution.sql}
(57%)
create mode 100644 dbt/seeds/metric_definitions.csv
copy dbt/{models/staging/stg_m_currency.sql =>
tests/assert_metric_catalog_covers_marts.sql} (59%)
create mode 100644 dbt/tests/assert_metric_consistency_across_marts.sql
copy dbt/{models/staging/stg_m_currency.sql =>
tests/assert_no_orphan_sources_in_marts.sql} (53%)
create mode 100644 dbt/tests/assert_repayment_due_matches_schedule.sql
create mode 100644 deploy/k8s/connect-and-sink.yaml
create mode 100644 deploy/k8s/kafka.yaml
copy docker/{dbt => airflow}/Dockerfile (59%)
create mode 100644 docker/ingestion/Dockerfile
create mode 100644 docs/cdc-ingestion.md
create mode 100644 ingestion/ingestion-core/pom.xml
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/cdc/ChangeEvent.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/cdc/DebeziumEnvelopeParser.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/cdc/PoisonEventException.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/cdc/ValueConverter.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/config/Env.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/contract/ColumnSpec.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/contract/Contract.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/contract/ContractLoader.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/contract/ContractVerifier.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/contract/TableContract.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/routing/OwnershipCache.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/routing/OwnershipSnapshot.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/routing/OwnershipView.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/routing/Route.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/sink/BatchWriter.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/sink/CsvEncoder.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/sink/FencedWriteException.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/sink/PartitionOffset.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/sink/RoutedEvent.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/sink/SkipCheck.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/sink/SqlFailures.java
create mode 100644
ingestion/ingestion-core/src/main/java/org/apache/fineract/bi/ingestion/sink/WarehouseWriter.java
create mode 100644
ingestion/ingestion-core/src/test/java/org/apache/fineract/bi/ingestion/cdc/DebeziumEnvelopeParserTest.java
create mode 100644
ingestion/ingestion-core/src/test/java/org/apache/fineract/bi/ingestion/cdc/ValueConverterTest.java
create mode 100644
ingestion/ingestion-core/src/test/java/org/apache/fineract/bi/ingestion/contract/ContractLoaderTest.java
create mode 100644
ingestion/ingestion-core/src/test/java/org/apache/fineract/bi/ingestion/routing/OwnershipSnapshotTest.java
create mode 100644
ingestion/ingestion-core/src/test/java/org/apache/fineract/bi/ingestion/sink/SqlFailuresTest.java
create mode 100644
ingestion/ingestion-core/src/test/java/org/apache/fineract/bi/ingestion/sink/WarehouseWriterEncodeTest.java
create mode 100644
ingestion/ingestion-core/src/test/java/org/apache/fineract/bi/ingestion/sink/WarehouseWriterSkipVerificationTest.java
create mode 100644 ingestion/ingestion-loadgen/pom.xml
create mode 100644
ingestion/ingestion-loadgen/src/main/java/org/apache/fineract/bi/ingestion/loadgen/Args.java
create mode 100644
ingestion/ingestion-loadgen/src/main/java/org/apache/fineract/bi/ingestion/loadgen/EventProducer.java
create mode 100644
ingestion/ingestion-loadgen/src/main/java/org/apache/fineract/bi/ingestion/loadgen/GroundTruth.java
create mode 100644
ingestion/ingestion-loadgen/src/main/java/org/apache/fineract/bi/ingestion/loadgen/LoadGenApplication.java
create mode 100644
ingestion/ingestion-loadgen/src/main/java/org/apache/fineract/bi/ingestion/loadgen/SourceWorkload.java
create mode 100644
ingestion/ingestion-loadgen/src/main/java/org/apache/fineract/bi/ingestion/loadgen/SyntheticPayloads.java
create mode 100644
ingestion/ingestion-loadgen/src/main/java/org/apache/fineract/bi/ingestion/loadgen/Verifier.java
copy CODE_OF_CONDUCT.md =>
ingestion/ingestion-loadgen/src/main/resources/log4j2.xml (60%)
create mode 100644
ingestion/ingestion-loadgen/src/test/java/org/apache/fineract/bi/ingestion/loadgen/GeneratorRoundTripTest.java
create mode 100644 ingestion/ingestion-sink/pom.xml
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/ConnectionSource.java
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/DeadLetterPublisher.java
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/DeadLetterSink.java
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/ProgressReporter.java
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/SinkApplication.java
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/SinkConfig.java
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/SinkMetrics.java
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/SinkWorker.java
create mode 100644
ingestion/ingestion-sink/src/main/java/org/apache/fineract/bi/ingestion/app/TopicProvisioner.java
copy CODE_OF_CONDUCT.md =>
ingestion/ingestion-sink/src/main/resources/log4j2.xml (60%)
create mode 100644
ingestion/ingestion-sink/src/test/java/org/apache/fineract/bi/ingestion/app/DeadLetterPublisherTest.java
create mode 100644
ingestion/ingestion-sink/src/test/java/org/apache/fineract/bi/ingestion/app/SinkConfigAndTopicPlanTest.java
create mode 100644
ingestion/ingestion-sink/src/test/java/org/apache/fineract/bi/ingestion/app/SinkKafkaPostgresTest.java
create mode 100644
ingestion/ingestion-sink/src/test/java/org/apache/fineract/bi/ingestion/app/SinkWorkerDeliveryTest.java
create mode 100644 ingestion/pom.xml
create mode 100644 loadtest/README.md
create mode 100644 loadtest/results/.gitkeep
create mode 100644 loadtest/run_stages.sh
create mode 100644 loadtest/verify_incremental_equivalence.sh
create mode 100644 scripts/airflow_trigger.py
create mode 100644 scripts/bootstrap_cdc_sources.sh
create mode 100644 scripts/demo_feature_flag.sh
create mode 100644 scripts/pipeline_status.sh
create mode 100644 tests/airflow/test_dag_integrity.py
create mode 100644 tests/integration/test_control_plane_hardening.py
create mode 100644 tests/integration/test_platform_write_path.py
copy tests/{ => unit/controlplane}/__init__.py (100%)
create mode 100644 tests/unit/controlplane/test_connectors_and_registry.py
create mode 100644 tests/unit/controlplane/test_contract_and_reconcile.py
create mode 100644 tests/unit/controlplane/test_ownership_state_machine.py
create mode 100644 tests/unit/controlplane/test_reconciler.py
delete mode 100644 tests/unit/extractor/test_cob_gate.py
delete mode 100644 tests/unit/extractor/test_config.py
delete mode 100644 tests/unit/extractor/test_helpers.py
create mode 100644 tests/unit/extractor/test_python_engine.py
delete mode 100644 tests/unit/extractor/test_reconcile.py
delete mode 100644 tests/unit/extractor/test_source_query.py
delete mode 100644 tests/unit/extractor/test_upsert_rows.py
delete mode 100644 tests/unit/extractor/test_watermark.py
create mode 100644 warehouse/schema/access_control.sql
delete mode 100644 warehouse/schema/mart_tables.sql
delete mode 100644 warehouse/schema/pipeline_state.sql
create mode 100644 warehouse/schema/platform.sql
delete mode 100644 warehouse/schema/raw_tables.sql
delete mode 100644 warehouse/schema/staging_tables.sql