Structured Logging in python

[Geddy Schellevis]

Hi all,

I would like to know if it is possible to have structured logging in
Dataflow.
In the attached file, you can find the code that I am trying to do.

I see the logs are appearing in gcp log explorer, but I cannot see the
extra fields.

Best regards,

from __future__ import annotations
import argparse
import logging
import google.cloud.logging
import apache_beam as beam

from apache_beam.options.pipeline_options import PipelineOptions
from dotenv import load_dotenv
from google.cloud.logging_v2.handlers import CloudLoggingHandler
from pythonjsonlogger import jsonlogger


def func():
    logger = logging.getLogger()
    logger.warning("check Logging", extra={"custom_field1": "value1", "custom_field2": 1, "A": {"B": 2}})
    logger.warning("check Logging", extra={"custom_field1": "value1", "custom_field2": 1})
    

def run(argv=None):
    parser = argparse.ArgumentParser()
    known_args, pipeline_args = parser.parse_known_args(argv)

    pipeline_options = PipelineOptions(pipeline_args)

    with beam.Pipeline(options=pipeline_options) as pipeline:
        (
                pipeline
                | "debug create data" >> beam.Create(["dummy_data"])
                | "dummy prints" >> beam.Map(func)
        )


if __name__ == "__main__":
    formatter = jsonlogger.JsonFormatter()

    # setup GCP handler
    logging_client = google.cloud.logging.Client()
    handler = CloudLoggingHandler(logging_client)
    handler.setFormatter(formatter)

    # setup logger
    logger = logging.getLogger()
    logger.addHandler(handler)
    logger.setLevel(logging.INFO)

    load_dotenv()
    run()