This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a change to branch jason/lang-sdk-e2e/02-importer-registry
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard a5d284bbc50 Skip the Python-only parse checks for non-Python Dag files
discard 0616bc80130 Queue the Dag files of non-Python Dag importers
add 5c2018283cf Share Dag cycle detection between the Task SDK and core
add 382f2e51300 Let a coordinator parse the Dag files of the bundles it
serves
add 395d9cb66a8 Check that a serialized Dag can be stored and loaded
add 2b43cb37f72 Move the Dag file processor's shared plumbing into a base
class
add caecd20c7ad Parse coordinator-claimed Dag files with their runtime
add 8ea7541c6e6 Fill a native Dag's unset settings from the Airflow config
add 8f7540a4429 Report a Lang-SDK parse past its import timeout as an
import error
add 6f91c3ee3e1 Report a native Dag's task that reaches a Python worker
add 6285308fdf9 Read JAR manifests per the JAR specification
add 99b72973144 Build the JVM command that parses a Dag JAR
add fbb91d882a0 Embed the Dag source in Java bundle JARs
add 578a7b4276e Parse native Java Dags with their coordinator
add e99973a104b Document native Java Dag parsing
add c9cbf7834e6 Treat an unreadable JAR like one without a manifest
add df3153f74dd Test the Java Dag source placeholder for a JAR without a
manifest
add 8ffd19e8bef Name the JAR manifest attributes the Java coordinator reads
add 18288e91081 Parse native TypeScript Dags with the Node coordinator
add 69b4b262dae Document parsing native TypeScript Dags
add fe683157090 Share the bundle section mutation helper between Node tests
add 7fe4da6e3cb Queue the Dag files of non-Python Dag importers
add c24e6fad843 Skip the Python-only parse checks for non-Python Dag files
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a5d284bbc50)
\
N -- N -- N refs/heads/jason/lang-sdk-e2e/02-importer-registry
(c24e6fad843)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
airflow-core/adr/lang-sdk/0004-dag-parsing.md | 8 +-
.../language-sdks/java.rst | 77 ++-
.../language-sdks/typescript.rst | 54 +-
.../src/airflow/cli/commands/dag_command.py | 9 +-
airflow-core/src/airflow/dag_processing/dagbag.py | 94 ++-
.../src/airflow/dag_processing/importer_routing.py | 26 +-
.../airflow/dag_processing/lang_sdk_processor.py | 521 ++++++++++++++++
airflow-core/src/airflow/dag_processing/manager.py | 34 +-
.../src/airflow/dag_processing/processor.py | 154 +++--
.../src/airflow/serialization/definitions/dag.py | 8 +-
.../airflow/serialization/serialized_objects.py | 67 +++
airflow-core/src/airflow/utils/cli.py | 12 +-
.../tests/unit/dag_processing/fake_lang_sdk.py | 127 ++++
.../tests/unit/dag_processing/test_dagbag.py | 181 ++++++
.../unit/dag_processing/test_importer_routing.py | 23 +-
.../unit/dag_processing/test_lang_sdk_processor.py | 593 ++++++++++++++++++
.../tests/unit/dag_processing/test_manager.py | 194 +++++-
.../tests/unit/dag_processing/test_processor.py | 17 +
.../unit/serialization/test_dag_serialization.py | 107 ++++
airflow-core/tests/unit/utils/test_cli_util.py | 56 ++
generated/known_sdk_imports_in_core.txt | 1 +
.../apache/airflow/sdk/plugin/AirflowSdkPlugin.kt | 58 ++
shared/dagnode/src/airflow_shared/dagnode/cycle.py | 73 +++
shared/dagnode/tests/dagnode/test_cycle.py | 87 +++
.../src/airflow/sdk/coordinators/_dag_importer.py | 111 ++++
.../src/airflow/sdk/coordinators/_subprocess.py | 164 ++++-
.../sdk/coordinators/executable/coordinator.py | 4 +-
.../airflow/sdk/coordinators/java/_dag_importer.py | 86 +++
.../airflow/sdk/coordinators/java/_jar_manifest.py | 71 +++
.../airflow/sdk/coordinators/java/coordinator.py | 67 ++-
.../sdk/coordinators/node/_bundle_reader.py | 29 +-
.../airflow/sdk/coordinators/node/_dag_importer.py | 71 +++
.../airflow/sdk/coordinators/node/coordinator.py | 25 +-
task-sdk/src/airflow/sdk/definitions/dag.py | 41 +-
.../src/airflow/sdk/execution_time/coordinator.py | 114 +++-
.../src/airflow/sdk/execution_time/task_runner.py | 23 +
task-sdk/src/airflow/sdk/importers/base.py | 47 +-
.../coordinators/fixtures/java_native.json | 151 +++++
.../coordinators/fixtures/ts_conformance.json | 661 +++++++++++++++++++++
.../task_sdk/coordinators/java/_jar_test_utils.py | 53 ++
.../task_sdk/coordinators/java/test_coordinator.py | 128 +++-
.../coordinators/java/test_dag_importer.py | 201 +++++++
.../coordinators/java/test_jar_manifest.py | 94 +++
.../coordinators/node/_bundle_test_utils.py | 5 +
.../coordinators/node/test_bundle_reader.py | 7 +
.../task_sdk/coordinators/node/test_coordinator.py | 57 ++
.../coordinators/node/test_dag_importer.py | 139 +++++
.../task_sdk/coordinators/test_dag_importer.py | 139 +++++
.../tests/task_sdk/coordinators/test_subprocess.py | 163 ++++-
.../task_sdk/execution_time/test_coordinator.py | 160 +++++
.../task_sdk/execution_time/test_task_runner.py | 92 +++
task-sdk/tests/task_sdk/importers/test_registry.py | 102 ++++
.../tests/task_sdk/importers/test_zip_importer.py | 3 +-
53 files changed, 5328 insertions(+), 261 deletions(-)
create mode 100644
airflow-core/src/airflow/dag_processing/lang_sdk_processor.py
create mode 100644 airflow-core/tests/unit/dag_processing/fake_lang_sdk.py
create mode 100644
airflow-core/tests/unit/dag_processing/test_lang_sdk_processor.py
create mode 100644 shared/dagnode/src/airflow_shared/dagnode/cycle.py
create mode 100644 shared/dagnode/tests/dagnode/test_cycle.py
create mode 100644 task-sdk/src/airflow/sdk/coordinators/_dag_importer.py
create mode 100644 task-sdk/src/airflow/sdk/coordinators/java/_dag_importer.py
create mode 100644 task-sdk/src/airflow/sdk/coordinators/java/_jar_manifest.py
create mode 100644 task-sdk/src/airflow/sdk/coordinators/node/_dag_importer.py
create mode 100644
task-sdk/tests/task_sdk/coordinators/fixtures/java_native.json
create mode 100644
task-sdk/tests/task_sdk/coordinators/fixtures/ts_conformance.json
create mode 100644 task-sdk/tests/task_sdk/coordinators/java/_jar_test_utils.py
create mode 100644
task-sdk/tests/task_sdk/coordinators/java/test_dag_importer.py
create mode 100644
task-sdk/tests/task_sdk/coordinators/java/test_jar_manifest.py
create mode 100644
task-sdk/tests/task_sdk/coordinators/node/test_dag_importer.py
create mode 100644 task-sdk/tests/task_sdk/coordinators/test_dag_importer.py
