phanikumv commented on code in PR #69545:
URL: https://github.com/apache/airflow/pull/69545#discussion_r3559391305
##########
dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py:
##########
@@ -2507,37 +2508,120 @@ def deploy_cluster(
"aws://test:test@/?region_name=us-east-1&"
"endpoint_url=http%3A%2F%2Flocalstack.airflow.svc.cluster.local%3A4566"
)
+# The Go/Java SDKs are always built from upstream main so branches with stale
or missing
+# go-sdk/java-sdk copies still test current SDK sources. See
kubernetes-tests/lang_sdk/README.md.
+LANG_SDK_UPSTREAM_GIT_URL = "https://github.com/apache/airflow.git";
+LANG_SDK_UPSTREAM_REF = "main"
-def _lang_sdk_build_go_bundle(staging: Path, output: Output | None, *, native:
bool = False) -> None:
+def _lang_sdk_fetch_upstream_sdk_sources(staging: Path, output: Output | None)
-> tuple[Path, Path]:
+ """Extract go-sdk/ and java-sdk/ from upstream main into a throwaway
staging dir.
+
+ Prefers the ``upstream`` remote when configured, falling back to the
canonical GitHub URL
+ (CI has no ``upstream`` and ``origin`` may be a fork). Shallow fetch +
``git archive`` never
+ touch the working tree or index.
+
+ The real, local task-sdk is symlinked alongside the extraction because
java-sdk's
+ ``sdk/build.gradle.kts`` reads a sibling ``../task-sdk/.../schema.json``.
The gradle wrapper
+ scripts and jar are ``export-ignore`` (ASF LEGAL-570) so ``git archive``
drops them;
+ ``git show`` restores them, re-marking ``gradlew`` executable.
+ """
+ remotes = run_command(
+ ["git", "remote"], cwd=AIRFLOW_ROOT_PATH, output=output,
capture_output=True, text=True, check=True
+ ).stdout.split()
+ fetch_source = "upstream" if "upstream" in remotes else
LANG_SDK_UPSTREAM_GIT_URL
+ get_console(output=output).print(
+ f"[info]Fetching {LANG_SDK_UPSTREAM_REF} from {fetch_source} for the
lang-SDK Go/Java sources"
+ )
+ run_command(
+ ["git", "fetch", "--depth=1", fetch_source, LANG_SDK_UPSTREAM_REF],
+ cwd=AIRFLOW_ROOT_PATH,
+ output=output,
+ check=True,
+ )
+ sha = run_command(
+ ["git", "rev-parse", "FETCH_HEAD"],
+ cwd=AIRFLOW_ROOT_PATH,
+ output=output,
+ capture_output=True,
+ text=True,
+ check=True,
+ ).stdout.strip()
+ get_console(output=output).print(f"[info]lang-SDK Go/Java sources pinned
to upstream main @ {sha}")
+ extracted = staging / "upstream_lang_sdk_sources"
+ extracted.mkdir(parents=True, exist_ok=True)
+ archive_path = staging / "upstream_lang_sdk_sources.tar"
+ run_command(
+ ["git", "archive", "--format=tar", f"--output={archive_path}", sha,
"--", "go-sdk", "java-sdk"],
+ cwd=AIRFLOW_ROOT_PATH,
+ output=output,
+ check=True,
+ )
+ run_command(["tar", "-xf", str(archive_path), "-C", str(extracted)],
output=output, check=True)
+ for rel_path, mode in (
+ ("java-sdk/gradlew", 0o755),
+ ("java-sdk/gradlew.bat", 0o644),
+ ("java-sdk/gradle/wrapper/gradle-wrapper.jar", 0o644),
+ ):
+ restored = extracted / rel_path
+ restored.parent.mkdir(parents=True, exist_ok=True)
+ restored.write_bytes(
+ run_command(
+ ["git", "show", f"{sha}:{rel_path}"],
+ cwd=AIRFLOW_ROOT_PATH,
+ output=output,
+ capture_output=True,
+ check=True,
+ ).stdout
+ )
Review Comment:
`breeze k8s setup-lang-sdk-test --dry-run` crashes here. In dry-run,
run_command returns stdout="" (a str),so `write_bytes("")` raises `TypeError: a
bytes-like object is required, not 'str'`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
