github-advanced-security[bot] commented on code in PR #40795:
URL: https://github.com/apache/airflow/pull/40795#discussion_r1937294032
##########
airflow/auth/managers/simple/simple_auth_manager.py:
##########
@@ -73,64 +82,68 @@
Default auth manager used in Airflow. This auth manager should not be used
in production.
This auth manager is very basic and only intended for development and
testing purposes.
-
- :param appbuilder: the flask app builder
"""
- # Cache containing the password associated to a username
- passwords: dict[str, str] = {}
-
@staticmethod
def get_generated_password_file() -> str:
return os.path.join(
os.getenv("AIRFLOW_AUTH_MANAGER_CREDENTIAL_DIRECTORY",
AIRFLOW_HOME),
"simple_auth_manager_passwords.json.generated",
)
- def init(self) -> None:
- if not self.appbuilder:
- return
+ @staticmethod
+ def get_users() -> list[dict[str, str]]:
+ users = [u.split(":") for u in conf.getlist("core",
"simple_auth_manager_users")]
+ return [{"username": username, "role": role} for username, role in
users]
+
+ @staticmethod
+ def get_passwords(users: list[dict[str, str]]) -> dict[str, str]:
user_passwords_from_file = {}
# Read passwords from file
- if os.path.isfile(self.get_generated_password_file()):
- with open(self.get_generated_password_file()) as file:
+ password_file = SimpleAuthManager.get_generated_password_file()
+ if os.path.isfile(password_file):
+ with open(password_file) as file:
passwords_str = file.read().strip()
user_passwords_from_file = json.loads(passwords_str)
- users =
self.appbuilder.get_app.config.get("SIMPLE_AUTH_MANAGER_USERS", [])
usernames = {user["username"] for user in users}
- self.passwords = {
+ return {
username: password
for username, password in user_passwords_from_file.items()
if username in usernames
}
+
+ def init(self) -> None:
+ users = self.get_users()
+ passwords = self.get_passwords(users)
for user in users:
- if user["username"] not in self.passwords:
+ if user["username"] not in passwords:
# User dot not exist in the file, adding it
- self.passwords[user["username"]] = self._generate_password()
+ passwords[user["username"]] = self._generate_password()
- self._print_output(f"Password for user '{user['username']}':
{self.passwords[user['username']]}")
+ self._print_output(f"Password for user '{user['username']}':
{passwords[user['username']]}")
with open(self.get_generated_password_file(), "w") as file:
- file.write(json.dumps(self.passwords))
+ file.write(json.dumps(passwords))
Review Comment:
## Clear-text storage of sensitive information
This expression stores [sensitive data (password)](1) as clear text.
This expression stores [sensitive data (password)](2) as clear text.
This expression stores [sensitive data (password)](3) as clear text.
This expression stores [sensitive data (password)](4) as clear text.
[Show more
details](https://github.com/apache/airflow/security/code-scanning/353)
##########
providers/edge/src/airflow/providers/edge/cli/edge_command.py:
##########
@@ -0,0 +1,457 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import logging
+import os
+import platform
+import signal
+import sys
+from dataclasses import dataclass
+from datetime import datetime
+from http import HTTPStatus
+from multiprocessing import Process
+from pathlib import Path
+from subprocess import Popen
+from time import sleep
+from typing import TYPE_CHECKING
+
+import psutil
+from lockfile.pidlockfile import read_pid_from_pidfile,
remove_existing_pidfile, write_pid_to_pidfile
+from requests import HTTPError
+
+from airflow import __version__ as airflow_version, settings
+from airflow.cli.cli_config import ARG_PID, ARG_VERBOSE, ActionCommand, Arg
+from airflow.configuration import conf
+from airflow.providers.edge import __version__ as edge_provider_version
+from airflow.providers.edge.cli.api_client import (
+ jobs_fetch,
+ jobs_set_state,
+ logs_logfile_path,
+ logs_push,
+ worker_register,
+ worker_set_state,
+)
+from airflow.providers.edge.models.edge_worker import EdgeWorkerState,
EdgeWorkerVersionException
+from airflow.providers.edge.version_compat import AIRFLOW_V_3_0_PLUS
+from airflow.utils import cli as cli_utils, timezone
+from airflow.utils.platform import IS_WINDOWS
+from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
+from airflow.utils.state import TaskInstanceState
+
+if TYPE_CHECKING:
+ from airflow.providers.edge.worker_api.datamodels import EdgeJobFetched
+
+logger = logging.getLogger(__name__)
+EDGE_WORKER_PROCESS_NAME = "edge-worker"
+EDGE_WORKER_HEADER = "\n".join(
+ [
+ r" ____ __ _ __ __",
+ r" / __/__/ /__ ____ | | /| / /__ ____/ /_____ ____",
+ r" / _// _ / _ `/ -_) | |/ |/ / _ \/ __/ '_/ -_) __/",
+ r"/___/\_,_/\_, /\__/ |__/|__/\___/_/ /_/\_\\__/_/",
+ r" /___/",
+ r"",
+ ]
+)
+
+
+@providers_configuration_loaded
+def force_use_internal_api_on_edge_worker():
+ """
+ Ensure that the environment is configured for the internal API without
needing to declare it outside.
+
+ This is only required for an Edge worker and must to be done before the
Click CLI wrapper is initiated.
+ That is because the CLI wrapper will attempt to establish a DB connection,
which will fail before the
+ function call can take effect. In an Edge worker, we need to "patch" the
environment before starting.
+ """
+ # export Edge API to be used for internal API
+ os.environ["_AIRFLOW__SKIP_DATABASE_EXECUTOR_COMPATIBILITY_CHECK"] = "1"
+ os.environ["AIRFLOW_ENABLE_AIP_44"] = "True"
+ if "airflow" in sys.argv[0] and sys.argv[1:3] == ["edge", "worker"]:
+ api_url = conf.get("edge", "api_url")
+ if not api_url:
+ raise SystemExit("Error: API URL is not configured, please correct
configuration.")
+ logger.info("Starting worker with API endpoint %s", api_url)
Review Comment:
## Clear-text logging of sensitive information
This expression logs [sensitive data (secret)](1) as clear text.
This expression logs [sensitive data (secret)](2) as clear text.
[Show more
details](https://github.com/apache/airflow/security/code-scanning/350)
##########
providers/edge/src/airflow/providers/edge/worker_api/routes/_v2_routes.py:
##########
@@ -0,0 +1,233 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Compatibility layer for Connexion API to Airflow v2.10 API routes."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import TYPE_CHECKING, Any
+from uuid import uuid4
+
+from flask import Response, request
+
+from airflow.exceptions import AirflowException
+from airflow.providers.edge.worker_api.auth import jwt_token_authorization,
jwt_token_authorization_rpc
+from airflow.providers.edge.worker_api.datamodels import (
+ EdgeJobFetched,
+ JsonRpcRequest,
+ PushLogsBody,
+ WorkerQueuesBody,
+ WorkerStateBody,
+)
+from airflow.providers.edge.worker_api.routes._v2_compat import HTTPException,
status
+from airflow.providers.edge.worker_api.routes.jobs import fetch, state as
state_api
+from airflow.providers.edge.worker_api.routes.logs import logfile_path,
push_logs
+from airflow.providers.edge.worker_api.routes.worker import register, set_state
+from airflow.serialization.serialized_objects import BaseSerialization
+from airflow.utils.session import NEW_SESSION, create_session, provide_session
+
+if TYPE_CHECKING:
+ from airflow.api_connexion.types import APIResponse
+ from airflow.utils.state import TaskInstanceState
+
+
+log = logging.getLogger(__name__)
+
+
+def error_response(message: str, status: int):
+ """Log the error and return the response as JSON object."""
+ error_id = uuid4()
+ server_message = f"{message} error_id={error_id}"
+ log.exception(server_message)
+ client_message = f"{message} The server side traceback may be identified
with error_id={error_id}"
+ return HTTPException(status, client_message)
+
+
+def rpcapi_v2(body: dict[str, Any]) -> APIResponse:
+ """Handle Edge Worker API `/edge_worker/v1/rpcapi` endpoint for Airflow
2.10."""
+ # Note: Except the method map this _was_ a 100% copy of internal API module
+ #
airflow.api_internal.endpoints.rpc_api_endpoint.internal_airflow_api()
+ # As of rework for FastAPI in Airflow 3.0, this is updated and to be
removed in the future.
+ from airflow.api_internal.endpoints.rpc_api_endpoint import ( # type:
ignore[attr-defined]
+ # Note: This is just for compatibility with Airflow 2.10, not working
for Airflow 3 / main as removed
+ initialize_method_map,
+ )
+
+ try:
+ if request.headers.get("Content-Type", "") != "application/json":
+ raise HTTPException(status.HTTP_403_FORBIDDEN, "Expected
Content-Type: application/json")
+ if request.headers.get("Accept", "") != "application/json":
+ raise HTTPException(status.HTTP_403_FORBIDDEN, "Expected Accept:
application/json")
+ auth = request.headers.get("Authorization", "")
+ request_obj = JsonRpcRequest(method=body["method"],
jsonrpc=body["jsonrpc"], params=body["params"])
+ jwt_token_authorization_rpc(request_obj, auth)
+ if request_obj.jsonrpc != "2.0":
+ raise error_response("Expected jsonrpc 2.0 request.",
status.HTTP_400_BAD_REQUEST)
+
+ log.debug("Got request for %s", request_obj.method)
+ methods_map = initialize_method_map()
+ if request_obj.method not in methods_map:
+ raise error_response(f"Unrecognized method:
{request_obj.method}.", status.HTTP_400_BAD_REQUEST)
+
+ handler = methods_map[request_obj.method]
+ params = {}
+ try:
+ if request_obj.params:
+ # Note, this is Airflow 2.10 specific, as it uses Pydantic
models for serialization
+ params = BaseSerialization.deserialize(request_obj.params,
use_pydantic_models=True) # type: ignore[call-arg]
+ except Exception:
+ raise error_response("Error deserializing parameters.",
status.HTTP_400_BAD_REQUEST)
+
+ log.debug("Calling method %s\nparams: %s", request_obj.method, params)
+ try:
+ # Session must be created there as it may be needed by serializer
for lazy-loaded fields.
+ with create_session() as session:
+ output = handler(**params, session=session)
+ # Note, this is Airflow 2.10 specific, as it uses Pydantic
models for serialization
+ output_json = BaseSerialization.serialize(output,
use_pydantic_models=True) # type: ignore[call-arg]
+ log.debug(
+ "Sending response: %s", json.dumps(output_json) if
output_json is not None else None
+ )
+ # In case of AirflowException or other selective known types,
transport the exception class back to caller
+ except (KeyError, AttributeError, AirflowException) as e:
+ # Note, this is Airflow 2.10 specific, as it uses Pydantic models
for serialization
+ output_json = BaseSerialization.serialize(e,
use_pydantic_models=True) # type: ignore[call-arg]
+ log.debug(
+ "Sending exception response: %s", json.dumps(output_json) if
output_json is not None else None
+ )
+ except Exception:
+ raise error_response(
+ f"Error executing method '{request_obj.method}'.",
status.HTTP_500_INTERNAL_SERVER_ERROR
+ )
+ response = json.dumps(output_json) if output_json is not None else None
+ return Response(response=response, headers={"Content-Type":
"application/json"})
Review Comment:
## Information exposure through an exception
[Stack trace information](1) flows to this location and may be exposed to an
external user.
[Show more
details](https://github.com/apache/airflow/security/code-scanning/351)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]