Taragolis commented on code in PR #34840: URL: https://github.com/apache/airflow/pull/34840#discussion_r1352033233
########## airflow/providers/papermill/operators/__init__.py: ########## @@ -15,3 +15,8 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from papermill.engines import papermill_engines + +from airflow.providers.papermill.operators.papermill import RemoteKernelEngine, REMOTE_KERNEL_ENGINE + +papermill_engines.register(REMOTE_KERNEL_ENGINE, RemoteKernelEngine) Review Comment: Do we need to register this every-time? Could we register only if we required to use Remote kernel? ########## airflow/providers/papermill/operators/papermill.py: ########## @@ -20,14 +20,97 @@ from typing import TYPE_CHECKING, ClassVar, Collection, Sequence import attr +from papermill.utils import remove_args, merge_kwargs +from pydantic import typing + import papermill as pm +from papermill.engines import NBClientEngine +from papermill.clientwrap import PapermillNotebookClient +from jupyter_client.manager import AsyncKernelManager +from jupyter_client.client import KernelClient +from traitlets import Unicode from airflow.lineage.entities import File from airflow.models import BaseOperator if TYPE_CHECKING: from airflow.utils.context import Context +JUPYTER_KERNEL_SHELL_PORT = 60316 +JUPYTER_KERNEL_IOPUB_PORT = 60317 +JUPYTER_KERNEL_STDIN_PORT = 60318 +JUPYTER_KERNEL_CONTROL_PORT = 60319 +JUPYTER_KERNEL_HB_PORT = 60320 +LOCALHOST = '127.0.0.1' +REMOTE_KERNEL_ENGINE = "remote_kernel_engine" + + +class RemoteKernelManager(AsyncKernelManager): Review Comment: I think this should utilise Airflow Connection + Hook capabilities -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org