This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch main
in repository
https://gitbox.apache.org/repos/asf/dolphinscheduler-sdk-python.git
The following commit(s) were added to refs/heads/main by this push:
new bd097dc fix: Task python created and gateway check error (#69)
bd097dc is described below
commit bd097dcdafd720fa3d6c4294259ab3232dd526ad
Author: Jay Chung <[email protected]>
AuthorDate: Mon Feb 6 18:00:44 2023 +0800
fix: Task python created and gateway check error (#69)
---
src/pydolphinscheduler/core/task.py | 1 -
src/pydolphinscheduler/core/yaml_workflow.py | 1 -
src/pydolphinscheduler/examples/local_parameter_example.py | 1 -
src/pydolphinscheduler/examples/task_mlflow_example.py | 1 -
src/pydolphinscheduler/examples/task_pytorch_example.py | 1 -
src/pydolphinscheduler/java_gateway.py | 6 ++++--
src/pydolphinscheduler/tasks/python.py | 2 +-
tests/tasks/test_python.py | 4 ----
8 files changed, 5 insertions(+), 12 deletions(-)
diff --git a/src/pydolphinscheduler/core/task.py
b/src/pydolphinscheduler/core/task.py
index f0d3e83..3341040 100644
--- a/src/pydolphinscheduler/core/task.py
+++ b/src/pydolphinscheduler/core/task.py
@@ -170,7 +170,6 @@ class Task(Base):
*args,
**kwargs,
):
-
super().__init__(name, description)
self.task_type = task_type
self.flag = flag
diff --git a/src/pydolphinscheduler/core/yaml_workflow.py
b/src/pydolphinscheduler/core/yaml_workflow.py
index b7a4744..e994d6d 100644
--- a/src/pydolphinscheduler/core/yaml_workflow.py
+++ b/src/pydolphinscheduler/core/yaml_workflow.py
@@ -168,7 +168,6 @@ class YamlWorkflow(YamlParser):
workflow_name = workflow_params["name"]
logger.info(f"Create workflow: {workflow_name}")
with Workflow(**workflow_params) as workflow:
-
# save dependencies between tasks
dependencies = {}
diff --git a/src/pydolphinscheduler/examples/local_parameter_example.py
b/src/pydolphinscheduler/examples/local_parameter_example.py
index f00d685..e280a9b 100644
--- a/src/pydolphinscheduler/examples/local_parameter_example.py
+++ b/src/pydolphinscheduler/examples/local_parameter_example.py
@@ -33,7 +33,6 @@ from pydolphinscheduler.core.workflow import Workflow
from pydolphinscheduler.tasks.shell import Shell
with Workflow(name="local_parameter_example", release_state="offline") as
workflow:
-
# [start parameter example]
# define a parameter "a", and use it in Shell task
example1_input_params = Shell(
diff --git a/src/pydolphinscheduler/examples/task_mlflow_example.py
b/src/pydolphinscheduler/examples/task_mlflow_example.py
index 46d6a86..95a732f 100644
--- a/src/pydolphinscheduler/examples/task_mlflow_example.py
+++ b/src/pydolphinscheduler/examples/task_mlflow_example.py
@@ -32,7 +32,6 @@ mlflow_tracking_uri = "http://127.0.0.1:5000";
with Workflow(
name="task_mlflow_example",
) as workflow:
-
# run custom mlflow project to train model
train_custom = MLFlowProjectsCustom(
name="train_xgboost_native",
diff --git a/src/pydolphinscheduler/examples/task_pytorch_example.py
b/src/pydolphinscheduler/examples/task_pytorch_example.py
index bfb4424..8e431d5 100644
--- a/src/pydolphinscheduler/examples/task_pytorch_example.py
+++ b/src/pydolphinscheduler/examples/task_pytorch_example.py
@@ -24,7 +24,6 @@ from pydolphinscheduler.tasks.pytorch import Pytorch
with Workflow(
name="task_pytorch_example",
) as workflow:
-
# run project with existing environment
task_existing_env = Pytorch(
name="task_existing_env",
diff --git a/src/pydolphinscheduler/java_gateway.py
b/src/pydolphinscheduler/java_gateway.py
index dcf4fe8..92ad19f 100644
--- a/src/pydolphinscheduler/java_gateway.py
+++ b/src/pydolphinscheduler/java_gateway.py
@@ -72,8 +72,10 @@ class GatewayEntryPoint:
# 1. Java gateway version is too old: doesn't have method
'getGatewayVersion()'
# 2. Error connecting to Java gateway
gateway_version = self.get_gateway_version()
- if not __version__.endswith("dev") and not version_match(
- Version.DS, gateway_version
+ if (
+ not __version__.endswith("dev")
+ and gateway_version
+ and not version_match(Version.DS, gateway_version)
):
warnings.warn(
f"Using unmatched version of pydolphinscheduler (version
{__version__}) "
diff --git a/src/pydolphinscheduler/tasks/python.py
b/src/pydolphinscheduler/tasks/python.py
index bec8336..818f1fb 100644
--- a/src/pydolphinscheduler/tasks/python.py
+++ b/src/pydolphinscheduler/tasks/python.py
@@ -55,7 +55,7 @@ class Python(Task):
want to execute.
"""
- _task_custom_attr = {"raw_script", "definition"}
+ _task_custom_attr = {"raw_script"}
ext: set = {".py"}
ext_attr: Union[str, types.FunctionType] = "_definition"
diff --git a/tests/tasks/test_python.py b/tests/tasks/test_python.py
index f237619..516513a 100644
--- a/tests/tasks/test_python.py
+++ b/tests/tasks/test_python.py
@@ -51,7 +51,6 @@ def setup_crt_first(request):
(
{"definition": "print(1)"},
{
- "definition": "print(1)",
"rawScript": "print(1)",
"localParams": [],
"resourceList": [],
@@ -63,7 +62,6 @@ def setup_crt_first(request):
(
{"definition": "def foo():\n print('I am foo')"},
{
- "definition": "def foo():\n print('I am foo')",
"rawScript": "def foo():\n print('I am foo')\nfoo()",
"localParams": [],
"resourceList": [],
@@ -75,7 +73,6 @@ def setup_crt_first(request):
(
{"definition": foo},
{
- "definition": foo,
"rawScript": 'def foo(): # noqa: D103\n print("hello
world.")\nfoo()',
"localParams": [],
"resourceList": [],
@@ -133,7 +130,6 @@ def test_python_get_define(name, script_code, raw):
code = 123
version = 1
expect_task_params = {
- "definition": script_code,
"resourceList": [],
"localParams": [],
"rawScript": raw,
