This is an automated email from the ASF dual-hosted git repository.
obristow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new ec44c53 Use click instead of argparse in py instance (#3598)
ec44c53 is described below
commit ec44c531042ac3eaa203bee78339ce76fee4457c
Author: Oliver Bristow <[email protected]>
AuthorDate: Tue Nov 3 15:33:38 2020 +0000
Use click instead of argparse in py instance (#3598)
* Use click instead of argparse in py instance
* Improve heron-python-instance option name consistency
* Rename instance/st_heron_instance.py to instance.py to avoid [this kind
of runtime
warning](https://stackoverflow.com/questions/43393764/python-3-6-project-structure-leads-to-runtimewarning).
---
heron/executor/src/python/heron_executor.py | 4 +-
heron/instance/src/python/BUILD | 5 +-
heron/instance/src/python/__init__.py | 16 ---
.../{instance/st_heron_instance.py => instance.py} | 107 ++++++++++++---------
heron/instance/src/python/instance/__init__.py | 20 ----
5 files changed, 66 insertions(+), 86 deletions(-)
diff --git a/heron/executor/src/python/heron_executor.py
b/heron/executor/src/python/heron_executor.py
index 491cbdc..193527a 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -709,8 +709,8 @@ class HeronExecutor:
'--stmgr_id=%s' % self.stmgr_ids[self.shard],
'--stmgr_port=%s' % self.tmaster_controller_port,
'--metricsmgr_port=%s' % self.metrics_manager_port,
- '--sys_config=%s' % self.heron_internals_config_file,
- '--override_config=%s' % self.override_config_file,
+ '--config_file=%s' % self.heron_internals_config_file,
+ '--override_config_file=%s' % self.override_config_file,
'--topology_pex=%s' % self.topology_binary_file,
'--max_ram=%s' %
str(self.component_ram_map[component_name])]
diff --git a/heron/instance/src/python/BUILD b/heron/instance/src/python/BUILD
index 8619c67..05f5022 100644
--- a/heron/instance/src/python/BUILD
+++ b/heron/instance/src/python/BUILD
@@ -15,10 +15,11 @@ pex_library(
# build binary for single thread python heron instance
pex_binary(
name = "heron-python-instance",
- srcs = ["instance/st_heron_instance.py"],
+ srcs = ["instance.py"],
reqs = [
- "colorlog==2.6.1",
"PyYAML==3.13",
+ "click==7.1.2",
+ "colorlog==2.6.1",
],
deps = [":instance-py"],
)
diff --git a/heron/instance/src/python/__init__.py
b/heron/instance/src/python/__init__.py
deleted file mode 100644
index a67d5ea..0000000
--- a/heron/instance/src/python/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/heron/instance/src/python/instance/st_heron_instance.py
b/heron/instance/src/python/instance.py
similarity index 84%
rename from heron/instance/src/python/instance/st_heron_instance.py
rename to heron/instance/src/python/instance.py
index 1c91c49..137eafc 100644
--- a/heron/instance/src/python/instance/st_heron_instance.py
+++ b/heron/instance/src/python/instance.py
@@ -19,7 +19,6 @@
# under the License.
'''module for single-thread Heron Instance in python'''
-import argparse
import collections
import logging
import os
@@ -38,12 +37,15 @@ from heron.instance.src.python.network import
MetricsManagerClient, SingleThread
from heron.instance.src.python.network import create_socket_options
from heron.instance.src.python.network import GatewayLooper
from heron.instance.src.python.basics import SpoutInstance, BoltInstance
-import heron.instance.src.python.utils.system_constants as constants
+from heron.instance.src.python.utils import system_constants as constants
from heron.instance.src.python.utils import system_config
-import heronpy.api.api_constants as api_constants
+from heronpy.api import api_constants
from heronpy.api.state.state import HashMapState
+import click
+
+
Log = log.Log
AssignedInstance = collections.namedtuple('AssignedInstance', 'is_spout,
protobuf, py_class')
@@ -324,44 +326,58 @@ def yaml_config_reader(config_path):
return config
-# pylint: disable=missing-docstring
-def main():
- parser = argparse.ArgumentParser(description='Heron Python Instance')
- parser.add_argument('--topology_name', required=True, help='Topology Name')
- parser.add_argument('--topology_id', required=True, help='Topology Id')
- parser.add_argument('--instance_id', required=True, help='Instance Id')
- parser.add_argument('--component_name', required=True, help='Component Name')
- parser.add_argument('--task_id', required=True, help='Task Id', type=int)
- parser.add_argument('--component_index', required=True, help='Component
Index', type=int)
- parser.add_argument('--stmgr_id', required=True, help='StMgr Id')
- parser.add_argument('--stmgr_port', required=True, help='StMgr Port',
type=int)
- parser.add_argument('--metricsmgr_port', required=True, help='MetricsMgr
Port', type=int)
- parser.add_argument('--sys_config', required=True, help='System Config File')
- parser.add_argument('--override_config', required=True, help='Override
Config File')
- parser.add_argument('--topology_pex', required=True, help='Topology Pex
File')
- parser.add_argument('--max_ram', required=True, help='Maximum RAM to limit',
type=int)
-
- args = parser.parse_args()
-
- sys_config = yaml_config_reader(args.sys_config)
- override_config = yaml_config_reader(args.override_config)
+
+# pylint: disable=too-many-arguments,too-many-locals
[email protected]()
[email protected]('--topology_name', required=True, help='Topology Name')
[email protected]('--topology_id', required=True, help='Topology Id')
[email protected]('--instance_id', required=True, help='Instance Id')
[email protected]('--component_name', required=True, help='Component Name')
[email protected]('--task_id', required=True, help='Task Id', type=int)
[email protected]('--component_index', required=True, help='Component Index',
type=int)
[email protected]('--stmgr_id', required=True, help='StMgr Id')
[email protected]('--stmgr_port', required=True, help='StMgr Port', type=int)
[email protected]('--metricsmgr_port', required=True, help='MetricsMgr Port',
type=int)
[email protected]('--config_file', required=True, help='System Config File')
[email protected]('--override_config_file', required=True, help='Override Config
File')
[email protected]('--topology_pex', required=True, help='Topology Pex File')
[email protected]('--max_ram', required=True, help='Maximum RAM to limit',
type=int)
+def cli(
+ topology_name: str,
+ topology_id: str,
+ instance_id: str,
+ component_name: str,
+ task_id: int,
+ component_index: int,
+ stmgr_id: str,
+ stmgr_port: int,
+ metricsmgr_port: int,
+ config_file: str,
+ override_config_file: str,
+ topology_pex: str,
+ max_ram: int,
+) -> None:
+ """Heron Python Instance."""
+
+ sys_config = yaml_config_reader(config_file)
+ override_config = yaml_config_reader(override_config_file)
system_config.set_sys_config(sys_config, override_config)
# get combined configuration
sys_config = system_config.get_sys_config()
# set resource limits
- set_resource_limit(args.max_ram)
+ set_resource_limit(max_ram)
# create the protobuf instance
instance_info = physical_plan_pb2.InstanceInfo()
- instance_info.task_id = args.task_id
- instance_info.component_index = args.component_index
- instance_info.component_name = args.component_name
+ instance_info.task_id = task_id
+ instance_info.component_index = component_index
+ instance_info.component_name = component_name
instance = physical_plan_pb2.Instance()
- instance.instance_id = args.instance_id
- instance.stmgr_id = args.stmgr_id
+ instance.instance_id = instance_id
+ instance.stmgr_id = stmgr_id
instance.info.MergeFrom(instance_info)
# Logging init
@@ -369,25 +385,24 @@ def main():
max_log_files = sys_config[constants.HERON_LOGGING_MAXIMUM_FILES]
max_log_bytes = sys_config[constants.HERON_LOGGING_MAXIMUM_SIZE_MB] *
constants.MB
- log_file = os.path.join(log_dir, args.instance_id + ".log.0")
+ log_file = os.path.join(log_dir, instance_id + ".log.0")
log.init_rotating_logger(level=logging.INFO, logfile=log_file,
max_files=max_log_files, max_bytes=max_log_bytes)
- Log.info("\nStarting instance: " + args.instance_id + " for topology: " +
args.topology_name +
- " and topologyId: " + args.topology_id + " for component: " +
args.component_name +
- " with taskId: " + str(args.task_id) + " and componentIndex: " +
- str(args.component_index) +
- " and stmgrId: " + args.stmgr_id + " and stmgrPort: " +
str(args.stmgr_port) +
- " and metricsManagerPort: " + str(args.metricsmgr_port) +
- "\n **Topology Pex file located at: " + args.topology_pex)
- Log.debug("System config: " + str(sys_config))
- Log.debug("Override config: " + str(override_config))
- Log.debug("Maximum RAM: " + str(args.max_ram))
-
- heron_instance = SingleThreadHeronInstance(args.topology_name,
args.topology_id, instance,
- args.stmgr_port,
args.metricsmgr_port,
- args.topology_pex)
+ Log.info(f"\nStarting instance: {instance_id} for topology: {topology_name}"
+ f" and topologyId: {topology_id} for component: {component_name}"
+ f" with taskId: {task_id} and componentIndex: {component_index}"
+ f" and stmgrId: {stmgr_id} and stmgrPort: {stmgr_port}"
+ f" and metricsManagerPort: {metricsmgr_port}"
+ f"\n **Topology Pex file located at: {topology_pex}")
+ Log.debug(f"System config: {sys_config}")
+ Log.debug(f"Override config: {override_config}")
+ Log.debug(f"Maximum RAM: {max_ram}")
+
+ heron_instance = SingleThreadHeronInstance(topology_name, topology_id,
instance,
+ stmgr_port, metricsmgr_port,
+ topology_pex)
heron_instance.start()
if __name__ == '__main__':
- main()
+ cli() # pylint: disable=no-value-for-parameter
diff --git a/heron/instance/src/python/instance/__init__.py
b/heron/instance/src/python/instance/__init__.py
deleted file mode 100644
index b80a7bf..0000000
--- a/heron/instance/src/python/instance/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-'''single thread heron instance module'''
-__all__ = ['st_heron_instance.py', 'st_stmgr_client.py']
-
-from .st_heron_instance import SingleThreadHeronInstance
