ARIA-215 Refactor plugin-related code into PluginManager Refactored plugin-related code from ProcessExecutor into PluginManager. Additionally, renamed plugin_prefix to plugin_dir in PluginManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/commit/3e1ed14c Tree: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/tree/3e1ed14c Diff: http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/diff/3e1ed14c Branch: refs/heads/ARIA-140-version-utils Commit: 3e1ed14c00ea2c83fafdca8ec0e37817bea1d5e8 Parents: 45c158e Author: Ran Ziv <r...@gigaspaces.com> Authored: Sun May 7 16:36:39 2017 +0300 Committer: Ran Ziv <r...@gigaspaces.com> Committed: Sun May 7 17:27:36 2017 +0300 ---------------------------------------------------------------------- aria/orchestrator/plugin.py | 36 +++++++++++- aria/orchestrator/workflows/executor/process.py | 60 ++++++-------------- aria/utils/process.py | 47 +++++++++++++++ tests/utils/test_plugin.py | 6 +- 4 files changed, 100 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/orchestrator/plugin.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/plugin.py b/aria/orchestrator/plugin.py index f99666c..8fbcf5a 100644 --- a/aria/orchestrator/plugin.py +++ b/aria/orchestrator/plugin.py @@ -23,6 +23,9 @@ from datetime import datetime import wagon from . import exceptions +from ..utils import process as process_utils + +_IS_WIN = os.name == 'nt' class PluginManager(object): @@ -62,11 +65,40 @@ class PluginManager(object): raise exceptions.PluginAlreadyExistsError( 'Plugin {0}, version {1} already exists'.format(plugin.package_name, plugin.package_version)) - self._install_wagon(source=source, prefix=self.get_plugin_prefix(plugin)) + self._install_wagon(source=source, prefix=self.get_plugin_dir(plugin)) self._model.plugin.put(plugin) return plugin - def get_plugin_prefix(self, plugin): + def load_plugin(self, plugin, env=None): + """ + Load the plugin into an environment. + Loading the plugin means the plugin's code and binaries paths will be appended to the + environment's PATH and PYTHONPATH, thereby allowing usage of the plugin. + :param plugin: The plugin to load + :param env: The environment to load the plugin into; If `None`, os.environ will be used. + """ + env = env or os.environ + plugin_dir = self.get_plugin_dir(plugin) + + # Update PATH environment variable to include plugin's bin dir + bin_dir = 'Scripts' if _IS_WIN else 'bin' + process_utils.append_to_path(os.path.join(plugin_dir, bin_dir), env=env) + + # Update PYTHONPATH environment variable to include plugin's site-packages + # directories + if _IS_WIN: + pythonpath_dirs = [os.path.join(plugin_dir, 'Lib', 'site-packages')] + else: + # In some linux environments, there will be both a lib and a lib64 directory + # with the latter, containing compiled packages. + pythonpath_dirs = [os.path.join( + plugin_dir, 'lib{0}'.format(b), + 'python{0}.{1}'.format(sys.version_info[0], sys.version_info[1]), + 'site-packages') for b in ('', '64')] + + process_utils.append_to_pythonpath(*pythonpath_dirs, env=env) + + def get_plugin_dir(self, plugin): return os.path.join( self._plugins_dir, '{0}-{1}'.format(plugin.package_name, plugin.package_version)) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/orchestrator/workflows/executor/process.py ---------------------------------------------------------------------- diff --git a/aria/orchestrator/workflows/executor/process.py b/aria/orchestrator/workflows/executor/process.py index 8481406..f3daf04 100644 --- a/aria/orchestrator/workflows/executor/process.py +++ b/aria/orchestrator/workflows/executor/process.py @@ -47,13 +47,12 @@ from aria.storage import instrumentation from aria.extension import process_executor from aria.utils import ( imports, - exceptions + exceptions, + process as process_utils ) from aria.modeling import types as modeling_types -_IS_WIN = os.name == 'nt' - _INT_FMT = 'I' _INT_SIZE = struct.calcsize(_INT_FMT) UPDATE_TRACKED_CHANGES_FAILED_STR = \ @@ -127,13 +126,7 @@ class ProcessExecutor(base.BaseExecutor): with open(arguments_json_path, 'wb') as f: f.write(pickle.dumps(self._create_arguments_dict(task))) - env = os.environ.copy() - # See _update_env for plugin_prefix usage - if task.plugin_fk and self._plugin_manager: - plugin_prefix = self._plugin_manager.get_plugin_prefix(task.plugin) - else: - plugin_prefix = None - self._update_env(env=env, plugin_prefix=plugin_prefix) + env = self._construct_subprocess_env(task=task) # Asynchronously start the operation in a subprocess subprocess.Popen( '{0} {1} {2}'.format(sys.executable, __file__, arguments_json_path), @@ -156,40 +149,19 @@ class ProcessExecutor(base.BaseExecutor): 'context': task.context.serialization_dict, } - def _update_env(self, env, plugin_prefix): - pythonpath_dirs = [] - # If this is a plugin operation, plugin prefix will point to where - # This plugin is installed. - # We update the environment variables that the subprocess will be started with based on it - if plugin_prefix: - - # Update PATH environment variable to include plugin's bin dir - bin_dir = 'Scripts' if _IS_WIN else 'bin' - env['PATH'] = '{0}{1}{2}'.format( - os.path.join(plugin_prefix, bin_dir), - os.pathsep, - env.get('PATH', '')) - - # Update PYTHONPATH environment variable to include plugin's site-packages - # directories - if _IS_WIN: - pythonpath_dirs = [os.path.join(plugin_prefix, 'Lib', 'site-packages')] - else: - # In some linux environments, there will be both a lib and a lib64 directory - # with the latter, containing compiled packages. - pythonpath_dirs = [os.path.join( - plugin_prefix, 'lib{0}'.format(b), - 'python{0}.{1}'.format(sys.version_info[0], sys.version_info[1]), - 'site-packages') for b in ('', '64')] - - # Add used supplied directories to injected PYTHONPATH - pythonpath_dirs.extend(self._python_path) - - if pythonpath_dirs: - env['PYTHONPATH'] = '{0}{1}{2}'.format( - os.pathsep.join(pythonpath_dirs), - os.pathsep, - env.get('PYTHONPATH', '')) + def _construct_subprocess_env(self, task): + env = os.environ.copy() + + if task.plugin_fk and self._plugin_manager: + # If this is a plugin operation, + # load the plugin on the subprocess env we're constructing + self._plugin_manager.load_plugin(task.plugin, env=env) + + # Add user supplied directories to injected PYTHONPATH + if self._python_path: + process_utils.append_to_pythonpath(*self._python_path, env=env) + + return env def _listener(self): # Notify __init__ method this thread has actually started http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/aria/utils/process.py ---------------------------------------------------------------------- diff --git a/aria/utils/process.py b/aria/utils/process.py new file mode 100644 index 0000000..9aeae67 --- /dev/null +++ b/aria/utils/process.py @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def append_to_path(*args, **kwargs): + """ + Appends one or more paths to the system path of an environment. + The environment will be that of the current process unless another is passed using the + 'env' keyword argument. + :param args: paths to append + :param kwargs: 'env' may be used to pass a custom environment to use + """ + _append_to_path('PATH', *args, **kwargs) + + +def append_to_pythonpath(*args, **kwargs): + """ + Appends one or more paths to the python path of an environment. + The environment will be that of the current process unless another is passed using the + 'env' keyword argument. + :param args: paths to append + :param kwargs: 'env' may be used to pass a custom environment to use + """ + _append_to_path('PYTHONPATH', *args, **kwargs) + + +def _append_to_path(path, *args, **kwargs): + env = kwargs.get('env') or os.environ + env[path] = '{0}{1}{2}'.format( + os.pathsep.join(args), + os.pathsep, + env.get(path, '') + ) http://git-wip-us.apache.org/repos/asf/incubator-ariatosca/blob/3e1ed14c/tests/utils/test_plugin.py ---------------------------------------------------------------------- diff --git a/tests/utils/test_plugin.py b/tests/utils/test_plugin.py index 3350247..c91d0c9 100644 --- a/tests/utils/test_plugin.py +++ b/tests/utils/test_plugin.py @@ -38,9 +38,9 @@ class TestPluginManager(object): assert plugin.package_name == PACKAGE_NAME assert plugin.package_version == PACKAGE_VERSION assert plugin == model.plugin.get(plugin.id) - plugin_prefix = os.path.join(plugins_dir, '{0}-{1}'.format(PACKAGE_NAME, PACKAGE_VERSION)) - assert os.path.isdir(plugin_prefix) - assert plugin_prefix == plugin_manager.get_plugin_prefix(plugin) + plugin_dir = os.path.join(plugins_dir, '{0}-{1}'.format(PACKAGE_NAME, PACKAGE_VERSION)) + assert os.path.isdir(plugin_dir) + assert plugin_dir == plugin_manager.get_plugin_dir(plugin) def test_install_already_exits(self, plugin_manager, mock_plugin): plugin_manager.install(mock_plugin)
