Repository: beam Updated Branches: refs/heads/master e2ee59557 -> 3785b5baf
[BEAM-1585] Fix the beam plugins installation Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/4c488152 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/4c488152 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/4c488152 Branch: refs/heads/master Commit: 4c488152c45ac7a6c344a21b67c968b97bf5066c Parents: e2ee595 Author: Sourabh Bajaj <sourabhba...@google.com> Authored: Thu Jun 22 08:33:31 2017 -0700 Committer: Ahmet Altay <al...@google.com> Committed: Thu Jun 22 10:46:50 2017 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/options/pipeline_options.py | 8 ++++++-- sdks/python/apache_beam/runners/dataflow/dataflow_runner.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/4c488152/sdks/python/apache_beam/options/pipeline_options.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index dab8ff2..ea996a3 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -552,13 +552,17 @@ class SetupOptions(PipelineOptions): 'worker will install the resulting package before running any custom ' 'code.')) parser.add_argument( - '--beam_plugins', + '--beam_plugin', '--beam_plugin', + dest='beam_plugins', + action='append', default=None, help= ('Bootstrap the python process before executing any code by importing ' 'all the plugins used in the pipeline. Please pass a comma separated' 'list of import paths to be included. This is currently an ' - 'experimental flag and provides no stability.')) + 'experimental flag and provides no stability. Multiple ' + '--beam_plugin options can be specified if more than one plugin ' + 'is needed.')) parser.add_argument( '--save_main_session', default=False, http://git-wip-us.apache.org/repos/asf/beam/blob/4c488152/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index 9395f16..f213b3b 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -247,7 +247,7 @@ class DataflowRunner(PipelineRunner): setup_options = pipeline._options.view_as(SetupOptions) plugins = BeamPlugin.get_all_plugin_paths() if setup_options.beam_plugins is not None: - plugins = list(set(plugins + setup_options.beam_plugins.split(','))) + plugins = list(set(plugins + setup_options.beam_plugins)) setup_options.beam_plugins = plugins self.job = apiclient.Job(pipeline._options)