[
https://issues.apache.org/jira/browse/BEAM-5194?focusedWorklogId=137529&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-137529
]
ASF GitHub Bot logged work on BEAM-5194:
----------------------------------------
Author: ASF GitHub Bot
Created on: 23/Aug/18 18:49
Start Date: 23/Aug/18 18:49
Worklog Time Spent: 10m
Work Description: pabloem closed pull request #6262: [BEAM-5194] Fix
Pipeline options with multi value deserialization from map
URL: https://github.com/apache/beam/pull/6262
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/options/pipeline_options.py
b/sdks/python/apache_beam/options/pipeline_options.py
index 2efbbdd3375..b0453864227 100644
--- a/sdks/python/apache_beam/options/pipeline_options.py
+++ b/sdks/python/apache_beam/options/pipeline_options.py
@@ -20,6 +20,7 @@
from __future__ import absolute_import
import argparse
+from builtins import list
from builtins import object
from apache_beam.options.value_provider import RuntimeValueProvider
@@ -182,6 +183,9 @@ def from_dictionary(cls, options):
if isinstance(v, bool):
if v:
flags.append('--%s' % k)
+ elif isinstance(v, list):
+ for i in v:
+ flags.append('--%s=%s' % (k, i))
else:
flags.append('--%s=%s' % (k, v))
diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py
b/sdks/python/apache_beam/options/pipeline_options_test.py
index 289e67f15fe..55e8ee88daf 100644
--- a/sdks/python/apache_beam/options/pipeline_options_test.py
+++ b/sdks/python/apache_beam/options/pipeline_options_test.py
@@ -38,14 +38,18 @@ def tearDown(self):
TEST_CASES = [
{'flags': ['--num_workers', '5'],
- 'expected': {'num_workers': 5, 'mock_flag': False, 'mock_option': None},
+ 'expected': {'num_workers': 5,
+ 'mock_flag': False,
+ 'mock_option': None,
+ 'mock_multi_option': None},
'display_data': [DisplayDataItemMatcher('num_workers', 5)]},
{
'flags': [
'--profile_cpu', '--profile_location', 'gs://bucket/',
'ignored'],
'expected': {
'profile_cpu': True, 'profile_location': 'gs://bucket/',
- 'mock_flag': False, 'mock_option': None},
+ 'mock_flag': False, 'mock_option': None,
+ 'mock_multi_option': None},
'display_data': [
DisplayDataItemMatcher('profile_cpu',
True),
@@ -53,32 +57,52 @@ def tearDown(self):
'gs://bucket/')]
},
{'flags': ['--num_workers', '5', '--mock_flag'],
- 'expected': {'num_workers': 5, 'mock_flag': True, 'mock_option': None},
+ 'expected': {'num_workers': 5,
+ 'mock_flag': True,
+ 'mock_option': None,
+ 'mock_multi_option': None},
'display_data': [
DisplayDataItemMatcher('num_workers', 5),
DisplayDataItemMatcher('mock_flag', True)]
},
{'flags': ['--mock_option', 'abc'],
- 'expected': {'mock_flag': False, 'mock_option': 'abc'},
+ 'expected': {'mock_flag': False,
+ 'mock_option': 'abc',
+ 'mock_multi_option': None},
'display_data': [
DisplayDataItemMatcher('mock_option', 'abc')]
},
{'flags': ['--mock_option', ' abc def '],
- 'expected': {'mock_flag': False, 'mock_option': ' abc def '},
+ 'expected': {'mock_flag': False,
+ 'mock_option': ' abc def ',
+ 'mock_multi_option': None},
'display_data': [
DisplayDataItemMatcher('mock_option', ' abc def ')]
},
{'flags': ['--mock_option= abc xyz '],
- 'expected': {'mock_flag': False, 'mock_option': ' abc xyz '},
+ 'expected': {'mock_flag': False,
+ 'mock_option': ' abc xyz ',
+ 'mock_multi_option': None},
'display_data': [
DisplayDataItemMatcher('mock_option', ' abc xyz ')]
},
- {'flags': ['--mock_option=gs://my bucket/my folder/my file'],
+ {'flags': ['--mock_option=gs://my bucket/my folder/my file',
+ '--mock_multi_option=op1',
+ '--mock_multi_option=op2'],
'expected': {'mock_flag': False,
- 'mock_option': 'gs://my bucket/my folder/my file'},
+ 'mock_option': 'gs://my bucket/my folder/my file',
+ 'mock_multi_option': ['op1', 'op2']},
'display_data': [
DisplayDataItemMatcher(
- 'mock_option', 'gs://my bucket/my folder/my file')]
+ 'mock_option', 'gs://my bucket/my folder/my file'),
+ DisplayDataItemMatcher('mock_multi_option', ['op1', 'op2'])]
+ },
+ {'flags': ['--mock_multi_option=op1', '--mock_multi_option=op2'],
+ 'expected': {'mock_flag': False,
+ 'mock_option': None,
+ 'mock_multi_option': ['op1', 'op2']},
+ 'display_data': [
+ DisplayDataItemMatcher('mock_multi_option', ['op1', 'op2'])]
},
]
@@ -89,6 +113,8 @@ class MockOptions(PipelineOptions):
def _add_argparse_args(cls, parser):
parser.add_argument('--mock_flag', action='store_true', help='mock flag')
parser.add_argument('--mock_option', help='mock option')
+ parser.add_argument(
+ '--mock_multi_option', action='append', help='mock multi option')
parser.add_argument('--option with space', help='mock option with space')
def test_display_data(self):
@@ -107,6 +133,9 @@ def test_get_all_options(self):
self.assertEqual(options.view_as(
PipelineOptionsTest.MockOptions).mock_option,
case['expected']['mock_option'])
+ self.assertEqual(options.view_as(
+ PipelineOptionsTest.MockOptions).mock_multi_option,
+ case['expected']['mock_multi_option'])
def test_from_dictionary(self):
for case in PipelineOptionsTest.TEST_CASES:
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 137529)
Time Spent: 40m (was: 0.5h)
> Pipeline options with multi value are not deserialized correctly from map
> -------------------------------------------------------------------------
>
> Key: BEAM-5194
> URL: https://issues.apache.org/jira/browse/BEAM-5194
> Project: Beam
> Issue Type: Bug
> Components: sdk-py-core
> Reporter: Ankur Goenka
> Assignee: Ankur Goenka
> Priority: Major
> Time Spent: 40m
> Remaining Estimate: 0h
>
> [https://github.com/apache/beam/blob/7c41e0a915083bd3b1fe52c2a417fa38a00e6463/sdks/python/apache_beam/options/pipeline_options.py#L171]
>
> Multiple options are converted to strings and added to flags which causes
> wrong deserialization.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
