[
https://issues.apache.org/jira/browse/BEAM-1251?focusedWorklogId=153408&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-153408
]
ASF GitHub Bot logged work on BEAM-1251:
----------------------------------------
Author: ASF GitHub Bot
Created on: 11/Oct/18 05:40
Start Date: 11/Oct/18 05:40
Worklog Time Spent: 10m
Work Description: manuzhang closed pull request #6650: [BEAM-1251] Make
it possible to unskip Py3 tests by setting an environment variable.
URL: https://github.com/apache/beam/pull/6650
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/sdks/python/apache_beam/pipeline_test.py
b/sdks/python/apache_beam/pipeline_test.py
index 6397820e6e9..41f749d2f79 100644
--- a/sdks/python/apache_beam/pipeline_test.py
+++ b/sdks/python/apache_beam/pipeline_test.py
@@ -21,6 +21,7 @@
import copy
import logging
+import os
import platform
import sys
import unittest
@@ -391,8 +392,9 @@ def process(self, element):
assert_that(pcoll, equal_to([11, 12]))
pipeline.run()
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_side_input_no_tag(self):
class TestDoFn(DoFn):
def process(self, element, prefix, suffix):
@@ -408,8 +410,9 @@ def process(self, element, prefix, suffix):
assert_that(result, equal_to(['zyx-%s-xyz' % x for x in words_list]))
pipeline.run()
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_side_input_tagged(self):
class TestDoFn(DoFn):
def process(self, element, prefix, suffix=DoFn.SideInputParam):
diff --git
a/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py
b/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py
index b21135c8717..767e06e527f 100644
--- a/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py
+++ b/sdks/python/apache_beam/runners/interactive/interactive_runner_test.py
@@ -24,6 +24,7 @@
from __future__ import division
from __future__ import print_function
+import os
import sys
import unittest
@@ -43,8 +44,9 @@ def printer(elem):
class InteractiveRunnerTest(unittest.TestCase):
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_basic(self):
p = beam.Pipeline(
runner=interactive_runner.InteractiveRunner(
@@ -60,8 +62,9 @@ def test_basic(self):
_ = pc0 | 'Print3' >> beam.Map(print_with_message('Run3'))
p.run().wait_until_finish()
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_wordcount(self):
class WordExtractingDoFn(beam.DoFn):
diff --git
a/sdks/python/apache_beam/runners/interactive/pipeline_analyzer_test.py
b/sdks/python/apache_beam/runners/interactive/pipeline_analyzer_test.py
index f2d82cd3288..caefbe04ba6 100644
--- a/sdks/python/apache_beam/runners/interactive/pipeline_analyzer_test.py
+++ b/sdks/python/apache_beam/runners/interactive/pipeline_analyzer_test.py
@@ -24,6 +24,7 @@
from __future__ import division
from __future__ import print_function
+import os
import sys
import unittest
@@ -87,8 +88,9 @@ def assertTransformEqual(self, pipeline_proto1, transform_id1,
self.assertSetEqual(set(transform_proto1.outputs),
set(transform_proto2.outputs))
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_basic(self):
p = beam.Pipeline(runner=self.runner)
@@ -136,8 +138,9 @@ def test_basic(self):
# No need to actually execute the second run.
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_word_count(self):
p = beam.Pipeline(runner=self.runner)
@@ -220,8 +223,9 @@ def test_write_cache_expansion(self):
self.assertPipelineEqual(analyzer.pipeline_proto_to_execute(),
expected_pipeline_proto)
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_read_cache_expansion(self):
p = beam.Pipeline(runner=self.runner)
diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py
b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py
index 73557dc7c66..a7dcbf32fe9 100644
--- a/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py
+++ b/sdks/python/apache_beam/runners/portability/fn_api_runner_test.py
@@ -147,8 +147,9 @@ def even_odd(elem):
assert_that(unnamed.even, equal_to([2]), label='unnamed.even')
assert_that(unnamed.odd, equal_to([1, 3]), label='unnamed.odd')
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_pardo_side_inputs(self):
def cross_product(elem, sides):
for side in sides:
@@ -160,8 +161,9 @@ def cross_product(elem, sides):
equal_to([('a', 'x'), ('b', 'x'), ('c', 'x'),
('a', 'y'), ('b', 'y'), ('c', 'y')]))
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_pardo_windowed_side_inputs(self):
with self.create_pipeline() as p:
# Now with some windowing.
@@ -189,8 +191,9 @@ def test_pardo_windowed_side_inputs(self):
(9, list(range(7, 10)))]),
label='windowed')
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_flattened_side_input(self):
with self.create_pipeline() as p:
main = p | 'main' >> beam.Create([None])
@@ -201,8 +204,9 @@ def test_flattened_side_input(self):
main | beam.Map(lambda a, b: (a, b), beam.pvalue.AsDict(side)),
equal_to([(None, {'a': 1, 'b': 2})]))
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_gbk_side_input(self):
with self.create_pipeline() as p:
main = p | 'main' >> beam.Create([None])
@@ -211,8 +215,9 @@ def test_gbk_side_input(self):
main | beam.Map(lambda a, b: (a, b), beam.pvalue.AsDict(side)),
equal_to([(None, {'a': [1]})]))
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_multimap_side_input(self):
with self.create_pipeline() as p:
main = p | 'main' >> beam.Create(['a', 'b'])
@@ -224,8 +229,9 @@ def test_multimap_side_input(self):
beam.pvalue.AsMultiMap(side)),
equal_to([('a', [1, 3]), ('b', [2])]))
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_pardo_unfusable_side_inputs(self):
def cross_product(elem, sides):
for side in sides:
@@ -352,8 +358,9 @@ def test_windowing(self):
| beam.Map(lambda k_vs1: (k_vs1[0], sorted(k_vs1[1]))))
assert_that(res, equal_to([('k', [1, 2]), ('k', [100, 101, 102])]))
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_large_elements(self):
with self.create_pipeline() as p:
big = (p
diff --git a/sdks/python/apache_beam/runners/worker/opcounters_test.py
b/sdks/python/apache_beam/runners/worker/opcounters_test.py
index 3e8a7e99333..d45213f3616 100644
--- a/sdks/python/apache_beam/runners/worker/opcounters_test.py
+++ b/sdks/python/apache_beam/runners/worker/opcounters_test.py
@@ -20,6 +20,7 @@
import logging
import math
+import os
import random
import sys
import unittest
@@ -161,8 +162,9 @@ def test_update_multiple(self):
total_size += coder.estimate_size(value)
self.verify_counters(opcounts, 3, (float(total_size) / 3))
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_should_sample(self):
# Order of magnitude more buckets than highest constant in code under test.
buckets = [0] * 300
diff --git
a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py
b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py
index 907ce98ec97..2b84dca8d01 100644
--- a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py
+++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py
@@ -19,6 +19,7 @@
from __future__ import absolute_import
+import os
import sys
import typing
import unittest
@@ -37,8 +38,9 @@ class _TestClass(object):
class NativeTypeCompatibilityTest(unittest.TestCase):
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_convert_to_beam_type(self):
test_cases = [
('raw bytes', bytes, bytes),
@@ -80,8 +82,9 @@ def test_convert_to_beam_type(self):
native_type_compatibility.convert_to_beam_type(typing_type),
beam_type, description)
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_convert_to_beam_types(self):
typing_types = [bytes, typing.List[bytes],
typing.List[typing.Tuple[bytes, int]],
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py
b/sdks/python/apache_beam/typehints/trivial_inference_test.py
index 1c6ebd64af6..e4a22fa53d6 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference_test.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py
@@ -19,6 +19,7 @@
from __future__ import absolute_import
+import os
import sys
import unittest
@@ -107,8 +108,9 @@ def testListComprehension(self):
lambda xs: [x for x in xs],
[typehints.Tuple[int, ...]])
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def testTupleListComprehension(self):
self.assertReturnType(
typehints.List[int],
diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py
b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
index fa98075fe26..22f58c09776 100644
--- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py
+++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
@@ -20,6 +20,7 @@
from __future__ import absolute_import
import inspect
+import os
import sys
import typing
import unittest
@@ -47,8 +48,9 @@ def repeat(s, times):
with self.assertRaises(typehints.TypeCheckError):
[1, 2, 3] | beam.Map(repeat, 3)
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_non_function(self):
result = ['a', 'bb', 'c'] | beam.Map(str.upper)
self.assertEqual(['A', 'BB', 'C'], sorted(result))
@@ -105,8 +107,9 @@ def process(self, element):
[1, 2, 3] | (beam.ParDo(my_do_fn) | 'again' >> beam.ParDo(my_do_fn))
-@unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+@unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
class NativeTypesTest(unittest.TestCase):
def test_good_main_input(self):
@@ -201,8 +204,9 @@ def repeat(s, *times):
# with self.assertRaises(typehints.TypeCheckError):
# ['a', 'bb', 'c'] | beam.Map(repeat, 'z')
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_deferred_side_inputs(self):
@typehints.with_input_types(str, int)
def repeat(s, times):
@@ -217,8 +221,9 @@ def repeat(s, times):
with self.assertRaises(typehints.TypeCheckError):
main_input | 'bis' >> beam.Map(repeat,
pvalue.AsSingleton(bad_side_input))
- @unittest.skipIf(sys.version_info[0] == 3, 'This test still needs to be '
- 'fixed on Python 3')
+ @unittest.skipIf(sys.version_info[0] == 3 and
+ os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+ 'This test still needs to be fixed on Python 3.')
def test_deferred_side_input_iterable(self):
@typehints.with_input_types(str, typehints.Iterable[str])
def concat(glue, items):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Issue Time Tracking
-------------------
Worklog Id: (was: 153408)
Time Spent: 22h (was: 21h 50m)
> Python 3 Support
> ----------------
>
> Key: BEAM-1251
> URL: https://issues.apache.org/jira/browse/BEAM-1251
> Project: Beam
> Issue Type: Improvement
> Components: sdk-py-core
> Reporter: Eyad Sibai
> Assignee: Robbe
> Priority: Major
> Time Spent: 22h
> Remaining Estimate: 0h
>
> I have been trying to use google datalab with python3. As I see there are
> several packages that does not support python3 yet which google datalab
> depends on. This is one of them.
> https://github.com/GoogleCloudPlatform/DataflowPythonSDK/issues/6
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
