This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 15d0341 [BEAM-5878] add tests for kwonly args in python 3 (#8505)
15d0341 is described below
commit 15d0341f847a708843a233623ec4195ba69f65d2
Author: Juta <[email protected]>
AuthorDate: Fri May 31 23:10:23 2019 +0200
[BEAM-5878] add tests for kwonly args in python 3 (#8505)
* [BEAM-5878] add tests for kwonly args in python 3
* [BEAM-5878] skip python3 sytntax tests in pylint and tox on python 2
---
.../transforms_keyword_only_args_py3_only_test.py | 106 +++++++++++++++++++++
sdks/python/scripts/run_pylint.sh | 16 +++-
sdks/python/tox.ini | 6 +-
3 files changed, 122 insertions(+), 6 deletions(-)
diff --git
a/sdks/python/apache_beam/transforms/transforms_keyword_only_args_py3_only_test.py
b/sdks/python/apache_beam/transforms/transforms_keyword_only_args_py3_only_test.py
new file mode 100644
index 0000000..a27a6f1
--- /dev/null
+++
b/sdks/python/apache_beam/transforms/transforms_keyword_only_args_py3_only_test.py
@@ -0,0 +1,106 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unit tests for side inputs."""
+
+from __future__ import absolute_import
+
+import logging
+import unittest
+
+import apache_beam as beam
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+
+
[email protected]('TODO BEAM-5878: support kwonly args in python 3')
+class KeywordOnlyArgsTests(unittest.TestCase):
+ # Enable nose tests running in parallel
+ _multiprocess_can_split_ = True
+
+ def test_side_input_keyword_only_args(self):
+ pipeline = TestPipeline()
+
+ def sort_with_side_inputs(x, *s, reverse=False):
+ for y in s:
+ yield sorted([x] + y, reverse=reverse)
+
+ pcol = pipeline | 'start' >> beam.Create([1, 2])
+ side = pipeline | 'side' >> beam.Create([3, 4]) # 2 values in side input.
+ result1 = pcol | 'compute1' >> beam.FlatMap(
+ sort_with_side_inputs,
+ beam.pvalue.AsList(side), reverse=True)
+ assert_that(result1, equal_to([[4,3,1], [4,3,2]]), label='assert1')
+
+ result2 = pcol | 'compute2' >> beam.FlatMap(
+ sort_with_side_inputs,
+ beam.pvalue.AsIter(side))
+ assert_that(result2, equal_to([[1,3,4], [2,3,4]]), label='assert2')
+
+ result3 = pcol | 'compute3' >> beam.FlatMap(
+ sort_with_side_inputs)
+ assert_that(result3, equal_to([]), label='assert3')
+
+ result4 = pcol | 'compute4' >> beam.FlatMap(
+ sort_with_side_inputs, reverse=True)
+ assert_that(result4, equal_to([]), label='assert4')
+
+ pipeline.run()
+
+ def test_combine_keyword_only_args(self):
+ pipeline = TestPipeline()
+
+ def bounded_sum(values, *s, bound=500):
+ return min(sum(values) + sum(s), bound)
+
+ pcoll = pipeline | 'start' >> beam.Create([6, 3, 1])
+ result1 = pcoll | 'sum1' >> beam.CombineGlobally(bounded_sum, 5, 8,
bound=20)
+ result2 = pcoll | 'sum2' >> beam.CombineGlobally(bounded_sum, 5, 8)
+ result3 = pcoll | 'sum3' >> beam.CombineGlobally(bounded_sum)
+ result4 = pcoll | 'sum4' >> beam.CombineGlobally(bounded_sum, bound=5)
+
+ assert_that(result1, equal_to([20]), label='assert1')
+ assert_that(result2, equal_to([23]), label='assert2')
+ assert_that(result3, equal_to([10]), label='assert3')
+ assert_that(result4, equal_to([5]), label='assert4')
+
+ pipeline.run()
+
+ def test_do_fn_keyword_only_args(self):
+ pipeline = TestPipeline()
+
+ class MyDoFn(beam.DoFn):
+ def process(self, element, *s, bound=500):
+ return [min(sum(s) + element, bound)]
+
+ pcoll = pipeline | 'start' >> beam.Create([6, 3, 1])
+ result1 = pcoll | 'sum1' >> beam.ParDo(MyDoFn(), 5, 8, bound=15)
+ result2 = pcoll | 'sum2' >> beam.ParDo(MyDoFn(), 5, 8)
+ result3 = pcoll | 'sum3' >> beam.ParDo(MyDoFn())
+ result4 = pcoll | 'sum4' >> beam.ParDo(MyDoFn(), bound=5)
+
+ assert_that(result1, equal_to([15,15,14]), label='assert1')
+ assert_that(result2, equal_to([19,16,14]), label='assert2')
+ assert_that(result3, equal_to([6,3,1]), label='assert3')
+ assert_that(result4, equal_to([5,3,1]), label='assert4')
+ pipeline.run()
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.DEBUG)
+ unittest.main()
diff --git a/sdks/python/scripts/run_pylint.sh
b/sdks/python/scripts/run_pylint.sh
index 2814b4f..7056e06 100755
--- a/sdks/python/scripts/run_pylint.sh
+++ b/sdks/python/scripts/run_pylint.sh
@@ -60,14 +60,22 @@ EXCLUDED_GENERATED_FILES=(
apache_beam/portability/api/*pb2*.py
)
+# TODO(BEAM-7372): Remove this list once Python 2 is no longer supported and
drop _py3_only_test suffix.
+# Following files contain python 3 syntax and are excluded from pylint in
python 2
+EXCLUDED_PYTHON3_FILES=(
+.*_py3_only_test.py
+)
+
+EXCLUDED_FILES=( "${EXCLUDED_GENERATED_FILES[@]}"
"${EXCLUDED_PYTHON3_FILES[@]}" )
+
FILES_TO_IGNORE=""
-for file in "${EXCLUDED_GENERATED_FILES[@]}"; do
+for file in "${EXCLUDED_FILES[@]}"; do
if test -z "$FILES_TO_IGNORE"
then FILES_TO_IGNORE="$(basename $file)"
else FILES_TO_IGNORE="$FILES_TO_IGNORE, $(basename $file)"
fi
done
-echo "Skipping lint for generated files: $FILES_TO_IGNORE"
+echo "Skipping lint for generated and python 3 files: $FILES_TO_IGNORE"
echo "Running pylint for module $MODULE:"
pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE"
@@ -75,7 +83,9 @@ echo "Running pycodestyle for module $MODULE:"
pycodestyle ${MODULE} --exclude="$FILES_TO_IGNORE"
echo "Running flake8 for module $MODULE:"
# TODO(BEAM-3959): Add F821 (undefined names) as soon as that test passes
-flake8 ${MODULE} --count --select=E9,F822,F823 --show-source --statistics
+# flake8 exclusions are not the same syntax as pylint
+FILES_TO_IGNORE=${FILES_TO_IGNORE//\.\*/*}
+flake8 ${MODULE} --count --select=E9,F822,F823 --show-source --statistics
--exclude="$FILES_TO_IGNORE"
echo "Running isort for module $MODULE:"
# Skip files where isort is behaving weirdly
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 2a221b0..1a34fcb 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -54,7 +54,7 @@ commands_post =
[testenv:py27]
commands =
python apache_beam/examples/complete/autocomplete_test.py
- python setup.py nosetests
+ python setup.py nosetests --ignore-files=.*_py3_only_test.py
[testenv:py35]
setenv =
@@ -85,7 +85,7 @@ commands =
platform = linux2
commands =
python apache_beam/examples/complete/autocomplete_test.py
- python setup.py nosetests
+ python setup.py nosetests --ignore-files=.*_py3_only_test.py
[testenv:py35-cython]
# cython tests are only expected to work in linux (2.x and 3.x)
@@ -127,7 +127,7 @@ commands =
extras = test,gcp
commands =
python apache_beam/examples/complete/autocomplete_test.py
- python setup.py nosetests
+ python setup.py nosetests --ignore-files=.*_py3_only_test.py
# Old and new Datastore client unit tests cannot be run in the same process
# due to conflicting protobuf modules.
# TODO(BEAM-4543): Remove these separate nosetests invocations once the
