This is an automated email from the ASF dual-hosted git repository. udim pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 9188bbf [BEAM-3713] Cleanup, remove nosetest references (#15245) 9188bbf is described below commit 9188bbf05c4bdd0d306257d1c38c501c61398577 Author: Benjamin Gonzalez <74670721+benw...@users.noreply.github.com> AuthorDate: Wed Aug 18 15:58:26 2021 -0500 [BEAM-3713] Cleanup, remove nosetest references (#15245) * [BEAM-3713] Fix lint * [BEAM-3713] Remove --pytest param because is no longer required Co-authored-by: Udi Meiri <u...@users.noreply.github.com> --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 2 - .../cookbook/bigquery_tornadoes_it_test.py | 3 -- sdks/python/apache_beam/testing/test_pipeline.py | 4 +- .../transforms/periodicsequence_test.py | 2 - .../apache_beam/transforms/ptransform_test.py | 3 -- .../apache_beam/transforms/sideinputs_test.py | 3 -- .../transforms_keyword_only_args_test.py | 4 -- sdks/python/container/base_image_requirements.txt | 2 - sdks/python/scripts/run_integration_test.sh | 45 +++++++------------- sdks/python/setup.py | 7 ---- sdks/python/test-suites/dataflow/common.gradle | 5 --- sdks/python/test-suites/direct/common.gradle | 3 -- sdks/python/test-suites/portable/common.gradle | 5 +-- sdks/python/test_config.py | 49 ---------------------- 14 files changed, 18 insertions(+), 119 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index f5a3c19..ce507db 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -2174,7 +2174,6 @@ class BeamModulePlugin implements Plugin<Project> { "pipeline_opts": config.pythonPipelineOptions + sdkLocationOpt, "test_opts": config.pytestOptions, "suite": "xlangValidateRunner", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed. "collect": config.pythonTestAttr ] def cmdArgs = project.project(':sdks:python').mapToArgString(beamPythonTestPipelineOptions) @@ -2221,7 +2220,6 @@ class BeamModulePlugin implements Plugin<Project> { "pipeline_opts": config.pythonPipelineOptions + sdkLocationOpt, "test_opts": config.pytestOptions, "suite": "xlangSqlValidateRunner", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed. "collect": "xlang_sql_expansion_service" ] def cmdArgs = project.project(':sdks:python').mapToArgString(beamPythonTestPipelineOptions) diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py index fa5f12c..8d44615 100644 --- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py +++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py @@ -35,9 +35,6 @@ from apache_beam.testing.test_pipeline import TestPipeline class BigqueryTornadoesIT(unittest.TestCase): - # Enable nose tests running in parallel - _multiprocess_can_split_ = True - # The default checksum is a SHA-1 hash generated from sorted rows reading # from expected Bigquery table. DEFAULT_CHECKSUM = 'd860e636050c559a16a791aff40d6ad809d4daf0' diff --git a/sdks/python/apache_beam/testing/test_pipeline.py b/sdks/python/apache_beam/testing/test_pipeline.py index 2ba273e..910f149 100644 --- a/sdks/python/apache_beam/testing/test_pipeline.py +++ b/sdks/python/apache_beam/testing/test_pipeline.py @@ -40,7 +40,7 @@ class TestPipeline(Pipeline): It has a functionality to parse arguments from command line and build pipeline options for tests who runs against a pipeline runner and utilizes resources of the pipeline runner. Those test functions are recommended to be tagged by - ``@attr("ValidatesRunner")`` annotation. + ``@pytest.mark.it_validatesrunner`` annotation. In order to configure the test with customized pipeline options from command line, system argument ``--test-pipeline-options`` can be used to obtains a @@ -48,7 +48,7 @@ class TestPipeline(Pipeline): For example, use following command line to execute all ValidatesRunner tests:: - python setup.py nosetests -a ValidatesRunner \\ + pytest -m it_validatesrunner \\ --test-pipeline-options="--runner=DirectRunner \\ --job_name=myJobName \\ --num_workers=1" diff --git a/sdks/python/apache_beam/transforms/periodicsequence_test.py b/sdks/python/apache_beam/transforms/periodicsequence_test.py index 7e6549e..b18bf75 100644 --- a/sdks/python/apache_beam/transforms/periodicsequence_test.py +++ b/sdks/python/apache_beam/transforms/periodicsequence_test.py @@ -35,8 +35,6 @@ from apache_beam.transforms.periodicsequence import PeriodicSequence class PeriodicSequenceTest(unittest.TestCase): - # Enable nose tests running in parallel - def test_periodicsequence_outputs_valid_sequence(self): start_offset = 1 start_time = time.time() + start_offset diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py index beb7a62..ec001e2 100644 --- a/sdks/python/apache_beam/transforms/ptransform_test.py +++ b/sdks/python/apache_beam/transforms/ptransform_test.py @@ -67,9 +67,6 @@ from apache_beam.utils.windowed_value import WindowedValue class PTransformTest(unittest.TestCase): - # Enable nose tests running in parallel - _multiprocess_can_split_ = True - def assertStartswith(self, msg, prefix): self.assertTrue( msg.startswith(prefix), '"%s" does not start with "%s"' % (msg, prefix)) diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py index 391ecdc..29a8845 100644 --- a/sdks/python/apache_beam/transforms/sideinputs_test.py +++ b/sdks/python/apache_beam/transforms/sideinputs_test.py @@ -38,9 +38,6 @@ from apache_beam.utils.timestamp import Timestamp class SideInputsTest(unittest.TestCase): - # Enable nose tests running in parallel - _multiprocess_can_split_ = True - def create_pipeline(self): return TestPipeline() diff --git a/sdks/python/apache_beam/transforms/transforms_keyword_only_args_test.py b/sdks/python/apache_beam/transforms/transforms_keyword_only_args_test.py index 80e0aff..28566ba 100644 --- a/sdks/python/apache_beam/transforms/transforms_keyword_only_args_test.py +++ b/sdks/python/apache_beam/transforms/transforms_keyword_only_args_test.py @@ -29,10 +29,6 @@ from apache_beam.testing.util import equal_to class KeywordOnlyArgsTests(unittest.TestCase): - - # Enable nose tests running in parallel - _multiprocess_can_split_ = True - def test_side_input_keyword_only_args(self): with TestPipeline() as pipeline: diff --git a/sdks/python/container/base_image_requirements.txt b/sdks/python/container/base_image_requirements.txt index c627b4e..6bfebfb 100644 --- a/sdks/python/container/base_image_requirements.txt +++ b/sdks/python/container/base_image_requirements.txt @@ -87,5 +87,3 @@ tensorflow==2.5.0 tenacity>=5.0.2 pyhamcrest<2.0,>=1.9 pytest==4.6.11 -# TODO(BEAM-3713): Remove once nose is no longer in use. -nose==1.3.7 diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh index c5baa89..b477e91 100755 --- a/sdks/python/scripts/run_integration_test.sh +++ b/sdks/python/scripts/run_integration_test.sh @@ -45,10 +45,10 @@ # using this flag. # # Test related flags: -# test_opts -> List of space separated options to configure Nose test -# during execution. Commonly used options like `--attr`, -# `--tests`, `--nologcapture`. More can be found in -# https://nose.readthedocs.io/en/latest/man.html#options +# test_opts -> List of space separated options to configure Pytest test +# during execution. Commonly used options like `--capture=no` +# `--collect-only`. More can be found in +# https://docs.pytest.org/en/latest/reference.html#command-line-flags # suite -> Namespace for this run of tests. Required if running # under Jenkins. Used to differentiate runs of the same # tests with different interpreters/dependencies/etc. @@ -58,7 +58,7 @@ # `$ ./run_integration_test.sh` # # - Run single integration test with default pipeline options: -# `$ ./run_integration_test.sh --test_opts --tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it` +# `$ ./run_integration_test.sh --test_opts apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it` # # - Run full set of PostCommit tests with customized pipeline options: # `$ ./run_integration_test.sh --project my-project --gcs_location gs://my-location` @@ -80,10 +80,10 @@ KMS_KEY_NAME="projects/apache-beam-testing/locations/global/keyRings/beam-it/cry SUITE="" COLLECT_MARKERS= -# Default test (nose) options. +# Default test (pytest) options. # Run WordCountIT.test_wordcount_it by default if no test options are # provided. -TEST_OPTS="--tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it --nocapture" +TEST_OPTS="apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it" while [[ $# -gt 0 ]] do @@ -164,11 +164,6 @@ case $key in shift # past argument shift # past value ;; - --pytest) - PYTEST="$2" - shift # past argument - shift # past value - ;; --collect) COLLECT_MARKERS="-m=$2" shift # past argument @@ -185,7 +180,6 @@ if [[ "$JENKINS_HOME" != "" && "$SUITE" == "" ]]; then echo "Argument --suite is required in a Jenkins environment." exit 1 fi -XUNIT_FILE="nosetests-$SUITE.xml" set -o errexit @@ -281,23 +275,12 @@ fi # Run tests and validate that jobs finish successfully. echo ">>> RUNNING integration tests with pipeline options: $PIPELINE_OPTS" -if [[ "$PYTEST" = true ]]; then - echo ">>> pytest options: $TEST_OPTS" - echo ">>> collect markers: $COLLECT_MARKERS" - ARGS="-o junit_suite_name=$SUITE --junitxml=pytest_$SUITE.xml $TEST_OPTS" - # Handle markers as an independient argument from $TEST_OPTS to prevent errors in space separeted flags - if [ -z "$COLLECT_MARKERS" ]; then - pytest $ARGS --test-pipeline-options="$PIPELINE_OPTS" - else - pytest $ARGS --test-pipeline-options="$PIPELINE_OPTS" "$COLLECT_MARKERS" - fi +echo ">>> pytest options: $TEST_OPTS" +echo ">>> collect markers: $COLLECT_MARKERS" +ARGS="-o junit_suite_name=$SUITE --junitxml=pytest_$SUITE.xml $TEST_OPTS" +# Handle markers as an independient argument from $TEST_OPTS to prevent errors in space separeted flags +if [ -z "$COLLECT_MARKERS" ]; then + pytest $ARGS --test-pipeline-options="$PIPELINE_OPTS" else - echo ">>> test options: $TEST_OPTS" - # TODO(BEAM-3713): Pass $SUITE once migrated to pytest. xunitmp doesn't - # support suite names. - python setup.py nosetests \ - --test-pipeline-options="$PIPELINE_OPTS" \ - --with-xunitmp --xunitmp-file=$XUNIT_FILE \ - --ignore-files '.*py3\d?\.py$' \ - $TEST_OPTS + pytest $ARGS --test-pipeline-options="$PIPELINE_OPTS" "$COLLECT_MARKERS" fi \ No newline at end of file diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 92d001a..5d72a9d 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -166,8 +166,6 @@ if sys.platform == 'win32' and sys.maxsize <= 2**32: REQUIRED_TEST_PACKAGES = [ 'freezegun>=0.3.12', 'mock>=1.0.1,<3.0.0', - 'nose>=1.3.7', - 'nose_xunitmp>=0.4.1', 'pandas>=1.0,<1.4.0', 'parameterized>=0.7.1,<0.8.0', 'pyhamcrest>=1.9,!=1.10.0,<2.0.0', @@ -290,7 +288,6 @@ setuptools.setup( ]), install_requires=REQUIRED_PACKAGES, python_requires=python_requires, - test_suite='nose.collector', # BEAM-8840: Do NOT use tests_require or setup_requires. extras_require={ 'docs': ['Sphinx>=1.5.2,<2.0'], @@ -317,10 +314,6 @@ setuptools.setup( ], license='Apache License, Version 2.0', keywords=PACKAGE_KEYWORDS, - entry_points={ - 'nose.plugins.0.10': [ - 'beam_test_plugin = test_config:BeamTestPlugin', - ]}, cmdclass={ 'build_py': generate_protos_first(build_py), 'develop': generate_protos_first(develop), diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 7d1cf2b..29373b5 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -65,7 +65,6 @@ def preCommitIT(String runScriptsDir, String envdir, Boolean streaming, Boolean "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar" : dataflowWorkerJar, "suite" : "preCommitIT-df${pythonSuffix}", - "pytest" : true, // TODO(BEAM-3713): Remove this once nose is removed. ] if (runnerV2) { @@ -115,7 +114,6 @@ task postCommitIT { "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "postCommitIT-df${pythonVersionSuffix}", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed. "collect": "it_postcommit" ] def cmdArgs = mapToArgString(argMap) @@ -137,7 +135,6 @@ task validatesRunnerBatchTests { "worker_jar" : dataflowWorkerJar, "sdk_location": files(configurations.distTarBall.files).singleFile, "suite" : "validatesRunnerBatchTests-df${pythonVersionSuffix}", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed. "collect": "it_validatesrunner and not no_sickbay_batch" ] @@ -172,7 +169,6 @@ task validatesRunnerStreamingTests { "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "validatesRunnerStreamingTests-df${pythonVersionSuffix}-xdist", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed "collect": "it_validatesrunner and not no_sickbay_streaming and not no_xdist" ] if (project.hasProperty('useRunnerV2')) { @@ -196,7 +192,6 @@ task validatesRunnerStreamingTests { "sdk_location": files(configurations.distTarBall.files).singleFile, "worker_jar": dataflowWorkerJar, "suite": "validatesRunnerStreamingTests-df${pythonVersionSuffix}-noxdist", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed "collect": "it_validatesrunner and not no_sickbay_streaming and no_xdist" ] if (project.hasProperty('useRunnerV2')) { diff --git a/sdks/python/test-suites/direct/common.gradle b/sdks/python/test-suites/direct/common.gradle index 50fe59a..3680bfa 100644 --- a/sdks/python/test-suites/direct/common.gradle +++ b/sdks/python/test-suites/direct/common.gradle @@ -48,7 +48,6 @@ task postCommitIT { def argMap = ["runner": "TestDirectRunner", "test_opts": testOpts, "suite": "postCommitIT-direct-py${pythonVersionSuffix}", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed. ] def batchCmdArgs = mapToArgString(argMap) exec { @@ -108,7 +107,6 @@ task directRunnerIT { def argMap = ["runner": "TestDirectRunner", "test_opts": batchTestOpts, "suite": "directRunnerIT-batch", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed. ] def batchCmdArgs = mapToArgString(argMap) exec { @@ -131,7 +129,6 @@ task directRunnerIT { "streaming": "true", "test_opts": streamingTestOpts, "suite": "directRunnerIT-streaming", - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed. ] def streamingCmdArgs = mapToArgString(argMap) exec { diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle index 20665f0..3b1654f 100644 --- a/sdks/python/test-suites/portable/common.gradle +++ b/sdks/python/test-suites/portable/common.gradle @@ -66,8 +66,8 @@ task flinkTriggerTranscript() { . ${envdir}/bin/activate \\ && cd ${pythonRootDir} \\ && pip install -e .[test] \\ - && python setup.py nosetests \\ - --tests apache_beam.transforms.trigger_test:WeakTestStreamTranscriptTest \\ + && pytest \\ + apache_beam/transforms/trigger_test.py::WeakTestStreamTranscriptTest \\ --test-pipeline-options='--runner=FlinkRunner --environment_type=LOOPBACK --flink_job_server_jar=${project(":runners:flink:${latestFlinkVersion}:job-server:").shadowJar.archivePath}' """ } @@ -220,7 +220,6 @@ project.task("postCommitPy${pythonVersionSuffix}IT") { "test_opts": testOpts, "suite": "postCommitIT-flink-py${pythonVersionSuffix}", "pipeline_opts": pipelineOpts.join(" "), - "pytest": true, // TODO(BEAM-3713): Remove this once nose is removed. ]) def kafkaJar = project(":sdks:java:testing:kafka-service:").buildTestKafkaServiceJar.archivePath exec { diff --git a/sdks/python/test_config.py b/sdks/python/test_config.py deleted file mode 100644 index d916a7c..0000000 --- a/sdks/python/test_config.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Test configurations for nose - -This module contains nose plugin hooks that configures Beam tests which -includes ValidatesRunner test and E2E integration test. - -TODO(BEAM-3713): Remove this module once nose is removed. -""" - -from nose.plugins import Plugin - - -class BeamTestPlugin(Plugin): - """A nose plugin for Beam testing that registers command line options - - This plugin is registered through setuptools in entry_points. - """ - - def options(self, parser, env): - """Add '--test-pipeline-options' and '--not_use-test-runner-api' - to command line option to avoid unrecognized option error thrown by nose. - - The value of this option will be processed by TestPipeline and used to - build customized pipeline for ValidatesRunner tests. - """ - parser.add_option('--test-pipeline-options', - action='store', - type=str, - help='providing pipeline options to run tests on runner') - parser.add_option('--not-use-test-runner-api', - action='store_true', - default=False, - help='whether not to use test-runner-api')