[
https://issues.apache.org/jira/browse/BEAM-4300?focusedWorklogId=102733&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-102733
]
ASF GitHub Bot logged work on BEAM-4300:
----------------------------------------
Author: ASF GitHub Bot
Created on: 17/May/18 00:07
Start Date: 17/May/18 00:07
Worklog Time Spent: 10m
Work Description: pabloem closed pull request #5372: [BEAM-4300] Fix
ValidatesRunner tests in Python: run with same mechanism as ITs
URL: https://github.com/apache/beam/pull/5372
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
index 392a93598c7..44611063633 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
@@ -20,9 +20,11 @@ import common_job_properties
// This job runs the suite of Python ValidatesRunner tests against the
// Dataflow runner.
-job('beam_PostCommit_Python_ValidatesRunner_Dataflow') {
+job('beam_PostCommit_Py_VR_Dataflow') {
description('Runs Python ValidatesRunner suite on the Dataflow runner.')
+ previousNames('beam_PostCommit_Python_ValidatesRunner_Dataflow')
+
// Set common parameters.
common_job_properties.setTopLevelMainJobProperties(delegate)
@@ -35,8 +37,12 @@ job('beam_PostCommit_Python_ValidatesRunner_Dataflow') {
'Google Cloud Dataflow Runner Python ValidatesRunner Tests',
'Run Python Dataflow ValidatesRunner')
- // Execute shell command to test Python SDK.
+ // Execute gradle task to test Python SDK.
steps {
- shell('cd ' + common_job_properties.checkoutDir + ' && bash
sdks/python/run_validatesrunner.sh')
+ gradle {
+ rootBuildScriptDir(common_job_properties.checkoutDir)
+ tasks(':beam-sdks-python:validatesRunnerTests')
+ common_job_properties.setGradleSwitches(delegate)
+ }
}
}
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
index 289e07892ec..2307371d380 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py
@@ -256,7 +256,7 @@ def _populate_requirements_cache(requirements_file,
cache_dir):
# the requirements file and will not download package dependencies.
cmd_args = [
_get_python_executable(), '-m', 'pip', 'download', '--dest', cache_dir,
- '-r', requirements_file,
+ '-r', requirements_file, '--exists-action', 'i',
# Download from PyPI source distributions.
'--no-binary', ':all:']
logging.info('Executing command: %s', cmd_args)
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index df5a09823c3..aa7d736a0d0 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -187,11 +187,20 @@ task localWordCount(dependsOn: 'installGcpTest') {
}
}
+task postCommitVRTests(dependsOn: 'installGcpTest') {
+ doLast {
+ exec {
+ executable 'sh'
+ args '-c', ". ${envdir}/bin/activate && ./run_postcommit.sh IT"
+ }
+ }
+}
+
task validatesRunnerTests(dependsOn: 'installGcpTest') {
doLast {
exec {
executable 'sh'
- args '-c', ". ${envdir}/bin/activate && ./run_postcommit.sh"
+ args '-c', ". ${envdir}/bin/activate && ./run_postcommit.sh
ValidatesRunner"
}
}
}
@@ -209,5 +218,5 @@ task postCommit() {
dependsOn "preCommit"
dependsOn "localWordCount"
dependsOn "hdfsIntegrationTest"
- dependsOn "validatesRunnerTests"
+ dependsOn "postCommitVRTests"
}
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index 582c432b7f5..d26a1c95b9b 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -19,11 +19,17 @@
# This script will be run by Jenkins as a post commit test. In order to run
# locally make the following changes:
#
-# LOCAL_PATH -> Path of tox and virtualenv if you have them already
installed.
# GCS_LOCATION -> Temporary location to use for service tests.
# PROJECT -> Project name to use for service jobs.
#
-# Execute from the root of the repository: sdks/python/run_postcommit.sh
+
+if [ -z "$1" ]; then
+ printf "Usage: \n$> ./run_postcommit.sh <test_type> [gcp_location]
[gcp_project]"
+ printf "\n\ttest_type: ValidatesRunner or IT"
+ printf "\n\tgcp_location: A gs:// path to stage artifacts and output results"
+ printf "\n\tgcp_project: A GCP project to run Dataflow pipelines\n"
+ exit 1
+fi
set -e
set -v
@@ -31,20 +37,24 @@ set -v
# Run tests on the service.
# Where to store integration test outputs.
-GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
+GCS_LOCATION=${2:-gs://temp-storage-for-end-to-end-tests}
-PROJECT=apache-beam-testing
+PROJECT=${3:-apache-beam-testing}
# Create a tarball
python setup.py sdist
SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
+# Install test dependencies for ValidatesRunner tests.
+echo "pyhamcrest" > postcommit_requirements.txt
+echo "mock" >> postcommit_requirements.txt
+
# Run integration tests on the Google Cloud Dataflow service
# and validate that jobs finish successfully.
echo ">>> RUNNING TEST DATAFLOW RUNNER it tests"
python setup.py nosetests \
- --attr IT \
+ --attr $1 \
--nocapture \
--processes=4 \
--process-timeout=1800 \
@@ -55,5 +65,6 @@ python setup.py nosetests \
--temp_location=$GCS_LOCATION/temp-it \
--output=$GCS_LOCATION/py-it-cloud/output \
--sdk_location=$SDK_LOCATION \
+ --requirements_file=postcommit_requirements.txt \
--num_workers=1 \
--sleep_secs=20"
diff --git a/sdks/python/run_validatesrunner.sh
b/sdks/python/run_validatesrunner.sh
deleted file mode 100755
index 6b2b80d026c..00000000000
--- a/sdks/python/run_validatesrunner.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script will be run by Jenkins as a post commit test. In order to run
-# locally make the following changes:
-#
-# GCS_LOCATION -> Temporary location to use for service tests.
-# PROJECT -> Project name to use for service jobs.
-#
-# Execute from the root of the repository: sdks/python/run_postcommit.sh
-
-set -e
-set -v
-
-# Virtualenv for the rest of the script to run setup & e2e tests
-/usr/bin/virtualenv sdks/python
-. sdks/python/bin/activate
-cd sdks/python
-pip install -e .[gcp,test]
-
-# Where to store integration test outputs.
-GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
-
-PROJECT=apache-beam-testing
-
-# Create a tarball
-python setup.py sdist
-
-SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
-
-# Install test dependencies for ValidatesRunner tests.
-echo "pyhamcrest" > postcommit_requirements.txt
-echo "mock" >> postcommit_requirements.txt
-
-# Run ValidatesRunner tests on Google Cloud Dataflow service
-echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS"
-python setup.py nosetests \
- --attr ValidatesRunner \
- --nocapture \
- --processes=4 \
- --process-timeout=900 \
- --test-pipeline-options=" \
- --runner=TestDataflowRunner \
- --project=$PROJECT \
- --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
- --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
- --sdk_location=$SDK_LOCATION \
- --requirements_file=postcommit_requirements.txt \
- --num_workers=1"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 102733)
Time Spent: 50m (was: 40m)
> Duplicated code for ValidatesRunner / PostCommit tests in Python
> ----------------------------------------------------------------
>
> Key: BEAM-4300
> URL: https://issues.apache.org/jira/browse/BEAM-4300
> Project: Beam
> Issue Type: Bug
> Components: sdk-py-core
> Reporter: Pablo Estrada
> Assignee: Pablo Estrada
> Priority: Major
> Time Spent: 50m
> Remaining Estimate: 0h
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
