This is an automated email from the ASF dual-hosted git repository.
tvalentyn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new a7f73d3 Update venv creation commands in release scripts. (#16381)
a7f73d3 is described below
commit a7f73d3d6af54df803923fe490046383c22a83ae
Author: tvalentyn <tvalen...@users.noreply.github.com>
AuthorDate: Wed Dec 29 12:33:01 2021 -0800
Update venv creation commands in release scripts. (#16381)
---
.../jenkins/dependency_check/generate_report.sh | 7 ++--
.test-infra/jenkins/job_00_seed.groovy | 3 +-
.test-infra/jenkins/job_seed_standalone.groovy | 5 ++-
.../python_release_automation_utils.sh | 3 +-
.../src/main/scripts/build_release_candidate.sh | 3 +-
release/src/main/scripts/deploy_pypi.sh | 1 +
.../main/scripts/deploy_release_candidate_pypi.sh | 2 +-
release/src/main/scripts/run_rc_validation.sh | 44 +++++++++++-----------
runners/portability/test_flink_uber_jar.sh | 1 +
runners/portability/test_pipeline_jar.sh | 1 +
.../container/license_scripts/license_script.sh | 3 +-
.../apache_beam/examples/kafkataxi/README.md | 5 ++-
.../apache_beam/runners/interactive/README.md | 31 +++++++--------
sdks/python/container/run_generate_requirements.sh | 3 +-
.../site/content/en/contribute/release-guide.md | 27 +++----------
.../en/documentation/sdks/python-dependencies.md | 2 +-
16 files changed, 65 insertions(+), 76 deletions(-)
diff --git a/.test-infra/jenkins/dependency_check/generate_report.sh
b/.test-infra/jenkins/dependency_check/generate_report.sh
index 825287b..4a8c879 100755
--- a/.test-infra/jenkins/dependency_check/generate_report.sh
+++ b/.test-infra/jenkins/dependency_check/generate_report.sh
@@ -44,10 +44,9 @@ REPORT_DESCRIPTION="
# Virtualenv for the rest of the script to run setup
$PYTHON -m venv dependency/check
-. dependency/check/bin/activate
-pip install --upgrade google-cloud-bigquery
-pip install --upgrade google-cloud-bigtable
-pip install --upgrade google-cloud-core
+. ./dependency/check/bin/activate
+pip install --upgrade pip setuptools wheel
+pip install --upgrade google-cloud-bigquery google-cloud-bigtable
google-cloud-core
rm -f build/dependencyUpdates/beam-dependency-check-report.txt
# Insall packages and run the unit tests of the report generator and the jira
manager
diff --git a/.test-infra/jenkins/job_00_seed.groovy
b/.test-infra/jenkins/job_00_seed.groovy
index e01bc34..820f3c4 100644
--- a/.test-infra/jenkins/job_00_seed.groovy
+++ b/.test-infra/jenkins/job_00_seed.groovy
@@ -108,7 +108,8 @@ job('beam_SeedJob') {
command("""
( cd .test-infra/jenkins/committers_list_generator &&
python3.8 -m venv ve3 && source ve3/bin/activate &&
- pip install -r requirements.txt &&
+ pip install --retries 10 --upgrade pip setuptools wheel &&
+ pip install --retries 10 -r requirements.txt &&
python main.py -o .. &&
deactivate ) ||
{ echo "ERROR: Failed to fetch committers"; exit 3; }
diff --git a/.test-infra/jenkins/job_seed_standalone.groovy
b/.test-infra/jenkins/job_seed_standalone.groovy
index 33ac831..6ae0cfb 100644
--- a/.test-infra/jenkins/job_seed_standalone.groovy
+++ b/.test-infra/jenkins/job_seed_standalone.groovy
@@ -107,8 +107,9 @@ job('beam_SeedJob_Standalone') {
shell {
command("""
( cd .test-infra/jenkins/committers_list_generator &&
- python3.8 -m venv ve3 && source ve3/bin/activate &&
- pip install -r requirements.txt &&
+ python3.8 -m venv ve3 && source ./ve3/bin/activate &&
+ pip install --retries 10 --upgrade pip setuptools wheel &&
+ pip install --retries 10 -r requirements.txt &&
python main.py -o .. &&
deactivate ) ||
{ echo "ERROR: Failed to fetch committers"; exit 3; }
diff --git a/release/src/main/python-release/python_release_automation_utils.sh
b/release/src/main/python-release/python_release_automation_utils.sh
index 507f50f..5d0e1c6 100644
--- a/release/src/main/python-release/python_release_automation_utils.sh
+++ b/release/src/main/python-release/python_release_automation_utils.sh
@@ -161,7 +161,8 @@ function install_sdk() {
gsutil version -l
rm -rf ./temp_virtualenv_${2}
$2 -m venv temp_virtualenv_${2}
- . temp_virtualenv_${2}/bin/activate
+ . ./temp_virtualenv_${2}/bin/activate
+ pip install --upgrade pip setuptools wheel
gcloud_version=$(gcloud --version | head -1 | awk '{print $4}')
if [[ "$gcloud_version" < "189" ]]; then
update_gcloud
diff --git a/release/src/main/scripts/build_release_candidate.sh
b/release/src/main/scripts/build_release_candidate.sh
index 0c8e7ae..7132ad2 100755
--- a/release/src/main/scripts/build_release_candidate.sh
+++ b/release/src/main/scripts/build_release_candidate.sh
@@ -271,7 +271,7 @@ if [[ $confirmation = "y" ]]; then
echo '-------------------Creating Python Virtualenv-----------------'
python3 -m venv "${LOCAL_PYTHON_VIRTUALENV}"
source "${LOCAL_PYTHON_VIRTUALENV}/bin/activate"
- pip install -U pip
+ pip install --upgrade pip setuptools wheel
pip install requests python-dateutil
echo '--------------Fetching GitHub Actions Artifacts--------------'
@@ -356,6 +356,7 @@ if [[ $confirmation = "y" ]]; then
echo "------------------Building Python Doc------------------------"
python3 -m venv "${LOCAL_PYTHON_VIRTUALENV}"
source "${LOCAL_PYTHON_VIRTUALENV}/bin/activate"
+ pip install --upgrade pip setuptools wheel
cd ${LOCAL_PYTHON_DOC}
pip install -U pip
pip install tox
diff --git a/release/src/main/scripts/deploy_pypi.sh
b/release/src/main/scripts/deploy_pypi.sh
index d16a6fd..38eef0c 100755
--- a/release/src/main/scripts/deploy_pypi.sh
+++ b/release/src/main/scripts/deploy_pypi.sh
@@ -43,6 +43,7 @@ cd ${LOCAL_CLONE_DIR}
python3 -m venv deploy_pypi_env
source ./deploy_pypi_env/bin/activate
+pip install --upgrade pip setuptools wheel
pip install twine
wget -r --no-parent -A zip,whl
"https://dist.apache.org/repos/dist/dev/beam/${RELEASE}/python";
diff --git a/release/src/main/scripts/deploy_release_candidate_pypi.sh
b/release/src/main/scripts/deploy_release_candidate_pypi.sh
index defdede..d1fef8c 100755
--- a/release/src/main/scripts/deploy_release_candidate_pypi.sh
+++ b/release/src/main/scripts/deploy_release_candidate_pypi.sh
@@ -129,7 +129,7 @@ echo "================Download python
artifacts======================"
PYTHON_ARTIFACTS_DIR="${LOCAL_CLONE_DIR_ROOT}/python"
python3 -m venv deploy_pypi_env
source ./deploy_pypi_env/bin/activate
-pip install -U pip
+pip install --upgrade pip setuptools wheel
pip install requests python-dateutil
python3 "${SCRIPT_DIR}/download_github_actions_artifacts.py" \
--github-user "${USER_GITHUB_ID}" \
diff --git a/release/src/main/scripts/run_rc_validation.sh
b/release/src/main/scripts/run_rc_validation.sh
index 28ba2eb..64914d8 100755
--- a/release/src/main/scripts/run_rc_validation.sh
+++ b/release/src/main/scripts/run_rc_validation.sh
@@ -98,7 +98,9 @@ echo "All environment and workflow configurations from
RC_VALIDATE_CONFIGS:"
for i in "${RC_VALIDATE_CONFIGS[@]}"; do
echo "$i = ${!i}"
done
-echo "[Confirmation Required] Are they all provided and correctly set? [y|N]"
+echo "TODO(BEAM-13054): parts of this script launch background processes with
gnome-terminal,"
+echo "It may not work well over ssh or within a tmux session. Using 'ssh -Y'
may help."
+echo "[Confirmation Required] Would you like to proceed with current settings?
[y|N]"
read confirmation
if [[ $confirmation != "y" ]]; then
echo "Please rerun this script and make sure you have the right
configurations."
@@ -174,12 +176,9 @@ if [[ -z `which gcloud` ]]; then
fi
gcloud --version
-echo "-----------------Checking Bigquery CLI-----------------"
-if [[ ! -f ~/.bigqueryrc ]]; then
- echo "-----------------Initialing Bigquery CLI-----------------"
- bq init
-fi
-bq version
+echo "-----Initializing gcloud default and application-default
credentials-----"
+gcloud auth login
+gcloud auth application-default login
echo "-----------------Checking gnome-terminal-----------------"
if [[ -z `which gnome-terminal` ]]; then
@@ -227,7 +226,7 @@ if [[ "$python_quickstart_mobile_game" = true && ! -z
`which hub` ]]; then
echo ""
echo "[NOTE] If there is no jenkins job started, please comment on $PR_URL
with: Run Python ReleaseCandidate"
else
- echo "* Skip Python Quickstart and MobileGame. Hub is required."
+ echo "* Skipping Python Quickstart and MobileGame. Hub is required."
fi
# TODO(BEAM-13220) Run the remaining tests on Jenkins.
@@ -250,9 +249,6 @@ if [[ ("$python_leaderboard_direct" = true \
echo "--------------------------Verifying
Hashes------------------------------------"
sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
- `which pip` install --upgrade pip
- `which pip` install --upgrade setuptools
-
echo "--------------------------Updating
~/.m2/settings.xml-------------------------"
cd ~
if [[ ! -d .m2 ]]; then
@@ -317,8 +313,9 @@ if [[ ("$python_leaderboard_direct" = true \
do
rm -rf ./beam_env_${py_version}
echo "--------------Setting up virtualenv with $py_version
interpreter----------------"
- $py_version -m venv beam_env_${py_version}
- . beam_env_${py_version}/bin/activate
+ $py_version -m venv beam_env_${py_version}
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
echo "--------------------------Installing Python
SDK-------------------------------"
pip install apache-beam-${RELEASE_VER}.zip[gcp]
@@ -355,7 +352,7 @@ if [[ ("$python_leaderboard_direct" = true \
bq head -n 10 ${LEADERBOARD_DIRECT_DATASET}.leader_board_teams
echo "***************************************************************"
else
- echo "* Skip Python Leaderboard with DirectRunner"
+ echo "* Skipping Python Leaderboard with DirectRunner"
fi
echo "----------------Starting Leaderboard with
DataflowRunner---------------------"
@@ -393,7 +390,7 @@ if [[ ("$python_leaderboard_direct" = true \
bq head -n 10 ${LEADERBOARD_DF_DATASET}.leader_board_teams
echo "***************************************************************"
else
- echo "* Skip Python Leaderboard with DataflowRunner"
+ echo "* Skipping Python Leaderboard with DataflowRunner"
fi
echo "------------------Starting GameStats with
DirectRunner-----------------------"
@@ -429,7 +426,7 @@ if [[ ("$python_leaderboard_direct" = true \
bq head -n 10 ${GAMESTATS_DIRECT_DATASET}.game_stats_sessions
echo "***************************************************************"
else
- echo "* Skip Python GameStats with DirectRunner"
+ echo "* Skipping Python GameStats with DirectRunner"
fi
echo "-------------------Starting GameStats with
DataflowRunner--------------------"
@@ -468,11 +465,11 @@ if [[ ("$python_leaderboard_direct" = true \
bq head -n 10 ${GAMESTATS_DF_DATASET}.game_stats_sessions
echo "***************************************************************"
else
- echo "* Skip Python GameStats with DataflowRunner"
+ echo "* Skipping Python GameStats with DataflowRunner"
fi
done # Loop over Python versions.
else
- echo "* Skip Python Leaderboard & GameStates Validations"
+ echo "* Skipping Python Leaderboard & GameStates Validations"
fi
echo ""
@@ -507,7 +504,7 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
echo "* Sleeping for 10 mins"
sleep 10m
else
- echo "* Skip Kafka cluster setup"
+ echo "* Skipping Kafka cluster setup"
fi
echo "-----------------------Building expansion service
jar------------------------"
@@ -521,7 +518,8 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
rm -rf ./beam_env_${py_version}
echo "--------------Setting up virtualenv with $py_version
interpreter----------------"
$py_version -m venv beam_env_${py_version}
- . beam_env_${py_version}/bin/activate
+ . ./beam_env_${py_version}/bin/activate
+ pip install --upgrade pip setuptools wheel
ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
echo "--------------------------Installing Python
SDK-------------------------------"
@@ -570,7 +568,7 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
fi
echo "***************************************************************"
else
- echo "* Skip Python XLang Kafka Taxi with DataflowRunner"
+ echo "* Skipping Python XLang Kafka Taxi with DataflowRunner"
fi
echo "----------------Starting XLang SQL Taxi with
DataflowRunner---------------------"
@@ -614,11 +612,11 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
fi
echo "***************************************************************"
else
- echo "* Skip Python XLang SQL Taxi with DataflowRunner"
+ echo "* Skipping Python XLang SQL Taxi with DataflowRunner"
fi
done # Loop over Python versions.
else
- echo "* Skip Python Cross-language Validations"
+ echo "* Skipping Python Cross-language Validations"
fi
echo "*************************************************************"
echo " NOTE: Streaming pipelines are not automatically canceled. "
diff --git a/runners/portability/test_flink_uber_jar.sh
b/runners/portability/test_flink_uber_jar.sh
index 042a514..d073de4 100755
--- a/runners/portability/test_flink_uber_jar.sh
+++ b/runners/portability/test_flink_uber_jar.sh
@@ -82,6 +82,7 @@ docker images --format "{{.Repository}}:{{.Tag}}" | grep
"$PYTHON_CONTAINER_IMAG
# Set up Python environment
python$PYTHON_VERSION -m venv "$ENV_DIR"
. $ENV_DIR/bin/activate
+pip install --retries 10 --upgrade pip setuptools wheel
pip install --retries 10 -e "$PYTHON_ROOT_DIR"
# Hacky python script to find a free port. Note there is a small chance the
chosen port could
diff --git a/runners/portability/test_pipeline_jar.sh
b/runners/portability/test_pipeline_jar.sh
index d2b8482..f01d67b 100755
--- a/runners/portability/test_pipeline_jar.sh
+++ b/runners/portability/test_pipeline_jar.sh
@@ -74,6 +74,7 @@ docker images --format "{{.Repository}}:{{.Tag}}" | grep
$PYTHON_CONTAINER_IMAGE
# Set up Python environment
python$PYTHON_VERSION -m venv $ENV_DIR
. $ENV_DIR/bin/activate
+pip install --retries 10 --upgrade pip setuptools wheel
pip install --retries 10 -e $PYTHON_ROOT_DIR
PIPELINE_PY="
diff --git a/sdks/java/container/license_scripts/license_script.sh
b/sdks/java/container/license_scripts/license_script.sh
index e731276..27560bf 100755
--- a/sdks/java/container/license_scripts/license_script.sh
+++ b/sdks/java/container/license_scripts/license_script.sh
@@ -41,9 +41,10 @@ mkdir -p "$DOWNLOAD_DIR"
cp -r "${EXISTING_LICENSE_DIR}"/*.jar "${DOWNLOAD_DIR}"
$PYTHON -m venv ${ENV_DIR} && . ${ENV_DIR}/bin/activate
+pip install --retries 10 --upgrade pip setuptools wheel
# install packages
-${ENV_DIR}/bin/pip install -r ${SCRIPT_DIR}/requirement.txt
+pip install --retries 10 -r ${SCRIPT_DIR}/requirement.txt
# pull licenses, notices and source code
FLAGS="--license_index=${INDEX_FILE} \
diff --git a/sdks/python/apache_beam/examples/kafkataxi/README.md
b/sdks/python/apache_beam/examples/kafkataxi/README.md
index e8e144d..dc086e3 100644
--- a/sdks/python/apache_beam/examples/kafkataxi/README.md
+++ b/sdks/python/apache_beam/examples/kafkataxi/README.md
@@ -149,8 +149,9 @@ instructions regarding setting up other types of Python
virtual environments.
```sh
cd .. # Creating the virtual environment in the top level work directory.
-python -m venv env
-source env/bin/activate
+python3 -m venv env
+source ./env/bin/activate
+pip install --upgrade pip setuptools wheel
```
Install Beam and dependencies and build a Beam distribution.
diff --git a/sdks/python/apache_beam/runners/interactive/README.md
b/sdks/python/apache_beam/runners/interactive/README.md
index 4c28a90..9d0da05 100644
--- a/sdks/python/apache_beam/runners/interactive/README.md
+++ b/sdks/python/apache_beam/runners/interactive/README.md
@@ -84,24 +84,13 @@ a quick reference). For a more general and complete getting
started guide, see
* Install [GraphViz](https://www.graphviz.org/download/) with your favorite
system package manager.
-- Install [JupyterLab](https://jupyter.org/install.html). You can use
- either **conda** or **pip**.
-
- * conda
- ```bash
- conda install -c conda-forge jupyterlab
- ```
- * pip
- ```bash
- pip install jupyterlab
- ```
-
-- Install, create and activate your
[venv](https://docs.python.org/3/library/venv.html).
+* Install, create and activate your
[venv](https://docs.python.org/3/library/venv.html).
(optional but recommended)
```bash
python3 -m venv /path/to/beam_venv_dir
source /path/to/beam_venv_dir/bin/activate
+ pip install --upgrade pip setuptools wheel
```
If you are using shells other than bash (e.g. fish, csh), check
@@ -114,6 +103,18 @@ a quick reference). For a more general and complete
getting started guide, see
which python # This sould point to beam_venv_dir/bin/python
```
+* Install [JupyterLab](https://jupyter.org/install.html). You can use
+ either **conda** or **pip**.
+
+ * conda
+ ```bash
+ conda install -c conda-forge jupyterlab
+ ```
+ * pip
+ ```bash
+ pip install jupyterlab
+ ```
+
* Set up Apache Beam Python. **Make sure the virtual environment is activated
when you run `setup.py`**
@@ -123,7 +124,7 @@ a quick reference). For a more general and complete getting
started guide, see
python setup.py install
```
-- Install an IPython kernel for the virtual environment you've just created.
+* Install an IPython kernel for the virtual environment you've just created.
**Make sure the virtual environment is activate when you do this.** You can
skip this step if not using venv.
@@ -139,7 +140,7 @@ a quick reference). For a more general and complete getting
started guide, see
jupyter kernelspec list
```
-- Extend JupyterLab through labextension. **Note**: labextension is
different from nbextension
+* Extend JupyterLab through labextension. **Note**: labextension is
different from nbextension
from pre-lab jupyter notebooks.
All jupyter labextensions need nodejs
diff --git a/sdks/python/container/run_generate_requirements.sh
b/sdks/python/container/run_generate_requirements.sh
index a3b8f61..9fb3979 100755
--- a/sdks/python/container/run_generate_requirements.sh
+++ b/sdks/python/container/run_generate_requirements.sh
@@ -55,8 +55,7 @@ ENV_PATH="$PWD/build/python${PY_VERSION/./}_requirements_gen"
rm -rf $ENV_PATH 2>/dev/null || true
python${PY_VERSION} -m venv $ENV_PATH
source $ENV_PATH/bin/activate
-pip install --upgrade pip
-pip install wheel
+pip install --upgrade pip setuptools wheel
# Install gcp extra deps since these deps are commonly used with Apache Beam.
# Install dataframe deps to add have Dataframe support in released images.
diff --git a/website/www/site/content/en/contribute/release-guide.md
b/website/www/site/content/en/contribute/release-guide.md
index 727c94b..bd93b1e 100644
--- a/website/www/site/content/en/contribute/release-guide.md
+++ b/website/www/site/content/en/contribute/release-guide.md
@@ -369,24 +369,8 @@ There are some projects that don't produce the artifacts,
e.g. `beam-test-tools`
To triage the failures and narrow things down you may want to look at
`settings.gradle.kts` and run the build only for the projects you're interested
at the moment, e.g. `./gradlew :runners:java-fn-execution`.
#### (Alternative) Run release build manually (locally)
-* **Pre-installation for python build**
- 1. Install pip
-
- ```
- curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
- python get-pip.py
- ```
- 1. Cython
-
- ```
- sudo pip install cython
- sudo apt-get install gcc
- sudo apt-get install python-dev
- sudo apt-get install python3-dev
- sudo apt-get install python3.5-dev
- sudo apt-get install python3.6-dev
- sudo apt-get install python3.7-dev
- ```
+You will need to have Python interpreters for all supported Python minor
+versions to run Python tests. See Python installation tips in [Developer
Wiki](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-InstallingPythoninterpreters).
* **Run gradle release build**
@@ -934,10 +918,9 @@ _Note_: -Prepourl and -Pver can be found in the RC vote
email sent by Release Ma
* **Setup virtual environment**
```
- pip install --upgrade pip
- pip install --upgrade setuptools
- python -m venv beam_env
- . beam_env/bin/activate
+ python3 -m venv beam_env
+ . ./beam_env/bin/activate
+ pip install --upgrade pip setuptools wheel
```
* **Install SDK**
diff --git
a/website/www/site/content/en/documentation/sdks/python-dependencies.md
b/website/www/site/content/en/documentation/sdks/python-dependencies.md
index 4e0bf0a..32d8ba6 100644
--- a/website/www/site/content/en/documentation/sdks/python-dependencies.md
+++ b/website/www/site/content/en/documentation/sdks/python-dependencies.md
@@ -44,7 +44,7 @@ You can also retrieve the dependency list from the command
line using the follow
1. Create a clean virtual environment on your local machine using a supported
python version.
```
- $ python -m venv env && source env/bin/activate
+ $ python3 -m venv env && source ./env/bin/activate && pip install
--upgrade pip setuptools wheel
```
2. [Install the Beam Python
SDK](/get-started/quickstart-py/#download-and-install).
