This is an automated email from the ASF dual-hosted git repository. derrickaw pushed a commit to branch 20260722_removeGsutil in repository https://gitbox.apache.org/repos/asf/beam.git
commit 4fced04bee394e6ef93d50294f4d4269c3e8becb Author: Derrick Williams <[email protected]> AuthorDate: Wed Jul 22 15:44:28 2026 +0000 remove gsutil usage --- .github/workflows/beam_PreCommit_Gsutil_Check.yml | 96 ++++++++++++++++++++++ .github/workflows/build_wheels.yml | 12 +-- .../workflows/run_rc_validation_go_wordcount.yml | 12 +-- .../run_rc_validation_python_mobile_gaming.yml | 6 +- .../workflows/run_rc_validation_python_yaml.yml | 6 +- .test-infra/dataproc/flink_cluster.sh | 2 +- .test-infra/metrics/influxdb/Dockerfile | 6 +- .test-infra/metrics/influxdb/gsutil/.boto | 24 ------ .test-infra/metrics/influxdb/gsutil/Dockerfile | 25 ------ .../kubernetes/beam-influxdb-autobackup.yaml | 7 +- .../beam/examples/complete/game/UserScore.java | 2 +- examples/multi-language/README.md | 6 +- .../beam-ml/automatic_model_refresh.ipynb | 4 +- .../get-started/learn_beam_basics_by_doing.ipynb | 4 +- .../learn_beam_transforms_by_doing.ipynb | 2 +- .../learn_beam_windowing_by_doing.ipynb | 2 +- .../notebooks/get-started/try-apache-beam-go.ipynb | 4 +- .../get-started/try-apache-beam-java.ipynb | 4 +- .../notebooks/get-started/try-apache-beam-py.ipynb | 4 +- .../main/groovy/mobilegaming-java-dataflow.groovy | 12 +-- .../groovy/mobilegaming-java-dataflowbom.groovy | 12 +-- .../main/groovy/quickstart-java-dataflow.groovy | 8 +- .../python_release_automation_utils.sh | 6 +- .../run_release_candidate_python_quickstart.sh | 6 +- sdks/go/README.md | 2 +- .../testing/benchmarks/chicago_taxi/run_chicago.sh | 6 +- sdks/python/scripts/run_snapshot_publish.sh | 4 +- website/Dockerfile | 2 +- website/build.gradle | 5 +- .../content/en/blog/apache-hop-with-dataflow.md | 12 +-- .../content/en/blog/beam-sql-with-notebooks.md | 4 +- .../site/content/en/documentation/runners/spark.md | 4 +- .../sdks/python-multi-language-pipelines.md | 2 +- .../site/content/en/get-started/quickstart-java.md | 4 +- 34 files changed, 182 insertions(+), 135 deletions(-) diff --git a/.github/workflows/beam_PreCommit_Gsutil_Check.yml b/.github/workflows/beam_PreCommit_Gsutil_Check.yml new file mode 100644 index 00000000000..aa315f5285d --- /dev/null +++ b/.github/workflows/beam_PreCommit_Gsutil_Check.yml @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: PreCommit Gsutil Check + +on: + push: + tags: ['v*'] + branches: ['master', 'release-*'] + pull_request_target: + branches: ['master', 'release-*'] + issue_comment: + types: [created] + schedule: + - cron: '15 3/6 * * *' + workflow_dispatch: + +# Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event +permissions: + actions: write + pull-requests: read + checks: read + contents: read + deployments: read + id-token: none + issues: read + discussions: read + packages: read + pages: read + repository-projects: read + security-events: read + statuses: read + +# This allows a subsequently queued workflow run to interrupt previous runs +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}' + cancel-in-progress: true + +env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} + GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} + +jobs: + beam_PreCommit_Gsutil_Check: + name: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + runs-on: [self-hosted, ubuntu-24.04, small] + strategy: + matrix: + job_name: [beam_PreCommit_Gsutil_Check] + job_phrase: [Run Gsutil Check PreCommit] + timeout-minutes: 10 + if: | + github.event_name == 'push' || + github.event_name == 'pull_request_target' || + (github.event_name == 'schedule' && github.repository == 'apache/beam') || + github.event_name == 'workflow_dispatch' || + github.event.comment.body == 'Run Gsutil Check PreCommit' + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Setup repository + uses: ./.github/actions/setup-action + with: + comment_phrase: ${{ matrix.job_phrase }} + github_token: ${{ secrets.GITHUB_TOKEN }} + github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) + - name: Check for gsutil references + run: | + echo "Checking codebase for gsutil..." + # Search for 'gsutil', excluding this workflow file itself to avoid false positives. + if git grep -n "gsutil" -- ':!.github/workflows/beam_PreCommit_Gsutil_Check.yml'; then + echo "ERROR: Found references to gsutil in the codebase. Please use gcloud storage instead." + exit 1 + elif [ "$(date +%Y%m)" -ge 202704 ]; then + echo "ERROR: Current date is April 2027 or later." + echo "Please verify gsutil deprecation date is still March 2027 (Reference: https://docs.cloud.google.com/storage/docs/gsutil)." + echo "If so, then delete this workflow." + exit 1 + else + echo "SUCCESS: No references to gsutil found." + fi + shell: bash diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 02b223943fe..2d0232d9f51 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -201,7 +201,7 @@ jobs: if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request' steps: - name: Remove existing files on GCS bucket - run: gsutil rm -r ${GCP_PATH} || true + run: gcloud storage rm -r ${GCP_PATH} || true upload_source_to_gcs: name: Upload python source distribution to GCS bucket @@ -217,7 +217,7 @@ jobs: name: source_zip path: source/ - name: Copy sources to GCS bucket - run: gsutil cp -r -a public-read source/* ${GCP_PATH} + run: gcloud storage cp -r --predefined-acl=publicRead source/* ${GCP_PATH} build_wheels: name: Build python ${{matrix.py_version}} wheels on ${{matrix.os_python.arch}} for ${{ matrix.os_python.os }} @@ -330,7 +330,7 @@ jobs: merge-multiple: true path: wheelhouse/ - name: Copy wheels to GCS bucket - run: gsutil cp -r -a public-read wheelhouse/* ${GCP_PATH} + run: gcloud storage cp -r --predefined-acl=publicRead wheelhouse/* ${GCP_PATH} - name: Create github action information file on GCS bucket run: | cat > github_action_info <<EOF @@ -348,9 +348,9 @@ jobs: GITHUB_BASE_REF=$GITHUB_BASE_REF EOF echo $(cat github_action_info) - gsutil cp -a public-read github_action_info ${GCP_PATH} + gcloud storage cp --predefined-acl=publicRead github_action_info ${GCP_PATH} - name: Upload GitHub event file to GCS bucket - run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${GCP_PATH} + run: gcloud storage cp --predefined-acl=publicRead ${GITHUB_EVENT_PATH} ${GCP_PATH} list_files_on_gcs: name: List files on Google Cloud Storage Bucket @@ -361,7 +361,7 @@ jobs: if: needs.check_env_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request' steps: - name: List file on Google Cloud Storage Bucket - run: gsutil ls "${GCP_PATH}*" + run: gcloud storage ls "${GCP_PATH}*" branch_repo_nightly: permissions: diff --git a/.github/workflows/run_rc_validation_go_wordcount.yml b/.github/workflows/run_rc_validation_go_wordcount.yml index ea1d63bd4c0..d0d1e6d3c28 100644 --- a/.github/workflows/run_rc_validation_go_wordcount.yml +++ b/.github/workflows/run_rc_validation_go_wordcount.yml @@ -74,7 +74,7 @@ jobs: # --- Prism Steps --- - name: Download Input File from GCS (Prism) working-directory: ./${{ steps.setup_go.outputs.work_dir }} - run: gsutil cp ${{ env.GCS_INPUT_PATH }} ./kinglear.txt + run: gcloud storage cp ${{ env.GCS_INPUT_PATH }} ./kinglear.txt - name: Run Go WordCount with PrismRunner working-directory: ./${{ steps.setup_go.outputs.work_dir }} @@ -118,17 +118,17 @@ jobs: --environment_config=apache/beam_go_sdk:${{ env.CONTAINER_TAG }} - name: Check Dataflow Output in GCS - working-directory: ./${{ steps.setup_go.outputs.work_dir }} # Added working directory for consistency, though not strictly needed for gsutil + working-directory: ./${{ steps.setup_go.outputs.work_dir }} # Added working directory for consistency, though not strictly needed for gcloud storage env: # Re-define the output path pattern for checking GCS_OUTPUT_PATH_PATTERN: ${{ env.GCS_OUTPUT_PREFIX }}/${{ env.RC_TAG }}/dataflow/output* run: | echo "Checking for Dataflow output files in GCS at: $GCS_OUTPUT_PATH_PATTERN" - # Use gsutil stat. The -q flag suppresses errors for non-existent files, - # allowing us to check the exit code. Exit code 0 means found, 1 means not found. - if gsutil -q stat $GCS_OUTPUT_PATH_PATTERN; then + # Use gcloud storage ls. Redirect stdout/stderr to suppress output. + # Exit code 0 means found, non-zero means not found. + if gcloud storage ls $GCS_OUTPUT_PATH_PATTERN > /dev/null 2>&1; then echo "Output files found in GCS." - FILE_COUNT=$(gsutil ls $GCS_OUTPUT_PATH_PATTERN | wc -l) + FILE_COUNT=$(gcloud storage ls $GCS_OUTPUT_PATH_PATTERN | wc -l) if [ "$FILE_COUNT" -gt 0 ]; then echo "Found $FILE_COUNT output file(s)."; else echo "Error: Output path exists but contains no files."; exit 1; fi else echo "Error: Output files not found in GCS at $GCS_OUTPUT_PATH_PATTERN" diff --git a/.github/workflows/run_rc_validation_python_mobile_gaming.yml b/.github/workflows/run_rc_validation_python_mobile_gaming.yml index da90d433d33..61bd4b1fb68 100644 --- a/.github/workflows/run_rc_validation_python_mobile_gaming.yml +++ b/.github/workflows/run_rc_validation_python_mobile_gaming.yml @@ -174,7 +174,7 @@ jobs: - name: Create GCS Bucket (if needed - reusing input bucket) run: | echo "Ensuring GCS Bucket exists: ${{ env.GCS_BUCKET }} in project ${{ env.GCP_PROJECT_ID }}" - gsutil mb -p ${{ env.GCP_PROJECT_ID }} ${{ env.GCS_BUCKET }} || echo "Bucket ${{ env.GCS_BUCKET }} likely already exists." + gcloud storage buckets create ${{ env.GCS_BUCKET }} --project=${{ env.GCP_PROJECT_ID }} || echo "Bucket ${{ env.GCS_BUCKET }} likely already exists." shell: bash - name: Create PubSub Topic @@ -533,8 +533,8 @@ jobs: if: always() run: | echo "Deleting objects in GCS Bucket: ${{ env.GCS_BUCKET }}/temp/" - gsutil -m rm -r "${{ env.GCS_BUCKET }}/temp/leaderboard/**" || echo "Failed to delete objects in GCS leaderboard temp folder." - gsutil -m rm -r "${{ env.GCS_BUCKET }}/temp/gamestats/**" || echo "Failed to delete objects in GCS gamestats temp folder." + gcloud storage rm -r "${{ env.GCS_BUCKET }}/temp/leaderboard/**" || echo "Failed to delete objects in GCS leaderboard temp folder." + gcloud storage rm -r "${{ env.GCS_BUCKET }}/temp/gamestats/**" || echo "Failed to delete objects in GCS gamestats temp folder." echo "Removing local log and jobid files..." rm -f leaderboard_dataflow_submit.log gamestats_dataflow_submit.log injector_run.log rm -f leaderboard_dataflow_jobid.txt # Remove Leaderboard jobid file here diff --git a/.github/workflows/run_rc_validation_python_yaml.yml b/.github/workflows/run_rc_validation_python_yaml.yml index 654add7a70d..dba20989b18 100644 --- a/.github/workflows/run_rc_validation_python_yaml.yml +++ b/.github/workflows/run_rc_validation_python_yaml.yml @@ -265,9 +265,9 @@ jobs: sleep 60 # Check if any files matching the pattern exist within the unique output folder. echo "Checking for files matching pattern: ${OUTPUT_PATTERN}" - if gsutil ls "${OUTPUT_PATTERN}" > /dev/null 2>&1; then + if gcloud storage ls "${OUTPUT_PATTERN}" > /dev/null 2>&1; then echo "SUCCESS: Found output files matching pattern in GCS." - gsutil ls "${OUTPUT_PATTERN}" # List found files + gcloud storage ls "${OUTPUT_PATTERN}" # List found files else echo "ERROR: No output files found matching pattern '${OUTPUT_PATTERN}' in GCS bucket." exit 1 @@ -280,7 +280,7 @@ jobs: run: | echo "Deleting unique run folder in GCS: ${GCS_UNIQUE_FOLDER_PREFIX}" # Delete the entire unique folder for this run, including temp, staging, and output - gsutil -m rm -r "${GCS_UNIQUE_FOLDER_PREFIX}" || echo "Failed to delete unique run folder ${GCS_UNIQUE_FOLDER_PREFIX} in GCS. Manual cleanup might be required." + gcloud storage rm -r "${GCS_UNIQUE_FOLDER_PREFIX}" || echo "Failed to delete unique run folder ${GCS_UNIQUE_FOLDER_PREFIX} in GCS. Manual cleanup might be required." echo "Removing local log, yaml, and jobid files..." rm -f yaml_dataflow_submit.log ${{ env.YAML_PIPELINE_FILE }} yaml_dataflow_jobid.txt diff --git a/.test-infra/dataproc/flink_cluster.sh b/.test-infra/dataproc/flink_cluster.sh index dac9f6972c5..e146aa4b6ce 100755 --- a/.test-infra/dataproc/flink_cluster.sh +++ b/.test-infra/dataproc/flink_cluster.sh @@ -91,7 +91,7 @@ YARN_APPLICATION_MASTER="" function upload_init_actions() { echo "Uploading initialization actions to GCS bucket: $GCS_BUCKET" - gsutil cp -r $INIT_ACTIONS_FOLDER_NAME/* $GCS_BUCKET/$INIT_ACTIONS_FOLDER_NAME + gcloud storage cp -r $INIT_ACTIONS_FOLDER_NAME/* $GCS_BUCKET/$INIT_ACTIONS_FOLDER_NAME } function get_leader() { diff --git a/.test-infra/metrics/influxdb/Dockerfile b/.test-infra/metrics/influxdb/Dockerfile index 7d08940fcb4..ff51055b392 100644 --- a/.test-infra/metrics/influxdb/Dockerfile +++ b/.test-infra/metrics/influxdb/Dockerfile @@ -16,13 +16,11 @@ # limitations under the License. ################################################################################ -FROM python:3.10-slim - -RUN pip install --no-cache-dir gsutil +FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:stable-slim WORKDIR / -RUN gsutil cp gs://apache-beam-testing-metrics/influxdb-backup.tar.gz . && \ +RUN gcloud storage cp gs://apache-beam-testing-metrics/influxdb-backup.tar.gz . && \ tar xzf influxdb-backup.tar.gz diff --git a/.test-infra/metrics/influxdb/gsutil/.boto b/.test-infra/metrics/influxdb/gsutil/.boto deleted file mode 100644 index b2eca06da5e..00000000000 --- a/.test-infra/metrics/influxdb/gsutil/.boto +++ /dev/null @@ -1,24 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -[GSUtil] -default_project_id = apache-beam-testing -default_api_version = 2 - -[GoogleCompute] -service_account = default diff --git a/.test-infra/metrics/influxdb/gsutil/Dockerfile b/.test-infra/metrics/influxdb/gsutil/Dockerfile deleted file mode 100644 index 87a46d4861c..00000000000 --- a/.test-infra/metrics/influxdb/gsutil/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -FROM python:3.10-slim - -# google-compute-engine package allows to obtain credentials for service -# account specified in .boto file. -RUN pip install --no-cache-dir gsutil google-compute-engine - -ADD .boto /etc/boto.cfg diff --git a/.test-infra/metrics/kubernetes/beam-influxdb-autobackup.yaml b/.test-infra/metrics/kubernetes/beam-influxdb-autobackup.yaml index e2ebd0c2c74..8f181b3879e 100644 --- a/.test-infra/metrics/kubernetes/beam-influxdb-autobackup.yaml +++ b/.test-infra/metrics/kubernetes/beam-influxdb-autobackup.yaml @@ -47,9 +47,12 @@ spec: - mountPath: /backup name: shared-data - name: copy-to-gsc-bucket - image: gcr.io/apache-beam-testing/gsutil + image: gcr.io/google.com/cloudsdktool/google-cloud-cli:stable-slim + env: + - name: CLOUDSDK_CORE_PROJECT + value: apache-beam-testing command: ['sh', '-c', 'tar czf influxdb-backup.tar.gz /backup - && gsutil cp influxdb-backup.tar.gz + && gcloud storage cp influxdb-backup.tar.gz gs://apache-beam-testing-metrics/'] volumeMounts: - mountPath: /backup diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java index 054ce7a5293..f0719932d5f 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/game/UserScore.java @@ -214,7 +214,7 @@ public class UserScore { day's worth (roughly) of data. Note: You may want to use a small sample dataset to test it locally/quickly : gs://apache-beam-samples/game/small/gaming_data.csv - You can also download it via the command line gsutil cp gs://apache-beam-samples/game/small/gaming_data.csv ./destination_folder/gaming_data.csv */ + You can also download it via the command line gcloud storage cp gs://apache-beam-samples/game/small/gaming_data.csv ./destination_folder/gaming_data.csv */ @Default.String("gs://apache-beam-samples/game/gaming_data*.csv") String getInput(); diff --git a/examples/multi-language/README.md b/examples/multi-language/README.md index f9905ca310e..9b01e4f8eba 100644 --- a/examples/multi-language/README.md +++ b/examples/multi-language/README.md @@ -124,7 +124,7 @@ mvn compile exec:java -Dexec.mainClass=org.apache.beam.examples.multilanguage.Sk the digit. The second item is the predicted label of the digit. ``` -gsutil cat gs://$GCP_BUCKET/multi-language-beam/output* +gcloud storage cat gs://$GCP_BUCKET/multi-language-beam/output* ``` #### Instructions for running the Java pipeline at HEAD (Beam 2.41.0 and 2.42.0). @@ -171,7 +171,7 @@ export GCP_REGION=<GCP region> export EXPANSION_SERVICE_PORT=<PORT> # This removes any existing output. -gsutil rm gs://$GCP_BUCKET/multi-language-beam/output* +gcloud storage rm gs://$GCP_BUCKET/multi-language-beam/output* ./gradlew :examples:multi-language:sklearnMinstClassification --args=" \ --runner=DataflowRunner \ @@ -188,7 +188,7 @@ gsutil rm gs://$GCP_BUCKET/multi-language-beam/output* of the digit. The second item is the predicted label of the digit. ``` -gsutil cat gs://$GCP_BUCKET/multi-language-beam/output* +gcloud storage cat gs://$GCP_BUCKET/multi-language-beam/output* ``` ### Python Dataframe Wordcount diff --git a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb index c29881ea72f..46d2d7e1eaa 100644 --- a/examples/notebooks/beam-ml/automatic_model_refresh.ipynb +++ b/examples/notebooks/beam-ml/automatic_model_refresh.ipynb @@ -298,7 +298,7 @@ "model = tf.keras.applications.resnet.ResNet101()\n", "model.save('resnet101_weights_tf_dim_ordering_tf_kernels.keras')\n", "# After saving the model locally, upload the model to GCS bucket and provide that gcs bucket `URI` as `model_uri` to the `TFModelHandler`\n", - "!gsutil cp resnet101_weights_tf_dim_ordering_tf_kernels.keras gs://${BUCKET_NAME}/dataflow/resnet101_weights_tf_dim_ordering_tf_kernels.keras" + "!gcloud storage cp resnet101_weights_tf_dim_ordering_tf_kernels.keras gs://${BUCKET_NAME}/dataflow/resnet101_weights_tf_dim_ordering_tf_kernels.keras" ] }, { @@ -603,7 +603,7 @@ "source": [ "model = tf.keras.applications.resnet.ResNet152()\n", "model.save('resnet152_weights_tf_dim_ordering_tf_kernels.keras')\n", - "!gsutil cp resnet152_weights_tf_dim_ordering_tf_kernels.keras gs://${BUCKET_NAME}/resnet152_weights_tf_dim_ordering_tf_kernels.keras" + "!gcloud storage cp resnet152_weights_tf_dim_ordering_tf_kernels.keras gs://${BUCKET_NAME}/resnet152_weights_tf_dim_ordering_tf_kernels.keras" ] }, { diff --git a/examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb b/examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb index 0a47a38197b..4259bad8239 100644 --- a/examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb +++ b/examples/notebooks/get-started/learn_beam_basics_by_doing.ipynb @@ -324,7 +324,7 @@ "source": [ "# Creates a data directory with our dataset SMSSpamCollection\n", "!mkdir -p data\n", - "!gsutil cp gs://apachebeamdt/SMSSpamCollection data/" + "!gcloud storage cp gs://apachebeamdt/SMSSpamCollection data/" ] }, { @@ -995,7 +995,7 @@ "source": [ "!pip install --quiet apache-beam\n", "!mkdir -p data\n", - "!gsutil cp gs://apachebeamdt/SMSSpamCollection data/" + "!gcloud storage cp gs://apachebeamdt/SMSSpamCollection data/" ] }, { diff --git a/examples/notebooks/get-started/learn_beam_transforms_by_doing.ipynb b/examples/notebooks/get-started/learn_beam_transforms_by_doing.ipynb index 4b5dc5d1599..15b2fa6f18a 100644 --- a/examples/notebooks/get-started/learn_beam_transforms_by_doing.ipynb +++ b/examples/notebooks/get-started/learn_beam_transforms_by_doing.ipynb @@ -324,7 +324,7 @@ "outputs": [], "source": [ "!mkdir -p data\n", - "!gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/" + "!gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/" ] }, { diff --git a/examples/notebooks/get-started/learn_beam_windowing_by_doing.ipynb b/examples/notebooks/get-started/learn_beam_windowing_by_doing.ipynb index 014a9ca7ff1..b716516c71b 100644 --- a/examples/notebooks/get-started/learn_beam_windowing_by_doing.ipynb +++ b/examples/notebooks/get-started/learn_beam_windowing_by_doing.ipynb @@ -173,7 +173,7 @@ "source": [ "# Copy the dataset file into the local file system from Google Cloud Storage.\n", "!mkdir -p data\n", - "!gsutil cp gs://batch-processing-example/air-quality-india.csv data/" + "!gcloud storage cp gs://batch-processing-example/air-quality-india.csv data/" ] }, { diff --git a/examples/notebooks/get-started/try-apache-beam-go.ipynb b/examples/notebooks/get-started/try-apache-beam-go.ipynb index 7e049af14c3..87130b59538 100644 --- a/examples/notebooks/get-started/try-apache-beam-go.ipynb +++ b/examples/notebooks/get-started/try-apache-beam-go.ipynb @@ -85,7 +85,7 @@ "\n", "# Copy the input file into the local filesystem.\n", "run('mkdir -p data')\n", - "run('gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/')" + "run('gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/')" ], "cell_type": "code", "execution_count": 1, @@ -98,7 +98,7 @@ "\n", ">> mkdir -p data\n", "\n", - ">> gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", + ">> gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", "Copying gs://dataflow-samples/shakespeare/kinglear.txt...\n", "/ [1 files][153.6 KiB/153.6 KiB] \n", "Operation completed over 1 objects/153.6 KiB. \n", diff --git a/examples/notebooks/get-started/try-apache-beam-java.ipynb b/examples/notebooks/get-started/try-apache-beam-java.ipynb index 46ab413083c..733d813e9cf 100644 --- a/examples/notebooks/get-started/try-apache-beam-java.ipynb +++ b/examples/notebooks/get-started/try-apache-beam-java.ipynb @@ -104,7 +104,7 @@ "\n", "# Copy the input file into the local filesystem.\n", "run('mkdir -p data')\n", - "run('gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/')" + "run('gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/')" ], "execution_count": 1, "outputs": [ @@ -113,7 +113,7 @@ "text": [ ">> mkdir -p data\n", "\n", - ">> gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", + ">> gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", "Copying gs://dataflow-samples/shakespeare/kinglear.txt...\n", "/ [1 files][153.6 KiB/153.6 KiB] \n", "Operation completed over 1 objects/153.6 KiB. \n", diff --git a/examples/notebooks/get-started/try-apache-beam-py.ipynb b/examples/notebooks/get-started/try-apache-beam-py.ipynb index f243547979d..656f87a2ebd 100644 --- a/examples/notebooks/get-started/try-apache-beam-py.ipynb +++ b/examples/notebooks/get-started/try-apache-beam-py.ipynb @@ -107,7 +107,7 @@ "\n", "# Copy the input file into the local file system.\n", "run('mkdir -p data')\n", - "run('gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/')" + "run('gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/')" ], "execution_count": 1, "outputs": [ @@ -118,7 +118,7 @@ "\n", ">> mkdir -p data\n", "\n", - ">> gsutil cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", + ">> gcloud storage cp gs://dataflow-samples/shakespeare/kinglear.txt data/\n", "Copying gs://dataflow-samples/shakespeare/kinglear.txt...\n", "/ [1 files][153.6 KiB/153.6 KiB] \n", "Operation completed over 1 objects/153.6 KiB. \n", diff --git a/release/src/main/groovy/mobilegaming-java-dataflow.groovy b/release/src/main/groovy/mobilegaming-java-dataflow.groovy index a2164aa6a01..51ea528a763 100644 --- a/release/src/main/groovy/mobilegaming-java-dataflow.groovy +++ b/release/src/main/groovy/mobilegaming-java-dataflow.groovy @@ -45,7 +45,7 @@ int waitTime = 15 // seconds def outputPath = "gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}" def outputFound = false for (int i = 0; i < retries; i++) { - def files = t.run("gsutil ls ${outputPath}*") + def files = t.run("gcloud storage ls ${outputPath}*") if (files?.trim()) { outputFound = true break @@ -58,10 +58,10 @@ if (!outputFound) { throw new RuntimeException("No output files found for HourlyTeamScore after ${retries * waitTime} seconds.") } -command_output_text = t.run "gsutil cat ${outputPath}* | grep user19_BananaWallaby" +command_output_text = t.run "gcloud storage cat ${outputPath}* | grep user19_BananaWallaby" t.see "total_score: 231, user: user19_BananaWallaby", command_output_text t.success("UserScore successfully run on DataflowRunner.") -t.run "gsutil rm gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}*" +t.run "gcloud storage rm gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}*" /** @@ -76,7 +76,7 @@ t.run(mobileGamingCommands.createPipelineCommand("HourlyTeamScore", runner)) outputPath = "gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}" outputFound = false for (int i = 0; i < retries; i++) { - def files = t.run("gsutil ls ${outputPath}*") + def files = t.run("gcloud storage ls ${outputPath}*") if (files?.trim()) { outputFound = true break @@ -89,10 +89,10 @@ if (!outputFound) { throw new RuntimeException("No output files found for UserScore after ${retries * waitTime} seconds.") } -command_output_text = t.run "gsutil cat ${outputPath}* | grep AzureBilby " +command_output_text = t.run "gcloud storage cat ${outputPath}* | grep AzureBilby " t.see "total_score: 2788, team: AzureBilby", command_output_text t.success("HourlyTeamScore successfully run on DataflowRunner.") -t.run "gsutil rm gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*" +t.run "gcloud storage rm gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*" /** diff --git a/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy b/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy index e156357c19d..b3f2adea1f6 100644 --- a/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy +++ b/release/src/main/groovy/mobilegaming-java-dataflowbom.groovy @@ -45,7 +45,7 @@ int waitTime = 15 // seconds def outputPath = "gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}" def outputFound = false for (int i = 0; i < retries; i++) { - def files = t.run("gsutil ls ${outputPath}*") + def files = t.run("gcloud storage ls ${outputPath}*") if (files?.trim()) { outputFound = true break @@ -58,10 +58,10 @@ if (!outputFound) { throw new RuntimeException("No output files found for UserScore after ${retries * waitTime} seconds.") } -command_output_text = t.run "gsutil cat ${outputPath}* | grep user19_BananaWallaby" +command_output_text = t.run "gcloud storage cat ${outputPath}* | grep user19_BananaWallaby" t.see "total_score: 231, user: user19_BananaWallaby", command_output_text t.success("UserScore successfully run on DataflowRunner.") -t.run "gsutil rm gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}*" +t.run "gcloud storage rm gs://${t.gcsBucket()}/${mobileGamingCommands.getUserScoreOutputName(runner)}*" /** @@ -76,7 +76,7 @@ t.run(mobileGamingCommands.createPipelineCommand("HourlyTeamScore", runner)) outputPath = "gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}" outputFound = false for (int i = 0; i < retries; i++) { - def files = t.run("gsutil ls ${outputPath}*") + def files = t.run("gcloud storage ls ${outputPath}*") if (files?.trim()) { outputFound = true break @@ -89,10 +89,10 @@ if (!outputFound) { throw new RuntimeException("No output files found for HourlyTeamScore after ${retries * waitTime} seconds.") } -command_output_text = t.run "gsutil cat ${outputPath}* | grep AzureBilby " +command_output_text = t.run "gcloud storage cat ${outputPath}* | grep AzureBilby " t.see "total_score: 2788, team: AzureBilby", command_output_text t.success("HourlyTeamScore successfully run on DataflowRunner.") -t.run "gsutil rm gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*" +t.run "gcloud storage rm gs://${t.gcsBucket()}/${mobileGamingCommands.getHourlyTeamScoreOutputName(runner)}*" new LeaderBoardRunner().run(runner, t, mobileGamingCommands, false) new LeaderBoardRunner().run(runner, t, mobileGamingCommands, true) diff --git a/release/src/main/groovy/quickstart-java-dataflow.groovy b/release/src/main/groovy/quickstart-java-dataflow.groovy index fb84fedca0b..5776185f8f1 100644 --- a/release/src/main/groovy/quickstart-java-dataflow.groovy +++ b/release/src/main/groovy/quickstart-java-dataflow.groovy @@ -31,7 +31,7 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow' t.intent 'Runs the WordCount Code with Dataflow runner' // Remove any count files - t.run """gsutil rm gs://${t.gcsBucket()}/count* || echo 'No files'""" + t.run """gcloud storage rm gs://${t.gcsBucket()}/count* || echo 'No files'""" // Run the wordcount example with the Dataflow runner t.run """mvn compile exec:java -q \ @@ -54,7 +54,7 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow' def outputPath = "gs://${t.gcsBucket()}/count" def outputFound = false for (int i = 0; i < retries; i++) { - def files = t.run("gsutil ls ${outputPath}*") + def files = t.run("gcloud storage ls ${outputPath}*") if (files?.trim()) { outputFound = true break @@ -68,11 +68,11 @@ t.describe 'Run Apache Beam Java SDK Quickstart - Dataflow' } // Verify wordcount text - String result = t.run """gsutil cat ${outputPath}* | grep Montague:""" + String result = t.run """gcloud storage cat ${outputPath}* | grep Montague:""" t.see "Montague: 47", result // Remove count files - t.run """gsutil rm gs://${t.gcsBucket()}/count*""" + t.run """gcloud storage rm gs://${t.gcsBucket()}/count*""" // Clean up t.done() diff --git a/release/src/main/python-release/python_release_automation_utils.sh b/release/src/main/python-release/python_release_automation_utils.sh index 7a0529482b6..3884b02c40f 100644 --- a/release/src/main/python-release/python_release_automation_utils.sh +++ b/release/src/main/python-release/python_release_automation_utils.sh @@ -160,7 +160,7 @@ function get_asc_name() { function install_sdk() { sdk_file=$(get_sdk_name $1) print_separator "Creating new virtualenv with $2 interpreter and installing the SDK from $sdk_file." - gsutil version -l + gcloud version rm -rf ./temp_virtualenv_${2} $2 -m venv temp_virtualenv_${2} . ./temp_virtualenv_${2}/bin/activate @@ -269,7 +269,7 @@ function verify_user_score() { expected_output_file_name="$USERSCORE_OUTPUT_PREFIX-$1-runner.txt" actual_output_files=$(ls) if [[ $1 = *"dataflow"* ]]; then - actual_output_files=$(gsutil ls gs://$BUCKET_NAME) + actual_output_files=$(gcloud storage ls gs://$BUCKET_NAME) expected_output_file_name="gs://$BUCKET_NAME/$expected_output_file_name" fi echo $actual_output_files @@ -281,7 +281,7 @@ function verify_user_score() { fi if [[ $1 = *"dataflow"* ]]; then - gsutil rm $expected_output_file_name* + gcloud storage rm $expected_output_file_name* fi echo "SUCCEED: user_score successfully run on $1-runner." } diff --git a/release/src/main/python-release/run_release_candidate_python_quickstart.sh b/release/src/main/python-release/run_release_candidate_python_quickstart.sh index 3af527acfa2..1cc815bcc41 100755 --- a/release/src/main/python-release/run_release_candidate_python_quickstart.sh +++ b/release/src/main/python-release/run_release_candidate_python_quickstart.sh @@ -75,7 +75,7 @@ function verify_hash() { wget https://dist.apache.org/repos/dist/dev/beam/KEYS gpg --import KEYS gpg --verify $ASC_FILE_NAME $BEAM_PYTHON_SDK - gsutil version -l + gcloud version } @@ -122,7 +122,7 @@ function verify_wordcount_dataflow() { # verify results. wordcount_output_in_gcs="gs://$BUCKET_NAME/$WORDCOUNT_OUTPUT" - gcs_pull_result=$(gsutil ls gs://$BUCKET_NAME) + gcs_pull_result=$(gcloud storage ls gs://$BUCKET_NAME) if [[ $gcs_pull_result != *$wordcount_output_in_gcs* ]]; then echo "ERROR: The wordcount example failed on DataflowRunner". complete "failed when running wordcount example with DataflowRunner." @@ -130,7 +130,7 @@ function verify_wordcount_dataflow() { fi # clean output files from GCS - gsutil rm gs://$BUCKET_NAME/$WORDCOUNT_OUTPUT-* + gcloud storage rm gs://$BUCKET_NAME/$WORDCOUNT_OUTPUT-* echo "SUCCEED: wordcount successfully run on DataflowRunner." } diff --git a/sdks/go/README.md b/sdks/go/README.md index 4d6a5047257..bb7f22934f6 100644 --- a/sdks/go/README.md +++ b/sdks/go/README.md @@ -93,7 +93,7 @@ $ go run wordcount.go --runner=dataflow --project=<YOUR_GCP_PROJECT> --region=<Y The output is a GCS file in this case: ```bash -$ gsutil cat <YOUR_GCS_LOCATION>/output* | head +$ gcloud storage cat <YOUR_GCS_LOCATION>/output* | head Blanket: 1 blot: 1 Kneeling: 3 diff --git a/sdks/python/apache_beam/testing/benchmarks/chicago_taxi/run_chicago.sh b/sdks/python/apache_beam/testing/benchmarks/chicago_taxi/run_chicago.sh index c170ea32df9..1103c0b0137 100755 --- a/sdks/python/apache_beam/testing/benchmarks/chicago_taxi/run_chicago.sh +++ b/sdks/python/apache_beam/testing/benchmarks/chicago_taxi/run_chicago.sh @@ -148,10 +148,10 @@ MODEL_DIR=${TRAIN_OUTPUT_PATH}/model_dir # Inputs TRAIN_FILE=${TFT_OUTPUT_PATH}/train_transformed-* TF_VERSION=1.14 -#workaround for boto in virtualenv, required for the gsutil commands to work: +#workaround for boto in virtualenv, required for the gcloud storage commands to work: export BOTO_CONFIG=/dev/null # Start clean, but don't fail if the path does not exist yet. -gsutil rm ${TRAIN_OUTPUT_PATH} || true +gcloud storage rm -r ${TRAIN_OUTPUT_PATH} || true # Options TRAIN_STEPS=10000 EVAL_STEPS=1000 @@ -177,7 +177,7 @@ gcloud ml-engine jobs submit training ${TRAINER_JOB_ID} \ # We evaluate with the last eval model written (hence tail -n1) EVAL_MODEL_DIR=${TRAIN_OUTPUT_PATH}/working_dir/eval_model_dir -LAST_EVAL_MODEL_DIR=$(gsutil ls ${EVAL_MODEL_DIR} | tail -n1) +LAST_EVAL_MODEL_DIR=$(gcloud storage ls ${EVAL_MODEL_DIR} | tail -n1) echo Eval model dir: ${EVAL_MODEL_DIR} diff --git a/sdks/python/scripts/run_snapshot_publish.sh b/sdks/python/scripts/run_snapshot_publish.sh index 0d7c7764748..8aa36a377e5 100755 --- a/sdks/python/scripts/run_snapshot_publish.sh +++ b/sdks/python/scripts/run_snapshot_publish.sh @@ -38,7 +38,7 @@ for file in "apache[-_]beam-$VERSION*.tar.gz"; do done # Upload to gcs bucket -gsutil cp $SNAPSHOT $BUCKET/$VERSION/ +gcloud storage cp $SNAPSHOT $BUCKET/$VERSION/ # Upload requirements.txt to gcs. -gsutil cp requirements.txt $DEP_SNAPSHOT_ROOT/$DEP_SNAPSHOT_FILE_NAME +gcloud storage cp requirements.txt $DEP_SNAPSHOT_ROOT/$DEP_SNAPSHOT_FILE_NAME diff --git a/website/Dockerfile b/website/Dockerfile index d275b00f367..fd0dc2e8b65 100644 --- a/website/Dockerfile +++ b/website/Dockerfile @@ -44,7 +44,7 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Install Google Cloud SDK (gsutil) +# Install Google Cloud SDK (gcloud storage) RUN apt-get update \ && apt-get install -y apt-transport-https \ # Download the key and save it directly to the keyring file (skipping apt-key) diff --git a/website/build.gradle b/website/build.gradle index 2aa04282e67..b9d721484d0 100644 --- a/website/build.gradle +++ b/website/build.gradle @@ -163,7 +163,6 @@ task downloadPerformanceLooks(type: Exec) { export HOME=/tmp export CLOUDSDK_CONFIG=/tmp/gcloud - export GSUTIL_CONFIG_DIR=/tmp/gsutil export GOOGLE_APPLICATION_CREDENTIALS="${gcpCredsInContainer}" gcloud auth activate-service-account --key-file="\$GOOGLE_APPLICATION_CREDENTIALS" --no-user-output-enabled @@ -172,7 +171,7 @@ task downloadPerformanceLooks(type: Exec) { echo "[performance_looks] Syncing looks..." - if gsutil -m rsync -r "\$SRC" "\$OUT_DIR"; then + if gcloud storage rsync -r "\$SRC" "\$OUT_DIR"; then echo "[performance_looks] Download completed successfully" else echo "[performance_looks][WARNING] Failed to sync looks from GCS" @@ -411,7 +410,7 @@ task stageWebsite { shell ". ${envdir}/bin/activate && python append_index_html_to_internal_links.py ${buildContentDir('gcs')}" // Copy the build website to GCS - shell "gsutil -m rsync -r -d ${buildContentDir('gcs')} ${gcs_path}" + shell "gcloud storage rsync -r --delete-unmatched-destination-objects ${buildContentDir('gcs')} ${gcs_path}" println "Website published to http://${gcs_bucket}."; + "storage.googleapis.com/${baseUrl}/index.html" diff --git a/website/www/site/content/en/blog/apache-hop-with-dataflow.md b/website/www/site/content/en/blog/apache-hop-with-dataflow.md index 0d023d6893b..ff841810015 100644 --- a/website/www/site/content/en/blog/apache-hop-with-dataflow.md +++ b/website/www/site/content/en/blog/apache-hop-with-dataflow.md @@ -226,7 +226,7 @@ For this example, I will use the region europe-west1 of GCP. Let's create a regi ``` -gsutil mb -c regional -l europe-west1 gs://ihr-apache-hop-blog +gcloud storage buckets create gs://ihr-apache-hop-blog --location=europe-west1 --default-storage-class=standard ``` @@ -234,7 +234,7 @@ Now let's upload the sample data to the GCS bucket, to test how the pipeline wou ``` - gsutil cp config/projects/samples/beam/input/customers-noheader-1k.txt gs://ihr-apache-hop-blog/data/ + gcloud storage cp config/projects/samples/beam/input/customers-noheader-1k.txt gs://ihr-apache-hop-blog/data/ ``` @@ -244,7 +244,7 @@ To make sure that you have uploaded the data correctly, check the contents of th ``` -gsutil ls gs://ihr-apache-hop-blog/data/ +gcloud storage ls gs://ihr-apache-hop-blog/data/ ``` @@ -411,11 +411,11 @@ When the pipeline starts running, you should see the graph of the pipeline in th alt="Dataflow pipeline graph"> -When the job finishes, there should be a file in the output location. You can check it out with `gsutil` +When the job finishes, there should be a file in the output location. You can check it out with `gcloud storage` ``` -% gsutil ls gs://ihr-apache-hop-blog/output +% gcloud storage ls gs://ihr-apache-hop-blog/output gs://ihr-apache-hop-blog/output/input-process-output-00000-of-00003.csv gs://ihr-apache-hop-blog/output/input-process-output-00001-of-00003.csv gs://ihr-apache-hop-blog/output/input-process-output-00002-of-00003.csv @@ -428,7 +428,7 @@ Let's explore the first lines of those files: ``` -gsutil cat "gs://ihr-apache-hop-blog/output/*csv"| head +gcloud storage cat "gs://ihr-apache-hop-blog/output/*csv"| head 12,wha-firstname,vnaov-name,egm-city,CALIFORNIA 25,ayl-firstname,bwkoe-name,rtw-city,CALIFORNIA 26,zio-firstname,rezku-name,nvt-city,CALIFORNIA diff --git a/website/www/site/content/en/blog/beam-sql-with-notebooks.md b/website/www/site/content/en/blog/beam-sql-with-notebooks.md index d7d80f4db7f..9941d65ef1e 100644 --- a/website/www/site/content/en/blog/beam-sql-with-notebooks.md +++ b/website/www/site/content/en/blog/beam-sql-with-notebooks.md @@ -744,7 +744,7 @@ Dataflow generates a one-shot job and it’s not interactive. A simple inspection of the data from the default output location: ``` -!gsutil cat 'gs://ningk-so-test/bq/staging/data_with_max_cases*' +!gcloud storage cat 'gs://ningk-so-test/bq/staging/data_with_max_cases*' ``` <img class="center-block" @@ -754,7 +754,7 @@ A simple inspection of the data from the default output location: The text file with encoded binary data written by your `WriteToText`: ``` -!gsutil cat 'gs://ningk-so-test/bq/encoded_max_data*' +!gcloud storage cat 'gs://ningk-so-test/bq/encoded_max_data*' ``` <img class="center-block" diff --git a/website/www/site/content/en/documentation/runners/spark.md b/website/www/site/content/en/documentation/runners/spark.md index 29ef5c28102..fce4927a82c 100644 --- a/website/www/site/content/en/documentation/runners/spark.md +++ b/website/www/site/content/en/documentation/runners/spark.md @@ -271,7 +271,7 @@ gcloud dataproc clusters create <b><i>CLUSTER_NAME</i></b> \ 2. Create a Cloud Storage bucket. <pre> -gsutil mb <b><i>BUCKET_NAME</i></b> +gcloud storage buckets create gs://<b><i>BUCKET_NAME</i></b> </pre> 3. Install the necessary Python libraries for the job in your local environment. @@ -313,7 +313,7 @@ gcloud dataproc jobs submit spark \ 6. Check that the results were written to your bucket. <pre> -gsutil cat gs://<b><i>BUCKET_NAME</b></i>/python-wordcount-out-<b><i>SHARD_ID</b></i> +gcloud storage cat gs://<b><i>BUCKET_NAME</b></i>/python-wordcount-out-<b><i>SHARD_ID</b></i> </pre> diff --git a/website/www/site/content/en/documentation/sdks/python-multi-language-pipelines.md b/website/www/site/content/en/documentation/sdks/python-multi-language-pipelines.md index 332a62901a2..2f4bfd7299f 100644 --- a/website/www/site/content/en/documentation/sdks/python-multi-language-pipelines.md +++ b/website/www/site/content/en/documentation/sdks/python-multi-language-pipelines.md @@ -203,7 +203,7 @@ export NUM_WORKERS="1" # other commands, e.g. changing into the appropriate directory -gsutil rm gs://$GCS_BUCKET/javaprefix/* +gcloud storage rm gs://$GCS_BUCKET/javaprefix/* python addprefix.py \ --runner DataflowRunner \ diff --git a/website/www/site/content/en/get-started/quickstart-java.md b/website/www/site/content/en/get-started/quickstart-java.md index d911918b9d2..b808f834ae4 100644 --- a/website/www/site/content/en/get-started/quickstart-java.md +++ b/website/www/site/content/en/get-started/quickstart-java.md @@ -344,7 +344,7 @@ ls /tmp/counts* ls counts* {{< /runner >}} {{< runner dataflow >}} -gsutil ls gs://<your-gcs-bucket>/counts* +gcloud storage ls gs://<your-gcs-bucket>/counts* {{< /runner >}} {{< runner nemo >}} ls counts* @@ -368,7 +368,7 @@ more /tmp/counts* more counts* {{< /runner >}} {{< runner dataflow >}} -gsutil cat gs://<your-gcs-bucket>/counts* +gcloud storage cat gs://<your-gcs-bucket>/counts* {{< /runner >}} {{< runner nemo >}} more counts*
