This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new d7264b7554 Decrease disk size for Dataproc cluster nodes in system 
tests (#33823)
d7264b7554 is described below

commit d7264b7554f869a3e0fc20ff19646201bf36b9df
Author: max <[email protected]>
AuthorDate: Mon Aug 28 15:12:30 2023 +0200

    Decrease disk size for Dataproc cluster nodes in system tests (#33823)
---
 .../google/cloud/dataproc/example_dataproc_cluster_deferrable.py    | 6 +++---
 .../google/cloud/dataproc/example_dataproc_cluster_generator.py     | 2 ++
 .../google/cloud/dataproc/example_dataproc_cluster_update.py        | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_hadoop.py      | 4 ++--
 .../system/providers/google/cloud/dataproc/example_dataproc_hive.py | 6 +++---
 .../system/providers/google/cloud/dataproc/example_dataproc_pig.py  | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_presto.py      | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_pyspark.py     | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_spark.py       | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_spark_async.py | 4 ++--
 .../google/cloud/dataproc/example_dataproc_spark_deferrable.py      | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_spark_sql.py   | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_sparkr.py      | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_trino.py       | 4 ++--
 .../providers/google/cloud/dataproc/example_dataproc_workflow.py    | 4 ++--
 .../google/cloud/dataproc/example_dataproc_workflow_deferrable.py   | 4 ++--
 16 files changed, 34 insertions(+), 32 deletions(-)

diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
index b14c5c00f1..3c39a99ae5 100644
--- 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
+++ 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
@@ -44,19 +44,19 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "secondary_worker_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
         "disk_config": {
             "boot_disk_type": "pd-standard",
-            "boot_disk_size_gb": 1024,
+            "boot_disk_size_gb": 32,
         },
         "is_preemptible": True,
         "preemptibility": "PREEMPTIBLE",
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
index f994247294..746ee9df0d 100644
--- 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
+++ 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
@@ -54,7 +54,9 @@ CLUSTER_GENERATOR_CONFIG = ClusterGenerator(
     project_id=PROJECT_ID,
     zone=ZONE,
     master_machine_type="n1-standard-4",
+    master_disk_size=32,
     worker_machine_type="n1-standard-4",
+    worker_disk_size=32,
     num_workers=2,
     storage_bucket=BUCKET_NAME,
     init_actions_uris=[f"gs://{BUCKET_NAME}/{INIT_FILE}"],
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py
 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py
index ed8e953239..ca7dff04af 100644
--- 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py
+++ 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py
index 4cd612c8ea..690561b66b 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py
@@ -48,12 +48,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 3,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py
index 99312b2011..1bc4e18a4b 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py
@@ -45,19 +45,19 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "secondary_worker_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
         "disk_config": {
             "boot_disk_type": "pd-standard",
-            "boot_disk_size_gb": 1024,
+            "boot_disk_size_gb": 32,
         },
         "is_preemptible": True,
         "preemptibility": "PREEMPTIBLE",
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py
index 1a0299bd10..c0157e5f96 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py
index 48e3d46614..a41b5077f5 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "software_config": {
         "optional_components": [
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py
index 001e989d7c..b73ae2742f 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py
index 170230abbf..f30c7224f6 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py
@@ -43,12 +43,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py
index a51f7c2e19..63485d2014 100644
--- 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py
+++ 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py
 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py
index 599eb4cf27..baa7875df2 100644
--- 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py
+++ 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py
index 3db2f881bc..ebf3333d25 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py
@@ -43,12 +43,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py
index 1a260ca290..b98fcde8c2 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py
index 828f596041..4d60157c26 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py
@@ -45,12 +45,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "software_config": {
         "optional_components": [
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py
index 35d3ee0ebe..dce5953182 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py
@@ -39,12 +39,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 PIG_JOB = {"query_list": {"queries": ["define sin HiveUDF('sin');"]}}
diff --git 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
index 843148bf9b..3e03e0e884 100644
--- 
a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
+++ 
b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
@@ -39,12 +39,12 @@ CLUSTER_CONFIG = {
     "master_config": {
         "num_instances": 1,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
     "worker_config": {
         "num_instances": 2,
         "machine_type_uri": "n1-standard-4",
-        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
1024},
+        "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb": 
32},
     },
 }
 PIG_JOB = {"query_list": {"queries": ["define sin HiveUDF('sin');"]}}

Reply via email to