This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d7264b7554 Decrease disk size for Dataproc cluster nodes in system
tests (#33823)
d7264b7554 is described below
commit d7264b7554f869a3e0fc20ff19646201bf36b9df
Author: max <[email protected]>
AuthorDate: Mon Aug 28 15:12:30 2023 +0200
Decrease disk size for Dataproc cluster nodes in system tests (#33823)
---
.../google/cloud/dataproc/example_dataproc_cluster_deferrable.py | 6 +++---
.../google/cloud/dataproc/example_dataproc_cluster_generator.py | 2 ++
.../google/cloud/dataproc/example_dataproc_cluster_update.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_hadoop.py | 4 ++--
.../system/providers/google/cloud/dataproc/example_dataproc_hive.py | 6 +++---
.../system/providers/google/cloud/dataproc/example_dataproc_pig.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_presto.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_pyspark.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_spark.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_spark_async.py | 4 ++--
.../google/cloud/dataproc/example_dataproc_spark_deferrable.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_spark_sql.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_sparkr.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_trino.py | 4 ++--
.../providers/google/cloud/dataproc/example_dataproc_workflow.py | 4 ++--
.../google/cloud/dataproc/example_dataproc_workflow_deferrable.py | 4 ++--
16 files changed, 34 insertions(+), 32 deletions(-)
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
index b14c5c00f1..3c39a99ae5 100644
---
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
+++
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
@@ -44,19 +44,19 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"secondary_worker_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
"disk_config": {
"boot_disk_type": "pd-standard",
- "boot_disk_size_gb": 1024,
+ "boot_disk_size_gb": 32,
},
"is_preemptible": True,
"preemptibility": "PREEMPTIBLE",
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
index f994247294..746ee9df0d 100644
---
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
+++
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_generator.py
@@ -54,7 +54,9 @@ CLUSTER_GENERATOR_CONFIG = ClusterGenerator(
project_id=PROJECT_ID,
zone=ZONE,
master_machine_type="n1-standard-4",
+ master_disk_size=32,
worker_machine_type="n1-standard-4",
+ worker_disk_size=32,
num_workers=2,
storage_bucket=BUCKET_NAME,
init_actions_uris=[f"gs://{BUCKET_NAME}/{INIT_FILE}"],
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py
index ed8e953239..ca7dff04af 100644
---
a/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py
+++
b/tests/system/providers/google/cloud/dataproc/example_dataproc_cluster_update.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py
index 4cd612c8ea..690561b66b 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_hadoop.py
@@ -48,12 +48,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 3,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py
index 99312b2011..1bc4e18a4b 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_hive.py
@@ -45,19 +45,19 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"secondary_worker_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
"disk_config": {
"boot_disk_type": "pd-standard",
- "boot_disk_size_gb": 1024,
+ "boot_disk_size_gb": 32,
},
"is_preemptible": True,
"preemptibility": "PREEMPTIBLE",
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py
index 1a0299bd10..c0157e5f96 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py
index 48e3d46614..a41b5077f5 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_presto.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"software_config": {
"optional_components": [
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py
index 001e989d7c..b73ae2742f 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_pyspark.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py
index 170230abbf..f30c7224f6 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark.py
@@ -43,12 +43,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py
index a51f7c2e19..63485d2014 100644
---
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py
+++
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_async.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py
index 599eb4cf27..baa7875df2 100644
---
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py
+++
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_deferrable.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py
index 3db2f881bc..ebf3333d25 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_spark_sql.py
@@ -43,12 +43,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py
index 1a260ca290..b98fcde8c2 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_sparkr.py
@@ -44,12 +44,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py
index 828f596041..4d60157c26 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_trino.py
@@ -45,12 +45,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"software_config": {
"optional_components": [
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py
index 35d3ee0ebe..dce5953182 100644
--- a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py
+++ b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow.py
@@ -39,12 +39,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
PIG_JOB = {"query_list": {"queries": ["define sin HiveUDF('sin');"]}}
diff --git
a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
index 843148bf9b..3e03e0e884 100644
---
a/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
+++
b/tests/system/providers/google/cloud/dataproc/example_dataproc_workflow_deferrable.py
@@ -39,12 +39,12 @@ CLUSTER_CONFIG = {
"master_config": {
"num_instances": 1,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
"worker_config": {
"num_instances": 2,
"machine_type_uri": "n1-standard-4",
- "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
1024},
+ "disk_config": {"boot_disk_type": "pd-standard", "boot_disk_size_gb":
32},
},
}
PIG_JOB = {"query_list": {"queries": ["define sin HiveUDF('sin');"]}}