This is an automated email from the ASF dual-hosted git repository. shahar1 pushed a commit to branch gha-arc-rebase in repository https://gitbox.apache.org/repos/asf/airflow-ci-infra.git
commit 827b0508bcf783d854dc9f0393e43dc230b5d1fb Author: Hussein Awala <[email protected]> AuthorDate: Thu Jul 20 02:09:37 2023 +0200 Create a new VPC and an EKS cluster with 3 small ng for testing --- terraform/README.rst | 66 +++++++++++++++++++++++++++ terraform/eks/eks.tf | 108 +++++++++++++++++++++++++++++++++++++++++++++ terraform/eks/provider.tf | 21 +++++++++ terraform/eks/state.tf | 27 ++++++++++++ terraform/eks/variables.tf | 22 +++++++++ terraform/eks/version.tf | 26 +++++++++++ terraform/eks/vpc.tf | 44 ++++++++++++++++++ 7 files changed, 314 insertions(+) diff --git a/terraform/README.rst b/terraform/README.rst new file mode 100644 index 0000000..b88607e --- /dev/null +++ b/terraform/README.rst @@ -0,0 +1,66 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +Terraform: Airflow infrastructure as code +======================================== + +In this folder, you will find the Terraform code to deploy the Airflow infrastructure on AWS. + +The `eks` folder contains the code to deploy the Airflow EKS cluster and all the required resources. + +Requirements +------------ + +In order to deploy the infrastructure, you need to have the following tools installed: + +- `tfenv <https://github.com/tfutils/tfenv>`_ to manage Terraform versions +- `AWS CLI <https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html>`_ to configure the AWS credentials + +Configure AWS credentials +~~~~~~~~~~~~~~~~~~~~~~~~~ + +To play with the terraform stack, you need to configure the AWS credentials. You need to create access & secret keys for your user, then create a profile in the AWS CLI configuration file. + +.. code-block:: bash + + aws configure --profile airflow + +You will be asked to provide the access and secret keys, the region, and the output format. For the region, you should choose `us-east-2`. + +Configure Terraform +~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + cd eks + tfenv install 1.5.3 + +Plan and deploy the infrastructure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # Initialize the Terraform modules and install the required providers, + # just the first time or when you add a new module + AWS_PROFILE=airflow terraform init + + # Plan the infrastructure + AWS_PROFILE=airflow terraform plan -out=terraform.tfplan + + # Deploy the infrastructure after reviewing the plan + AWS_PROFILE=airflow terraform apply terraform.tfplan diff --git a/terraform/eks/eks.tf b/terraform/eks/eks.tf new file mode 100644 index 0000000..e130df7 --- /dev/null +++ b/terraform/eks/eks.tf @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +data "aws_availability_zones" "available" {} + +locals { + cluster_name = "airflow" +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "19.15.3" + + cluster_name = local.cluster_name + cluster_version = "1.27" + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + cluster_endpoint_public_access = true + + eks_managed_node_group_defaults = { + ami_type = "AL2_ARM_64" + } + + eks_managed_node_groups = { + + default_nodes = { + name = "default" + + instance_types = ["t4g.small"] + + min_size = 1 + max_size = 3 + + capacity_type = "SPOT" + + labels = { + "node-type" = "default" + } + } + + GHA_runners_small = { + name = "gha-runners-small" + + instance_types = ["t4g.small"] + + min_size = 0 + max_size = 3 + desired_size = 0 + + capacity_type = "SPOT" + + labels = { + "node-type" = "gha-runners" + "size" = "small" + } + + taints = [ + { + "key" = "node-type" + "value" = "gha-runners" + "effect" = "NO_SCHEDULE" + } + ] + } + + GHA_runners_medium = { + name = "gha-runners-medium" + + instance_types = ["t4g.medium"] + + min_size = 0 + max_size = 3 + desired_size = 0 + + capacity_type = "SPOT" + + labels = { + "node-type" = "gha-runners" + "size" = "medium" + } + + taints = [ + { + "key" = "node-type" + "value" = "gha-runners" + "effect" = "NO_SCHEDULE" + } + ] + } + } +} + diff --git a/terraform/eks/provider.tf b/terraform/eks/provider.tf new file mode 100644 index 0000000..c4c55f9 --- /dev/null +++ b/terraform/eks/provider.tf @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +provider "aws" { + profile = "airflow" + region = "us-east-2" +} \ No newline at end of file diff --git a/terraform/eks/state.tf b/terraform/eks/state.tf new file mode 100644 index 0000000..025e912 --- /dev/null +++ b/terraform/eks/state.tf @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +terraform { + + backend "s3" { + region = "us-east-2" + bucket = "airflow-ci-tfstate" + key = "tf-state/eks.tfstate" + encrypt = true + dynamodb_table = "TerraformLocks" + } +} diff --git a/terraform/eks/variables.tf b/terraform/eks/variables.tf new file mode 100644 index 0000000..eaf84b6 --- /dev/null +++ b/terraform/eks/variables.tf @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +variable "region" { + description = "AWS region" + type = string + default = "us-east-2" +} diff --git a/terraform/eks/version.tf b/terraform/eks/version.tf new file mode 100644 index 0000000..4b1549d --- /dev/null +++ b/terraform/eks/version.tf @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.7.0" + } + } + required_version = "~> 1.3" +} \ No newline at end of file diff --git a/terraform/eks/vpc.tf b/terraform/eks/vpc.tf new file mode 100644 index 0000000..121b054 --- /dev/null +++ b/terraform/eks/vpc.tf @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# TODO: use the existing VPC? +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "5.0.0" + + name = "airflow-vpc" + + cidr = "10.1.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + private_subnets = ["10.1.1.0/24", "10.1.2.0/24", "10.1.3.0/24"] + public_subnets = ["10.1.4.0/24", "10.1.5.0/24", "10.1.6.0/24"] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } +} \ No newline at end of file
