This is an automated email from the ASF dual-hosted git repository.
membphis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/master by this push:
new a38cbf7 feat: chaos test on route could still works when etcd is down
(#3404)
a38cbf7 is described below
commit a38cbf78b7223ac5017fdff4df45ae45380cb55d
Author: Shuyang Wu <wosoyo...@gmail.com>
AuthorDate: Fri Jan 29 19:22:43 2021 +0800
feat: chaos test on route could still works when etcd is down (#3404)
---
.github/workflows/chaos.yml | 52 ++++++++++
kubernetes/deployment.yaml | 3 +-
t/chaos/go.mod | 8 ++
t/chaos/go.sum | 97 ++++++++++++++++++
t/chaos/kill-etcd.yaml | 29 ++++++
t/chaos/kill-etcd_test.go | 240 ++++++++++++++++++++++++++++++++++++++++++++
t/chaos/utils.sh | 62 ++++++++++++
7 files changed, 490 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/chaos.yml b/.github/workflows/chaos.yml
new file mode 100644
index 0000000..1b31730
--- /dev/null
+++ b/.github/workflows/chaos.yml
@@ -0,0 +1,52 @@
+name: Chaos Test
+
+on: [pull_request]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: setup go
+ uses: actions/setup-go@v2.1.3
+ with:
+ go-version: "1.14"
+
+ - name: Creating minikube cluster
+ run: |
+ bash ./t/chaos/utils.sh start_minikube
+
+ - name: Print cluster information
+ run: |
+ kubectl config view
+ kubectl cluster-info
+ kubectl get nodes
+ kubectl get pods -n kube-system
+ kubectl version
+
+ # TODO: should not use personal repo, while it's hard to modify a lot in
an archived repo. Could we use api7's repo?
+ - name: Deploy Etcd Operator
+ run: |
+ git clone https://github.com/yiyiyimu/etcd-operator.git --depth 1
+ bash etcd-operator/example/rbac/create_role.sh
+ kubectl create -f etcd-operator/example/deployment.yaml
+ bash ./t/chaos/utils.sh ensure_pods_ready etcd-operator "True" 30
+ kubectl create -f etcd-operator/example/example-etcd-cluster.yaml
+ bash ./t/chaos/utils.sh ensure_pods_ready etcd "True True True" 30
+
+ - name: Deploy APISIX
+ run: |
+ bash ./t/chaos/utils.sh modify_config
+ kubectl create configmap apisix-gw-config.yaml
--from-file=./conf/config.yaml
+ kubectl apply -f ./kubernetes/deployment.yaml
+ kubectl apply -f ./kubernetes/service.yaml
+ bash ./t/chaos/utils.sh ensure_pods_ready apisix-gw "True" 30
+ nohup kubectl port-forward svc/apisix-gw-lb 9080:9080 >/dev/null
2>&1 &
+
+ - name: Deploy Chaos mesh
+ run: |
+ curl -sSL https://mirrors.chaos-mesh.org/v1.1.1/install.sh | bash
+
+ - name: run test
+ working-directory: ./t/chaos
+ run: go test -v
diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml
index 34f2091..0413db4 100644
--- a/kubernetes/deployment.yaml
+++ b/kubernetes/deployment.yaml
@@ -23,7 +23,7 @@ metadata:
name: apisix-gw-deployment
# namespace: default
spec:
- replicas: 2
+ replicas: 1
selector:
matchLabels:
app: apisix-gw
@@ -57,6 +57,7 @@ spec:
sysctl -w fs.inotify.max_user_watches=524288
sysctl -w fs.inotify.max_queued_events=16384
image: busybox:latest
+ imagePullPolicy: IfNotPresent
name: init-sysctl
resources: {}
securityContext:
diff --git a/t/chaos/go.mod b/t/chaos/go.mod
new file mode 100644
index 0000000..b24abf3
--- /dev/null
+++ b/t/chaos/go.mod
@@ -0,0 +1,8 @@
+module github.com/apache/apisix/t/chaos
+
+require (
+ github.com/gavv/httpexpect/v2 v2.1.0
+ github.com/onsi/gomega v1.9.0
+)
+
+go 1.14
diff --git a/t/chaos/go.sum b/t/chaos/go.sum
new file mode 100644
index 0000000..5d88022
--- /dev/null
+++ b/t/chaos/go.sum
@@ -0,0 +1,97 @@
+github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
+github.com/ajg/form v1.5.1/go.mod
h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/davecgh/go-spew v1.1.0
h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod
h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fasthttp/websocket v1.4.2
h1:AU/zSiIIAuJjBMf5o+vO0syGOnEfvZRu40xIhW/3RuM=
+github.com/fasthttp/websocket v1.4.2/go.mod
h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y=
+github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU=
+github.com/fatih/structs v1.0.0/go.mod
h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fsnotify/fsnotify v1.4.7
h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod
h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/gavv/httpexpect/v2 v2.1.0
h1:Q7xnFuKqBY2si4DsqxdbWBt9rfrbVTT2/9YSomc9tEw=
+github.com/gavv/httpexpect/v2 v2.1.0/go.mod
h1:lnd0TqJLrP+wkJk3SFwtrpSlOAZQ7HaaIFuOYbgqgUM=
+github.com/golang/protobuf v1.2.0
h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod
h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-querystring v1.0.0
h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod
h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/gorilla/websocket v1.0.0
h1:J/mA+d2LqcDKjAEhQjXDHt9/e7Cnm+oBUwgHp5C6XDg=
+github.com/gorilla/websocket v1.0.0/go.mod
h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod
h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imkira/go-interpol v1.0.0
h1:HrmLyvOLJyjR0YofMw8QGdCIuYOs4TJUBDNU5sJC09E=
+github.com/imkira/go-interpol v1.0.0/go.mod
h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88
h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod
h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
+github.com/klauspost/compress v1.8.2
h1:Bx0qjetmNjdFXASH02NSAREKpiaDwkO1DRZ3dV2KCcs=
+github.com/klauspost/compress v1.8.2/go.mod
h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.1
h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
+github.com/klauspost/cpuid v1.2.1/go.mod
h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/mattn/go-colorable v0.1.2
h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
+github.com/mattn/go-colorable v0.1.2/go.mod
h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8
h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod
h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod
h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo=
+github.com/onsi/ginkgo v1.10.1/go.mod
h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod
h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
+github.com/onsi/gomega v1.9.0/go.mod
h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/pmezard/go-difflib v1.0.0
h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod
h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f
h1:PgA+Olipyj258EIEYnpFFONrrCcAIWNUNoFhUfMqAGY=
+github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod
h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod
h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/stretchr/objx v0.1.0/go.mod
h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0
h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod
h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/valyala/bytebufferpool v1.0.0
h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod
h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.9.0
h1:hNpmUdy/+ZXYpGy0OBfm7K0UQTzb73W0T0U4iJIVrMw=
+github.com/valyala/fasthttp v1.9.0/go.mod
h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod
h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f
h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod
h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415
h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod
h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.1.0
h1:ngVtJC9TY/lg0AA/1k48FYhBrhRoFlEmWzsehpNAaZg=
+github.com/xeipuuv/gojsonschema v1.1.0/go.mod
h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod
h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
+github.com/yudai/gojsondiff v1.0.0
h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA=
+github.com/yudai/gojsondiff v1.0.0/go.mod
h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82
h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M=
+github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod
h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
+github.com/yudai/pp v2.0.1+incompatible
h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI=
+github.com/yudai/pp v2.0.1+incompatible/go.mod
h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod
h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd
h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod
h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod
h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod
h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e
h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod
h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod
h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223
h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod
h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7
h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod
h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405
h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod
h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod
h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e
h1:C7q+e9M5nggAvWfVg9Nl66kebKeuJlP3FD58V4RR5wo=
+moul.io/http2curl v1.0.1-0.20190925090545-5cd742060b0e/go.mod
h1:nejbQVfXh96n9dSF6cH3Jsk/QI1Z2oEL7sSI2ifXFNA=
diff --git a/t/chaos/kill-etcd.yaml b/t/chaos/kill-etcd.yaml
new file mode 100644
index 0000000..98c41f6
--- /dev/null
+++ b/t/chaos/kill-etcd.yaml
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: chaos-mesh.org/v1alpha1
+kind: PodChaos
+metadata:
+ name: kill-etcd
+spec:
+ action: pod-kill
+ mode: all
+ selector:
+ labelSelectors:
+ "app": "etcd"
+ scheduler:
+ cron: "@every 10m"
diff --git a/t/chaos/kill-etcd_test.go b/t/chaos/kill-etcd_test.go
new file mode 100644
index 0000000..204249c
--- /dev/null
+++ b/t/chaos/kill-etcd_test.go
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package chaos
+
+import (
+ "fmt"
+ "net/http"
+ "os/exec"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/gavv/httpexpect/v2"
+ . "github.com/onsi/gomega"
+)
+
+var (
+ token = "edd1c9f034335f136f87ad84b625c8f1"
+ host = "http://127.0.0.1:9080";
+)
+
+type httpTestCase struct {
+ E *httpexpect.Expect
+ Method string
+ Path string
+ Body string
+ Headers map[string]string
+ ExpectStatus int
+ ExpectBody string
+}
+
+func caseCheck(tc httpTestCase) *httpexpect.Response {
+ e := tc.E
+ var req *httpexpect.Request
+ switch tc.Method {
+ case http.MethodGet:
+ req = e.GET(tc.Path)
+ case http.MethodPut:
+ req = e.PUT(tc.Path)
+ default:
+ }
+
+ if req == nil {
+ panic("fail to init request")
+ }
+ for key, val := range tc.Headers {
+ req.WithHeader(key, val)
+ }
+ if tc.Body != "" {
+ req.WithText(tc.Body)
+ }
+
+ resp := req.Expect()
+ if tc.ExpectStatus != 0 {
+ resp.Status(tc.ExpectStatus)
+ }
+
+ if tc.ExpectBody != "" {
+ resp.Body().Contains(tc.ExpectBody)
+ }
+
+ return resp
+}
+
+func setRoute(e *httpexpect.Expect, expectStatus int) {
+ caseCheck(httpTestCase{
+ E: e,
+ Method: http.MethodPut,
+ Path: "/apisix/admin/routes/1",
+ Headers: map[string]string{"X-API-KEY": token},
+ Body: `{
+ "uri": "/hello",
+ "host": "foo.com",
+ "plugins": {
+ "prometheus": {}
+ },
+ "upstream": {
+ "nodes": {
+ "bar.org": 1
+ },
+ "type": "roundrobin"
+ }
+ }`,
+ ExpectStatus: expectStatus,
+ })
+}
+
+func getRoute(e *httpexpect.Expect, expectStatus int) {
+ caseCheck(httpTestCase{
+ E: e,
+ Method: http.MethodGet,
+ Path: "/hello",
+ Headers: map[string]string{"Host": "foo.com"},
+ ExpectStatus: expectStatus,
+ })
+}
+
+func deleteRoute(e *httpexpect.Expect, expectStatus int) {
+ caseCheck(httpTestCase{
+ E: e,
+ Method: http.MethodDelete,
+ Path: "/apisix/admin/routes/1",
+ Headers: map[string]string{"X-API-KEY": token},
+ ExpectStatus: expectStatus,
+ })
+}
+
+func testPrometheusEtcdMetric(e *httpexpect.Expect, expectEtcd int) {
+ caseCheck(httpTestCase{
+ E: e,
+ Method: http.MethodGet,
+ Path: "/apisix/prometheus/metrics",
+ ExpectBody: fmt.Sprintf("apisix_etcd_reachable %d", expectEtcd),
+ })
+}
+
+// get the first line which contains the key
+func getPrometheusMetric(e *httpexpect.Expect, g *WithT, key string) string {
+ resp := caseCheck(httpTestCase{
+ E: e,
+ Method: http.MethodGet,
+ Path: "/apisix/prometheus/metrics",
+ })
+ resps := strings.Split(resp.Body().Raw(), "\n")
+ var targetLine string
+ for _, line := range resps {
+ if strings.Contains(line, key) {
+ targetLine = line
+ break
+ }
+ }
+ targetSlice := strings.Fields(targetLine)
+ g.Expect(len(targetSlice) == 2).To(BeTrue())
+ return targetSlice[1]
+}
+
+func getIngressBandwidthPerSecond(e *httpexpect.Expect, g *WithT) float64 {
+ key := "apisix_bandwidth{type=\"ingress\","
+ bandWidthString := getPrometheusMetric(e, g, key)
+ bandWidthStart, err := strconv.ParseFloat(bandWidthString, 64)
+ g.Expect(err).To(BeNil())
+ // after etcd got killed, it would take longer time to get the metrics
+ // so need to calculate the duration
+ timeStart := time.Now()
+
+ time.Sleep(10 * time.Second)
+ bandWidthString = getPrometheusMetric(e, g, key)
+ bandWidthEnd, err := strconv.ParseFloat(bandWidthString, 64)
+ g.Expect(err).To(BeNil())
+ duration := time.Now().Sub(timeStart)
+
+ return (bandWidthEnd - bandWidthStart) / duration.Seconds()
+}
+
+func runCommand(t *testing.T, cmd string) string {
+ out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
+ if err != nil {
+ t.Fatalf("fail to run command %s: %s, %s", cmd, err.Error(),
out)
+ }
+ return string(out)
+}
+
+func roughCompare(a float64, b float64) bool {
+ ratio := a / b
+ if ratio < 1.3 && ratio > 0.7 {
+ return true
+ }
+ return false
+}
+
+func TestGetSuccessWhenEtcdKilled(t *testing.T) {
+ g := NewWithT(t)
+ e := httpexpect.New(t, host)
+
+ // check if everything works
+ setRoute(e, http.StatusCreated)
+
+ // to avoid route haven't been set yet
+ time.Sleep(1 * time.Second)
+ getRoute(e, http.StatusOK)
+ testPrometheusEtcdMetric(e, 1)
+
+ // run in background
+ go func() {
+ for {
+ go getRoute(e, http.StatusOK)
+ time.Sleep(1000 * time.Millisecond)
+ }
+ }()
+
+ // wait 5 second to let first route access returns
+ time.Sleep(5 * time.Second)
+ bpsBefore := getIngressBandwidthPerSecond(e, g)
+ g.Expect(bpsBefore).NotTo(BeZero())
+
+ podName := runCommand(t, "kubectl get pod -l app=apisix-gw -o
'jsonpath={..metadata.name}'")
+ t.Run("error log not contains etcd error", func(t *testing.T) {
+ errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s --
cat logs/error.log", podName))
+ g.Expect(strings.Contains(errorLog, "failed to fetch data from
etcd")).To(BeFalse())
+ })
+
+ // TODO: use client-go
+ // apply chaos to kill all etcd pods
+ t.Log("kill all etcd pods")
+ _ = runCommand(t, "kubectl apply -f kill-etcd.yaml")
+ time.Sleep(3 * time.Second)
+
+ // fail to set route since etcd is all killed
+ // while get route could still succeed
+ setRoute(e, http.StatusInternalServerError)
+ getRoute(e, http.StatusOK)
+ testPrometheusEtcdMetric(e, 0)
+
+ t.Run("error log contains etcd error", func(t *testing.T) {
+ errorLog := runCommand(t, fmt.Sprintf("kubectl exec -it %s --
cat logs/error.log", podName))
+ g.Expect(strings.Contains(errorLog, "failed to fetch data from
etcd")).To(BeTrue())
+ })
+
+ bpsAfter := getIngressBandwidthPerSecond(e, g)
+ t.Run("ingress bandwidth per second not change much", func(t
*testing.T) {
+ t.Logf("bps before: %f, after: %f", bpsBefore, bpsAfter)
+ g.Expect(roughCompare(bpsBefore, bpsAfter)).To(BeTrue())
+ })
+}
diff --git a/t/chaos/utils.sh b/t/chaos/utils.sh
new file mode 100755
index 0000000..7a250b8
--- /dev/null
+++ b/t/chaos/utils.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+start_minikube() {
+ curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl
-s
https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
+ chmod +x ./kubectl
+ sudo mv ./kubectl /usr/local/bin/kubectl
+
+ curl -LO
https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
+ sudo dpkg -i --force-architecture minikube_latest_amd64.deb
+ minikube start
+}
+
+modify_config() {
+ DNS_IP=$(kubectl get svc -n kube-system -l k8s-app=kube-dns -o
'jsonpath={..spec.clusterIP}')
+ echo "dns_resolver:
+ - ${DNS_IP}
+etcd:
+ host:
+ - \"http://etcd-cluster-client.default.svc.cluster.local:2379\"; " >
./conf/config.yaml
+}
+
+ensure_pods_ready() {
+ local app=$1
+ local status=$2
+ local retries=$3
+
+ count=0
+ while [[ $(kubectl get pods -l app=${app} -o
'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != ${status} ]];
+ do
+ echo "Waiting for pod running" && sleep 10;
+
+ ((count=count+1))
+ if [ $count -gt ${retries} ]; then
+ printf "Waiting for pod status running timeout\n"
+ kubectl describe pod -l app=${app}
+ printf "\n\n"
+ kubectl logs -l app=${app}
+ exit 1
+ fi
+ done
+}
+
+"$@"
