This is an automated email from the ASF dual-hosted git repository. markusthoemmes pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push: new 66c9dbc Add tests to check high-availability of the controller. (#2661) 66c9dbc is described below commit 66c9dbc094234c6851731c2f858ea9e5977b5409 Author: Christian Bickel <git...@cbickel.de> AuthorDate: Mon Sep 11 10:14:21 2017 +0200 Add tests to check high-availability of the controller. (#2661) --- tests/src/test/scala/ha/ShootComponentsTests.scala | 147 +++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/tests/src/test/scala/ha/ShootComponentsTests.scala b/tests/src/test/scala/ha/ShootComponentsTests.scala new file mode 100644 index 0000000..4b28ded --- /dev/null +++ b/tests/src/test/scala/ha/ShootComponentsTests.scala @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ha + +import java.io.File +import java.time.Instant + +import scala.concurrent.Future +import scala.concurrent.duration.DurationInt +import scala.util.Try + +import org.junit.runner.RunWith +import org.scalatest.FlatSpec +import org.scalatest.Matchers +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.junit.JUnitRunner + +import akka.http.scaladsl.Http +import akka.http.scaladsl.model.HttpRequest +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.unmarshalling.Unmarshal +import akka.stream.ActorMaterializer +import common.TestUtils +import common.WhiskProperties +import common.Wsk +import common.WskActorSystem +import common.WskProps +import common.WskTestHelpers +import whisk.core.WhiskConfig +import whisk.utils.retry + +@RunWith(classOf[JUnitRunner]) +class ShootComponentsTests extends FlatSpec with Matchers with WskTestHelpers with ScalaFutures with WskActorSystem { + + implicit val wskprops = WskProps() + val wsk = new Wsk + val defaultAction = Some(TestUtils.getTestActionFilename("hello.js")) + + implicit val materializer = ActorMaterializer() + implicit val testConfig = PatienceConfig(1.minute) + + val controller0DockerHost = WhiskProperties.getBaseControllerHost() + ":" + WhiskProperties.getProperty( + WhiskConfig.dockerPort) + + def restartComponent(host: String, component: String) = { + def file(path: String) = Try(new File(path)).filter(_.exists).map(_.getAbsolutePath).toOption + val docker = (file("/usr/bin/docker") orElse file("/usr/local/bin/docker")).getOrElse("docker") + + val cmd = Seq(docker, "--host", host, "restart", component) + println(s"Running command: ${cmd.mkString(" ")}") + + TestUtils.runCmd(0, new File("."), cmd: _*) + } + + def ping(host: String, port: Int) = { + val response = Try { Http().singleRequest(HttpRequest(uri = s"http://$host:$port/ping")).futureValue }.toOption + + response.map { res => + (res.status, Unmarshal(res).to[String].futureValue) + } + } + + def isControllerAlive(instance: Int): Boolean = { + require(instance >= 0 && instance < 2, "Controller instance not known.") + + val host = WhiskProperties.getProperty("controller.hosts").split(",")(instance) + val port = WhiskProperties.getControllerBasePort + instance + + val res = ping(host, port) + res == Some((StatusCodes.OK, "pong")) + } + + def doRequests(amount: Int, actionName: String): Seq[(Int, Int)] = { + (0 until amount).map { i => + val start = Instant.now + + // Do POSTs and GETs + val invokeExit = Future { wsk.action.invoke(actionName, expectedExitCode = TestUtils.DONTCARE_EXIT).exitCode } + val getExit = Future { wsk.action.get(actionName, expectedExitCode = TestUtils.DONTCARE_EXIT).exitCode } + + println(s"Done rerquests with responses: invoke: ${invokeExit.futureValue} and get: ${getExit.futureValue}") + + // Do at most one action invocation per second to avoid getting 429s. (60 req/min - limit) + val wait = 1000 - (Instant.now.toEpochMilli - start.toEpochMilli) + Thread.sleep(if (wait < 0) 0L else if (wait > 1000) 1000L else wait) + (invokeExit.futureValue, getExit.futureValue) + } + } + + behavior of "Controllers hot standby" + + it should "use controller1 if controller0 goes down" in withAssetCleaner(wskprops) { (wp, assetHelper) => + if (WhiskProperties.getProperty(WhiskConfig.controllerInstances).toInt >= 2) { + val actionName = "shootcontroller" + + assetHelper.withCleaner(wsk.action, actionName) { (action, _) => + action.create(actionName, defaultAction) + } + + // Produce some load on the system for 100 seconds (each second one request). Kill the controller after 4 requests + val totalRequests = 100 + + val requestsBeforeRestart = doRequests(4, actionName) + + // Kill the controller + restartComponent(controller0DockerHost, "controller0") + // Wait until down + retry({ + isControllerAlive(0) shouldBe false + }, 100, Some(100.milliseconds)) + // Check that second controller is still up + isControllerAlive(1) shouldBe true + + val requestsAfterRestart = doRequests(totalRequests - 4, actionName) + + val requests = requestsBeforeRestart ++ requestsAfterRestart + + val unsuccessfulInvokes = requests.map(_._1).count(_ != TestUtils.SUCCESS_EXIT) + // Allow 3 failures for the 90 seconds + unsuccessfulInvokes should be <= 3 + + val unsuccessfulGets = requests.map(_._2).count(_ != TestUtils.SUCCESS_EXIT) + // Only allow 1 failure in GET requests, because they are idempotent and they should be passed to the next controller if one crashes + unsuccessfulGets shouldBe 0 + + // Check that both controllers are up + // controller0 + isControllerAlive(0) shouldBe true + //controller1 + isControllerAlive(1) shouldBe true + } + } +} -- To stop receiving notification emails like this one, please contact ['"commits@openwhisk.apache.org" <commits@openwhisk.apache.org>'].