Repository: incubator-s2graph Updated Branches: refs/heads/master ef0b6e51a -> 19254301d (forced update)
s2http initial commit Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/a16ecb08 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/a16ecb08 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/a16ecb08 Branch: refs/heads/master Commit: a16ecb08819ea01f4498dfdd371717de56387a72 Parents: 66b656f Author: daewon <dae...@apache.org> Authored: Mon Nov 26 17:11:36 2018 +0900 Committer: daewon <dae...@apache.org> Committed: Mon Nov 26 17:32:33 2018 +0900 ---------------------------------------------------------------------- build.sbt | 8 +- s2core/src/main/resources/reference.conf | 10 +- s2http/README.md | 47 +++++++ s2http/build.sbt | 41 +++++++ s2http/src/main/resources/application.conf | 27 +++++ s2http/src/main/resources/log4j.properties | 26 ++++ .../apache/s2graph/http/S2GraphAdminRoute.scala | 121 +++++++++++++++++++ .../s2graph/http/S2GraphTraversalRoute.scala | 65 ++++++++++ .../scala/org/apache/s2graph/http/Server.scala | 70 +++++++++++ .../apache/s2graph/http/AdminRouteSpec.scala | 61 ++++++++++ 10 files changed, 467 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/build.sbt ---------------------------------------------------------------------- diff --git a/build.sbt b/build.sbt index 03e9bd0..c4d2c35 100755 --- a/build.sbt +++ b/build.sbt @@ -46,7 +46,7 @@ lazy val s2rest_play = project.enablePlugins(PlayScala).disablePlugins(PlayLogba .settings(dockerSettings: _*) .enablePlugins(sbtdocker.DockerPlugin, JavaAppPackaging) -lazy val s2rest_netty = project +lazy val s2http = project .dependsOn(s2core) .settings(commonSettings: _*) .settings(dockerSettings: _*) @@ -78,8 +78,8 @@ lazy val s2graph_gremlin = project.dependsOn(s2core) .settings(commonSettings: _*) lazy val root = (project in file(".")) - .aggregate(s2core, s2rest_play, s2jobs) - .dependsOn(s2rest_play, s2rest_netty, s2jobs, s2counter_loader, s2graphql) // this enables packaging on the root project + .aggregate(s2core, s2rest_play, s2jobs, s2http) + .dependsOn(s2rest_play, s2http, s2jobs, s2counter_loader, s2graphql) // this enables packaging on the root project .settings(commonSettings: _*) lazy val dockerSettings = Seq( @@ -124,7 +124,7 @@ Packager.defaultSettings publishSigned := { (publishSigned in s2rest_play).value - (publishSigned in s2rest_netty).value + (publishSigned in s2http).value (publishSigned in s2core).value (publishSigned in spark).value (publishSigned in s2jobs).value http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2core/src/main/resources/reference.conf ---------------------------------------------------------------------- diff --git a/s2core/src/main/resources/reference.conf b/s2core/src/main/resources/reference.conf index 73c949a..e0681dc 100644 --- a/s2core/src/main/resources/reference.conf +++ b/s2core/src/main/resources/reference.conf @@ -53,9 +53,9 @@ db.default.url = "jdbc:h2:file:./var/metastore;MODE=MYSQL" db.default.user = "sa" db.default.password = "sa" - -akka { - loggers = ["akka.event.slf4j.Slf4jLogger"] - loglevel = "DEBUG" -} +// +//akka { +// loggers = ["akka.event.slf4j.Slf4jLogger"] +// loglevel = "DEBUG" +//} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/README.md ---------------------------------------------------------------------- diff --git a/s2http/README.md b/s2http/README.md new file mode 100644 index 0000000..18f43a2 --- /dev/null +++ b/s2http/README.md @@ -0,0 +1,47 @@ +<!--- +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +---> +# S2Graph HTTP Layer + +## Development Setup + +### Run Server +Let's run http server. + +```bash +sbt 'project s2http' '~reStart' +``` + +When the server is running, connect to `http://localhost:8000`. If it works normally, you can see the following screen. + +```json +{ + "port": 8000, + "started_at": 1543218853354 +} +``` + +### API testing +```test +sbt 'project s2http' "test-only *s2graph.http*" +``` + +### API List + - link to S2GraphDocument http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/build.sbt ---------------------------------------------------------------------- diff --git a/s2http/build.sbt b/s2http/build.sbt new file mode 100644 index 0000000..944493c --- /dev/null +++ b/s2http/build.sbt @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +lazy val akkaHttpVersion = "10.1.5" +lazy val akkaVersion = "2.5.18" + +name := "s2http" + +version := "0.1" + +description := "s2graph http server" + +scalacOptions ++= Seq("-deprecation", "-feature") + +libraryDependencies ++= Seq( + "com.typesafe.akka" %% "akka-http" % akkaHttpVersion, + "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion, + "com.typesafe.akka" %% "akka-stream" % akkaVersion, + + "com.typesafe.akka" %% "akka-http-testkit" % akkaHttpVersion % Test, + "com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion % Test, + + "org.scalatest" %% "scalatest" % "3.0.5" % Test +) + +Revolver.settings http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/resources/application.conf ---------------------------------------------------------------------- diff --git a/s2http/src/main/resources/application.conf b/s2http/src/main/resources/application.conf new file mode 100644 index 0000000..7019e02 --- /dev/null +++ b/s2http/src/main/resources/application.conf @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +//db.default.url="jdbc:h2:file:./var/metastore;MODE=MYSQL", +//db.default.password = sa +//db.default.user = sa +//s2graph.storage.backend = rocks +//rocks.storage.file.path = rocks_db +//rocks.storage.mode = production +//rocks.storage.ttl = -1 +//rocks.storage.read.only = false http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/s2http/src/main/resources/log4j.properties b/s2http/src/main/resources/log4j.properties new file mode 100644 index 0000000..2070d82 --- /dev/null +++ b/s2http/src/main/resources/log4j.properties @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Set root logger level to DEBUG and its only appender to A1. +log4j.rootLogger=INFO, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala ---------------------------------------------------------------------- diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala new file mode 100644 index 0000000..c2b832e --- /dev/null +++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphAdminRoute.scala @@ -0,0 +1,121 @@ +package org.apache.s2graph.http + +import akka.http.scaladsl.model._ +import org.apache.s2graph.core.rest.RequestParser +import org.apache.s2graph.core.{Management, S2Graph} + +import akka.http.scaladsl.server.Route +import akka.http.scaladsl.server.Directives._ + +import org.slf4j.LoggerFactory +import play.api.libs.json._ + +import scala.util.Try + +object S2GraphAdminRoute { + + import scala.util._ + + trait AdminMessageFormatter[T] { + def toJson(msg: T): JsValue + } + + import scala.language.reflectiveCalls + + type ToPlayJson = {def toJson: JsValue} + + object AdminMessageFormatter { + + implicit def toPlayJson[A <: ToPlayJson] = new AdminMessageFormatter[A] { + def toJson(js: A) = js.toJson + } + + implicit def fromPlayJson[T <: JsValue] = new AdminMessageFormatter[T] { + def toJson(js: T) = js + } + } + + def toHttpEntity[A: AdminMessageFormatter](opt: Option[A], status: StatusCode = StatusCodes.OK, message: String = ""): HttpResponse = { + val ev = implicitly[AdminMessageFormatter[A]] + val res = opt.map(ev.toJson).getOrElse(Json.obj("message" -> message)) + + HttpResponse( + status = status, + entity = HttpEntity(ContentTypes.`application/json`, res.toString) + ) + } + + def toHttpEntity[A: AdminMessageFormatter](opt: Try[A]): HttpResponse = { + val ev = implicitly[AdminMessageFormatter[A]] + val (status, res) = opt match { + case Success(m) => StatusCodes.Created -> Json.obj("status" -> "ok", "message" -> ev.toJson(m)) + case Failure(e) => StatusCodes.OK -> Json.obj("status" -> "failure", "message" -> e.toString) + } + + toHttpEntity(Option(res), status = status) + } +} + +trait S2GraphAdminRoute { + + import S2GraphAdminRoute._ + + val s2graph: S2Graph + val logger = LoggerFactory.getLogger(this.getClass) + + lazy val management: Management = s2graph.management + lazy val requestParser: RequestParser = new RequestParser(s2graph) + + // routes impl + lazy val getLabel = path("getLabel" / Segment) { labelName => + val labelOpt = Management.findLabel(labelName) + + complete(toHttpEntity(labelOpt, message = s"Label not found: ${labelName}")) + } + + lazy val getService = path("getService" / Segment) { serviceName => + val serviceOpt = Management.findService(serviceName) + + complete(toHttpEntity(serviceOpt, message = s"Service not found: ${serviceName}")) + } + + lazy val createService = path("createService") { + entity(as[String]) { body => + val params = Json.parse(body) + + val parseTry = Try(requestParser.toServiceElements(params)) + val serviceTry = for { + (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) <- parseTry + service <- management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) + } yield service + + complete(toHttpEntity(serviceTry)) + } + } + + lazy val createLabel = path("createLabel") { + entity(as[String]) { body => + val params = Json.parse(body) + + val labelTry = for { + label <- requestParser.toLabelElements(params) + } yield label + + complete(toHttpEntity(labelTry)) + } + } + + // expose routes + lazy val adminRoute: Route = + get { + concat( + getService, + getLabel + ) + } ~ post { + concat( + createService, + createLabel + ) + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala ---------------------------------------------------------------------- diff --git a/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala new file mode 100644 index 0000000..1814436 --- /dev/null +++ b/s2http/src/main/scala/org/apache/s2graph/http/S2GraphTraversalRoute.scala @@ -0,0 +1,65 @@ +package org.apache.s2graph.http + +import org.apache.s2graph.core.S2Graph +import org.apache.s2graph.core.rest.RestHandler.CanLookup +import org.slf4j.LoggerFactory +import akka.http.scaladsl.server.Route +import akka.http.scaladsl.server.Directives._ +import akka.http.scaladsl.model.headers.RawHeader +import akka.http.scaladsl.model.{HttpHeader, StatusCodes} +import org.apache.s2graph.core.GraphExceptions.{BadQueryException, JsonParseException} +import org.apache.s2graph.core.rest.RestHandler + + +object S2GraphTraversalRoute { + + import scala.collection._ + + implicit val akkHttpHeaderLookup = new CanLookup[immutable.Seq[HttpHeader]] { + override def lookup(m: immutable.Seq[HttpHeader], key: String): Option[String] = m.find(_.name() == key).map(_.value()) + } +} + +trait S2GraphTraversalRoute { + + import S2GraphTraversalRoute._ + + val s2graph: S2Graph + val logger = LoggerFactory.getLogger(this.getClass) + + implicit lazy val ec = s2graph.ec + lazy val restHandler = new RestHandler(s2graph) + + // The `/graphs/*` APIs are implemented to be branched from the existing restHandler.doPost. + // Implement it first by delegating that function. + lazy val delegated: Route = { + entity(as[String]) { body => + logger.info(body) + + extractRequest { request => + val result = restHandler.doPost(request.uri.toRelative.toString(), body, request.headers) + val responseHeaders = result.headers.toList.map { case (k, v) => RawHeader(k, v) } + + val f = result.body.map(StatusCodes.OK -> _.toString).recover { + case BadQueryException(msg, _) => StatusCodes.BadRequest -> msg + case JsonParseException(msg) => StatusCodes.BadRequest -> msg + case e: Exception => StatusCodes.InternalServerError -> e.toString + } + + respondWithHeaders(responseHeaders) { + complete(f) + } + } + } + } + + // expose routes + lazy val traversalRoute: Route = + post { + concat( + delegated // getEdges, experiments, etc. + ) + } + +} + http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/main/scala/org/apache/s2graph/http/Server.scala ---------------------------------------------------------------------- diff --git a/s2http/src/main/scala/org/apache/s2graph/http/Server.scala b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala new file mode 100644 index 0000000..82d2343 --- /dev/null +++ b/s2http/src/main/scala/org/apache/s2graph/http/Server.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.s2graph.http + +import scala.language.postfixOps +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.duration.Duration +import scala.util.{Failure, Success} +import akka.actor.ActorSystem +import akka.http.scaladsl.Http +import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes} +import akka.http.scaladsl.server.Route +import akka.http.scaladsl.server.Directives._ +import akka.stream.ActorMaterializer +import com.typesafe.config.ConfigFactory +import org.apache.s2graph.core.S2Graph +import org.slf4j.LoggerFactory + +object Server extends App + with S2GraphTraversalRoute + with S2GraphAdminRoute { + + implicit val system: ActorSystem = ActorSystem("S2GraphHttpServer") + implicit val materializer: ActorMaterializer = ActorMaterializer() + implicit val executionContext: ExecutionContext = system.dispatcher + + val config = ConfigFactory.load() + + override val s2graph = new S2Graph(config) + override val logger = LoggerFactory.getLogger(this.getClass) + + val port = sys.props.get("http.port").fold(8000)(_.toInt) + val serverStatus = s""" { "port": ${port}, "started_at": ${System.currentTimeMillis()} }""" + + val health = HttpResponse(status = StatusCodes.OK, entity = HttpEntity(ContentTypes.`application/json`, serverStatus)) + + // Allows you to determine routes to expose according to external settings. + lazy val routes: Route = concat( + pathPrefix("graphs")(traversalRoute), + pathPrefix("admin")(adminRoute), + get(complete(health)) + ) + + val binding: Future[Http.ServerBinding] = Http().bindAndHandle(routes, "localhost", port) + binding.onComplete { + case Success(bound) => logger.info(s"Server online at http://${bound.localAddress.getHostString}:${bound.localAddress.getPort}/";) + case Failure(e) => + logger.error(s"Server could not start!", e) + system.terminate() + } + + Await.result(system.whenTerminated, Duration.Inf) +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/a16ecb08/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala ---------------------------------------------------------------------- diff --git a/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala new file mode 100644 index 0000000..7e5d794 --- /dev/null +++ b/s2http/src/test/scala/org/apache/s2graph/http/AdminRouteSpec.scala @@ -0,0 +1,61 @@ +package org.apache.s2graph.http + +import akka.http.scaladsl.marshalling.Marshal +import akka.http.scaladsl.model._ +import akka.http.scaladsl.testkit.ScalatestRouteTest +import com.typesafe.config.ConfigFactory +import org.apache.s2graph.core.S2Graph +import org.scalatest.concurrent.ScalaFutures +import org.scalatest.{BeforeAndAfterAll, Matchers, WordSpec} +import org.slf4j.LoggerFactory +import play.api.libs.json.{JsString, Json} + +class AdminRoutesSpec extends WordSpec with Matchers with ScalaFutures with ScalatestRouteTest with S2GraphAdminRoute with BeforeAndAfterAll { + val config = ConfigFactory.load() + val s2graph = new S2Graph(config) + override val logger = LoggerFactory.getLogger(this.getClass) + + override def afterAll(): Unit = { + s2graph.shutdown() + } + + import akka.http.scaladsl.server.Directives._ + + lazy val routes = adminRoute + + "AdminRoute" should { + "be able to create service (POST /createService)" in { + val serviceParam = Json.obj( + "serviceName" -> "kakaoFavorites", + "compressionAlgorithm" -> "gz" + ) + + val serviceEntity = Marshal(serviceParam.toString).to[MessageEntity].futureValue + val request = Post("/createService").withEntity(serviceEntity) + + request ~> routes ~> check { + status should ===(StatusCodes.Created) + contentType should ===(ContentTypes.`application/json`) + + val response = Json.parse(entityAs[String]) + + (response \\ "name").head should ===(JsString("kakaoFavorites")) + (response \\ "status").head should ===(JsString("ok")) + } + } + + "return service if present (GET /getService/{serviceName})" in { + val request = HttpRequest(uri = "/getService/kakaoFavorites") + + request ~> routes ~> check { + status should ===(StatusCodes.OK) + contentType should ===(ContentTypes.`application/json`) + + val response = Json.parse(entityAs[String]) + + (response \\ "name").head should ===(JsString("kakaoFavorites")) + } + } + } +} +
