Repository: incubator-s2graph Updated Branches: refs/heads/master 42077bf8d -> 66b656f69
bug fix on offset/limit with schema v3. - note that HBase version 1.2.8, 1.3.2.1, 1.4.8 have bug on ColumnPaginationFilter. - details on HBASE-20565. Project: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/commit/646860f0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/tree/646860f0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/diff/646860f0 Branch: refs/heads/master Commit: 646860f0e12ab0fce0edc7743f2bad379e5d97eb Parents: f53661e Author: DO YUNG YOON <steams...@apache.org> Authored: Sat Nov 3 06:52:42 2018 +0900 Committer: DO YUNG YOON <steams...@apache.org> Committed: Sat Nov 3 06:52:42 2018 +0900 ---------------------------------------------------------------------- .travis.yml | 2 +- .travis/install-hbase.sh | 21 +++ project/Common.scala | 1 + s2core/build.sbt | 2 +- .../core/storage/hbase/AsynchbaseStorage.scala | 2 +- .../s2graph/core/Integrate/ExampleTest.scala | 130 +++++++++++++++++++ .../graphql/repository/GraphRepository.scala | 4 +- .../s2graph/graphql/types/FieldResolver.scala | 2 +- 8 files changed, 158 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index bc5e6cb..8664e15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ language: scala env: - - HBASE_VERSION=1.2.6.1 + - HBASE_VERSION=1.4.8 cache: directories: http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/.travis/install-hbase.sh ---------------------------------------------------------------------- diff --git a/.travis/install-hbase.sh b/.travis/install-hbase.sh new file mode 100755 index 0000000..5744fd8 --- /dev/null +++ b/.travis/install-hbase.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#set -xe +# +#if [ ! -d "$HOME/hbase-$HBASE_VERSION/bin" ]; then +# cd $HOME && wget -q -O - http://mirror.apache-kr.org/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz | tar xz +#fi http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/project/Common.scala ---------------------------------------------------------------------- diff --git a/project/Common.scala b/project/Common.scala index e4a1b61..42628f0 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -26,6 +26,7 @@ object Common { val specs2Version = "3.8.5" val hbaseVersion = "1.2.6.1" + val asynchbaseVersion = "1.7.2" val hadoopVersion = "2.7.3" val tinkerpopVersion = "3.2.5" http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2core/build.sbt ---------------------------------------------------------------------- diff --git a/s2core/build.sbt b/s2core/build.sbt index 0368715..76ec221 100644 --- a/s2core/build.sbt +++ b/s2core/build.sbt @@ -41,7 +41,7 @@ libraryDependencies ++= Seq( "com.h2database" % "h2" % "1.4.192", "com.stumbleupon" % "async" % "1.4.1", "io.netty" % "netty" % "3.9.4.Final" force(), - "org.hbase" % "asynchbase" % "1.7.2" excludeLogging(), + "org.hbase" % "asynchbase" % asynchbaseVersion excludeLogging(), "net.bytebuddy" % "byte-buddy" % "1.4.26", "org.apache.tinkerpop" % "gremlin-core" % tinkerpopVersion excludeLogging(), "org.apache.tinkerpop" % "gremlin-test" % tinkerpopVersion % "test", http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala ---------------------------------------------------------------------- diff --git a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala index f670e9c..0c49e71 100644 --- a/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala +++ b/s2core/src/main/scala/org/apache/s2graph/core/storage/hbase/AsynchbaseStorage.scala @@ -297,7 +297,7 @@ object AsynchbaseStorage { get.setMaxTimestamp(maxTs) get.setTimeout(queryParam.rpcTimeout) - val pagination = new ColumnPaginationFilter(queryParam.limit, queryParam.offset) + val pagination = new ColumnPaginationFilter(queryParam.innerLimit, queryParam.innerOffset) val columnRangeFilterOpt = queryParam.intervalOpt.map { interval => new ColumnRangeFilter(intervalMaxBytes, true, intervalMinBytes, true) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2core/src/test/scala/org/apache/s2graph/core/Integrate/ExampleTest.scala ---------------------------------------------------------------------- diff --git a/s2core/src/test/scala/org/apache/s2graph/core/Integrate/ExampleTest.scala b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/ExampleTest.scala new file mode 100644 index 0000000..76e30d9 --- /dev/null +++ b/s2core/src/test/scala/org/apache/s2graph/core/Integrate/ExampleTest.scala @@ -0,0 +1,130 @@ +package org.apache.s2graph.core.Integrate + +import org.apache.s2graph.core.schema._ +import play.api.libs.json.{JsObject, Json} + +import scala.concurrent.Await +import scala.concurrent.duration.Duration + +class ExampleTest extends IntegrateCommon { + + import TestUtil._ + + private def prepareTestData() = { + // create service KakaoFavorites + val createServicePayload = Json.parse( + """ + |{"serviceName": "KakaoFavorites", "compressionAlgorithm" : "gz"} + |""".stripMargin) + + val createFriendsPayload = Json.parse( + s"""{ + | "label": "friends", + | "srcServiceName": "KakaoFavorites", + | "srcColumnName": "userName", + | "srcColumnType": "string", + | "tgtServiceName": "KakaoFavorites", + | "tgtColumnName": "userName", + | "tgtColumnType": "string", + | "isDirected": "false", + | "indices": [], + | "props": [], + | "consistencyLevel": "strong" + |}""".stripMargin) + + val createPostPayload = Json.parse( + """{ + | "label": "post", + | "srcServiceName": "KakaoFavorites", + | "srcColumnName": "userName", + | "srcColumnType": "string", + | "tgtServiceName": "KakaoFavorites", + | "tgtColumnName": "url", + | "tgtColumnType": "string", + | "isDirected": "true", + | "indices": [], + | "props": [], + | "consistencyLevel": "strong" + |}""".stripMargin) + + val insertFriendsPayload = Json.parse( + """[ + | {"from":"Elmo","to":"Big Bird","label":"friends","props":{},"timestamp":1444360152477}, + | {"from":"Elmo","to":"Ernie","label":"friends","props":{},"timestamp":1444360152478}, + | {"from":"Elmo","to":"Bert","label":"friends","props":{},"timestamp":1444360152479}, + | + | {"from":"Cookie Monster","to":"Grover","label":"friends","props":{},"timestamp":1444360152480}, + | {"from":"Cookie Monster","to":"Kermit","label":"friends","props":{},"timestamp":1444360152481}, + | {"from":"Cookie Monster","to":"Oscar","label":"friends","props":{},"timestamp":1444360152482} + |]""".stripMargin) + + val insertPostPayload = Json.parse( + """[ + | {"from":"Big Bird","to":"www.kakaocorp.com/en/main","label":"post","props":{},"timestamp":1444360152477}, + | {"from":"Big Bird","to":"github.com/kakao/s2graph","label":"post","props":{},"timestamp":1444360152478}, + | {"from":"Ernie","to":"groups.google.com/forum/#!forum/s2graph","label":"post","props":{},"timestamp":1444360152479}, + | {"from":"Grover","to":"hbase.apache.org/forum/#!forum/s2graph","label":"post","props":{},"timestamp":1444360152480}, + | {"from":"Kermit","to":"www.playframework.com","label":"post","props":{},"timestamp":1444360152481}, + | {"from":"Oscar","to":"www.scala-lang.org","label":"post","props":{},"timestamp":1444360152482} + |]""".stripMargin) + + + val (serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) = parser.toServiceElements(createServicePayload) + management.createService(serviceName, cluster, tableName, preSplitSize, ttl, compressionAlgorithm) + + Service.findByName("KakaoFavorites", useCache = false) + + Label.findByName("friends", useCache = false).foreach { label => + Label.delete(label.id.get) + } + parser.toLabelElements(createFriendsPayload) + + Label.findByName("post", useCache = false).foreach { label => + Label.delete(label.id.get) + } + + parser.toLabelElements(createPostPayload) + Await.result(graph.mutateEdges(parser.parseJsonFormat(insertFriendsPayload, operation = "insert").map(_._1), withWait = true), Duration("10 seconds")) + Await.result(graph.mutateEdges(parser.parseJsonFormat(insertPostPayload, operation = "insert").map(_._1), withWait = true), Duration("10 seconds")) + } + + override def initTestData(): Unit = { + prepareTestData() + } + + test("[S2GRAPH-243]: Limit bug on 'graph/getEdges' offset 0, limit 3") { + val queryJson = Json.parse( + """{ + | "select": ["to"], + | "srcVertices": [{"serviceName": "KakaoFavorites", "columnName": "userName", "id":"Elmo"}], + | "steps": [ + | {"step": [{"label": "friends", "direction": "out", "offset": 0, "limit": 3}]} + | ] + |}""".stripMargin) + + val response = getEdgesSync(queryJson) + + val expectedFriends = Seq("Bert", "Ernie", "Big Bird") + val friends = (response \ "results").as[Seq[JsObject]].map(obj => (obj \ "to").as[String]) + + friends shouldBe expectedFriends + } + + test("[S2GRAPH-243]: Limit bug on 'graph/getEdges' offset 1, limit 2") { + val queryJson = Json.parse( + """{ + | "select": ["to"], + | "srcVertices": [{"serviceName": "KakaoFavorites", "columnName": "userName", "id":"Elmo"}], + | "steps": [ + | {"step": [{"label": "friends", "direction": "out", "offset": 1, "limit": 2}]} + | ] + |}""".stripMargin) + + val response = getEdgesSync(queryJson) + + val expectedFriends = Seq("Ernie", "Big Bird") + val friends = (response \ "results").as[Seq[JsObject]].map(obj => (obj \ "to").as[String]) + + friends shouldBe expectedFriends + } +} http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala ---------------------------------------------------------------------- diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala index d5212c8..460a817 100644 --- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala +++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/repository/GraphRepository.scala @@ -193,14 +193,14 @@ class GraphRepository(val graph: S2GraphLike) { def getEdges(vertices: Seq[S2VertexLike], queryParam: QueryParam): Future[Seq[S2EdgeLike]] = { val step = Step(Seq(queryParam)) - val q = Query(vertices, steps = Vector(step)) + val q = Query(vertices, steps = Vector(step), QueryOption(returnDegree = false)) graph.getEdges(q).map(_.edgeWithScores.map(_.edge)) } def getEdges(vertex: S2VertexLike, queryParam: QueryParam): Future[Seq[S2EdgeLike]] = { val step = Step(Seq(queryParam)) - val q = Query(Seq(vertex), steps = Vector(step)) + val q = Query(Seq(vertex), steps = Vector(step), QueryOption(returnDegree = false)) graph.getEdges(q).map(_.edgeWithScores.map(_.edge)) } http://git-wip-us.apache.org/repos/asf/incubator-s2graph/blob/646860f0/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala ---------------------------------------------------------------------- diff --git a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala index 1994e5e..4de1b2e 100644 --- a/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala +++ b/s2graphql/src/main/scala/org/apache/s2graph/graphql/types/FieldResolver.scala @@ -53,7 +53,7 @@ object FieldResolver { val vertex = c.value.asInstanceOf[S2VertexLike] val dir = c.arg[String]("direction") - val offset = c.arg[Int]("offset") + 1 // +1 for skip degree edge: currently not support + val offset = c.arg[Int]("offset") // +1 for skip degree edge: currently not support val limit = c.arg[Int]("limit") val whereClauseOpt = c.argOpt[String]("filter") val where = c.ctx.parser.extractWhere(label, whereClauseOpt)