Repository: incubator-atlas Updated Branches: refs/heads/master 539f24314 -> 7660c9b29
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala b/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala index ea0b9bb..b2654e6 100755 --- a/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala +++ b/repository/src/test/scala/org/apache/atlas/query/GremlinTest2.scala @@ -19,7 +19,10 @@ package org.apache.atlas.query import com.thinkaurelius.titan.core.TitanGraph +import com.thinkaurelius.titan.core.util.TitanCleanup +import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy import org.apache.atlas.query.Expressions._ +import org.apache.atlas.repository.graph.{TitanGraphProvider, GraphBackedMetadataRepository} import org.apache.atlas.typesystem.types.TypeSystem import org.junit.runner.RunWith import org.scalatest._ @@ -29,15 +32,25 @@ import org.scalatest.junit.JUnitRunner class GremlinTest2 extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { var g: TitanGraph = null + var gProvider:TitanGraphProvider = null; + var gp:GraphPersistenceStrategies = null; override def beforeAll() { TypeSystem.getInstance().reset() QueryTestsUtils.setupTypes - g = QueryTestsUtils.setupTestGraph + gProvider = new TitanGraphProvider(); + gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider)) + g = QueryTestsUtils.setupTestGraph(gProvider) } override def afterAll() { g.shutdown() + try { + TitanCleanup.clear(g); + } catch { + case ex: Exception => + print("Could not clear the graph ", ex); + } } test("testTraitSelect") { http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala b/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala index c662847..67ce12b 100755 --- a/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala +++ b/repository/src/test/scala/org/apache/atlas/query/HiveTitanSample.scala @@ -24,7 +24,8 @@ import java.util.{Date, UUID} import javax.script.{Bindings, ScriptEngine, ScriptEngineManager} import com.thinkaurelius.titan.core.TitanGraph -import com.typesafe.config.ConfigFactory +import org.apache.atlas.repository.Constants +import org.apache.atlas.repository.graph.TitanGraphProvider import org.apache.atlas.TestUtils import org.apache.commons.io.FileUtils @@ -39,33 +40,41 @@ object HiveTitanSample { val _id: String def id = _id - val version = 0 - val guid = s"""${UUID.randomUUID()}""".stripMargin + val __version = 0 + val __guid = s"""${UUID.randomUUID()}""".stripMargin - def addEdge(to: Vertex, label: String, edges: ArrayBuffer[String]): Unit = { + def addEdge(to: Vertex, label: String, edges: ArrayBuffer[String]): Int = { + val edgeId = nextEdgeId.incrementAndGet(); edges += - s"""{"_id" : "${nextEdgeId.incrementAndGet()}", "_type" : "edge", "_inV" : "${to.id}", "_outV" : "$id", "_label" : "$label"}""" + s"""{"_id" : "${edgeId}", "_type" : "edge", "_inV" : "${to.id}", "_outV" : "$id", "_label" : "$label"}""" + edgeId } def toGSon(vertices: ArrayBuffer[String], edges: ArrayBuffer[String]): Unit = { val sb = new StringBuilder - sb.append( s"""{"typeName" : "${this.getClass.getSimpleName}", "_type" : "vertex"""") + sb.append( s"""{"${Constants.ENTITY_TYPE_PROPERTY_KEY}" : "${this.getClass.getSimpleName}", "_type" : "vertex"""") this.getClass.getDeclaredFields filter (_.getName != "traits") foreach { f => f.setAccessible(true) val fV = f.get(this) val convertedVal = fV match { case _: String => s""""$fV"""" + case ls: List[_] if isPrimitiveType(ls) => + s"""["${ls.mkString(",")}"]""" case d: Date => d.getTime case _ => fV } convertedVal match { - case x: Vertex => addEdge(x, s"${this.getClass.getSimpleName}.${f.getName}", edges) - case l: List[_] => l.foreach(x => addEdge(x.asInstanceOf[Vertex], - s"${this.getClass.getSimpleName}.${f.getName}", edges)) + case x: Vertex => addEdge(x, s"__${this.getClass.getSimpleName}.${f.getName}", edges) + case l: List[_] => val edgeList = l.map(x => + s""""${addEdge(x.asInstanceOf[Vertex], s"__${this.getClass.getSimpleName}.${f.getName}", edges)}"""" + ) + if(l.head.isInstanceOf[Struct]) { + sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : ${edgeList.mkString("[", ",", "]")}""") + } case _ => sb.append( s""", "${f.getName}" : $convertedVal""") sb.append( s""", "${this.getClass.getSimpleName}.${f.getName}" : $convertedVal""") } @@ -77,13 +86,29 @@ object HiveTitanSample { if (traits.isDefined) { val fV = traits.get.map(_.getClass.getSimpleName).mkString(",") - sb.append( s""", "traitNames" : "$fV"""") + sb.append( s""", "${Constants.TRAIT_NAMES_PROPERTY_KEY}" : "$fV"""") } } sb.append("}") vertices += sb.toString() } + + def isPrimitiveType(ls: List[_]) : Boolean = { + ls.head match { + case _: String => true + case _: Byte => true + case _: Short => true + case _: Int => true + case _: Long => true + case _: Float => true + case _: Double => true + case _: BigDecimal => true + case _: BigInt => true + case _: Boolean => true + case default => false + } + } } trait Trait extends Vertex @@ -118,11 +143,21 @@ object HiveTitanSample { case class ETL(_id: String = "" + nextVertexId.incrementAndGet()) extends Trait - case class DB(name: String, owner: String, createTime: Int, traits: Option[List[Trait]] = None, + case class DB(name: String, owner: String, createTime: Int, clusterName: String, traits: Option[List[Trait]] = None, _id: String = "" + nextVertexId.incrementAndGet()) extends Instance + case class HiveOrder(col: String, order: Int, + _id: String = "" + nextVertexId.incrementAndGet()) extends Struct + case class StorageDescriptor(inputFormat: String, outputFormat: String, - _id: String = "" + nextVertexId.incrementAndGet()) extends Struct + sortCols: List[Struct], _id: String = "" + nextVertexId.incrementAndGet()) extends Struct { + + override def toGSon(vertices: ArrayBuffer[String], + edges: ArrayBuffer[String]): Unit = { + sortCols.foreach(_.toGSon(vertices, edges)) + super.toGSon(vertices, edges) + } + } case class Column(name: String, dataType: String, sd: StorageDescriptor, traits: Option[List[Trait]] = None, @@ -133,7 +168,7 @@ object HiveTitanSample { traits: Option[List[Trait]] = None, _id: String = "" + nextVertexId.incrementAndGet()) extends Instance - case class TableDef(name: String, db: DB, inputFormat: String, outputFormat: String, + case class TableDef(name: String, db: DB, sd: StorageDescriptor, columns: List[(String, String, Option[List[Trait]])], traits: Option[List[Trait]] = None, created: Option[Date] = None) { @@ -141,7 +176,7 @@ object HiveTitanSample { case Some(x) => x case None => new Date(TestUtils.TEST_DATE_IN_LONG) } - val sd = StorageDescriptor(inputFormat, outputFormat) + val colDefs = columns map { c => Column(c._1, c._2, sd, c._3) } @@ -157,6 +192,9 @@ object HiveTitanSample { } } + case class Partition(values: List[String], table: Table, traits: Option[List[Trait]] = None, + _id: String = "" + nextVertexId.incrementAndGet()) extends Instance + case class LoadProcess(name: String, inputTables: List[Vertex], outputTable: Vertex, traits: Option[List[Trait]] = None, @@ -166,11 +204,11 @@ object HiveTitanSample { traits: Option[List[Trait]] = None, _id: String = "" + nextVertexId.incrementAndGet()) extends Instance - val salesDB = DB("Sales", "John ETL", 1000) + val salesDB = DB("Sales", "John ETL", 1000, "test") val salesFact = TableDef("sales_fact", salesDB, - "TextInputFormat", - "TextOutputFormat", + StorageDescriptor("TextInputFormat", + "TextOutputFormat", List(HiveOrder("customer_id", 0))), List( ("time_id", "int", None), ("product_id", "int", None), @@ -180,8 +218,8 @@ object HiveTitanSample { )) val productDim = TableDef("product_dim", salesDB, - "TextInputFormat", - "TextOutputFormat", + StorageDescriptor("TextInputFormat", + "TextOutputFormat", List(HiveOrder("product_id", 0))), List( ("product_id", "int", None), ("product_name", "string", None), @@ -190,8 +228,8 @@ object HiveTitanSample { Some(List(Dimension()))) val timeDim = TableDef("time_dim", salesDB, - "TextInputFormat", - "TextOutputFormat", + StorageDescriptor("TextInputFormat", + "TextOutputFormat", List(HiveOrder("time_id", 0))), List( ("time_id", "int", None), ("dayOfYear", "int", None), @@ -200,8 +238,8 @@ object HiveTitanSample { Some(List(Dimension()))) val customerDim = TableDef("customer_dim", salesDB, - "TextInputFormat", - "TextOutputFormat", + StorageDescriptor("TextInputFormat", + "TextOutputFormat", List(HiveOrder("customer_id", 0))), List( ("customer_id", "int", None), ("name", "int", None), @@ -209,11 +247,11 @@ object HiveTitanSample { ), Some(List(Dimension()))) - val reportingDB = DB("Reporting", "Jane BI", 1500) + val reportingDB = DB("Reporting", "Jane BI", 1500, "test") val salesFactDaily = TableDef("sales_fact_daily_mv", reportingDB, - "TextInputFormat", - "TextOutputFormat", + StorageDescriptor("TextInputFormat", + "TextOutputFormat", List(HiveOrder("customer_id", 0))), List( ("time_id", "int", None), ("product_id", "int", None), @@ -235,8 +273,8 @@ object HiveTitanSample { val salesFactMonthly = TableDef("sales_fact_monthly_mv", reportingDB, - "TextInputFormat", - "TextOutputFormat", + StorageDescriptor("TextInputFormat", + "TextOutputFormat", List(HiveOrder("customer_id", 0))), List( ("time_id", "int", None), ("product_id", "int", None), @@ -247,6 +285,8 @@ object HiveTitanSample { List(salesFactDaily.tablDef), salesFactMonthly.tablDef, Some(List(ETL()))) + val salesDailyPartition = Partition(List("2015-01-01"),salesFactDaily.tablDef) + val vertices: ArrayBuffer[String] = new ArrayBuffer[String]() val edges: ArrayBuffer[String] = new ArrayBuffer[String]() @@ -264,6 +304,7 @@ object HiveTitanSample { customerDimView.toGSon(vertices, edges) salesFactMonthly.toGSon(vertices, edges) loadSalesFactMonthly.toGSon(vertices, edges) + salesDailyPartition.toGSon(vertices, edges) def toGSon(): String = { s"""{ @@ -328,9 +369,7 @@ object HiveTitanSample { object TestApp extends App with GraphUtils { - var conf = ConfigFactory.load() - conf = conf.getConfig("graphRepo") - val g: TitanGraph = titanGraph(conf) + val g: TitanGraph = TitanGraphProvider.getGraphInstance val manager: ScriptEngineManager = new ScriptEngineManager val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy") val bindings: Bindings = engine.createBindings http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala b/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala index 0b4dd2b..f1108e4 100755 --- a/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala +++ b/repository/src/test/scala/org/apache/atlas/query/LineageQueryTest.scala @@ -19,7 +19,10 @@ package org.apache.atlas.query import com.thinkaurelius.titan.core.TitanGraph +import com.thinkaurelius.titan.core.util.TitanCleanup +import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy import org.apache.atlas.query.Expressions._ +import org.apache.atlas.repository.graph.{GraphBackedMetadataRepository, TitanGraphProvider} import org.apache.atlas.typesystem.types.TypeSystem import org.junit.runner.RunWith import org.scalatest.junit.JUnitRunner @@ -29,21 +32,31 @@ import org.scalatest.{Assertions, BeforeAndAfterAll, FunSuite} class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinTest { var g: TitanGraph = null + var gProvider:TitanGraphProvider = null; + var gp:GraphPersistenceStrategies = null; override def beforeAll() { - TypeSystem.getInstance().reset() - QueryTestsUtils.setupTypes - g = QueryTestsUtils.setupTestGraph + TypeSystem.getInstance().reset() + QueryTestsUtils.setupTypes + gProvider = new TitanGraphProvider(); + gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider)) + g = QueryTestsUtils.setupTestGraph(gProvider) } override def afterAll() { - g.shutdown() + g.shutdown() + try { + TitanCleanup.clear(g); + } catch { + case ex: Exception => + print("Could not clear the graph ", ex); + } } val PREFIX_SPACES_REGEX = ("\\n\\s*").r test("testInputTables") { - val r = QueryProcessor.evaluate(_class("LoadProcess").field("inputTables"), g) + val r = QueryProcessor.evaluate(_class("LoadProcess").field("inputTables"), g, gp) val x = r.toJson validateJson(r,"""{ | "query":"LoadProcess inputTables", @@ -82,7 +95,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | { | "name":"sd", - | "dataTypeName":"StorageDesc", + | "dataTypeName":"StorageDescriptor", | "multiplicity":{ | "lower":1, | "upper":1, @@ -117,7 +130,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -134,7 +147,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -156,7 +169,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -171,12 +184,12 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT } test("testLoadProcessOut") { - val r = QueryProcessor.evaluate(_class("Table").field("LoadProcess").field("outputTable"), g) + val r = QueryProcessor.evaluate(_class("Table").field("LoadProcess").field("outputTable"), g, gp) validateJson(r, null) } test("testLineageAll") { - val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")), g) + val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable")), g, gp) validateJson(r, """{ | "query":"Table as _loop0 loop (LoadProcess outputTable)", | "dataType":{ @@ -214,7 +227,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | { | "name":"sd", - | "dataTypeName":"StorageDesc", + | "dataTypeName":"StorageDescriptor", | "multiplicity":{ | "lower":1, | "upper":1, @@ -244,13 +257,12 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | { | "$typeName$":"Table", | "$id$":{ - | "id":"9216", | "$typeName$":"Table", | "version":0 | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -267,7 +279,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -284,7 +296,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -301,7 +313,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -318,7 +330,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -333,7 +345,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT test("testLineageAllSelect") { val r = QueryProcessor.evaluate(_class("Table").as("src").loop(id("LoadProcess").field("outputTable")).as("dest"). - select(id("src").field("name").as("srcTable"), id("dest").field("name").as("destTable")), g) + select(id("src").field("name").as("srcTable"), id("dest").field("name").as("destTable")), g, gp) validateJson(r, """{ "query":"Table as src loop (LoadProcess outputTable) as dest select src.name as srcTable, dest.name as destTable", "dataType":{ @@ -398,7 +410,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT } test("testLineageFixedDepth") { - val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable"), int(1)), g) + val r = QueryProcessor.evaluate(_class("Table").loop(id("LoadProcess").field("outputTable"), int(1)), g, gp) validateJson(r, """{ | "query":"Table as _loop0 loop (LoadProcess outputTable) times 1", | "dataType":{ @@ -436,7 +448,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | { | "name":"sd", - | "dataTypeName":"StorageDesc", + | "dataTypeName":"StorageDescriptor", | "multiplicity":{ | "lower":1, | "upper":1, @@ -471,7 +483,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -488,7 +500,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ @@ -505,7 +517,7 @@ class LineageQueryTest extends FunSuite with BeforeAndAfterAll with BaseGremlinT | }, | "created":"2014-12-11T02:35:58.440Z", | "sd":{ - | "$typeName$":"StorageDesc", + | "$typeName$":"StorageDescriptor", | "version":0 | }, | "db":{ http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala b/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala index 602b2b5..452438a 100755 --- a/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala +++ b/repository/src/test/scala/org/apache/atlas/query/ParserTest.scala @@ -27,9 +27,7 @@ class ParserTest extends BaseTest { @Before override def setup { super.setup - QueryTestsUtils.setupTypes - } @Test def testFrom: Unit = { @@ -98,4 +96,13 @@ class ParserTest extends BaseTest { ) } + @Test def testList: Unit = { + val p = new QueryParser + println(p( + "Partition as p where values = ['2015-01-01']," + + " table where name = 'tableoq8ty'," + + " db where name = 'default' and clusterName = 'test'").right.get.toString + ) + } + } http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala ---------------------------------------------------------------------- diff --git a/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala b/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala index 3c66da0..5076162 100755 --- a/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala +++ b/repository/src/test/scala/org/apache/atlas/query/QueryTestsUtils.scala @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList import com.thinkaurelius.titan.core.{TitanFactory, TitanGraph} import com.tinkerpop.blueprints.Vertex import com.typesafe.config.{Config, ConfigFactory} +import org.apache.atlas.repository.graph.TitanGraphProvider import org.apache.atlas.typesystem.types._ import org.apache.commons.configuration.{Configuration, ConfigurationException, MapConfiguration} import org.apache.commons.io.FileUtils @@ -49,9 +50,9 @@ trait GraphUtils { } - def titanGraph(conf: Config) = { + def titanGraph(conf: Configuration) = { try { - val g = TitanFactory.open(getConfiguration(conf)) + val g = TitanFactory.open(conf) val mgmt = g.getManagementSystem val typname = mgmt.makePropertyKey("typeName").dataType(classOf[String]).make() mgmt.buildIndex("byTypeName", classOf[Vertex]).addKey(typname).buildCompositeIndex() @@ -79,30 +80,44 @@ object QueryTestsUtils extends GraphUtils { Array( attrDef("name", DataTypes.STRING_TYPE), attrDef("owner", DataTypes.STRING_TYPE), - attrDef("createTime", DataTypes.INT_TYPE) + attrDef("createTime", DataTypes.INT_TYPE), + attrDef("clusterName", DataTypes.STRING_TYPE) )) - def storageDescClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "StorageDesc", null, + def hiveOrderDef = new StructTypeDefinition("HiveOrder", + Array( + attrDef("col", DataTypes.STRING_TYPE), + attrDef("order", DataTypes.INT_TYPE) + )) + + def storageDescClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "StorageDescriptor", null, Array( attrDef("inputFormat", DataTypes.STRING_TYPE), - attrDef("outputFormat", DataTypes.STRING_TYPE) + attrDef("outputFormat", DataTypes.STRING_TYPE), + new AttributeDefinition("sortCols", DataTypes.arrayTypeName("HiveOrder"), Multiplicity.REQUIRED, false, null) )) def columnClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Column", null, Array( attrDef("name", DataTypes.STRING_TYPE), attrDef("dataType", DataTypes.STRING_TYPE), - new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null) + new AttributeDefinition("sd", "StorageDescriptor", Multiplicity.REQUIRED, false, null) )) def tblClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Table", null, Array( attrDef("name", DataTypes.STRING_TYPE), new AttributeDefinition("db", "DB", Multiplicity.REQUIRED, false, null), - new AttributeDefinition("sd", "StorageDesc", Multiplicity.REQUIRED, false, null), + new AttributeDefinition("sd", "StorageDescriptor", Multiplicity.REQUIRED, false, null), attrDef("created", DataTypes.DATE_TYPE) )) + def partitionClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "Partition", null, + Array( + new AttributeDefinition("values", DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName), Multiplicity.REQUIRED, false, null), + new AttributeDefinition("table", "Table", Multiplicity.REQUIRED, false, null) + )) + def loadProcessClsDef = new HierarchicalTypeDefinition[ClassType](classOf[ClassType], "LoadProcess", null, Array( attrDef("name", DataTypes.STRING_TYPE), @@ -128,19 +143,17 @@ object QueryTestsUtils extends GraphUtils { Array[AttributeDefinition]()) TypeSystem.getInstance().defineTypes(ImmutableList.of[EnumTypeDefinition], - ImmutableList.of[StructTypeDefinition], + ImmutableList.of[StructTypeDefinition](hiveOrderDef), ImmutableList.of[HierarchicalTypeDefinition[TraitType]](dimTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef), ImmutableList.of[HierarchicalTypeDefinition[ClassType]](dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, - loadProcessClsDef, viewClsDef)) + partitionClsDef, loadProcessClsDef, viewClsDef)) () } - def setupTestGraph: TitanGraph = { - var conf = ConfigFactory.load() - conf = conf.getConfig("graphRepo") - val g = titanGraph(conf) + def setupTestGraph(gp: TitanGraphProvider): TitanGraph = { + val g = gp.get val manager: ScriptEngineManager = new ScriptEngineManager val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy") val bindings: Bindings = engine.createBindings http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/7660c9b2/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java ---------------------------------------------------------------------- diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java index e35453e..4c55ce7 100755 --- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java +++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java @@ -48,9 +48,9 @@ public class DataTypes { public static BigDecimalType BIGDECIMAL_TYPE = new BigDecimalType(); public static DateType DATE_TYPE = new DateType(); public static StringType STRING_TYPE = new StringType(); - static String ARRAY_TYPE_PREFIX = "array<"; + public static String ARRAY_TYPE_PREFIX = "array<"; static String ARRAY_TYPE_SUFFIX = ">"; - static String MAP_TYPE_PREFIX = "map<"; + public static String MAP_TYPE_PREFIX = "map<"; static String MAP_TYPE_SUFFIX = ">"; public static String arrayTypeName(String elemTypeName) {
