http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/Expressions.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/Expressions.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/Expressions.scala deleted file mode 100755 index aec424c..0000000 --- a/repository/src/main/scala/org/apache/hadoop/metadata/query/Expressions.scala +++ /dev/null @@ -1,739 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.query - -import org.apache.hadoop.metadata.MetadataException -import org.apache.hadoop.metadata.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory} -import org.apache.hadoop.metadata.typesystem.types._ - -object Expressions { - - import TypeUtils._ - - class ExpressionException(val e: Expression, message: String, cause: Throwable, enableSuppression: Boolean, - writableStackTrace: Boolean) - extends MetadataException(message, cause, enableSuppression, writableStackTrace) { - - def this(e: Expression, message: String) { - this(e, message, null, false, false) - } - - def this(e: Expression, message: String, cause: Throwable) { - this(e, message, cause, false, false) - } - - def this(e: Expression, cause: Throwable) { - this(e, null, cause, false, false) - } - - override def getMessage: String = { - val eString = e.toString - s"${super.getMessage}, expression:${if (eString contains "\n") "\n" else " "}$e" - } - - } - - class UnresolvedException(expr: Expression, function: String) extends - ExpressionException(expr, s"Unresolved $function") - - def attachExpression[A](e: Expression, msg: String = "")(f: => A): A = { - try f catch { - case eex: ExpressionException => throw eex - case ex: Exception => throw new ExpressionException(e, msg, ex) - } - } - - trait Expression { - self: Product => - - def children: Seq[Expression] - - /** - * Returns `true` if the schema for this expression and all its children have been resolved. - * The default logic is that an Expression is resolve if all its children are resolved. - */ - lazy val resolved: Boolean = childrenResolved - - /** - * Returns the output [[IDataType[_]] of this expression. Expressions that are unresolved will - * throw if this method is invoked. - */ - def dataType: IDataType[_] - - /** - * Returns true if all the children have been resolved. - */ - def childrenResolved = !children.exists(!_.resolved) - - - /** - * the aliases that are present in this Expression Tree - */ - def namedExpressions: Map[String, Expression] = Map() - - def fastEquals(other: Expression): Boolean = { - this.eq(other) || this == other - } - - def makeCopy(newArgs: Array[AnyRef]): this.type = attachExpression(this, "makeCopy") { - try { - val defaultCtor = getClass.getConstructors.find(_.getParameterTypes.size != 0).head - defaultCtor.newInstance(newArgs: _*).asInstanceOf[this.type] - } catch { - case e: java.lang.IllegalArgumentException => - throw new ExpressionException( - this, s"Failed to copy node. Reason: ${e.getMessage}.") - } - } - - def transformChildrenDown(rule: PartialFunction[Expression, Expression]): this.type = { - var changed = false - val newArgs = productIterator.map { - case arg: Expression if children contains arg => - val newChild = arg.asInstanceOf[Expression].transformDown(rule) - if (!(newChild fastEquals arg)) { - changed = true - newChild - } else { - arg - } - case Some(arg: Expression) if children contains arg => - val newChild = arg.asInstanceOf[Expression].transformDown(rule) - if (!(newChild fastEquals arg)) { - changed = true - Some(newChild) - } else { - Some(arg) - } - case m: Map[_, _] => m - case args: Traversable[_] => args.map { - case arg: Expression if children contains arg => - val newChild = arg.asInstanceOf[Expression].transformDown(rule) - if (!(newChild fastEquals arg)) { - changed = true - newChild - } else { - arg - } - case other => other - } - case nonChild: AnyRef => nonChild - case null => null - }.toArray - if (changed) makeCopy(newArgs) else this - } - - def transformDown(rule: PartialFunction[Expression, Expression]): Expression = { - val afterRule = rule.applyOrElse(this, identity[Expression]) - // Check if unchanged and then possibly return old copy to avoid gc churn. - if (this fastEquals afterRule) { - transformChildrenDown(rule) - } else { - afterRule.transformChildrenDown(rule) - } - } - - def traverseChildren(traverseFunc: (Expression, PartialFunction[Expression, Unit]) => Unit) - (rule: PartialFunction[Expression, Unit]): Unit = { - productIterator.foreach { - case arg: Expression if children contains arg => - traverseFunc(arg.asInstanceOf[Expression], rule) - case Some(arg: Expression) if children contains arg => - traverseFunc(arg.asInstanceOf[Expression], rule) - case m: Map[_, _] => m - case args: Traversable[_] => args.map { - case arg: Expression if children contains arg => - traverseFunc(arg.asInstanceOf[Expression], rule) - case other => other - } - case nonChild: AnyRef => nonChild - case null => null - } - } - - def traverseChildrenDown = traverseChildren(_traverseDown) _ - - private def _traverseDown(e: Expression, rule: PartialFunction[Expression, Unit]): Unit = { - if (rule.isDefinedAt(e)) { - rule.apply(e) - } - e.traverseChildrenDown(rule) - } - - def traverseDown(rule: PartialFunction[Expression, Unit]): Unit = { - _traverseDown(this, rule) - } - - def traverseChildrenUp = traverseChildren(_traverseUp) _ - - private def _traverseUp(e: Expression, rule: PartialFunction[Expression, Unit]): Unit = { - e.traverseChildrenUp(rule) - if (rule.isDefinedAt(e)) { - rule.apply(e) - } - } - - def traverseUp(rule: PartialFunction[Expression, Unit]): Unit = { - _traverseUp(this, rule) - } - - def transformUp(rule: PartialFunction[Expression, Expression]): Expression = { - val afterRuleOnChildren = transformChildrenUp(rule); - if (this fastEquals afterRuleOnChildren) { - rule.applyOrElse(this, identity[Expression]) - } else { - rule.applyOrElse(afterRuleOnChildren, identity[Expression]) - } - } - - def transformChildrenUp(rule: PartialFunction[Expression, Expression]): this.type = { - var changed = false - val newArgs = productIterator.map { - case arg: Expression if children contains arg => - val newChild = arg.asInstanceOf[Expression].transformUp(rule) - if (!(newChild fastEquals arg)) { - changed = true - newChild - } else { - arg - } - case Some(arg: Expression) if children contains arg => - val newChild = arg.asInstanceOf[Expression].transformUp(rule) - if (!(newChild fastEquals arg)) { - changed = true - Some(newChild) - } else { - Some(arg) - } - case m: Map[_, _] => m - case args: Traversable[_] => args.map { - case arg: Expression if children contains arg => - val newChild = arg.asInstanceOf[Expression].transformUp(rule) - if (!(newChild fastEquals arg)) { - changed = true - newChild - } else { - arg - } - case other => other - } - case nonChild: AnyRef => nonChild - case null => null - }.toArray - if (changed) makeCopy(newArgs) else this - } - - /* - * treeString methods - */ - def nodeName = getClass.getSimpleName - - def argString: String = productIterator.flatMap { - case e: Expression if children contains e => Nil - case e: Expression if e.toString contains "\n" => s"(${e.simpleString})" :: Nil - case seq: Seq[_] => seq.mkString("[", ",", "]") :: Nil - case set: Set[_] => set.mkString("{", ",", "}") :: Nil - case f: IDataType[_] => f.getName :: Nil - case other => other :: Nil - }.mkString(", ") - - /** String representation of this node without any children */ - def simpleString = s"$nodeName $argString" - - protected def generateTreeString(depth: Int, builder: StringBuilder): StringBuilder = { - builder.append(" " * depth) - builder.append(simpleString) - builder.append("\n") - children.foreach(_.generateTreeString(depth + 1, builder)) - builder - } - - def treeString = generateTreeString(0, new StringBuilder).toString - - /* - * Fluent API methods - */ - def field(fieldName: String) = new UnresolvedFieldExpression(this, fieldName) - - def join(fieldName: String) = field(fieldName) - - def `.`(fieldName: String) = field(fieldName) - - def as(alias: String) = new AliasExpression(this, alias) - - def arith(op: String)(rightExpr: Expression) = new ArithmeticExpression(op, this, rightExpr) - - def + = arith("+") _ - - def - = arith("-") _ - - def * = arith("*") _ - - def / = arith("/") _ - - def % = arith("%") _ - - def isTrait(name: String) = new isTraitUnaryExpression(name, this) - - def hasField(name: String) = new hasFieldUnaryExpression(name, this) - - def compareOp(op: String)(rightExpr: Expression) = new ComparisonExpression(op, this, rightExpr) - - def `=` = compareOp("=") _ - - def `!=` = compareOp("!=") _ - - def `>` = compareOp(">") _ - - def `>=` = compareOp(">=") _ - - def `<` = compareOp("<") _ - - def `<=` = compareOp("=") _ - - def logicalOp(op: String)(rightExpr: Expression) = new LogicalExpression(op, List(this, rightExpr)) - - def and = logicalOp("and") _ - - def or = logicalOp("or") _ - - def where(condExpr: Expression) = new FilterExpression(this, condExpr) - - def select(selectList: Expression*) = new SelectExpression(this, selectList.toList) - - def loop(loopingExpr: Expression) = new LoopExpression(this, loopingExpr, None) - - def loop(loopingExpr: Expression, times: Literal[Integer]) = - new LoopExpression(this, loopingExpr, Some(times)) - - def traitInstance() = new TraitInstanceExpression(this) - def instance() = new InstanceExpression(this) - - def path() = new PathExpression(this) - } - - trait BinaryNode { - self: Expression => - def left: Expression - - def right: Expression - - def children = Seq(left, right) - - override def namedExpressions = left.namedExpressions ++ right.namedExpressions - } - - trait LeafNode { - def children = Nil - } - - trait UnaryNode { - self: Expression => - def child: Expression - - override def namedExpressions = child.namedExpressions - - def children = child :: Nil - } - - abstract class BinaryExpression extends Expression with BinaryNode { - self: Product => - def symbol: String - - override def toString = s"($left $symbol $right)" - } - - case class ClassExpression(clsName: String) extends Expression with LeafNode { - val dataType = typSystem.getDataType(classOf[ClassType], clsName) - - override def toString = clsName - } - - def _class(name: String): Expression = new ClassExpression(name) - - case class TraitExpression(traitName: String) extends Expression with LeafNode { - val dataType = typSystem.getDataType(classOf[TraitType], traitName) - - override def toString = traitName - } - - def _trait(name: String) = new TraitExpression(name) - - case class IdExpression(name: String) extends Expression with LeafNode { - override def toString = name - - override lazy val resolved = false - - override def dataType = throw new UnresolvedException(this, "id") - } - - def id(name: String) = new IdExpression(name) - - case class UnresolvedFieldExpression(child: Expression, fieldName: String) extends Expression - with UnaryNode { - override def toString = s"${child}.$fieldName" - - override lazy val resolved = false - - override def dataType = throw new UnresolvedException(this, "field") - } - - case class FieldExpression(fieldName: String, fieldInfo: FieldInfo, child: Option[Expression]) - extends Expression { - - def elemType(t: IDataType[_]): IDataType[_] = { - if (t.getTypeCategory == TypeCategory.ARRAY) { - val aT = t.asInstanceOf[ArrayType] - if (aT.getElemType.getTypeCategory == TypeCategory.CLASS || - aT.getElemType.getTypeCategory == TypeCategory.STRUCT) { - return aT.getElemType - } - } - t - } - - val children = if (child.isDefined) List(child.get) else Nil - import scala.language.existentials - lazy val dataType = { - val t = { - if (fieldInfo.traitName != null ) { - typSystem.getDataType(classOf[TraitType], fieldInfo.traitName) - } else if (!fieldInfo.isReverse) { - fieldInfo.attrInfo.dataType() - } else { - fieldInfo.reverseDataType - } - } - elemType(t) - } - override lazy val resolved: Boolean = true - - override def namedExpressions = if (child.isDefined) child.get.namedExpressions else Map() - - override def toString = { - if (child.isDefined) { - val sep = if (dataType.isInstanceOf[ClassType]) " " else "." - s"${child.get}${sep}$fieldName" - } else { - fieldName - } - } - } - - case class AliasExpression(child: Expression, alias: String) extends Expression with UnaryNode { - override def namedExpressions = child.namedExpressions + (alias -> child) - - override def toString = s"$child as $alias" - - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved child") - } - child.dataType - } - } - - case class BackReference(alias: String, reference: Expression, child: Option[Expression]) extends Expression { - val children = if (child.isDefined) List(child.get) else Nil - val dataType = reference.dataType - - override def namedExpressions = if (child.isDefined) child.get.namedExpressions else Map() - - override def toString = if (child.isDefined) s"${child.get} $alias" else alias - } - - case class Literal[T](dataType: PrimitiveType[T], rawValue: Any) extends Expression with LeafNode { - val value = if (rawValue == null) dataType.nullValue() else dataType.convert(rawValue, Multiplicity.REQUIRED) - - override def toString = value match { - case s: String => s""""$s"""" - case x => x.toString - } - } - - def literal[T](typ: PrimitiveType[T], rawValue: Any) = new Literal[T](typ, rawValue) - - def boolean(rawValue: Any) = literal(DataTypes.BOOLEAN_TYPE, rawValue) - - def byte(rawValue: Any) = literal(DataTypes.BYTE_TYPE, rawValue) - - def short(rawValue: Any) = literal(DataTypes.SHORT_TYPE, rawValue) - - def int(rawValue: Any) = literal(DataTypes.INT_TYPE, rawValue) - - def long(rawValue: Any) = literal(DataTypes.LONG_TYPE, rawValue) - - def float(rawValue: Any) = literal(DataTypes.FLOAT_TYPE, rawValue) - - def double(rawValue: Any) = literal(DataTypes.DOUBLE_TYPE, rawValue) - - def bigint(rawValue: Any) = literal(DataTypes.BIGINTEGER_TYPE, rawValue) - - def bigdecimal(rawValue: Any) = literal(DataTypes.BIGDECIMAL_TYPE, rawValue) - - def string(rawValue: Any) = literal(DataTypes.STRING_TYPE, rawValue) - - case class ArithmeticExpression(symbol: String, - left: Expression, - right: Expression) - extends BinaryExpression { - - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved children") - } - TypeUtils.combinedType(left.dataType, right.dataType) - } - } - - case class isTraitLeafExpression(traitName: String, classExpression: Option[Expression] = None) - extends Expression with LeafNode { - // validate TraitName - try { - typSystem.getDataType(classOf[TraitType], traitName) - } catch { - case me: MetadataException => throw new ExpressionException(this, "not a TraitType", me) - } - - override lazy val resolved = classExpression.isDefined - lazy val dataType = { - - if (!resolved) { - throw new UnresolvedException(this, - s"cannot resolve isTrait application") - } - - if (!classExpression.get.dataType.isInstanceOf[ClassType]) { - throw new ExpressionException(this, - s"Cannot apply isTrait on ${classExpression.get.dataType.getName}, it is not a ClassType") - } - DataTypes.BOOLEAN_TYPE - } - - override def toString = s"${classExpression.getOrElse("")} is $traitName" - } - - def isTrait(name: String) = new isTraitLeafExpression(name) - - case class isTraitUnaryExpression(traitName: String, child: Expression) - extends Expression with UnaryNode { - // validate TraitName - typSystem.getDataType(classOf[TraitType], traitName) - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved child") - } - if (!child.dataType.isInstanceOf[ClassType]) { - throw new ExpressionException(this, - s"Cannot apply isTrait on ${child.dataType.getName}, it is not a ClassType") - } - DataTypes.BOOLEAN_TYPE - } - - override def toString = s"$child is $traitName" - } - - case class hasFieldLeafExpression(fieldName: String, classExpression: Option[Expression] = None) - extends Expression with LeafNode { - - override lazy val resolved = classExpression.isDefined - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"Cannot apply hasField on ${classExpression.get.dataType.getName}, it is not a ClassType") - } - if (classExpression.isDefined && !TypeUtils.fieldMapping(classExpression.get.dataType).isDefined) { - throw new ExpressionException(this, s"Cannot apply hasField on ${classExpression.get.dataType.getName}") - } - DataTypes.BOOLEAN_TYPE - } - - override def toString = s"${classExpression.getOrElse("")} has $fieldName" - } - - def hasField(name: String) = new hasFieldLeafExpression(name) - - case class hasFieldUnaryExpression(fieldName: String, child: Expression) - extends Expression with UnaryNode { - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved child") - } - if (!TypeUtils.fieldMapping(child.dataType).isDefined) { - throw new MetadataException(s"Cannot apply hasField on ${child.dataType.getName}") - } - DataTypes.BOOLEAN_TYPE - } - - override def toString = s"$child has $fieldName" - } - - case class ComparisonExpression(symbol: String, - left: Expression, - right: Expression) - extends BinaryExpression { - - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved children") - } - if (left.dataType != DataTypes.STRING_TYPE || right.dataType != DataTypes.STRING_TYPE) { - TypeUtils.combinedType(left.dataType, right.dataType) - } - DataTypes.BOOLEAN_TYPE - } - } - - case class LogicalExpression(symbol: String, children: List[Expression]) - extends Expression { - assert(children.size > 0) - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved children") - } - children.foreach { childExpr => - if (childExpr.dataType != DataTypes.BOOLEAN_TYPE) { - throw new MetadataException( - s"Cannot apply logical operator '$symbol' on input of type '${childExpr.dataType}") - } - } - DataTypes.BOOLEAN_TYPE - } - - override def toString = children.mkString("", s" $symbol ", "") - } - - case class FilterExpression(val child: Expression, val condExpr: Expression) extends Expression { - val children = List(child, condExpr) - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved children") - } - if (condExpr.dataType != DataTypes.BOOLEAN_TYPE) { - throw new ExpressionException(this, s"Filter condition '$condExpr' is not a boolean expression") - } - child.dataType - } - - override def namedExpressions = child.namedExpressions ++ condExpr.namedExpressions - - override def toString = s"$child where $condExpr" - } - - val GEN_COL_ALIAS_PREFIX = "_col" - - case class SelectExpression(child: Expression, selectList: List[Expression]) extends Expression { - - val children = List(child) ::: selectList - lazy val selectListWithAlias = selectList.zipWithIndex map { - case (s: AliasExpression, _) => s - case (x, i) => new AliasExpression(x, s"${GEN_COL_ALIAS_PREFIX}_$i") - } - - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved children") - } - TypeUtils.createStructType(selectListWithAlias) - } - - override def namedExpressions = child.namedExpressions ++ (selectList.flatMap(_.namedExpressions)) - - override def toString = s"""$child select ${selectListWithAlias.mkString("", ", ", "")}""" - } - - case class LoopExpression(val input: Expression, val loopingExpression: Expression, - val times: Option[Literal[Integer]]) extends Expression { - val children = List(input, loopingExpression) - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved children") - } - if (input.dataType.getTypeCategory != TypeCategory.CLASS) { - throw new ExpressionException(this, s"Loop Expression applied to type : '${input.dataType.getName}';" + - " loop can only be applied to Class Expressions") - } - if (input.dataType != loopingExpression.dataType) { - throw new ExpressionException(this, - s"Invalid Loop Expression; input and loopExpression dataTypes don't match: " + - s"(${input.dataType.getName},${loopingExpression.dataType.getName}})") - } - input.dataType - } - - override def namedExpressions = input.namedExpressions - - override def toString = { - if (times.isDefined) s"$input loop ($loopingExpression) times ${times.get.value}" - else s"$input loop ($loopingExpression)" - } - } - - case class TraitInstanceExpression(child: Expression) - extends Expression with UnaryNode { - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved child") - } - if (!child.dataType.isInstanceOf[TraitType]) { - throw new ExpressionException(this, - s"Cannot apply instance on ${child.dataType.getName}, it is not a TraitType") - } - typSystem.getIdType.getStructType - } - - override def toString = s"$child traitInstance" - } - - case class InstanceExpression(child: Expression) - extends Expression with UnaryNode { - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved child") - } - typSystem.getIdType.getStructType - } - - override def toString = s"$child instance" - } - - case class PathExpression(child: Expression) - extends Expression with UnaryNode { - lazy val dataType = { - if (!resolved) { - throw new UnresolvedException(this, - s"datatype. Can not resolve due to unresolved child") - } - TypeUtils.ResultWithPathStruct.createType(this, child.dataType) - } - - override def toString = s"$child withPath" - } -}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala deleted file mode 100755 index 5684365..0000000 --- a/repository/src/main/scala/org/apache/hadoop/metadata/query/GraphPersistenceStrategies.scala +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.query - -import com.thinkaurelius.titan.core.TitanVertex -import com.tinkerpop.blueprints.Direction -import org.apache.hadoop.metadata.query.Expressions.{ComparisonExpression, ExpressionException} -import org.apache.hadoop.metadata.query.TypeUtils.FieldInfo -import org.apache.hadoop.metadata.typesystem.persistence.Id -import org.apache.hadoop.metadata.typesystem.types.DataTypes._ -import org.apache.hadoop.metadata.typesystem.types._ -import org.apache.hadoop.metadata.typesystem.{ITypedInstance, ITypedReferenceableInstance} - -import scala.collection.JavaConversions._ - -/** - * Represents the Bridge between the QueryProcessor and the Graph Persistence scheme used. - * Some of the behaviors captured are: - * - how is type and id information stored in the Vertex that represents an [[ITypedReferenceableInstance]] - * - how are edges representing trait and attribute relationships labelled. - * - how are attribute names mapped to Property Keys in Vertices. - * - * This is a work in progress. - */ -trait GraphPersistenceStrategies { - /** - * Name of attribute used to store typeName in vertex - */ - def typeAttributeName: String - - /** - * Name of attribute used to store super type names in vertex. - */ - def superTypeAttributeName: String - - /** - * Name of attribute used to store guid in vertex - */ - def idAttributeName : String - - /** - * Given a dataType and a reference attribute, how is edge labeled - */ - def edgeLabel(iDataType: IDataType[_], aInfo: AttributeInfo): String - - def traitLabel(cls: IDataType[_], traitName: String): String - - def instanceToTraitEdgeDirection : String = "out" - def traitToInstanceEdgeDirection = instanceToTraitEdgeDirection match { - case "out" => "in" - case "in" => "out" - case x => x - } - - /** - * The propertyKey used to store the attribute in a Graph Vertex. - * @param dataType - * @param aInfo - * @return - */ - def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo): String - - /** - * from a vertex for an [[ITypedReferenceableInstance]] get the traits that it has. - * @param v - * @return - */ - def traitNames(v: TitanVertex): java.util.List[String] - - def edgeLabel(fInfo: FieldInfo): String = fInfo match { - case FieldInfo(dataType, aInfo, null, null) => edgeLabel(dataType, aInfo) - case FieldInfo(dataType, aInfo, reverseDataType, null) => edgeLabel(reverseDataType, aInfo) - case FieldInfo(dataType, null, null, traitName) => traitLabel(dataType, traitName) - } - - def fieldPrefixInSelect: String - - /** - * extract the Id from a Vertex. - * @param dataTypeNm the dataType of the instance that the given vertex represents - * @param v - * @return - */ - def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id - - def constructInstance[U](dataType: IDataType[U], v: java.lang.Object): U - - def gremlinCompOp(op: ComparisonExpression) = op.symbol match { - case "=" => "T.eq" - case "!=" => "T.neq" - case ">" => "T.gt" - case ">=" => "T.gte" - case "<" => "T.lt" - case "<=" => "T.lte" - case _ => throw new ExpressionException(op, "Comparison operator not supported in Gremlin") - } - - def loopObjectExpression(dataType: IDataType[_]) = { - _typeTestExpression(dataType.getName, "it.object") - } - - def addGraphVertexPrefix(preStatements : Traversable[String]) = !collectTypeInstancesIntoVar - - /** - * Controls behavior of how instances of a Type are discovered. - * - query is generated in a way that indexes are exercised using a local set variable across multiple lookups - * - query is generated using an 'or' expression. - * - * '''This is a very bad idea: controlling query execution behavior via query generation.''' But our current - * knowledge of seems to indicate we have no choice. See - * [[https://groups.google.com/forum/#!topic/gremlin-users/n1oV86yr4yU discussion in Gremlin group]]. - * Also this seems a fragile solution, dependend on the memory requirements of the Set variable. - * For now enabling via the '''collectTypeInstancesIntoVar''' behavior setting. Reverting back would require - * setting this to false. - * - * Long term have to get to the bottom of Gremlin: - * - there doesn't seem to be way to see the physical query plan. Maybe we should directly interface with Titan. - * - At least from querying perspective a columnar db maybe a better route. Daniel Abadi did some good work - * on showing how to use a columnar store as a Graph Db. - * - * - * @return - */ - def collectTypeInstancesIntoVar = true - - def typeTestExpression(typeName : String, intSeq : IntSequence) : Seq[String] = { - if (collectTypeInstancesIntoVar) - typeTestExpressionMultiStep(typeName, intSeq) - else - typeTestExpressionUsingFilter(typeName) - } - - private def typeTestExpressionUsingFilter(typeName : String) : Seq[String] = { - Seq(s"""filter${_typeTestExpression(typeName, "it")}""") - } - - private def _typeTestExpression(typeName: String, itRef: String): String = { - s"""{(${itRef}.'${typeAttributeName}' == '${typeName}') | - |(${itRef}.'${superTypeAttributeName}' ? - |${itRef}.'${superTypeAttributeName}'.contains('${typeName}') : false)}""". - stripMargin.replace(System.getProperty("line.separator"), "") - } - - private def typeTestExpressionMultiStep(typeName : String, intSeq : IntSequence) : Seq[String] = { - - val varName = s"_var_${intSeq.next}" - Seq( - newSetVar(varName), - fillVarWithTypeInstances(typeName, varName), - fillVarWithSubTypeInstances(typeName, varName), - s"$varName._()" - ) - } - - private def newSetVar(varName : String) = s"$varName = [] as Set" - - private def fillVarWithTypeInstances(typeName : String, fillVar : String) = { - s"""g.V().has("${typeAttributeName}", "${typeName}").fill($fillVar)""" - } - - private def fillVarWithSubTypeInstances(typeName : String, fillVar : String) = { - s"""g.V().has("${superTypeAttributeName}", "${typeName}").fill($fillVar)""" - } -} - -object GraphPersistenceStrategy1 extends GraphPersistenceStrategies { - val typeAttributeName = "typeName" - val superTypeAttributeName = "superTypeNames" - val idAttributeName = "guid" - - def edgeLabel(dataType: IDataType[_], aInfo: AttributeInfo) = s"${dataType.getName}.${aInfo.name}" - - val fieldPrefixInSelect = "it" - - def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName" - - def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = aInfo.name - - def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id = - new Id(v.getId.toString, 0, dataTypeNm) - - def traitNames(v: TitanVertex): java.util.List[String] = { - val s = v.getProperty[String]("traitNames") - if (s != null) { - Seq[String](s.split(","): _*) - } else { - Seq() - } - } - - def constructInstance[U](dataType: IDataType[U], v: AnyRef): U = { - dataType.getTypeCategory match { - case DataTypes.TypeCategory.PRIMITIVE => dataType.convert(v, Multiplicity.OPTIONAL) - case DataTypes.TypeCategory.STRUCT - if dataType.getName == TypeSystem.getInstance().getIdType.getName => { - val sType = dataType.asInstanceOf[StructType] - val sInstance = sType.createInstance() - val tV = v.asInstanceOf[TitanVertex] - sInstance.set(TypeSystem.getInstance().getIdType.typeNameAttrName, - tV.getProperty[java.lang.String](typeAttributeName)) - sInstance.set(TypeSystem.getInstance().getIdType.idAttrName, - tV.getProperty[java.lang.String](idAttributeName)) - dataType.convert(sInstance, Multiplicity.OPTIONAL) - } - case DataTypes.TypeCategory.STRUCT => { - val sType = dataType.asInstanceOf[StructType] - val sInstance = sType.createInstance() - loadStructInstance(sType, sInstance, v.asInstanceOf[TitanVertex]) - dataType.convert(sInstance, Multiplicity.OPTIONAL) - } - case DataTypes.TypeCategory.TRAIT => { - val tType = dataType.asInstanceOf[TraitType] - val tInstance = tType.createInstance() - /* - * this is not right, we should load the Instance associated with this trait. - * for now just loading the trait struct. - */ - loadStructInstance(tType, tInstance, v.asInstanceOf[TitanVertex]) - dataType.convert(tInstance, Multiplicity.OPTIONAL) - } - case DataTypes.TypeCategory.CLASS => { - val cType = dataType.asInstanceOf[ClassType] - val cInstance = constructClassInstance(dataType.asInstanceOf[ClassType], v.asInstanceOf[TitanVertex]) - dataType.convert(cInstance, Multiplicity.OPTIONAL) - } - case DataTypes.TypeCategory.ENUM => dataType.convert(v, Multiplicity.OPTIONAL) - case x => throw new UnsupportedOperationException(s"load for ${dataType} not supported") - } - } - - def loadStructInstance(dataType: IConstructableType[_, _ <: ITypedInstance], - typInstance: ITypedInstance, v: TitanVertex): Unit = { - import scala.collection.JavaConversions._ - dataType.fieldMapping().fields.foreach { t => - val fName = t._1 - val aInfo = t._2 - loadAttribute(dataType, aInfo, typInstance, v) - } - } - - def constructClassInstance(dataType: ClassType, v: TitanVertex): ITypedReferenceableInstance = { - val id = getIdFromVertex(dataType.name, v) - val tNms = traitNames(v) - val cInstance = dataType.createInstance(id, tNms: _*) - // load traits - tNms.foreach { tNm => - val tLabel = traitLabel(dataType, tNm) - val edges = v.getEdges(Direction.OUT, tLabel) - val tVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex] - val tType = TypeSystem.getInstance().getDataType[TraitType](classOf[TraitType], tNm) - val tInstance = cInstance.getTrait(tNm).asInstanceOf[ITypedInstance] - loadStructInstance(tType, tInstance, tVertex) - } - loadStructInstance(dataType, cInstance, v) - cInstance - } - - def loadAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex): Unit = { - aInfo.dataType.getTypeCategory match { - case DataTypes.TypeCategory.PRIMITIVE => loadPrimitiveAttribute(dataType, aInfo, i, v) - case DataTypes.TypeCategory.ENUM => loadEnumAttribute(dataType, aInfo, i, v) - case DataTypes.TypeCategory.ARRAY => - throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported") - case DataTypes.TypeCategory.MAP => - throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported") - case DataTypes.TypeCategory.STRUCT => loadStructAttribute(dataType, aInfo, i, v) - case DataTypes.TypeCategory.TRAIT => - throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported") - case DataTypes.TypeCategory.CLASS => loadStructAttribute(dataType, aInfo, i, v) - } - } - - private def loadEnumAttribute(dataType: IDataType[_], aInfo: AttributeInfo, i: ITypedInstance, v: TitanVertex) - : Unit = { - val fName = fieldNameInVertex(dataType, aInfo) - i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName)) - } - - private def loadPrimitiveAttribute(dataType: IDataType[_], aInfo: AttributeInfo, - i: ITypedInstance, v: TitanVertex): Unit = { - val fName = fieldNameInVertex(dataType, aInfo) - aInfo.dataType() match { - case x: BooleanType => i.setBoolean(aInfo.name, v.getProperty[java.lang.Boolean](fName)) - case x: ByteType => i.setByte(aInfo.name, v.getProperty[java.lang.Byte](fName)) - case x: ShortType => i.setShort(aInfo.name, v.getProperty[java.lang.Short](fName)) - case x: IntType => i.setInt(aInfo.name, v.getProperty[java.lang.Integer](fName)) - case x: LongType => i.setLong(aInfo.name, v.getProperty[java.lang.Long](fName)) - case x: FloatType => i.setFloat(aInfo.name, v.getProperty[java.lang.Float](fName)) - case x: DoubleType => i.setDouble(aInfo.name, v.getProperty[java.lang.Double](fName)) - case x: StringType => i.setString(aInfo.name, v.getProperty[java.lang.String](fName)) - case _ => throw new UnsupportedOperationException(s"load for ${aInfo.dataType()} not supported") - } - } - - private def loadStructAttribute(dataType: IDataType[_], aInfo: AttributeInfo, - i: ITypedInstance, v: TitanVertex): Unit = { - val eLabel = edgeLabel(FieldInfo(dataType, aInfo, null)) - val edges = v.getEdges(Direction.OUT, eLabel) - val sVertex = edges.iterator().next().getVertex(Direction.IN).asInstanceOf[TitanVertex] - if (aInfo.dataType().getTypeCategory == DataTypes.TypeCategory.STRUCT) { - val sType = aInfo.dataType().asInstanceOf[StructType] - val sInstance = sType.createInstance() - loadStructInstance(sType, sInstance, sVertex) - i.set(aInfo.name, sInstance) - } else { - val cInstance = constructClassInstance(aInfo.dataType().asInstanceOf[ClassType], sVertex) - i.set(aInfo.name, cInstance) - } - } -} - http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinEvaluator.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinEvaluator.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinEvaluator.scala deleted file mode 100755 index 6a77bc3..0000000 --- a/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinEvaluator.scala +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.query - -import javax.script.{Bindings, ScriptEngine, ScriptEngineManager} - -import com.thinkaurelius.titan.core.TitanGraph -import com.tinkerpop.pipes.util.structures.Row -import org.apache.hadoop.metadata.query.TypeUtils.ResultWithPathStruct -import org.apache.hadoop.metadata.typesystem.json._ -import org.apache.hadoop.metadata.typesystem.types._ -import org.json4s._ -import org.json4s.native.Serialization._ - -import scala.language.existentials - -case class GremlinQueryResult(query: String, - resultDataType: IDataType[_], - rows: List[_]) { - def toJson = JsonHelper.toJson(this) -} - -class GremlinEvaluator(qry: GremlinQuery, persistenceStrategy: GraphPersistenceStrategies, g: TitanGraph) { - - val manager: ScriptEngineManager = new ScriptEngineManager - val engine: ScriptEngine = manager.getEngineByName("gremlin-groovy") - val bindings: Bindings = engine.createBindings - bindings.put("g", g) - - /** - * - * @param gResultObj is the object returned from gremlin. This must be a List - * @param qryResultObj is the object constructed for the output w/o the Path. - * @return a ResultWithPathStruct - */ - def addPathStruct(gResultObj : AnyRef, qryResultObj : Any) : Any = { - if ( !qry.isPathExpresion) { - qryResultObj - } else { - import scala.collection.JavaConversions._ - import scala.collection.JavaConverters._ - val iPaths = gResultObj.asInstanceOf[java.util.List[AnyRef]].init - - val oPaths = iPaths.map { p => - persistenceStrategy.constructInstance(TypeSystem.getInstance().getIdType.getStructType, p) - }.toList.asJava - val sType = qry.expr.dataType.asInstanceOf[StructType] - val sInstance = sType.createInstance() - sInstance.set(ResultWithPathStruct.pathAttrName, oPaths) - sInstance.set(ResultWithPathStruct.resultAttrName, qryResultObj) - sInstance - } - } - - def instanceObject(v : AnyRef) : AnyRef = { - if ( qry.isPathExpresion ) { - import scala.collection.JavaConversions._ - v.asInstanceOf[java.util.List[AnyRef]].last - } else { - v - } - } - - def evaluate(): GremlinQueryResult = { - import scala.collection.JavaConversions._ - val rType = qry.expr.dataType - val oType = if (qry.isPathExpresion) qry.expr.children(0).dataType else rType - val rawRes = engine.eval(qry.queryStr, bindings) - - if (!qry.hasSelectList) { - val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { v => - val iV = instanceObject(v) - val o = persistenceStrategy.constructInstance(oType, iV) - addPathStruct(v, o) - } - GremlinQueryResult(qry.expr.toString, rType, rows.toList) - } else { - val sType = oType.asInstanceOf[StructType] - val rows = rawRes.asInstanceOf[java.util.List[AnyRef]].map { r => - val rV = instanceObject(r).asInstanceOf[Row[java.util.List[AnyRef]]] - val sInstance = sType.createInstance() - val selExpr = - (if (qry.isPathExpresion) qry.expr.children(0) else qry.expr). - asInstanceOf[Expressions.SelectExpression] - selExpr.selectListWithAlias.foreach { aE => - val cName = aE.alias - val (src, idx) = qry.resultMaping(cName) - val v = rV.getColumn(src).get(idx) - sInstance.set(cName, persistenceStrategy.constructInstance(aE.dataType, v)) - } - addPathStruct(r, sInstance) - } - GremlinQueryResult(qry.expr.toString, rType, rows.toList) - } - - } -} - -object JsonHelper { - - class GremlinQueryResultSerializer() - extends Serializer[GremlinQueryResult] { - def deserialize(implicit format: Formats) = { - throw new UnsupportedOperationException("Deserialization of GremlinQueryResult not supported") - } - - def serialize(implicit f: Formats) = { - case GremlinQueryResult(query, rT, rows) => - JObject(JField("query", JString(query)), - JField("dataType", TypesSerialization.toJsonValue(rT)(f)), - JField("rows", Extraction.decompose(rows)(f)) - ) - } - } - - implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer + - new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer + - new GremlinQueryResultSerializer - - def toJson(r: GremlinQueryResult): String = { - writePretty(r) - } -} http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinQuery.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinQuery.scala deleted file mode 100755 index 1395f6d..0000000 --- a/repository/src/main/scala/org/apache/hadoop/metadata/query/GremlinQuery.scala +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.query - -import org.apache.hadoop.metadata.query.Expressions._ -import org.apache.hadoop.metadata.typesystem.types.DataTypes.TypeCategory -import org.apache.hadoop.metadata.typesystem.types.TypeSystem - -import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer - -trait IntSequence { - def next: Int -} - -case class GremlinQuery(expr: Expression, queryStr: String, resultMaping: Map[String, (String, Int)]) { - - def hasSelectList = resultMaping != null - - def isPathExpresion = expr.isInstanceOf[PathExpression] -} - -trait SelectExpressionHandling { - - /** - * To aide in gremlinQuery generation add an alias to the input of SelectExpressions - */ - class AddAliasToSelectInput extends PartialFunction[Expression, Expression] { - - private var idx = 0 - - def isDefinedAt(e: Expression) = true - - class DecorateFieldWithAlias(aliasE: AliasExpression) - extends PartialFunction[Expression, Expression] { - def isDefinedAt(e: Expression) = true - - def apply(e: Expression) = e match { - case fe@FieldExpression(fieldName, fInfo, None) => - FieldExpression(fieldName, fInfo, Some(BackReference(aliasE.alias, aliasE.child, None))) - case _ => e - } - } - - def apply(e: Expression) = e match { - case SelectExpression(aliasE@AliasExpression(_, _), selList) => { - idx = idx + 1 - SelectExpression(aliasE, selList.map(_.transformUp(new DecorateFieldWithAlias(aliasE)))) - } - case SelectExpression(child, selList) => { - idx = idx + 1 - val aliasE = AliasExpression(child, s"_src$idx") - SelectExpression(aliasE, selList.map(_.transformUp(new DecorateFieldWithAlias(aliasE)))) - } - case _ => e - } - } - - def getSelectExpressionSrc(e: Expression): List[String] = { - val l = ArrayBuffer[String]() - e.traverseUp { - case BackReference(alias, _, _) => l += alias - } - l.toSet.toList - } - - def validateSelectExprHaveOneSrc: PartialFunction[Expression, Unit] = { - case SelectExpression(_, selList) => { - selList.foreach { se => - val srcs = getSelectExpressionSrc(se) - if (srcs.size > 1) { - throw new GremlinTranslationException(se, "Only one src allowed in a Select Expression") - } - } - } - } - - def groupSelectExpressionsBySrc(sel: SelectExpression): mutable.LinkedHashMap[String, List[Expression]] = { - val m = mutable.LinkedHashMap[String, List[Expression]]() - sel.selectListWithAlias.foreach { se => - val l = getSelectExpressionSrc(se.child) - if (!m.contains(l(0))) { - m(l(0)) = List() - } - m(l(0)) = m(l(0)) :+ se.child - } - m - } - - /** - * For each Output Column in the SelectExpression compute the ArrayList(Src) this maps to and the position within - * this list. - * @param sel - * @return - */ - def buildResultMapping(sel: SelectExpression): Map[String, (String, Int)] = { - val srcToExprs = groupSelectExpressionsBySrc(sel) - val m = new mutable.HashMap[String, (String, Int)] - sel.selectListWithAlias.foreach { se => - val src = getSelectExpressionSrc(se.child)(0) - val srcExprs = srcToExprs(src) - var idx = srcExprs.indexOf(se.child) - m(se.alias) = (src, idx) - } - m.toMap - } - -} - -class GremlinTranslationException(expr: Expression, reason: String) extends -ExpressionException(expr, s"Unsupported Gremlin translation: $reason") - -class GremlinTranslator(expr: Expression, - gPersistenceBehavior: GraphPersistenceStrategies) - extends SelectExpressionHandling { - - val preStatements = ArrayBuffer[String]() - val postStatements = ArrayBuffer[String]() - - val wrapAndRule: PartialFunction[Expression, Expression] = { - case f: FilterExpression if !f.condExpr.isInstanceOf[LogicalExpression] => - FilterExpression(f.child, new LogicalExpression("and", List(f.condExpr))) - } - - val validateComparisonForm: PartialFunction[Expression, Unit] = { - case c@ComparisonExpression(_, left, right) => - if (!left.isInstanceOf[FieldExpression]) { - throw new GremlinTranslationException(c, s"lhs of comparison is not a field") - } - if (!right.isInstanceOf[Literal[_]]) { - throw new GremlinTranslationException(c, - s"rhs of comparison is not a literal") - } - () - } - - val counter = new IntSequence { - var i: Int = -1; - - def next: Int = { - i += 1; i - } - } - - def addAliasToLoopInput(c: IntSequence = counter): PartialFunction[Expression, Expression] = { - case l@LoopExpression(aliasE@AliasExpression(_, _), _, _) => l - case l@LoopExpression(inputExpr, loopExpr, t) => { - val aliasE = AliasExpression(inputExpr, s"_loop${c.next}") - LoopExpression(aliasE, loopExpr, t) - } - } - - def instanceClauseToTop(topE : Expression) : PartialFunction[Expression, Expression] = { - case le : LogicalExpression if (le fastEquals topE) => { - le.instance() - } - case ce : ComparisonExpression if (ce fastEquals topE) => { - ce.instance() - } - case he : hasFieldUnaryExpression if (he fastEquals topE) => { - he.instance() - } - } - - def traitClauseWithInstanceForTop(topE : Expression) : PartialFunction[Expression, Expression] = { - case te : TraitExpression if (te fastEquals topE) => { - val theTrait = te.as("theTrait") - val theInstance = theTrait.traitInstance().as("theInstance") - val outE = - theInstance.select(id("theTrait").as("traitDetails"), - id("theInstance").as("instanceInfo")) - QueryProcessor.validate(outE) - } - } - - def typeTestExpression(typeName : String) : String = { - val stats = gPersistenceBehavior.typeTestExpression(typeName, counter) - preStatements ++= stats.init - stats.last - } - - private def genQuery(expr: Expression, inSelect: Boolean): String = expr match { - case ClassExpression(clsName) => - typeTestExpression(clsName) - case TraitExpression(clsName) => - typeTestExpression(clsName) - case fe@FieldExpression(fieldName, fInfo, child) if fe.dataType.getTypeCategory == TypeCategory.PRIMITIVE => { - val fN = "\"" + gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo) + "\"" - child match { - case Some(e) => s"${genQuery(e, inSelect)}.$fN" - case None => s"$fN" - } - } - case fe@FieldExpression(fieldName, fInfo, child) - if fe.dataType.getTypeCategory == TypeCategory.CLASS || fe.dataType.getTypeCategory == TypeCategory.STRUCT => { - val direction = if (fInfo.isReverse) "in" else "out" - val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo) - val step = s"""$direction("$edgeLbl")""" - child match { - case Some(e) => s"${genQuery(e, inSelect)}.$step" - case None => step - } - } - case fe@FieldExpression(fieldName, fInfo, child) - if fInfo.traitName != null => { - val direction = gPersistenceBehavior.instanceToTraitEdgeDirection - val edgeLbl = gPersistenceBehavior.edgeLabel(fInfo) - val step = s"""$direction("$edgeLbl")""" - child match { - case Some(e) => s"${genQuery(e, inSelect)}.$step" - case None => step - } - } - case c@ComparisonExpression(symb, f@FieldExpression(fieldName, fInfo, ch), l) => { - val fieldGremlinExpr = s"${gPersistenceBehavior.fieldNameInVertex(fInfo.dataType, fInfo.attrInfo)}" - ch match { - case Some(child) => { - s"""${genQuery(child, inSelect)}.has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)""" - } - case None => s"""has("$fieldGremlinExpr", ${gPersistenceBehavior.gremlinCompOp(c)}, $l)""" - } - } - case fil@FilterExpression(child, condExpr) => { - s"${genQuery(child, inSelect)}.${genQuery(condExpr, inSelect)}" - } - case l@LogicalExpression(symb, children) => { - s"""$symb${children.map("_()." + genQuery(_, inSelect)).mkString("(", ",", ")")}""" - } - case sel@SelectExpression(child, selList) => { - val m = groupSelectExpressionsBySrc(sel) - var srcNamesList: List[String] = List() - var srcExprsList: List[List[String]] = List() - val it = m.iterator - while (it.hasNext) { - val (src, selExprs) = it.next - srcNamesList = srcNamesList :+ s""""$src"""" - srcExprsList = srcExprsList :+ selExprs.map { selExpr => - genQuery(selExpr, true) - } - } - val srcNamesString = srcNamesList.mkString("[", ",", "]") - val srcExprsStringList = srcExprsList.map { - _.mkString("[", ",", "]") - } - val srcExprsString = srcExprsStringList.foldLeft("")(_ + "{" + _ + "}") - s"${genQuery(child, inSelect)}.select($srcNamesString)$srcExprsString" - } - case loop@LoopExpression(input, loopExpr, t) => { - val inputQry = genQuery(input, inSelect) - val loopingPathGExpr = genQuery(loopExpr, inSelect) - val loopGExpr = s"""loop("${input.asInstanceOf[AliasExpression].alias}")""" - val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{true}" - val loopObjectGExpr = gPersistenceBehavior.loopObjectExpression(input.dataType) - s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}""" - } - case BackReference(alias, _, _) => - if (inSelect) gPersistenceBehavior.fieldPrefixInSelect else s"""back("$alias")""" - case AliasExpression(child, alias) => s"""${genQuery(child, inSelect)}.as("$alias")""" - case isTraitLeafExpression(traitName, Some(clsExp)) => - s"""out("${gPersistenceBehavior.traitLabel(clsExp.dataType, traitName)}")""" - case isTraitUnaryExpression(traitName, child) => - s"""out("${gPersistenceBehavior.traitLabel(child.dataType, traitName)}")""" - case hasFieldLeafExpression(fieldName, Some(clsExp)) => - s"""has("$fieldName")""" - case hasFieldUnaryExpression(fieldName, child) => - s"""${genQuery(child, inSelect)}.has("$fieldName")""" - case ArithmeticExpression(symb, left, right) => s"${genQuery(left, inSelect)} $symb ${genQuery(right, inSelect)}" - case l: Literal[_] => l.toString - case in@TraitInstanceExpression(child) => { - val direction = gPersistenceBehavior.traitToInstanceEdgeDirection - s"${genQuery(child, inSelect)}.$direction()" - } - case in@InstanceExpression(child) => { - s"${genQuery(child, inSelect)}" - } - case pe@PathExpression(child) => { - s"${genQuery(child, inSelect)}.path" - } - case x => throw new GremlinTranslationException(x, "expression not yet supported") - } - - def genFullQuery(expr: Expression): String = { - var q = genQuery(expr, false) - - if(gPersistenceBehavior.addGraphVertexPrefix(preStatements)) { - q = s"g.V.$q" - } - - q = s"$q.toList()" - - q = (preStatements ++ Seq(q) ++ postStatements).mkString("", ";", "") - /* - * the L:{} represents a groovy code block; the label is needed - * to distinguish it from a groovy closure. - */ - s"L:{$q}" - } - - def translate(): GremlinQuery = { - var e1 = expr.transformUp(wrapAndRule) - - e1.traverseUp(validateComparisonForm) - - e1 = e1.transformUp(new AddAliasToSelectInput) - e1.traverseUp(validateSelectExprHaveOneSrc) - e1 = e1.transformUp(addAliasToLoopInput()) - e1 = e1.transformUp(instanceClauseToTop(e1)) - e1 = e1.transformUp(traitClauseWithInstanceForTop(e1)) - - e1 match { - case e1: SelectExpression => { - val rMap = buildResultMapping(e1) - GremlinQuery(e1, genFullQuery(e1), rMap) - } - case pe@PathExpression(se@SelectExpression(child, selectList)) => { - val rMap = buildResultMapping(se) - GremlinQuery(e1, genFullQuery(e1), rMap) - } - case e1 => GremlinQuery(e1, genFullQuery(e1), null) - } - - } - - /* - * Translation Issues: - * 1. back references in filters. For e.g. testBackreference: 'DB as db Table where (db.name = "Reporting")' - * this is translated to: - * g.V.has("typeName","DB").as("db").in("Table.db").and(_().back("db").has("name", T.eq, "Reporting")).map().toList() - * But the '_().back("db") within the and is ignored, the has condition is applied on the current element. - * The solution is to to do predicate pushdown and apply the filter immediately on top of the referred Expression. - */ - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryParser.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryParser.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryParser.scala deleted file mode 100755 index 66628c0..0000000 --- a/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryParser.scala +++ /dev/null @@ -1,398 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.query - -import org.apache.hadoop.metadata.query.Expressions._ - -import scala.util.parsing.combinator.lexical.StdLexical -import scala.util.parsing.combinator.syntactical.StandardTokenParsers -import scala.util.parsing.combinator.{ImplicitConversions, PackratParsers} -import scala.util.parsing.input.CharArrayReader._ - -trait QueryKeywords { - this: StandardTokenParsers => - - import scala.language.implicitConversions - - protected case class Keyword(str: String) - - protected implicit def asParser(k: Keyword): Parser[String] = k.str - - protected val LPAREN = Keyword("(") - protected val RPAREN = Keyword(")") - protected val EQ = Keyword("=") - protected val LT = Keyword("<") - protected val GT = Keyword(">") - protected val NEQ = Keyword("!=") - protected val LTE = Keyword("<=") - protected val GTE = Keyword(">=") - protected val COMMA = Keyword(",") - protected val AND = Keyword("and") - protected val OR = Keyword("or") - protected val PLUS = Keyword("+") - protected val MINUS = Keyword("-") - protected val STAR = Keyword("*") - protected val DIV = Keyword("/") - protected val DOT = Keyword(".") - - protected val SELECT = Keyword("select") - protected val FROM = Keyword("from") - protected val WHERE = Keyword("where") - protected val GROUPBY = Keyword("groupby") - protected val LOOP = Keyword("loop") - protected val ISA = Keyword("isa") - protected val IS = Keyword("is") - protected val HAS = Keyword("has") - protected val AS = Keyword("as") - protected val TIMES = Keyword("times") - protected val WITHPATH = Keyword("withPath") -} - -trait ExpressionUtils { - - def loop(input: Expression, l: (Expression, Option[Literal[Integer]], Option[String])) = l match { - case (c, None, None) => input.loop(c) - case (c, t, None) => input.loop(c, t.get) - case (c, None, Some(a)) => input.loop(c).as(a) - case (c, t, Some(a)) => input.loop(c, t.get).as(a) - } - - def select(input: Expression, s: List[(Expression, Option[String])]) = { - val selList = s.map { t => - t._2 match { - case None => t._1 - case _ => t._1.as(t._2.get) - } - } - input.select(selList: _*) - } - - def leftmostId(e: Expression) = { - var le: IdExpression = null - e.traverseUp { case i: IdExpression if le == null => le = i} - le - } - - def notIdExpression = new PartialFunction[Expression, Expression] { - def isDefinedAt(x: Expression): Boolean = !x.isInstanceOf[IdExpression] - - def apply(e: Expression) = e - } - - def replaceIdWithField(id: IdExpression, fe: UnresolvedFieldExpression): PartialFunction[Expression, Expression] = { - case e: IdExpression if e == id => fe - } - - def merge(snglQuery1: Expression, sngQuery2: Expression): Expression = { - val leftSrcId = leftmostId(sngQuery2) - sngQuery2.transformUp(replaceIdWithField(leftSrcId, snglQuery1.field(leftSrcId.name))) - } -} - -class QueryParser extends StandardTokenParsers with QueryKeywords with ExpressionUtils with PackratParsers { - - import scala.language.higherKinds - - private val reservedWordsDelims: Seq[String] = this. - getClass.getMethods.filter(_.getReturnType == classOf[Keyword]).map(_.invoke(this).asInstanceOf[Keyword].str) - - private val (queryreservedWords: Seq[String], querydelims: Seq[String]) = - reservedWordsDelims.partition(s => s.charAt(0).isLetter) - - override val lexical = new QueryLexer(queryreservedWords, querydelims) - - def apply(input: String): Either[NoSuccess, Expression] = { - phrase(queryWithPath)(new lexical.Scanner(input)) match { - case Success(r, x) => Right(r) - case f@Failure(m, x) => Left(f) - case e@Error(m, x) => Left(e) - } - } - - def queryWithPath = query ~ opt(WITHPATH) ^^ { - case q ~ None => q - case q ~ p => q.path() - } - - def query: Parser[Expression] = rep1sep(singleQuery, opt(COMMA)) ^^ { l => l match { - case h :: Nil => h - case h :: t => t.foldLeft(h)(merge(_, _)) - } - } - - def singleQuery = singleQrySrc ~ opt(loopExpression) ~ opt(selectClause) ^^ { - case s ~ None ~ None => s - case s ~ l ~ None => loop(s, l.get) - case s ~ l ~ sel if l.isDefined => select(loop(s, l.get), sel.get) - case s ~ None ~ sel => select(s, sel.get) - } - - /** - * A SingleQuerySrc can have the following forms: - * 1. FROM id [WHERE] [expr] -> from optionally followed by a filter - * 2. WHERE expr -> where clause, FROM is assumed to be the leftmost Id in the where clause - * 3. expr (that is not an IdExpression) -> where clause, FROM is assumed to be the leftmost Id in the expr - * 4. Id [WHERE] [expr] -> from optionally followed by a filter - * - * @return - */ - def singleQrySrc: Parser[Expression] = FROM ~ fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) ^^ { - case f ~ i ~ w ~ None => i - case f ~ i ~ w ~ c => i.where(c.get) - } | - WHERE ~ (expr ^? notIdExpression) ^^ { case w ~ e => { - val lId = leftmostId(e) - if (lId == null) { - failure("cannot infer Input from the where clause") - } - lId.where(e) - } - } | - expr ^? notIdExpression ^^ { case e => { - val lId = leftmostId(e) - if (lId == null) { - failure("cannot infer Input from the where clause") - } - lId.where(e) - } - } | - fromSrc ~ opt(WHERE) ~ opt(expr ^? notIdExpression) ^^ { - case i ~ w ~ None => i - case i ~ w ~ c => i.where(c.get) - } - - def fromSrc = identifier ~ AS ~ alias ^^ { case s ~ a ~ al => s.as(al)} | - identifier - - - def loopExpression: Parser[(Expression, Option[Literal[Integer]], Option[String])] = - LOOP ~ (LPAREN ~> query <~ RPAREN) ~ opt(intConstant <~ TIMES) ~ opt(AS ~> alias) ^^ { - case l ~ e ~ None ~ a => (e, None, a) - case l ~ e ~ Some(i) ~ a => (e, Some(int(i)), a) - } - - def selectClause: Parser[List[(Expression, Option[String])]] = SELECT ~ rep1sep(selectExpression, COMMA) ^^ { - case s ~ cs => cs - } - - def selectExpression: Parser[(Expression, Option[String])] = expr ~ opt(AS ~> alias) ^^ { - case e ~ a => (e, a) - } - - def expr: Parser[Expression] = compE ~ opt(rep(exprRight)) ^^ { - case l ~ None => l - case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.logicalOp(r._1)(r._2)} - } - - def exprRight = (AND | OR) ~ compE ^^ { case op ~ c => (op, c)} - - def compE = - arithE ~ (LT | LTE | EQ | NEQ | GT | GTE) ~ arithE ^^ { case l ~ op ~ r => l.compareOp(op)(r)} | - arithE ~ (ISA | IS) ~ ident ^^ { case l ~ i ~ t => l.isTrait(t)} | - arithE ~ HAS ~ ident ^^ { case l ~ i ~ f => l.hasField(f)} | - arithE - - def arithE = multiE ~ opt(rep(arithERight)) ^^ { - case l ~ None => l - case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.arith(r._1)(r._2)} - } - - def arithERight = (PLUS | MINUS) ~ multiE ^^ { case op ~ r => (op, r)} - - def multiE = atomE ~ opt(rep(multiERight)) ^^ { - case l ~ None => l - case l ~ Some(r) => r.foldLeft(l) { (l, r) => l.arith(r._1)(r._2)} - } - - def multiERight = (STAR | DIV) ~ atomE ^^ { case op ~ r => (op, r)} - - - def atomE = literal | identifier | LPAREN ~> expr <~ RPAREN - - def identifier = rep1sep(ident, DOT) ^^ { l => l match { - case h :: Nil => id(h) - case h :: t => { - t.foldLeft(id(h).asInstanceOf[Expression])(_.field(_)) - } - } - } - - def alias = ident | stringLit - - def literal = booleanConstant ^^ { - boolean(_) - } | - intConstant ^^ { - int(_) - } | - longConstant ^^ { - long(_) - } | - floatConstant ^^ { - float(_) - } | - doubleConstant ^^ { - double(_) - } | - stringLit ^^ { - string(_) - } - - def booleanConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.BooleanLiteral]) ^^ (_.chars) - - def intConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.IntLiteral]) ^^ (_.chars) - - def longConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.LongLiteral]) ^^ (_.chars) - - def floatConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.FloatLiteral]) ^^ (_.chars) - - def doubleConstant: Parser[String] = - elem("int", _.isInstanceOf[lexical.DoubleLiteral]) ^^ (_.chars) - -} - -class QueryLexer(val keywords: Seq[String], val delims: Seq[String]) extends StdLexical with ImplicitConversions { - - case class BooleanLiteral(chars: String) extends Token { - override def toString = chars - } - - case class IntLiteral(chars: String) extends Token { - override def toString = chars - } - - case class LongLiteral(chars: String) extends Token { - override def toString = chars - } - - case class FloatLiteral(chars: String) extends Token { - override def toString = chars - } - - case class DoubleLiteral(chars: String) extends Token { - override def toString = chars - } - - reserved ++= keywords.flatMap(w => allCaseVersions(w)) - - delimiters ++= delims - - override lazy val token: Parser[Token] = - ( - (trueP | falseP) - | longConstant ^^ LongLiteral - | intConstant ^^ IntLiteral - | floatConstant ^^ FloatLiteral - | dubConstant ^^ DoubleLiteral - | identifier ^^ processIdent - | string ^^ StringLit - | EofCh ^^^ EOF - | '\'' ~> failure("unclosed string literal") - | '"' ~> failure("unclosed string literal") - | delim - | '.' ^^^ new Keyword(".") - | failure("illegal character") - ) - - override def identChar = letter | elem('_') - - def identifier = identChar ~ (identChar | digit).* ^^ { case first ~ rest => (first :: rest).mkString} | - '`' ~> chrExcept('`', '\n', EofCh).* <~ '`' ^^ { - _ mkString "" - } - - override def whitespace: Parser[Any] = - (whitespaceChar - | '/' ~ '*' ~ comment - | '/' ~ '/' ~ chrExcept(EofCh, '\n').* - | '#' ~ chrExcept(EofCh, '\n').* - | '/' ~ '*' ~ failure("unclosed comment") - ).* - - protected override def comment: Parser[Any] = ( - commentChar.* ~ '*' ~ '/' - ) - - protected def commentChar = chrExcept(EofCh, '*') | '*' ~ not('/') - - def string = '\"' ~> chrExcept('\"', '\n', EofCh).* <~ '\"' ^^ { - _ mkString "" - } | - '\'' ~> chrExcept('\'', '\n', EofCh).* <~ '\'' ^^ { - _ mkString "" - } - - def zero: Parser[String] = '0' ^^^ "0" - - def nonzero = elem("nonzero digit", d => d.isDigit && d != '0') - - def sign = elem("sign character", d => d == '-' || d == '+') - - def exponent = elem("exponent character", d => d == 'e' || d == 'E') - - - def intConstant = opt(sign) ~> zero | intList - - def intList = opt(sign) ~ nonzero ~ rep(digit) ^^ { case s ~ x ~ y => (optString("", s) :: x :: y) mkString ""} - - def fracPart: Parser[String] = '.' ~> rep(digit) ^^ { r => - "." + (r mkString "") - } - - def expPart = exponent ~ opt(sign) ~ rep1(digit) ^^ { case e ~ s ~ d => - e.toString + optString("", s) + d.mkString("") - } - - def dubConstant = opt(sign) ~ digit.+ ~ fracPart ~ opt(expPart) ^^ { - case s ~ i ~ f ~ e => { - optString("", s) + (i mkString "") + f + optString("", e) - } - } - - def floatConstant = opt(sign) ~ digit.* ~ fracPart ~ 'f' ^^ { case s ~ i ~ fr ~ f => - optString("", s) + i + fr - } | opt(sign) ~ digit.+ ~ opt(fracPart) ~ 'f' ^^ { case s ~ i ~ fr ~ f => - optString("", s) + i + optString("", fr) - } - - def longConstant = intConstant ~ 'l' ^^ { case i ~ l => i} - - def trueP = 't' ~ 'r' ~ 'u' ~ 'e' ^^^ BooleanLiteral("true") - - def falseP = 'f' ~ 'a' ~ 'l' ~ 's' ~ 'e' ^^^ BooleanLiteral("false") - - private def optString[A](pre: String, a: Option[A]) = a match { - case Some(x) => pre + x.toString - case None => "" - } - - /** Generate all variations of upper and lower case of a given string */ - def allCaseVersions(s: String, prefix: String = ""): Stream[String] = { - if (s.isEmpty) { - Stream(prefix) - } else { - allCaseVersions(s.tail, prefix + s.head.toLower) #::: - allCaseVersions(s.tail, prefix + s.head.toUpper) - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryProcessor.scala ---------------------------------------------------------------------- diff --git a/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryProcessor.scala b/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryProcessor.scala deleted file mode 100755 index 538fe04..0000000 --- a/repository/src/main/scala/org/apache/hadoop/metadata/query/QueryProcessor.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.metadata.query - -import com.thinkaurelius.titan.core.TitanGraph -import org.apache.hadoop.metadata.query.Expressions._ -import org.slf4j.{LoggerFactory, Logger} - -object QueryProcessor { - val LOG : Logger = LoggerFactory.getLogger("org.apache.hadoop.metadata.query.QueryProcessor") - - def evaluate(e: Expression, g: TitanGraph, gP : GraphPersistenceStrategies = GraphPersistenceStrategy1): - GremlinQueryResult = { - val e1 = validate(e) - val q = new GremlinTranslator(e1, gP).translate() - LOG.debug("Query: " + e1) - LOG.debug("Expression Tree:\n" + e1.treeString) - LOG.debug("Gremlin Query: " + q.queryStr) - new GremlinEvaluator(q, gP, g).evaluate() - } - - def validate(e: Expression): Expression = { - val e1 = e.transformUp(new Resolver()) - - e1.traverseUp { - case x: Expression if !x.resolved => - throw new ExpressionException(x, s"Failed to resolved expression $x") - } - - /* - * trigger computation of dataType of expression tree - */ - e1.dataType - - /* - * ensure fieldReferences match the input expression's dataType - */ - val e2 = e1.transformUp(FieldValidator) - val e3 = e2.transformUp(new Resolver()) - - e3.dataType - - e3 - } -}
