[ https://issues.apache.org/jira/browse/SPARK-6213?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Littlestar updated SPARK-6213: ------------------------------ Description: sql.sources.CatalystScan# public RDD<Row> buildScan(Seq<Attribute> requiredColumns, Seq<Expression> filters) I use java to extends BaseRelation, but sql.catalyst.expressions.Expression is not friendly to java, it's can't iterated by java, such as NodeName, NodeType, FuncName, FuncArgs..... DataSourceStrategy.scala#selectFilters {noformat} /** * Selects Catalyst predicate [[Expression]]s which are convertible into data source [[Filter]]s, * and convert them. */ protected[sql] def selectFilters(filters: Seq[Expression]) = { def translate(predicate: Expression): Option[Filter] = predicate match { case expressions.EqualTo(a: Attribute, Literal(v, _)) => Some(sources.EqualTo(a.name, v)) case expressions.EqualTo(Literal(v, _), a: Attribute) => Some(sources.EqualTo(a.name, v)) case expressions.GreaterThan(a: Attribute, Literal(v, _)) => Some(sources.GreaterThan(a.name, v)) case expressions.GreaterThan(Literal(v, _), a: Attribute) => Some(sources.LessThan(a.name, v)) case expressions.LessThan(a: Attribute, Literal(v, _)) => Some(sources.LessThan(a.name, v)) case expressions.LessThan(Literal(v, _), a: Attribute) => Some(sources.GreaterThan(a.name, v)) case expressions.GreaterThanOrEqual(a: Attribute, Literal(v, _)) => Some(sources.GreaterThanOrEqual(a.name, v)) case expressions.GreaterThanOrEqual(Literal(v, _), a: Attribute) => Some(sources.LessThanOrEqual(a.name, v)) case expressions.LessThanOrEqual(a: Attribute, Literal(v, _)) => Some(sources.LessThanOrEqual(a.name, v)) case expressions.LessThanOrEqual(Literal(v, _), a: Attribute) => Some(sources.GreaterThanOrEqual(a.name, v)) case expressions.InSet(a: Attribute, set) => Some(sources.In(a.name, set.toArray)) case expressions.IsNull(a: Attribute) => Some(sources.IsNull(a.name)) case expressions.IsNotNull(a: Attribute) => Some(sources.IsNotNull(a.name)) case expressions.And(left, right) => (translate(left) ++ translate(right)).reduceOption(sources.And) case expressions.Or(left, right) => for { leftFilter <- translate(left) rightFilter <- translate(right) } yield sources.Or(leftFilter, rightFilter) case expressions.Not(child) => translate(child).map(sources.Not) case _ => None } filters.flatMap(translate) } {noformat} was: sql.sources.CatalystScan# public RDD<Row> buildScan(Seq<Attribute> requiredColumns, Seq<Expression> filters) I use java to extends BaseRelation, but sql.catalyst.expressions.Expression is not friendly to java, it's can't iterated by java, such as NodeName, NodeType, FunctionName, FuncArgs..... DataSourceStrategy.scala#selectFilters {noformat} /** * Selects Catalyst predicate [[Expression]]s which are convertible into data source [[Filter]]s, * and convert them. */ protected[sql] def selectFilters(filters: Seq[Expression]) = { def translate(predicate: Expression): Option[Filter] = predicate match { case expressions.EqualTo(a: Attribute, Literal(v, _)) => Some(sources.EqualTo(a.name, v)) case expressions.EqualTo(Literal(v, _), a: Attribute) => Some(sources.EqualTo(a.name, v)) case expressions.GreaterThan(a: Attribute, Literal(v, _)) => Some(sources.GreaterThan(a.name, v)) case expressions.GreaterThan(Literal(v, _), a: Attribute) => Some(sources.LessThan(a.name, v)) case expressions.LessThan(a: Attribute, Literal(v, _)) => Some(sources.LessThan(a.name, v)) case expressions.LessThan(Literal(v, _), a: Attribute) => Some(sources.GreaterThan(a.name, v)) case expressions.GreaterThanOrEqual(a: Attribute, Literal(v, _)) => Some(sources.GreaterThanOrEqual(a.name, v)) case expressions.GreaterThanOrEqual(Literal(v, _), a: Attribute) => Some(sources.LessThanOrEqual(a.name, v)) case expressions.LessThanOrEqual(a: Attribute, Literal(v, _)) => Some(sources.LessThanOrEqual(a.name, v)) case expressions.LessThanOrEqual(Literal(v, _), a: Attribute) => Some(sources.GreaterThanOrEqual(a.name, v)) case expressions.InSet(a: Attribute, set) => Some(sources.In(a.name, set.toArray)) case expressions.IsNull(a: Attribute) => Some(sources.IsNull(a.name)) case expressions.IsNotNull(a: Attribute) => Some(sources.IsNotNull(a.name)) case expressions.And(left, right) => (translate(left) ++ translate(right)).reduceOption(sources.And) case expressions.Or(left, right) => for { leftFilter <- translate(left) rightFilter <- translate(right) } yield sources.Or(leftFilter, rightFilter) case expressions.Not(child) => translate(child).map(sources.Not) case _ => None } filters.flatMap(translate) } {noformat} > sql.catalyst.expressions.Expression is not friendly to java > ----------------------------------------------------------- > > Key: SPARK-6213 > URL: https://issues.apache.org/jira/browse/SPARK-6213 > Project: Spark > Issue Type: Improvement > Components: SQL > Affects Versions: 1.3.0 > Reporter: Littlestar > Priority: Minor > > sql.sources.CatalystScan# > public RDD<Row> buildScan(Seq<Attribute> requiredColumns, Seq<Expression> > filters) > I use java to extends BaseRelation, but sql.catalyst.expressions.Expression > is not friendly to java, it's can't iterated by java, such as NodeName, > NodeType, FuncName, FuncArgs..... > DataSourceStrategy.scala#selectFilters > {noformat} > /** > * Selects Catalyst predicate [[Expression]]s which are convertible into > data source [[Filter]]s, > * and convert them. > */ > protected[sql] def selectFilters(filters: Seq[Expression]) = { > def translate(predicate: Expression): Option[Filter] = predicate match { > case expressions.EqualTo(a: Attribute, Literal(v, _)) => > Some(sources.EqualTo(a.name, v)) > case expressions.EqualTo(Literal(v, _), a: Attribute) => > Some(sources.EqualTo(a.name, v)) > case expressions.GreaterThan(a: Attribute, Literal(v, _)) => > Some(sources.GreaterThan(a.name, v)) > case expressions.GreaterThan(Literal(v, _), a: Attribute) => > Some(sources.LessThan(a.name, v)) > case expressions.LessThan(a: Attribute, Literal(v, _)) => > Some(sources.LessThan(a.name, v)) > case expressions.LessThan(Literal(v, _), a: Attribute) => > Some(sources.GreaterThan(a.name, v)) > case expressions.GreaterThanOrEqual(a: Attribute, Literal(v, _)) => > Some(sources.GreaterThanOrEqual(a.name, v)) > case expressions.GreaterThanOrEqual(Literal(v, _), a: Attribute) => > Some(sources.LessThanOrEqual(a.name, v)) > case expressions.LessThanOrEqual(a: Attribute, Literal(v, _)) => > Some(sources.LessThanOrEqual(a.name, v)) > case expressions.LessThanOrEqual(Literal(v, _), a: Attribute) => > Some(sources.GreaterThanOrEqual(a.name, v)) > case expressions.InSet(a: Attribute, set) => > Some(sources.In(a.name, set.toArray)) > case expressions.IsNull(a: Attribute) => > Some(sources.IsNull(a.name)) > case expressions.IsNotNull(a: Attribute) => > Some(sources.IsNotNull(a.name)) > case expressions.And(left, right) => > (translate(left) ++ translate(right)).reduceOption(sources.And) > case expressions.Or(left, right) => > for { > leftFilter <- translate(left) > rightFilter <- translate(right) > } yield sources.Or(leftFilter, rightFilter) > case expressions.Not(child) => > translate(child).map(sources.Not) > case _ => None > } > filters.flatMap(translate) > } > {noformat} -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org