[
https://issues.apache.org/jira/browse/CARBONDATA-50?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15381054#comment-15381054
]
ASF GitHub Bot commented on CARBONDATA-50:
------------------------------------------
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/incubator-carbondata/pull/35#discussion_r71075502
--- Diff:
integration/spark/src/main/scala/org/apache/spark/sql/CodeGenerateFactory.scala
---
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.SparkContext
+import org.apache.spark.sql.catalyst.CatalystConf
+import org.apache.spark.sql.catalyst.optimizer.Optimizer
+import org.apache.spark.sql.catalyst.plans.logical.{Expand, LogicalPlan}
+import org.apache.spark.util.{ScalaCompilerUtil, Utils}
+
+class CodeGenerateFactory(version: String) {
+
+ val optimizerFactory = if (version.equals("1.6.2")) {
+
ScalaCompilerUtil.compiledCode(CodeTemplates.spark1_6_2_OptimizerString)
+ .asInstanceOf[AbstractCarbonOptimizerFactory]
+ } else {
+ ScalaCompilerUtil.compiledCode(CodeTemplates.defaultOptimizerString)
+ .asInstanceOf[AbstractCarbonOptimizerFactory]
+ }
+
+ val expandFactory = if (version.startsWith("1.5")) {
+ ScalaCompilerUtil.compiledCode(CodeTemplates.spark1_5ExpandString)
+ .asInstanceOf[AbstractCarbonExpandFactory]
+ } else {
+ new AbstractCarbonExpandFactory {
+ override def createExpand(expand: Expand, child: LogicalPlan):
Expand = {
+ val loader = Utils.getContextOrSparkClassLoader
+ try {
+ val cons =
loader.loadClass("org.apache.spark.sql.catalyst.plans.logical.Expand")
+ .getDeclaredConstructors
+ cons.head.setAccessible(true)
+ cons.head.newInstance(expand.projections, expand.output,
child).asInstanceOf[Expand]
+ } catch {
+ case e: Exception => null
+ }
+ }
+ }
+ }
+
+ def createDefaultOptimizer(conf: CatalystConf, sc: SparkContext):
Optimizer = {
+ val name = "org.apache.spark.sql.catalyst.optimizer.DefaultOptimizer"
+ val loader = Utils.getContextOrSparkClassLoader
+ try {
+ val cons = loader.loadClass(name + "$").getDeclaredConstructors
+ cons.head.setAccessible(true)
+ cons.head.newInstance().asInstanceOf[Optimizer]
+ } catch {
+ case e: Exception =>
+ loader.loadClass(name).getConstructor(classOf[CatalystConf])
+ .newInstance(conf).asInstanceOf[Optimizer]
+ }
+ }
+
+}
+
+object CodeGenerateFactory {
+
+ private var codeGenerateFactory: CodeGenerateFactory = _
+
+ def init(version: String): Unit = {
+ if (codeGenerateFactory == null) {
+ codeGenerateFactory = new CodeGenerateFactory(version)
+ }
+ }
+
+ def getInstance(): CodeGenerateFactory = {
+ codeGenerateFactory
+ }
+
+ def getInstance(version: String): CodeGenerateFactory = {
+ init(version)
+ codeGenerateFactory
+ }
+
+}
+
+object CodeTemplates {
+
+ val spark1_6_2_OptimizerString =
+ s"""
+ import org.apache.spark.sql._;
+ import org.apache.spark.sql.optimizer._;
+ import org.apache.spark.sql.catalyst.plans.logical._;
+ import org.apache.spark.sql.catalyst._;
+ import org.apache.spark.sql.catalyst.optimizer.Optimizer;
+
+ new AbstractCarbonOptimizerFactory {
+
+ override def createOptimizer(optimizer1: Optimizer, conf1:
CarbonSQLConf): Optimizer = {
--- End diff --
suggest rename input parameter `optimizer1` to `optimizer`. the same in
defaultOptimizerString.
> Support Spark 1.6.2 in CarbonData
> ---------------------------------
>
> Key: CARBONDATA-50
> URL: https://issues.apache.org/jira/browse/CARBONDATA-50
> Project: CarbonData
> Issue Type: New Feature
> Reporter: Ravindra Pesala
> Assignee: Ravindra Pesala
>
> Current carbon cannot support latest Spark version 1.6.2. It should be
> supported with new maven profile like below
> {code}
> mvn clean -Pspark-1.6.2 package
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
