[jira] [Created] (SPARK-21557) Debug issues for SparkML(scala.Predef$.any2ArrowAssoc)

[prabir bhowmick (JIRA)]

prabir bhowmick created SPARK-21557:
---------------------------------------

             Summary: Debug issues for SparkML(scala.Predef$.any2ArrowAssoc)
                 Key: SPARK-21557
                 URL: https://issues.apache.org/jira/browse/SPARK-21557
             Project: Spark
          Issue Type: Bug
          Components: MLlib
    Affects Versions: 2.1.1
            Reporter: prabir bhowmick
            Priority: Critical
             Fix For: 2.1.2


Hi Team,

Can you please see the below error ,when I am running the below program using 
below mvn config.Kindly tell me which version I have to use.I am running this 
program from eclipse neon.

Error at Runtime:- 

Exception in thread "main" java.lang.NoSuchMethodError: 
scala.Predef$.any2ArrowAssoc(Ljava/lang/Object;)Ljava/lang/Object;
        at 
org.apache.spark.sql.SparkSession$Builder.config(SparkSession.scala:750)
        at 
org.apache.spark.sql.SparkSession$Builder.appName(SparkSession.scala:741)
        at com.MLTest.JavaPCAExample.main(JavaPCAExample.java:20)

Java Class:-

package com.MLTest;

import org.apache.spark.sql.SparkSession;

import java.util.Arrays;
import java.util.List;
import org.apache.spark.ml.feature.PCA;
import org.apache.spark.ml.feature.PCAModel;
import org.apache.spark.ml.linalg.VectorUDT;
import org.apache.spark.ml.linalg.Vectors;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.types.Metadata;
import org.apache.spark.sql.types.StructField;
import org.apache.spark.sql.types.StructType;

public class JavaPCAExample {
        public static void main(String[] args) {
                SparkSession spark = 
SparkSession.builder().appName("JavaPCAExample3")
                                .config("spark.some.config.option", 
"some-value").getOrCreate();

                List<Row> data = Arrays.asList(
                                RowFactory.create(Vectors.sparse(5, new int[] { 
1, 3 }, new double[] { 1.0, 7.0 })),
                                RowFactory.create(Vectors.dense(2.0, 0.0, 3.0, 
4.0, 5.0)),
                                RowFactory.create(Vectors.dense(4.0, 0.0, 0.0, 
6.0, 7.0)));

                StructType schema = new StructType(
                                new StructField[] { new StructField("features", 
new VectorUDT(), false, Metadata.empty()), });

                Dataset<Row> df = spark.createDataFrame(data, schema);

                PCAModel pca = new 
PCA().setInputCol("features").setOutputCol("pcaFeatures").setK(3).fit(df);

                Dataset<Row> result = pca.transform(df).select("pcaFeatures");
                result.show(false);

                spark.stop();
        }
}

pom.xml:-

<project xmlns="http://maven.apache.org/POM/4.0.0"; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
        xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
        <modelVersion>4.0.0</modelVersion>
        <groupId>SparkMLTest</groupId>
        <artifactId>SparkMLTest</artifactId>
        <version>0.0.1-SNAPSHOT</version>
        <build>
                <sourceDirectory>src</sourceDirectory>
                <plugins>
                        <plugin>
                                <artifactId>maven-compiler-plugin</artifactId>
                                <version>3.5.1</version>
                                <configuration>
                                        <source>1.8</source>
                                        <target>1.8</target>
                                </configuration>
                        </plugin>
                </plugins>
        </build>
        <dependencies>
                <dependency>
                        <groupId>org.apache.spark</groupId>
                        <artifactId>spark-core_2.10</artifactId>
                        <version>2.2.0</version>
                </dependency>
                <dependency>
                        <groupId>org.apache.spark</groupId>
                        <artifactId>spark-streaming_2.10</artifactId>
                        <version>2.1.1</version>
                </dependency>
                <dependency>
                        <groupId>org.apache.spark</groupId>
                        <artifactId>spark-mllib_2.10</artifactId>
                        <version>2.1.1</version>
                        <scope>provided</scope>
                </dependency>
                <dependency>
                        <groupId>org.apache.spark</groupId>
                        <artifactId>spark-sql_2.10</artifactId>
                        <version>2.1.1</version>
                </dependency>
                <dependency>
                        <groupId>org.scala-lang</groupId>
                        <artifactId>scala-library</artifactId>
                        <version>2.13.0-M1</version>
                </dependency>
                <dependency>
                        <groupId>org.apache.parquet</groupId>
                        <artifactId>parquet-hadoop-bundle</artifactId>
                        <version>1.8.1</version>
                </dependency>
        </dependencies>
</project>







--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org