[
https://issues.apache.org/jira/browse/SPARK-21557?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Sean Owen resolved SPARK-21557.
-------------------------------
Resolution: Invalid
Fix Version/s: (was: 2.1.2)
Target Version/s: (was: 2.2.0)
JIRA isn't for questions - stackoverflow maybe.
> Debug issues for SparkML(scala.Predef$.any2ArrowAssoc)
> ------------------------------------------------------
>
> Key: SPARK-21557
> URL: https://issues.apache.org/jira/browse/SPARK-21557
> Project: Spark
> Issue Type: Bug
> Components: MLlib
> Affects Versions: 2.1.1
> Reporter: prabir bhowmick
> Priority: Critical
>
> Hi Team,
> Can you please see the below error ,when I am running the below program using
> below mvn config.Kindly tell me which version I have to use.I am running this
> program from eclipse neon.
> Error at Runtime:-
> Exception in thread "main" java.lang.NoSuchMethodError:
> scala.Predef$.any2ArrowAssoc(Ljava/lang/Object;)Ljava/lang/Object;
> at
> org.apache.spark.sql.SparkSession$Builder.config(SparkSession.scala:750)
> at
> org.apache.spark.sql.SparkSession$Builder.appName(SparkSession.scala:741)
> at com.MLTest.JavaPCAExample.main(JavaPCAExample.java:20)
> Java Class:-
> package com.MLTest;
> import org.apache.spark.sql.SparkSession;
> import java.util.Arrays;
> import java.util.List;
> import org.apache.spark.ml.feature.PCA;
> import org.apache.spark.ml.feature.PCAModel;
> import org.apache.spark.ml.linalg.VectorUDT;
> import org.apache.spark.ml.linalg.Vectors;
> import org.apache.spark.sql.Dataset;
> import org.apache.spark.sql.Row;
> import org.apache.spark.sql.RowFactory;
> import org.apache.spark.sql.types.Metadata;
> import org.apache.spark.sql.types.StructField;
> import org.apache.spark.sql.types.StructType;
> public class JavaPCAExample {
> public static void main(String[] args) {
> SparkSession spark =
> SparkSession.builder().appName("JavaPCAExample3")
> .config("spark.some.config.option",
> "some-value").getOrCreate();
> List<Row> data = Arrays.asList(
> RowFactory.create(Vectors.sparse(5, new int[] {
> 1, 3 }, new double[] { 1.0, 7.0 })),
> RowFactory.create(Vectors.dense(2.0, 0.0, 3.0,
> 4.0, 5.0)),
> RowFactory.create(Vectors.dense(4.0, 0.0, 0.0,
> 6.0, 7.0)));
> StructType schema = new StructType(
> new StructField[] { new StructField("features",
> new VectorUDT(), false, Metadata.empty()), });
> Dataset<Row> df = spark.createDataFrame(data, schema);
> PCAModel pca = new
> PCA().setInputCol("features").setOutputCol("pcaFeatures").setK(3).fit(df);
> Dataset<Row> result = pca.transform(df).select("pcaFeatures");
> result.show(false);
> spark.stop();
> }
> }
> pom.xml:-
> <project xmlns="http://maven.apache.org/POM/4.0.0";
> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
> http://maven.apache.org/xsd/maven-4.0.0.xsd";>
> <modelVersion>4.0.0</modelVersion>
> <groupId>SparkMLTest</groupId>
> <artifactId>SparkMLTest</artifactId>
> <version>0.0.1-SNAPSHOT</version>
> <build>
> <sourceDirectory>src</sourceDirectory>
> <plugins>
> <plugin>
> <artifactId>maven-compiler-plugin</artifactId>
> <version>3.5.1</version>
> <configuration>
> <source>1.8</source>
> <target>1.8</target>
> </configuration>
> </plugin>
> </plugins>
> </build>
> <dependencies>
> <dependency>
> <groupId>org.apache.spark</groupId>
> <artifactId>spark-core_2.10</artifactId>
> <version>2.2.0</version>
> </dependency>
> <dependency>
> <groupId>org.apache.spark</groupId>
> <artifactId>spark-streaming_2.10</artifactId>
> <version>2.1.1</version>
> </dependency>
> <dependency>
> <groupId>org.apache.spark</groupId>
> <artifactId>spark-mllib_2.10</artifactId>
> <version>2.1.1</version>
> <scope>provided</scope>
> </dependency>
> <dependency>
> <groupId>org.apache.spark</groupId>
> <artifactId>spark-sql_2.10</artifactId>
> <version>2.1.1</version>
> </dependency>
> <dependency>
> <groupId>org.scala-lang</groupId>
> <artifactId>scala-library</artifactId>
> <version>2.13.0-M1</version>
> </dependency>
> <dependency>
> <groupId>org.apache.parquet</groupId>
> <artifactId>parquet-hadoop-bundle</artifactId>
> <version>1.8.1</version>
> </dependency>
> </dependencies>
> </project>
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
