[ https://issues.apache.org/jira/browse/SPARK-12123?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Sean Owen resolved SPARK-12123. ------------------------------- Resolution: Invalid [~michael_han] This should be a question on user@, not a JIRA. Have a look at https://cwiki.apache.org/confluence/display/SPARK/Contributing+to+Spark You need to narrow down your question more anyway, not just post your code. > Spark ava.lang.NullPointerException > ----------------------------------- > > Key: SPARK-12123 > URL: https://issues.apache.org/jira/browse/SPARK-12123 > Project: Spark > Issue Type: Question > Affects Versions: 1.5.2 > Reporter: Michael Han > > Hi, > I'm fresh to study Spark. > I download Spark 1.5.2 into my windows C disk. > Download the latest Eclipse and create a Java project with maven, > The only java class is: > package com.qad; > import org.apache.spark.api.java.*; > import java.io.BufferedWriter; > import java.io.File; > import java.io.FileWriter; > import org.apache.spark.SparkConf; > import org.apache.spark.api.java.function.Function; > public class SparkTest1 { > public static void main(String[] args) { > String logFile = "README.md"; // Should be some file on your system > SparkConf conf = new > SparkConf().setMaster("spark://192.168.79.1:7077").setAppName("Simple > Application"); > JavaSparkContext sc = new JavaSparkContext(conf); > > // example 1 > JavaRDD<String> logData = sc.textFile(logFile); > long numAs, numBs; > numAs = 0; > numBs = 0; > > JavaRDD<String> logData2 = logData.filter(new > Function<String,Boolean>() { > > private static final long serialVersionUID = 1L; > public Boolean call(String s) { return > s.contains("Spark is a fast"); } > }); > numAs = logData2.count(); > > String content = "Lines with a: " + numAs + ", lines with b: " + > numBs; > System.out.println(content); > WriteText(content,"mh6log.txt"); > > sc.close(); > > } > > private static void WriteText(String content,String fileName) > { > try { > > File logFile=new File(fileName); > BufferedWriter writer = new BufferedWriter(new > FileWriter(logFile)); > writer.write (content); > //Close writer > writer.close(); > } catch(Exception e) { > e.printStackTrace(); > } > } > } > The pom are: > <project xmlns="http://maven.apache.org/POM/4.0.0" > xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" > xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 > http://maven.apache.org/xsd/maven-4.0.0.xsd"> > <modelVersion>4.0.0</modelVersion> > <groupId>Spark-Test</groupId> > <artifactId>Spark-Test</artifactId> > <version>1.0</version> > <build> > <sourceDirectory>src</sourceDirectory> > <plugins> > <plugin> > <artifactId>maven-compiler-plugin</artifactId> > <version>3.3</version> > <configuration> > <source>1.8</source> > <target>1.8</target> > </configuration> > </plugin> > </plugins> > </build> > <dependencies> > <dependency> > <groupId>org.apache.spark</groupId> > <artifactId>spark-core_2.10</artifactId> > <version>1.5.2</version> > </dependency> > </dependencies> > </project> > I can run this java class correctly in Eclipse, but exceptions when I using > the following command to commit it: > spark-submit --master local --class com.qad.SparkTest1 Spark-Test-1.0.jar > Who knows which step I was wrong? Thank you. > The exceptions are: > Exception in thread "main" org.apache.spark.SparkException: Job aborted due > to s > tage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost > task > 0.0 in stage 0.0 (TID 0, localhost): java.lang.NullPointerException > at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012) > at org.apache.hadoop.util.Shell.runCommand(Shell.java:482) > at org.apache.hadoop.util.Shell.run(Shell.java:455) > at > org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java: > 715) > at org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:873) > at org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:853) > at org.apache.spark.util.Utils$.fetchFile(Utils.scala:381) > at > org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor > $Executor$$updateDependencies$5.apply(Executor.scala:405) > at > org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor > $Executor$$updateDependencies$5.apply(Executor.scala:397) > at > scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply( > TraversableLike.scala:772) > at > scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.sca > la:98) > at > scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.sca > la:98) > at > scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala > :226) > at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39) > at scala.collection.mutable.HashMap.foreach(HashMap.scala:98) > at > scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.s > cala:771) > at > org.apache.spark.executor.Executor.org$apache$spark$executor$Executor > $$updateDependencies(Executor.scala:397) > at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:193) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor. > java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor > .java:617) > at java.lang.Thread.run(Thread.java:745) > Driver stacktrace: > at > org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DA > GScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1283) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(D > AGScheduler.scala:1271) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(D > AGScheduler.scala:1270) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray. > scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) > at > org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala > :1270) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$ > 1.apply(DAGScheduler.scala:697) > at > org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$ > 1.apply(DAGScheduler.scala:697) > at scala.Option.foreach(Option.scala:236) > at > org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGSchedu > ler.scala:697) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(D > AGScheduler.scala:1496) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAG > Scheduler.scala:1458) > at > org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAG > Scheduler.scala:1447) > at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) > at > org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:567 > ) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1824) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1837) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1850) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1921) > at org.apache.spark.rdd.RDD.count(RDD.scala:1125) > at > org.apache.spark.api.java.JavaRDDLike$class.count(JavaRDDLike.scala:4 > 45) > at > org.apache.spark.api.java.AbstractJavaRDDLike.count(JavaRDDLike.scala > :47) > at com.qad.SparkTest1.main(SparkTest1.java:29) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl. > java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAcces > sorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSub > mit$$runMain(SparkSubmit.scala:674) > at > org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:18 > 0) > at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205) > at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:120) > at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) > Caused by: java.lang.NullPointerException > at java.lang.ProcessBuilder.start(ProcessBuilder.java:1012) > at org.apache.hadoop.util.Shell.runCommand(Shell.java:482) > at org.apache.hadoop.util.Shell.run(Shell.java:455) > at > org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java: > 715) > at org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:873) > at org.apache.hadoop.fs.FileUtil.chmod(FileUtil.java:853) > at org.apache.spark.util.Utils$.fetchFile(Utils.scala:381) > at > org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor > $Executor$$updateDependencies$5.apply(Executor.scala:405) > at > org.apache.spark.executor.Executor$$anonfun$org$apache$spark$executor > $Executor$$updateDependencies$5.apply(Executor.scala:397) > at > scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply( > TraversableLike.scala:772) > at > scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.sca > la:98) > at > scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.sca > la:98) > at > scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala > :226) > at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:39) > at scala.collection.mutable.HashMap.foreach(HashMap.scala:98) > at > scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.s > cala:771) > at > org.apache.spark.executor.Executor.org$apache$spark$executor$Executor > $$updateDependencies(Executor.scala:397) > at > org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:193) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor. > java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor > .java:617) > at java.lang.Thread.run(Thread.java:745) -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org