Hi All , I am just trying to use scala test for testing a small spark code . But spark context is not getting initialized , while I am running test file . I have given code, pom and exception I am getting in mail , please help me to understand what mistake I am doing , so that Spark context is not getting initialized
Code:- import org.apache.log4j.LogManager import org.apache.spark.SharedSparkContext import org.scalatest.FunSuite import org.apache.spark.{SparkContext, SparkConf} /** * Created by PSwain on 4/5/2017. */ class Test extends FunSuite with SharedSparkContext { test("test initializing spark context") { val list = List(1, 2, 3, 4) val rdd = sc.parallelize(list) assert(list.length === rdd.count()) } } POM File:- <?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>tesing.loging</groupId> <artifactId>logging</artifactId> <version>1.0-SNAPSHOT</version> <repositories> <repository> <id>central</id> <name>central</name> <url>http://repo1.maven.org/maven/</url> </repository> </repositories> <dependencies> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> <version>1.6.0</version> <type>test-jar</type> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_2.10</artifactId> <version>1.6.0</version> </dependency> <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest_2.10</artifactId> <version>2.2.6</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-hive_2.10</artifactId> <version>1.5.0</version> <scope>provided</scope> </dependency> <dependency> <groupId>com.databricks</groupId> <artifactId>spark-csv_2.10</artifactId> <version>1.3.0</version> </dependency> <dependency> <groupId>com.rxcorp.bdf.logging</groupId> <artifactId>loggingframework</artifactId> <version>1.0-SNAPSHOT</version> </dependency> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.6</version> <scope>provided</scope> </dependency> <!-- Scala --> <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> <version>2.10.5</version> <scope>compile</scope> <optional>true</optional> </dependency> <dependency> <groupId>org.scalatest</groupId> <artifactId>scalatest</artifactId> <version>1.4.RC2</version> </dependency> <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>1.2.17</version> </dependency> <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-compiler</artifactId> <version>2.10.5</version> <scope>compile</scope> <optional>true</optional> </dependency> <!-- END Scala --> </dependencies> <build> <sourceDirectory>src/main/scala</sourceDirectory> <plugins> <plugin> <artifactId>maven-assembly-plugin</artifactId> <version>2.2.1</version> <configuration> <descriptorRefs> <descriptorRef>jar-with-dependencies</descriptorRef> </descriptorRefs> </configuration> <executions> <execution> <id>make-assembly</id> <phase>package</phase> <goals> <goal>single</goal> </goals> </execution> </executions> </plugin> <plugin> <groupId>net.alchim31.maven</groupId> <artifactId>scala-maven-plugin</artifactId> <version>3.2.0</version> <executions> <execution> <goals> <goal>compile</goal> <goal>testCompile</goal> </goals> </execution> </executions> <configuration> <sourceDir>src/main/scala</sourceDir> <jvmArgs> <jvmArg>-Xms64m</jvmArg> <jvmArg>-Xmx1024m</jvmArg> </jvmArgs> </configuration> </plugin> </plugins> </build> </project> Exception:- An exception or error caused a run to abort. java.lang.ExceptionInInitializerError at org.apache.spark.Logging$class.initializeLogging(Logging.scala:121) at org.apache.spark.Logging$class.initializeIfNecessary(Logging.scala:106) at org.apache.spark.Logging$class.log(Logging.scala:50) at org.apache.spark.SparkContext.log(SparkContext.scala:79) at org.apache.spark.Logging$class.logInfo(Logging.scala:58) at org.apache.spark.SparkContext.logInfo(SparkContext.scala:79) at org.apache.spark.SparkContext.<init>(SparkContext.scala:211) at org.apache.spark.SparkContext.<init>(SparkContext.scala:147) at org.apache.spark.SharedSparkContext$class.beforeAll(SharedSparkContext.scala:33) at Test.beforeAll(Test.scala:10) at org.scalatest.BeforeAndAfterAll$class.beforeAll(BeforeAndAfterAll.scala:187) at Test.beforeAll(Test.scala:10) at org.scalatest.BeforeAndAfterAll$class.run(BeforeAndAfterAll.scala:253) at Test.run(Test.scala:10) at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:55) at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2563) at org.scalatest.tools.Runner$$anonfun$doRunRunRunDaDoRunRun$3.apply(Runner.scala:2557) at scala.collection.immutable.List.foreach(List.scala:318) at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:2557) at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1044) at org.scalatest.tools.Runner$$anonfun$runOptionallyWithPassFailReporter$2.apply(Runner.scala:1043) at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:2722) at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:1043) at org.scalatest.tools.Runner$.run(Runner.scala:883) at org.scalatest.tools.Runner.run(Runner.scala) at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2(ScalaTestRunner.java:138) at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:28) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at com.intellij.rt.execution.application.AppMain.main(AppMain.java:144) Caused by: java.lang.IllegalStateException: FileNamePattern [-.log] does not contain a valid date format specifier at org.apache.log4j.rolling.TimeBasedRollingPolicy.activateOptions(TimeBasedRollingPolicy.java:172) at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307) at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172) at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:149) at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104) at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842) at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768) at org.apache.log4j.PropertyConfigurator.configureRootCategory(PropertyConfigurator.java:648) at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:514) at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:580) at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:526) at org.apache.log4j.LogManager.<clinit>(LogManager.java:127) ... 32 more Process finished with exit code 0 ********************** IMPORTANT--PLEASE READ ************************ This electronic message, including its attachments, is CONFIDENTIAL and may contain PROPRIETARY or LEGALLY PRIVILEGED or PROTECTED information and is intended for the authorized recipient of the sender. If you are not the intended recipient, you are hereby notified that any use, disclosure, copying, or distribution of this message or any of the information included in it is unauthorized and strictly prohibited. If you have received this message in error, please immediately notify the sender by reply e-mail and permanently delete this message and its attachments, along with any copies thereof, from all locations received (e.g., computer, mobile device, etc.). Thank you. ********************************************************************