[
https://issues.apache.org/jira/browse/SPARK-36684?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Yang Jie updated SPARK-36684:
-----------------------------
Description:
[https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-maven-hadoop-2.7/2493/consoleFull]
There are no failed test cases, but warning logs as follows when test sql/core
module with hadoop-2.7 profile:
{code:java}
17:12:17.605 WARN org.apache.hadoop.fs.FileSystem: Cannot load filesystem
java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem: Provider
org.apache.hadoop.hdfs.web.WebHdfsFileSystem could not be instantiated
at java.util.ServiceLoader.fail(ServiceLoader.java:232)
at java.util.ServiceLoader.access$100(ServiceLoader.java:185)
at
java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384)
at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)
at java.util.ServiceLoader$1.next(ServiceLoader.java:480)
at org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2631)
at
org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2650)
at
org.apache.hadoop.fs.FsUrlStreamHandlerFactory.<init>(FsUrlStreamHandlerFactory.java:62)
at
org.apache.spark.sql.internal.SharedState$.liftedTree2$1(SharedState.scala:193)
at
org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$setFsUrlStreamHandlerFactory(SharedState.scala:192)
at
org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:54)
at
org.apache.spark.sql.SparkSession.$anonfun$sharedState$1(SparkSession.scala:139)
at scala.Option.getOrElse(Option.scala:189)
at
org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:139)
at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:138)
at
org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:158)
at scala.Option.getOrElse(Option.scala:189)
at
org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:156)
at
org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:153)
at org.apache.spark.sql.SparkSession.udf(SparkSession.scala:223)
at
test.org.apache.spark.sql.JavaUDFSuite.sourceTest(JavaUDFSuite.java:148)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at
org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
at
org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548)
Caused by: java.lang.NoClassDefFoundError: org/codehaus/jackson/map/ObjectMapper
at
org.apache.hadoop.hdfs.web.WebHdfsFileSystem.<clinit>(WebHdfsFileSystem.java:129)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at java.lang.Class.newInstance(Class.java:442)
at
java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380)
... 48 more
Caused by: java.lang.ClassNotFoundException:
org.codehaus.jackson.map.ObjectMapper
at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
... 55 more
17:12:17.612 WARN org.apache.hadoop.fs.FileSystem: Cannot load filesystem
java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem: Provider
org.apache.hadoop.hdfs.web.SWebHdfsFileSystem could not be instantiated
at java.util.ServiceLoader.fail(ServiceLoader.java:232)
at java.util.ServiceLoader.access$100(ServiceLoader.java:185)
at
java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384)
at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)
at java.util.ServiceLoader$1.next(ServiceLoader.java:480)
at org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2631)
at
org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2650)
at
org.apache.hadoop.fs.FsUrlStreamHandlerFactory.<init>(FsUrlStreamHandlerFactory.java:62)
at
org.apache.spark.sql.internal.SharedState$.liftedTree2$1(SharedState.scala:193)
at
org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$setFsUrlStreamHandlerFactory(SharedState.scala:192)
at
org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:54)
at
org.apache.spark.sql.SparkSession.$anonfun$sharedState$1(SparkSession.scala:139)
at scala.Option.getOrElse(Option.scala:189)
at
org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:139)
at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:138)
at
org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:158)
at scala.Option.getOrElse(Option.scala:189)
at
org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:156)
at
org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:153)
at org.apache.spark.sql.SparkSession.udf(SparkSession.scala:223)
at
test.org.apache.spark.sql.JavaUDFSuite.sourceTest(JavaUDFSuite.java:148)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at
org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at
org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
at
org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562)
at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548)
Caused by: java.lang.NoClassDefFoundError:
org.apache.hadoop.hdfs.web.WebHdfsFileSystem
at java.lang.Class.getDeclaredConstructors0(Native Method)
at java.lang.Class.privateGetDeclaredConstructors(Class.java:2671)
at java.lang.Class.getConstructor0(Class.java:3075)
at java.lang.Class.newInstance(Class.java:412)
at
java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380)
... 48 more
{code}
was:
[https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-maven-hadoop-2.7/2493/consoleFull]
There are no failed test cases, but warning logs as follows when test sql/core
module with hadoop-2.7 profile:
{code:java}
00:45:36.768 WARN org.apache.hadoop.fs.FileSystem: Cannot load
filesystem00:45:36.768 WARN org.apache.hadoop.fs.FileSystem: Cannot load
filesystemjava.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem:
Provider org.apache.hadoop.hdfs.web.WebHdfsFileSystem could not be instantiated
at java.util.ServiceLoader.fail(ServiceLoader.java:232) at
java.util.ServiceLoader.access$100(ServiceLoader.java:185) at
java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384) at
java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404) at
java.util.ServiceLoader$1.next(ServiceLoader.java:480) at
org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2631) at
org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2650) at
org.apache.hadoop.fs.FsUrlStreamHandlerFactory.<init>(FsUrlStreamHandlerFactory.java:62)
at
org.apache.spark.sql.internal.SharedState$.liftedTree2$1(SharedState.scala:193)
at
org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$setFsUrlStreamHandlerFactory(SharedState.scala:192)
at org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:54) at
org.apache.spark.sql.SparkSession.$anonfun$sharedState$1(SparkSession.scala:139)
at scala.Option.getOrElse(Option.scala:189) at
org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:139)
at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:138) at
org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:158)
at scala.Option.getOrElse(Option.scala:189) at
org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:156)
at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:153) at
org.apache.spark.sql.SparkSession.$anonfun$new$3(SparkSession.scala:113) at
scala.Option.map(Option.scala:230) at
org.apache.spark.sql.SparkSession.$anonfun$new$1(SparkSession.scala:113) at
org.apache.spark.sql.internal.SQLConf$.get(SQLConf.scala:230) at
org.apache.spark.sql.catalyst.util.CharVarcharUtils$.failIfHasCharVarchar(CharVarcharUtils.scala:63)
at org.apache.spark.sql.SparkSession.createDataFrame(SparkSession.scala:373)
at
test.org.apache.spark.sql.JavaApplySchemaSuite.applySchema(JavaApplySchemaSuite.java:105)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at
org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at
org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at
org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at
org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at
org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at
org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at
org.junit.runners.ParentRunner.run(ParentRunner.java:413) at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162) at
org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562) at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548)Caused
by: java.lang.NoClassDefFoundError: org/codehaus/jackson/map/ObjectMapper at
org.apache.hadoop.hdfs.web.WebHdfsFileSystem.<clinit>(WebHdfsFileSystem.java:129)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at
java.lang.Class.newInstance(Class.java:442) at
java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380) ... 53
moreCaused by: java.lang.ClassNotFoundException:
org.codehaus.jackson.map.ObjectMapper at
java.net.URLClassLoader.findClass(URLClassLoader.java:382) at
java.lang.ClassLoader.loadClass(ClassLoader.java:418) at
sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) at
java.lang.ClassLoader.loadClass(ClassLoader.java:351) ... 60 more00:45:36.773
WARN org.apache.hadoop.fs.FileSystem: Cannot load
filesystemjava.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem:
Provider org.apache.hadoop.hdfs.web.SWebHdfsFileSystem could not be
instantiated at java.util.ServiceLoader.fail(ServiceLoader.java:232) at
java.util.ServiceLoader.access$100(ServiceLoader.java:185) at
java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384) at
java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404) at
java.util.ServiceLoader$1.next(ServiceLoader.java:480) at
org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2631) at
org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2650) at
org.apache.hadoop.fs.FsUrlStreamHandlerFactory.<init>(FsUrlStreamHandlerFactory.java:62)
at
org.apache.spark.sql.internal.SharedState$.liftedTree2$1(SharedState.scala:193)
at
org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$setFsUrlStreamHandlerFactory(SharedState.scala:192)
at org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:54) at
org.apache.spark.sql.SparkSession.$anonfun$sharedState$1(SparkSession.scala:139)
at scala.Option.getOrElse(Option.scala:189) at
org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:139)
at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:138) at
org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:158)
at scala.Option.getOrElse(Option.scala:189) at
org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:156)
at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:153) at
org.apache.spark.sql.SparkSession.$anonfun$new$3(SparkSession.scala:113) at
scala.Option.map(Option.scala:230) at
org.apache.spark.sql.SparkSession.$anonfun$new$1(SparkSession.scala:113) at
org.apache.spark.sql.internal.SQLConf$.get(SQLConf.scala:230) at
org.apache.spark.sql.catalyst.util.CharVarcharUtils$.failIfHasCharVarchar(CharVarcharUtils.scala:63)
at org.apache.spark.sql.SparkSession.createDataFrame(SparkSession.scala:373)
at
test.org.apache.spark.sql.JavaApplySchemaSuite.applySchema(JavaApplySchemaSuite.java:105)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at
org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at
org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at
org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at
org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at
org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
at
org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at
org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at
org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at
org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at
org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at
org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at
org.junit.runners.ParentRunner.run(ParentRunner.java:413) at
org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272)
at
org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237)
at
org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158)
at
org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428)
at
org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162) at
org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562) at
org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548)Caused
by: java.lang.NoClassDefFoundError:
org.apache.hadoop.hdfs.web.WebHdfsFileSystem at
java.lang.Class.getDeclaredConstructors0(Native Method) at
java.lang.Class.privateGetDeclaredConstructors(Class.java:2671) at
java.lang.Class.getConstructor0(Class.java:3075) at
java.lang.Class.newInstance(Class.java:412) at
java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380) ... 53
more
{code}
> Clean up `Cannot load filesystem` warning log when test the sql/core module
> with hadoop-2.7 module
> --------------------------------------------------------------------------------------------------
>
> Key: SPARK-36684
> URL: https://issues.apache.org/jira/browse/SPARK-36684
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 3.2.0, 3.3.0
> Reporter: Yang Jie
> Priority: Minor
>
> [https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-maven-hadoop-2.7/2493/consoleFull]
> There are no failed test cases, but warning logs as follows when test
> sql/core module with hadoop-2.7 profile:
> {code:java}
> 17:12:17.605 WARN org.apache.hadoop.fs.FileSystem: Cannot load filesystem
> java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem:
> Provider org.apache.hadoop.hdfs.web.WebHdfsFileSystem could not be
> instantiated
> at java.util.ServiceLoader.fail(ServiceLoader.java:232)
> at java.util.ServiceLoader.access$100(ServiceLoader.java:185)
> at
> java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384)
> at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)
> at java.util.ServiceLoader$1.next(ServiceLoader.java:480)
> at org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2631)
> at
> org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2650)
> at
> org.apache.hadoop.fs.FsUrlStreamHandlerFactory.<init>(FsUrlStreamHandlerFactory.java:62)
> at
> org.apache.spark.sql.internal.SharedState$.liftedTree2$1(SharedState.scala:193)
> at
> org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$setFsUrlStreamHandlerFactory(SharedState.scala:192)
> at
> org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:54)
> at
> org.apache.spark.sql.SparkSession.$anonfun$sharedState$1(SparkSession.scala:139)
> at scala.Option.getOrElse(Option.scala:189)
> at
> org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:139)
> at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:138)
> at
> org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:158)
> at scala.Option.getOrElse(Option.scala:189)
> at
> org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:156)
> at
> org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:153)
> at org.apache.spark.sql.SparkSession.udf(SparkSession.scala:223)
> at
> test.org.apache.spark.sql.JavaUDFSuite.sourceTest(JavaUDFSuite.java:148)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at
> org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
> at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
> at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158)
> at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428)
> at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
> at
> org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562)
> at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548)
> Caused by: java.lang.NoClassDefFoundError:
> org/codehaus/jackson/map/ObjectMapper
> at
> org.apache.hadoop.hdfs.web.WebHdfsFileSystem.<clinit>(WebHdfsFileSystem.java:129)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
> at java.lang.Class.newInstance(Class.java:442)
> at
> java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380)
> ... 48 more
> Caused by: java.lang.ClassNotFoundException:
> org.codehaus.jackson.map.ObjectMapper
> at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
> ... 55 more
> 17:12:17.612 WARN org.apache.hadoop.fs.FileSystem: Cannot load filesystem
> java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem:
> Provider org.apache.hadoop.hdfs.web.SWebHdfsFileSystem could not be
> instantiated
> at java.util.ServiceLoader.fail(ServiceLoader.java:232)
> at java.util.ServiceLoader.access$100(ServiceLoader.java:185)
> at
> java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384)
> at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)
> at java.util.ServiceLoader$1.next(ServiceLoader.java:480)
> at org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2631)
> at
> org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2650)
> at
> org.apache.hadoop.fs.FsUrlStreamHandlerFactory.<init>(FsUrlStreamHandlerFactory.java:62)
> at
> org.apache.spark.sql.internal.SharedState$.liftedTree2$1(SharedState.scala:193)
> at
> org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$setFsUrlStreamHandlerFactory(SharedState.scala:192)
> at
> org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:54)
> at
> org.apache.spark.sql.SparkSession.$anonfun$sharedState$1(SparkSession.scala:139)
> at scala.Option.getOrElse(Option.scala:189)
> at
> org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:139)
> at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:138)
> at
> org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:158)
> at scala.Option.getOrElse(Option.scala:189)
> at
> org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:156)
> at
> org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:153)
> at org.apache.spark.sql.SparkSession.udf(SparkSession.scala:223)
> at
> test.org.apache.spark.sql.JavaUDFSuite.sourceTest(JavaUDFSuite.java:148)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
> at
> org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
> at
> org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
> at
> org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
> at
> org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
> at
> org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at
> org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
> at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
> at
> org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
> at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
> at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
> at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
> at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
> at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
> at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
> at org.junit.runners.ParentRunner.run(ParentRunner.java:413)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237)
> at
> org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158)
> at
> org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428)
> at
> org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162)
> at
> org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562)
> at
> org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548)
> Caused by: java.lang.NoClassDefFoundError:
> org.apache.hadoop.hdfs.web.WebHdfsFileSystem
> at java.lang.Class.getDeclaredConstructors0(Native Method)
> at java.lang.Class.privateGetDeclaredConstructors(Class.java:2671)
> at java.lang.Class.getConstructor0(Class.java:3075)
> at java.lang.Class.newInstance(Class.java:412)
> at
> java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380)
> ... 48 more
> {code}
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
