[ https://issues.apache.org/jira/browse/SPARK-36684?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17411307#comment-17411307 ]
Apache Spark commented on SPARK-36684: -------------------------------------- User 'LuciferYang' has created a pull request for this issue: https://github.com/apache/spark/pull/33926 > Clean up `Cannot load filesystem` warning log when test the sql/core module > with hadoop-2.7 profile > --------------------------------------------------------------------------------------------------- > > Key: SPARK-36684 > URL: https://issues.apache.org/jira/browse/SPARK-36684 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 3.2.0, 3.3.0 > Reporter: Yang Jie > Priority: Minor > > [https://amplab.cs.berkeley.edu/jenkins/job/spark-master-test-maven-hadoop-2.7/2493/consoleFull] > There are no failed test cases, but warning logs as follows when test > sql/core module with hadoop-2.7 profile: > {code:java} > 17:12:17.605 WARN org.apache.hadoop.fs.FileSystem: Cannot load filesystem > java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem: > Provider org.apache.hadoop.hdfs.web.WebHdfsFileSystem could not be > instantiated > at java.util.ServiceLoader.fail(ServiceLoader.java:232) > at java.util.ServiceLoader.access$100(ServiceLoader.java:185) > at > java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384) > at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404) > at java.util.ServiceLoader$1.next(ServiceLoader.java:480) > at org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2631) > at > org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2650) > at > org.apache.hadoop.fs.FsUrlStreamHandlerFactory.<init>(FsUrlStreamHandlerFactory.java:62) > at > org.apache.spark.sql.internal.SharedState$.liftedTree2$1(SharedState.scala:193) > at > org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$setFsUrlStreamHandlerFactory(SharedState.scala:192) > at > org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:54) > at > org.apache.spark.sql.SparkSession.$anonfun$sharedState$1(SparkSession.scala:139) > at scala.Option.getOrElse(Option.scala:189) > at > org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:139) > at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:138) > at > org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:158) > at scala.Option.getOrElse(Option.scala:189) > at > org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:156) > at > org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:153) > at org.apache.spark.sql.SparkSession.udf(SparkSession.scala:223) > at > test.org.apache.spark.sql.JavaUDFSuite.sourceTest(JavaUDFSuite.java:148) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at > org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) > at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) > at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at org.junit.runners.ParentRunner.run(ParentRunner.java:413) > at > org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364) > at > org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272) > at > org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237) > at > org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158) > at > org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428) > at > org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162) > at > org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562) > at > org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548) > Caused by: java.lang.NoClassDefFoundError: > org/codehaus/jackson/map/ObjectMapper > at > org.apache.hadoop.hdfs.web.WebHdfsFileSystem.<clinit>(WebHdfsFileSystem.java:129) > at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) > at > sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) > at > sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) > at java.lang.reflect.Constructor.newInstance(Constructor.java:423) > at java.lang.Class.newInstance(Class.java:442) > at > java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380) > ... 48 more > Caused by: java.lang.ClassNotFoundException: > org.codehaus.jackson.map.ObjectMapper > at java.net.URLClassLoader.findClass(URLClassLoader.java:382) > at java.lang.ClassLoader.loadClass(ClassLoader.java:418) > at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) > at java.lang.ClassLoader.loadClass(ClassLoader.java:351) > ... 55 more > 17:12:17.612 WARN org.apache.hadoop.fs.FileSystem: Cannot load filesystem > java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem: > Provider org.apache.hadoop.hdfs.web.SWebHdfsFileSystem could not be > instantiated > at java.util.ServiceLoader.fail(ServiceLoader.java:232) > at java.util.ServiceLoader.access$100(ServiceLoader.java:185) > at > java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384) > at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404) > at java.util.ServiceLoader$1.next(ServiceLoader.java:480) > at org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2631) > at > org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2650) > at > org.apache.hadoop.fs.FsUrlStreamHandlerFactory.<init>(FsUrlStreamHandlerFactory.java:62) > at > org.apache.spark.sql.internal.SharedState$.liftedTree2$1(SharedState.scala:193) > at > org.apache.spark.sql.internal.SharedState$.org$apache$spark$sql$internal$SharedState$$setFsUrlStreamHandlerFactory(SharedState.scala:192) > at > org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:54) > at > org.apache.spark.sql.SparkSession.$anonfun$sharedState$1(SparkSession.scala:139) > at scala.Option.getOrElse(Option.scala:189) > at > org.apache.spark.sql.SparkSession.sharedState$lzycompute(SparkSession.scala:139) > at org.apache.spark.sql.SparkSession.sharedState(SparkSession.scala:138) > at > org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:158) > at scala.Option.getOrElse(Option.scala:189) > at > org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:156) > at > org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:153) > at org.apache.spark.sql.SparkSession.udf(SparkSession.scala:223) > at > test.org.apache.spark.sql.JavaUDFSuite.sourceTest(JavaUDFSuite.java:148) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at > org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) > at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) > at > org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) > at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) > at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) > at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) > at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) > at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at org.junit.runners.ParentRunner.run(ParentRunner.java:413) > at > org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:364) > at > org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:272) > at > org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:237) > at > org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:158) > at > org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:428) > at > org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162) > at > org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:562) > at > org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:548) > Caused by: java.lang.NoClassDefFoundError: > org.apache.hadoop.hdfs.web.WebHdfsFileSystem > at java.lang.Class.getDeclaredConstructors0(Native Method) > at java.lang.Class.privateGetDeclaredConstructors(Class.java:2671) > at java.lang.Class.getConstructor0(Class.java:3075) > at java.lang.Class.newInstance(Class.java:412) > at > java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380) > ... 48 more > {code} > -- This message was sent by Atlassian Jira (v8.3.4#803005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org