[ https://issues.apache.org/jira/browse/HUDI-3744?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ethan Guo reassigned HUDI-3744: ------------------------------- Assignee: Raymond Xu (was: Ethan Guo) > NoSuchMethodError of getReadStatistics with Spark 3.2/Hadoop 3.2 using HBase > ----------------------------------------------------------------------------- > > Key: HUDI-3744 > URL: https://issues.apache.org/jira/browse/HUDI-3744 > Project: Apache Hudi > Issue Type: Bug > Reporter: Ethan Guo > Assignee: Raymond Xu > Priority: Blocker > Fix For: 0.11.0 > > > Environment: Hadoop 3.2.1 & Spark-3.2.1 > hudi compile from commit f2a93ead3b5a6964a72b3543ada58aa334edef9c > just use spark-sql and default job configuration to execute "show partitions > [hudi_table_name];" > {code:java} > // command > spark-sql --conf spark.serializer=org.apache.spark.serializer.KryoSerializer > --conf > spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension > --conf > spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog > // spark-sql > spark-sql> show partitions hudi_partition_table; > {code} > // code placeholderjava.lang.NoSuchMethodError: > org.apache.hadoop.hdfs.client.HdfsDataInputStream.getReadStatistics()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics; > at > org.apache.hudi.org.apache.hadoop.hbase.io.FSDataInputStreamWrapper.updateInputStreamStatistics(FSDataInputStreamWrapper.java:249) > at > org.apache.hudi.org.apache.hadoop.hbase.io.FSDataInputStreamWrapper.close(FSDataInputStreamWrapper.java:296) > at > org.apache.hudi.org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.closeStreams(HFileBlock.java:1825) > at > org.apache.hudi.org.apache.hadoop.hbase.io.hfile.HFilePreadReader.close(HFilePreadReader.java:107) > at > org.apache.hudi.org.apache.hadoop.hbase.io.hfile.HFileReaderImpl.close(HFileReaderImpl.java:1421) > at > org.apache.hudi.io.storage.HoodieHFileReader.close(HoodieHFileReader.java:423) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.close(HoodieBackedTableMetadata.java:435) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeys$0(HoodieBackedTableMetadata.java:162) > at java.util.HashMap.forEach(HashMap.java:1290) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:138) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:128) > at > org.apache.hudi.metadata.BaseTableMetadata.fetchAllPartitionPaths(BaseTableMetadata.java:281) > at > org.apache.hudi.metadata.BaseTableMetadata.getAllPartitionPaths(BaseTableMetadata.java:111) > at > org.apache.hudi.common.fs.FSUtils.getAllPartitionPaths(FSUtils.java:308) > at > org.apache.spark.sql.hudi.HoodieSqlCommonUtils$.getAllPartitionPaths(HoodieSqlCommonUtils.scala:81) > at > org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable.getPartitionPaths(HoodieCatalogTable.scala:157) > at > org.apache.spark.sql.hudi.command.ShowHoodieTablePartitionsCommand.run(ShowHoodieTablePartitionsCommand.scala:51) > at > org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:75) > at > org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:73) > at > org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:84) > at > org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.$anonfun$applyOrElse$1(QueryExecution.scala:110) > at > org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(SQLExecution.scala:103) > at > org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:163) > at > org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:90) > at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:775) > at > org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:64) > at > org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:110) > at > org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:106) -- This message was sent by Atlassian Jira (v8.20.1#820001)