[ https://issues.apache.org/jira/browse/HUDI-3141?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
sivabalan narayanan updated HUDI-3141: -------------------------------------- Status: Resolved (was: Patch Available) > Metadata table getAllFilesInPartition() crashes with NullPointerException > ------------------------------------------------------------------------- > > Key: HUDI-3141 > URL: https://issues.apache.org/jira/browse/HUDI-3141 > Project: Apache Hudi > Issue Type: Task > Reporter: Manoj Govindassamy > Assignee: Manoj Govindassamy > Priority: Blocker > Labels: pull-request-available > Fix For: 0.11.0, 0.10.1 > > > When doing getAllFilesInPartition(), metadata table crashes with > NullPointerException. This is in the context of multiple clients requesting > "datafiles/range/latest/", thats is, getLatestDataFilesBeforeOrOn() from the > table. > > {noformat} > java.lang.NullPointerException > at > org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.processQueuedBlocksForInstant(AbstractHoodieLogRecordReader.java:422) > at > org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:238) > at > org.apache.hudi.metadata.HoodieMetadataMergedLogRecordReader.getRecordsByKeys(HoodieMetadataMergedLogRecordReader.java:126) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.readLogRecords(HoodieBackedTableMetadata.java:188) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeys$0(HoodieBackedTableMetadata.java:152) > at java.util.HashMap.forEach(HashMap.java:1290) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:139) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:128) > at > org.apache.hudi.metadata.BaseTableMetadata.fetchAllFilesInPartition(BaseTableMetadata.java:285) > at > org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:131) > at > org.apache.hudi.metadata.HoodieMetadataFileSystemView.listPartition(HoodieMetadataFileSystemView.java:65) > at > org.apache.hudi.common.table.view.AbstractTableFileSystemView.lambda$ensurePartitionLoadedCorrectly$9(AbstractTableFileSystemView.java:304) > at > java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660) > at > org.apache.hudi.common.table.view.AbstractTableFileSystemView.ensurePartitionLoadedCorrectly(AbstractTableFileSystemView.java:295) > at > org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestBaseFilesBeforeOrOn(AbstractTableFileSystemView.java:489) > at > org.apache.hudi.timeline.service.handlers.BaseFileHandler.getLatestDataFilesBeforeOrOn(BaseFileHandler.java:60) > at > org.apache.hudi.timeline.service.RequestHandler.lambda$registerDataFilesAPI$6(RequestHandler.java:268) > at > org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:497) > at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22) > at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606) > at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46) > at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17) > at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143) > at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41) > at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107) > at > io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) > at > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) > at > org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) > at > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) > at > org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61) > at > org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) > at org.eclipse.jetty.server.Server.handle(Server.java:502) > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370) > at > org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267) > at > org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) > at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) > at > org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) > at > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) > at > org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) > at java.lang.Thread.run(Thread.java:748) > 24424 [qtp1929693831-818] ERROR > org.apache.hudi.timeline.service.RequestHandler - Got runtime exception > servicing request > partition=2015%2F03%2F16&maxinstant=0000003&basepath=%2Fvar%2Ffolders%2Ft7%2Fkr69rlvx5rdd824m61zjqkjr0000gn%2FT%2Fjunit875461810838100905%2Fdataset&lastinstantts=0000003&timelinehash=feda55fdfbaedc993ac653b3410a8951cce50961df61f4b1db9062da7abc95e1 > org.apache.hudi.exception.HoodieMetadataException: Failed to retrieve files > in partition > /var/folders/t7/kr69rlvx5rdd824m61zjqkjr0000gn/T/junit875461810838100905/dataset/2015/03/16 > from metadata > at > org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:133) > at > org.apache.hudi.metadata.HoodieMetadataFileSystemView.listPartition(HoodieMetadataFileSystemView.java:65) > at > org.apache.hudi.common.table.view.AbstractTableFileSystemView.lambda$ensurePartitionLoadedCorrectly$9(AbstractTableFileSystemView.java:304) > at > java.util.concurrent.ConcurrentHashMap.computeIfAbsent(ConcurrentHashMap.java:1660) > at > org.apache.hudi.common.table.view.AbstractTableFileSystemView.ensurePartitionLoadedCorrectly(AbstractTableFileSystemView.java:295) > at > org.apache.hudi.common.table.view.AbstractTableFileSystemView.getLatestBaseFilesBeforeOrOn(AbstractTableFileSystemView.java:489) > at > org.apache.hudi.timeline.service.handlers.BaseFileHandler.getLatestDataFilesBeforeOrOn(BaseFileHandler.java:60) > at > org.apache.hudi.timeline.service.RequestHandler.lambda$registerDataFilesAPI$6(RequestHandler.java:268) > at > org.apache.hudi.timeline.service.RequestHandler$ViewHandler.handle(RequestHandler.java:497) > at io.javalin.security.SecurityUtil.noopAccessManager(SecurityUtil.kt:22) > at io.javalin.Javalin.lambda$addHandler$0(Javalin.java:606) > at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:46) > at io.javalin.core.JavalinServlet$service$2$1.invoke(JavalinServlet.kt:17) > at io.javalin.core.JavalinServlet$service$1.invoke(JavalinServlet.kt:143) > at io.javalin.core.JavalinServlet$service$2.invoke(JavalinServlet.kt:41) > at io.javalin.core.JavalinServlet.service(JavalinServlet.kt:107) > at > io.javalin.core.util.JettyServerUtil$initialize$httpHandler$1.doHandle(JettyServerUtil.kt:72) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) > at > org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480) > at > org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1668) > at > org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) > at > org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247) > at > org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) > at > org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:61) > at > org.eclipse.jetty.server.handler.StatisticsHandler.handle(StatisticsHandler.java:174) > at > org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) > at org.eclipse.jetty.server.Server.handle(Server.java:502) > at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370) > at > org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267) > at > org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) > at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) > at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) > at > org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) > at > org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) > at > org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) > at > org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) > at java.lang.Thread.run(Thread.java:748) > Caused by: org.apache.hudi.exception.HoodieException: Exception when reading > log file > at > org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:332) > at > org.apache.hudi.metadata.HoodieMetadataMergedLogRecordReader.getRecordsByKeys(HoodieMetadataMergedLogRecordReader.java:126) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.readLogRecords(HoodieBackedTableMetadata.java:188) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.lambda$getRecordsByKeys$0(HoodieBackedTableMetadata.java:152) > at java.util.HashMap.forEach(HashMap.java:1290) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordsByKeys(HoodieBackedTableMetadata.java:139) > at > org.apache.hudi.metadata.HoodieBackedTableMetadata.getRecordByKey(HoodieBackedTableMetadata.java:128) > at > org.apache.hudi.metadata.BaseTableMetadata.fetchAllFilesInPartition(BaseTableMetadata.java:285) > at > org.apache.hudi.metadata.BaseTableMetadata.getAllFilesInPartition(BaseTableMetadata.java:131) > ... 39 more > Caused by: java.lang.NullPointerException > at > org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.processQueuedBlocksForInstant(AbstractHoodieLogRecordReader.java:422) > at > org.apache.hudi.common.table.log.AbstractHoodieLogRecordReader.scan(AbstractHoodieLogRecordReader.java:238) > ... 47 more{noformat} -- This message was sent by Atlassian Jira (v8.20.1#820001)