刘珍 created IOTDB-5639: ------------------------- Summary: [compaction]The NPE exception needs to be handled Key: IOTDB-5639 URL: https://issues.apache.org/jira/browse/IOTDB-5639 Project: Apache IoTDB Issue Type: Improvement Components: Core/Compaction, mpp-cluster Reporter: 刘珍 Assignee: 周沛辰 Attachments: comp1_aligned.conf, comp2_aligned.conf, run_2_comp_aligned.sh, test.sh
master 0302_33cd749 对齐序列合并过程中,有NPE 异常,可以处理一下: 2023-03-07 09:57:23,123 [pool-62-IoTDB-Compaction_Schedule-root.test.g_0-9-1] ERROR o.a.i.t.f.f.LocalFSFactory:98 - Failed to get buffered input stream for /data1/iotdb/./opt_0302_33cd749/sbin/../data/datanode/data/sequence/root.test.g_0/9/253/1678092688862-303-0-0.tsfile.resource. java.io.FileNotFoundException: /data1/iotdb/./opt_0302_33cd749/sbin/../data/datanode/data/sequence/root.test.g_0/9/253/1678092688862-303-0-0.tsfile.resource (No such file or directory) at java.io.FileInputStream.open0(Native Method) at java.io.FileInputStream.open(FileInputStream.java:195) at java.io.FileInputStream.<init>(FileInputStream.java:138) at java.io.FileInputStream.<init>(FileInputStream.java:93) at org.apache.iotdb.tsfile.fileSystem.fsFactory.LocalFSFactory.getBufferedInputStream(LocalFSFactory.java:96) at org.apache.iotdb.db.engine.storagegroup.TsFileResource.buildDeviceTimeIndex(TsFileResource.java:423) at org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate$TsFileResourceCandidate.prepareDeviceInfos(CrossSpaceCompactionCandidate.java:209) at org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate$TsFileResourceCandidate.containsDevice(CrossSpaceCompactionCandidate.java:241) at org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate.prepareNextSplit(CrossSpaceCompactionCandidate.java:86) at org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate.hasNextSplit(CrossSpaceCompactionCandidate.java:68) at org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector.executeTaskResourceSelection(RewriteCrossSpaceCompactionSelector.java:152) at org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector.selectOneTaskResources(RewriteCrossSpaceCompactionSelector.java:117) at org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector.selectCrossSpaceTask(RewriteCrossSpaceCompactionSelector.java:240) at org.apache.iotdb.db.engine.compaction.schedule.CompactionScheduler.tryToSubmitCrossSpaceCompactionTask(CompactionScheduler.java:165) at org.apache.iotdb.db.engine.compaction.schedule.CompactionScheduler.scheduleCompaction(CompactionScheduler.java:61) at org.apache.iotdb.db.engine.storagegroup.DataRegion.executeCompaction(DataRegion.java:2150) at org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil.lambda$scheduleWithFixedDelay$1(ScheduledExecutorUtil.java:177) at org.apache.iotdb.commons.concurrent.WrappedRunnable$1.runMayThrow(WrappedRunnable.java:44) at org.apache.iotdb.commons.concurrent.WrappedRunnable.run(WrappedRunnable.java:29) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2023-03-07 09:57:23,153 [pool-62-IoTDB-Compaction_Schedule-root.test.g_0-9-1] ERROR o.a.i.d.e.c.s.i.RewriteCrossSpaceCompactionSelector:269 - root.test.g_0 cannot select file for cross space compaction org.apache.iotdb.db.exception.MergeException: Can't read file /data1/iotdb/./opt_0302_33cd749/sbin/../data/datanode/data/sequence/root.test.g_0/9/253/1678092688862-303-0-0.tsfile.resource from disk at org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector.selectOneTaskResources(RewriteCrossSpaceCompactionSelector.java:121) at org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector.selectCrossSpaceTask(RewriteCrossSpaceCompactionSelector.java:240) at org.apache.iotdb.db.engine.compaction.schedule.CompactionScheduler.tryToSubmitCrossSpaceCompactionTask(CompactionScheduler.java:165) at org.apache.iotdb.db.engine.compaction.schedule.CompactionScheduler.scheduleCompaction(CompactionScheduler.java:61) at org.apache.iotdb.db.engine.storagegroup.DataRegion.executeCompaction(DataRegion.java:2150) at org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil.lambda$scheduleWithFixedDelay$1(ScheduledExecutorUtil.java:177) at org.apache.iotdb.commons.concurrent.WrappedRunnable$1.runMayThrow(WrappedRunnable.java:44) at org.apache.iotdb.commons.concurrent.WrappedRunnable.run(WrappedRunnable.java:29) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.io.IOException: Can't read file /data1/iotdb/./opt_0302_33cd749/sbin/../data/datanode/data/sequence/root.test.g_0/9/253/1678092688862-303-0-0.tsfile.resource from disk at org.apache.iotdb.db.engine.storagegroup.TsFileResource.buildDeviceTimeIndex(TsFileResource.java:432) at org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate$TsFileResourceCandidate.prepareDeviceInfos(CrossSpaceCompactionCandidate.java:209) at org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate$TsFileResourceCandidate.containsDevice(CrossSpaceCompactionCandidate.java:241) at org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate.prepareNextSplit(CrossSpaceCompactionCandidate.java:86) at org.apache.iotdb.db.engine.compaction.selector.utils.CrossSpaceCompactionCandidate.hasNextSplit(CrossSpaceCompactionCandidate.java:68) at org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector.executeTaskResourceSelection(RewriteCrossSpaceCompactionSelector.java:152) at org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector.selectOneTaskResources(RewriteCrossSpaceCompactionSelector.java:117) ... 14 common frames omitted Caused by: java.lang.NullPointerException: null at org.apache.iotdb.tsfile.utils.ReadWriteIOUtils.readByte(ReadWriteIOUtils.java:477) at org.apache.iotdb.db.engine.storagegroup.TsFileResource.buildDeviceTimeIndex(TsFileResource.java:424) ... 20 common frames omitted 测试环境 192.168.130.1 16CPU 32GB内存 ~关合并生成数据备份 备份路径在:/data1/iotdb/m_0305_1df6a76/data_xx CN :MAX_HEAP_SIZE="2G" DN:MAX_HEAP_SIZE="20G" MAX_DIRECT_MEMORY_SIZE="6G" COMMON : time_partition_interval=6048000000 max_waiting_time_when_insert_blocked=3600000 enable_seq_space_compaction=false enable_unseq_space_compaction=false enable_cross_space_compaction=false benchmark配置见附件,运行脚本:run_2_comp_aligned.sh ~开合并 ,测试合并 CN : MAX_HEAP_SIZE="2G" DN : MAX_HEAP_SIZE="20G" MAX_DIRECT_MEMORY_SIZE="6G" COMMON: time_partition_interval=6048000000 query_timeout_threshold=36000000 max_waiting_time_when_insert_blocked=3600000 enable_seq_space_compaction=true enable_unseq_space_compaction=true enable_cross_space_compaction=true compaction_write_throughput_mb_per_sec=1024 -- This message was sent by Atlassian Jira (v8.20.10#820010)