This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch cp_upgrade_commit in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b426016678e41467268752a70c1f903bb46ed4a2 Author: Haonan <[email protected]> AuthorDate: Sat Jun 5 01:04:01 2021 +0800 [To rel/0.12] Fix upgrade NPE and DeadLock (#3329) * Fix upgrade NPE and DeadLock * Fix get ungrade file num deadlock --- .../iotdb/db/engine/upgrade/UpgradeTask.java | 15 +++++--------- .../org/apache/iotdb/db/service/UpgradeSevice.java | 24 +++++----------------- .../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 3 ++- 3 files changed, 12 insertions(+), 30 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java index 3d4aa76..76b1bac 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/upgrade/UpgradeTask.java @@ -61,19 +61,14 @@ public class UpgradeTask extends WrappedRunnable { logger.info("find upgraded file for {}", upgradeResource.getTsFile()); upgradedResources = findUpgradedFiles(); } - upgradeResource.writeLock(); - try { - upgradeResource.setUpgradedResources(upgradedResources); - upgradeResource.getUpgradeTsFileResourceCallBack().call(upgradeResource); - } finally { - upgradeResource.writeUnlock(); - } - UpgradeSevice.setCntUpgradeFileNum(UpgradeSevice.getCntUpgradeFileNum() - 1); + upgradeResource.setUpgradedResources(upgradedResources); + upgradeResource.getUpgradeTsFileResourceCallBack().call(upgradeResource); + UpgradeSevice.getTotalUpgradeFileNum().getAndAdd(-1); logger.info( "Upgrade completes, file path:{} , the remaining upgraded file num: {}", oldTsfilePath, - UpgradeSevice.getCntUpgradeFileNum()); - if (UpgradeSevice.getCntUpgradeFileNum() == 0) { + UpgradeSevice.getTotalUpgradeFileNum().get()); + if (UpgradeSevice.getTotalUpgradeFileNum().get() == 0) { logger.info("Start delete empty tmp folders"); clearTmpFolders(DirectoryManager.getInstance().getAllSequenceFileFolders()); clearTmpFolders(DirectoryManager.getInstance().getAllUnSequenceFileFolders()); diff --git a/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java b/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java index 4468208..faa3c39 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java +++ b/server/src/main/java/org/apache/iotdb/db/service/UpgradeSevice.java @@ -38,7 +38,7 @@ public class UpgradeSevice implements IService { private ExecutorService upgradeThreadPool; private AtomicInteger threadCnt = new AtomicInteger(); - private static int cntUpgradeFileNum; + private static AtomicInteger cntUpgradeFileNum = new AtomicInteger(); private UpgradeSevice() {} @@ -63,7 +63,7 @@ public class UpgradeSevice implements IService { updateThreadNum, r -> new Thread(r, "UpgradeThread-" + threadCnt.getAndIncrement())); UpgradeLog.createUpgradeLog(); countUpgradeFiles(); - if (cntUpgradeFileNum == 0) { + if (cntUpgradeFileNum.get() == 0) { stop(); return; } @@ -87,22 +87,8 @@ public class UpgradeSevice implements IService { return ServiceType.UPGRADE_SERVICE; } - public static void setCntUpgradeFileNum(int cntUpgradeFileNum) { - UpgradeUtils.getCntUpgradeFileLock().writeLock().lock(); - try { - UpgradeSevice.cntUpgradeFileNum = cntUpgradeFileNum; - } finally { - UpgradeUtils.getCntUpgradeFileLock().writeLock().unlock(); - } - } - - public static int getCntUpgradeFileNum() { - UpgradeUtils.getCntUpgradeFileLock().readLock().lock(); - try { - return cntUpgradeFileNum; - } finally { - UpgradeUtils.getCntUpgradeFileLock().readLock().unlock(); - } + public static AtomicInteger getTotalUpgradeFileNum() { + return cntUpgradeFileNum; } public void submitUpgradeTask(UpgradeTask upgradeTask) { @@ -110,7 +96,7 @@ public class UpgradeSevice implements IService { } private static void countUpgradeFiles() { - cntUpgradeFileNum = StorageEngine.getInstance().countUpgradeFiles(); + cntUpgradeFileNum.addAndGet(StorageEngine.getInstance().countUpgradeFiles()); logger.info("finish counting upgrading files, total num:{}", cntUpgradeFileNum); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java index 078df52..7204a81 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java @@ -299,7 +299,8 @@ public class ChunkWriterImpl implements IChunkWriter { if (numOfPages == 0) { // record the firstPageStatistics this.firstPageStatistics = pageWriter.getStatistics(); this.sizeWithoutStatistic = pageWriter.writePageHeaderAndDataIntoBuff(pageBuffer, true); - } else if (numOfPages == 1) { // put the firstPageStatistics into pageBuffer + } else if (numOfPages == 1 + && firstPageStatistics != null) { // put the firstPageStatistics into pageBuffer byte[] b = pageBuffer.toByteArray(); pageBuffer.reset(); pageBuffer.write(b, 0, this.sizeWithoutStatistic);
