This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 153b2ce970423d9fa0f02f83951d9b854b4732e6 Author: Murtadha Hubail <mhub...@apache.org> AuthorDate: Sun Apr 7 21:11:52 2019 +0300 [NO ISSUE][STO] Ensure Invalid Merged Components Are Deleted - user model changes: no - storage format changes: no - interface changes: no Details: - When cleaning up an index's invalid components, check the components end sequence in addition to the start sequence to ensure invalid merged components with are cleaned too. - Add test case. Change-Id: I80d72b1b614718e2e283bc72a874c140d178d1e0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/3331 Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Till Westmann <ti...@apache.org> --- .../PersistentLocalResourceRepositoryTest.java | 30 ++++++++++++---------- .../PersistentLocalResourceRepository.java | 8 +++--- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java index 54ae683..00d2d3d 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/PersistentLocalResourceRepositoryTest.java @@ -19,6 +19,7 @@ package org.apache.asterix.test.storage; import java.io.File; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -127,19 +128,9 @@ public class PersistentLocalResourceRepositoryTest { String invalidComponentRange = invalidComponentId + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId; FileReference indexDirRef = ncAppCtx.getIoManager().resolve(indexPath); String indexDir = indexDirRef.getFile().getAbsolutePath(); - // create the invalid component files - Path btreePath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER - + AbstractLSMIndexFileManager.BTREE_SUFFIX); - Path filterPath = Paths.get(indexDir, invalidComponentRange + AbstractLSMIndexFileManager.DELIMITER - + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX); - Files.createFile(btreePath); - Files.createFile(filterPath); - - // clean up the index partition - localResourceRepository.cleanup(lr.getPartition()); - // ensure that the invalid component was deleted - Assert.assertFalse(btreePath.toFile().exists()); - Assert.assertFalse(filterPath.toFile().exists()); + ensureInvalidComponentDeleted(indexDir, invalidComponentRange, localResourceRepository, lr); + String invalidMergeComponentRange = "0" + AbstractLSMIndexFileManager.DELIMITER + invalidComponentId; + ensureInvalidComponentDeleted(indexDir, invalidMergeComponentRange, localResourceRepository, lr); // ensure that valid components still exist // find index valid component timestamp from checkpoint @@ -179,4 +170,17 @@ public class PersistentLocalResourceRepositoryTest { Assert.assertFalse(indexMetadataFile.exists()); Assert.assertFalse(indexMetadataMaskFile.exists()); } + + private void ensureInvalidComponentDeleted(String indexDir, String componentSeq, + PersistentLocalResourceRepository localResourceRepository, DatasetLocalResource lr) throws IOException { + Path btreePath = Paths.get(indexDir, + componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BTREE_SUFFIX); + Path filterPath = Paths.get(indexDir, + componentSeq + AbstractLSMIndexFileManager.DELIMITER + AbstractLSMIndexFileManager.BLOOM_FILTER_SUFFIX); + Files.createFile(btreePath); + Files.createFile(filterPath); + localResourceRepository.cleanup(lr.getPartition()); + Assert.assertFalse(btreePath.toFile().exists()); + Assert.assertFalse(filterPath.toFile().exists()); + } } diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java index 8f870c0..aef7bbd 100644 --- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java +++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java @@ -497,10 +497,12 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito } final long validComponentSequence = getIndexCheckpointManager(index).getValidComponentSequence(); for (File componentFile : indexComponentFiles) { - // delete any file with start sequence > valid component sequence + // delete any file with start or end sequence > valid component sequence final long fileStart = IndexComponentFileReference.of(componentFile.getName()).getSequenceStart(); - if (fileStart > validComponentSequence) { - LOGGER.info(() -> "Deleting invalid component file: " + componentFile.getAbsolutePath()); + final long fileEnd = IndexComponentFileReference.of(componentFile.getName()).getSequenceEnd(); + if (fileStart > validComponentSequence || fileEnd > validComponentSequence) { + LOGGER.warn(() -> "Deleting invalid component file " + componentFile.getAbsolutePath() + + " based on valid sequence " + validComponentSequence); Files.delete(componentFile.toPath()); } }