abdullah alamoudi has uploaded a new change for review.
https://asterix-gerrit.ics.uci.edu/1849
Change subject: [ASTERIXDB-1953][STO] Cleanup LSM File Managers
......................................................................
[ASTERIXDB-1953][STO] Cleanup LSM File Managers
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- use FileReference instead of String absolute path
- user error codes
Change-Id: I97bab76888790ca282ad9508ce8416f7c7a52fb7
---
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
M
hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
M
hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
17 files changed, 185 insertions(+), 313 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/49/1849/1
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 8a91547..78bdc5d 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -116,6 +116,10 @@
public static final int CANNOT_CREATE_EXISTING_INDEX = 80;
public static final int FILE_ALREADY_MAPPED = 81;
public static final int FILE_ALREADY_EXISTS = 82;
+ public static final int FOUND_OVERLAPPING_LSM_FILES = 83;
+ public static final int FOUND_MULTIPLE_TRANSACTIONS = 84;
+ public static final int UNRECOGNIZED_INDEX_COMPONENT_FILE = 85;
+ public static final int UNEQUAL_NUM_FILTERS_TREES = 86;
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
index b48ba14..0375e9e 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java
@@ -86,4 +86,8 @@
public String getAbsolutePath() {
return file.getAbsolutePath();
}
+
+ public FileReference getChild(String name) {
+ return new FileReference(dev, path + File.separator + name);
+ }
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 9f983a7..89e0d01 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -99,5 +99,9 @@
80 = Cannot create index because it already exists
81 = File %1$s is already mapped
82 = Failed to create the file %1$s because it already exists
+83 = Files with overlapping non-contained timestamp intervals were found in
%1$s
+84 = Found more than one transaction file in %1$s
+85 = Found an unrecognized index file %1$s
+86 = Unequal number of trees and filters found in %1$s
10000 = The given rule collection %1$s is not an instance of the List class.
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index c06edfd..86d926f 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -197,7 +197,7 @@
ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(),
opCtx, cursorFactory);
ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor,
mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(),
relMergeFileRefs.getBloomFilterFileReference(),
- callback, fileManager.getBaseDir()));
+ callback, fileManager.getBaseDir().getAbsolutePath()));
}
// This function should only be used when a transaction fail. it doesn't
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 4245687..7462c7a 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -366,7 +366,8 @@
ioScheduler.scheduleOperation(new
LSMBTreeWithBuddyMergeOperation(accessor, mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(),
relMergeFileRefs.getDeleteIndexFileReference(),
- relMergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir(), keepDeleteTuples));
+ relMergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir().getAbsolutePath(),
+ keepDeleteTuples));
}
// This method creates the appropriate opContext for the targeted version
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index b437fda..d241dc6 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -470,7 +470,7 @@
ILSMIOOperationCallback callback) {
ILSMIndexAccessor accessor = createAccessor(opCtx);
return new LSMBTreeFlushOperation(accessor, flushingComponent,
componentFileRefs.getInsertIndexFileReference(),
- componentFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir());
+ componentFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir().getAbsolutePath());
}
@Override
@@ -625,6 +625,6 @@
ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx,
returnDeletedTuples);
return new LSMBTreeMergeOperation(accessor, mergingComponents, cursor,
mergeFileRefs.getInsertIndexFileReference(),
mergeFileRefs.getBloomFilterFileReference(), callback,
- fileManager.getBaseDir());
+ fileManager.getBaseDir().getAbsolutePath());
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 432a2b8..54bc1fe 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -22,17 +22,17 @@
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -40,6 +40,8 @@
public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
+ private static final FilenameFilter btreeFilter =
+ (dir, name) -> !name.startsWith(".") &&
name.endsWith(BTREE_SUFFIX);
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
private final boolean hasBloomFilter;
@@ -53,10 +55,10 @@
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws
HyracksDataException {
String ts = getCurrentTimestamp();
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = ts + DELIMITER + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName +
DELIMITER + BTREE_SUFFIX), null,
- hasBloomFilter ? createFlushFile(baseName + DELIMITER +
BLOOM_FILTER_SUFFIX) : null);
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + BTREE_SUFFIX), null,
+ hasBloomFilter ? baseDir.getChild(baseName + DELIMITER +
BLOOM_FILTER_SUFFIX) : null);
}
@Override
@@ -65,30 +67,19 @@
String[] firstTimestampRange = firstFileName.split(DELIMITER);
String[] lastTimestampRange = lastFileName.split(DELIMITER);
- String baseName = baseDir + firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1];
+ String baseName = firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName +
DELIMITER + BTREE_SUFFIX), null,
- hasBloomFilter ? createMergeFile(baseName + DELIMITER +
BLOOM_FILTER_SUFFIX) : null);
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + BTREE_SUFFIX), null,
+ hasBloomFilter ? baseDir.getChild(baseName + DELIMITER +
BLOOM_FILTER_SUFFIX) : null);
}
-
- private static FilenameFilter btreeFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
- }
- };
@Override
public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws
HyracksDataException {
List<LSMComponentFileReferences> validFiles = new ArrayList<>();
ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
-
// create transaction filter <to hide transaction files>
FilenameFilter transactionFilter = getTransactionFileFilter(false);
-
- // Gather files
-
// List of valid BTree files.
cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter,
btreeFilter), btreeFactory, allBTreeFiles);
HashSet<String> btreeFilesSet = new HashSet<>();
@@ -102,8 +93,7 @@
null);
// Sanity check.
if (allBTreeFiles.size() != allBloomFilterFiles.size()) {
- throw new HyracksDataException(
- "Unequal number of valid BTree and bloom filter files
found. Aborting cleanup.");
+ throw
HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
}
}
@@ -137,7 +127,7 @@
validComparableBloomFilterFiles.add(lastBloomFilter);
}
- ComparableFileName currentBTree = null;
+ ComparableFileName currentBTree;
ComparableFileName currentBloomFilter = null;
for (int i = 1; i < allBTreeFiles.size(); i++) {
currentBTree = allBTreeFiles.get(i);
@@ -158,16 +148,13 @@
&& (!hasBloomFilter ||
(currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0))) {
// Invalid files are completely contained in last interval.
- File invalidBTreeFile = new File(currentBTree.fullPath);
- invalidBTreeFile.delete();
+ IoUtil.delete(new File(currentBTree.fullPath));
if (hasBloomFilter) {
- File invalidBloomFilterFile = new
File(currentBloomFilter.fullPath);
- invalidBloomFilterFile.delete();
+ IoUtil.delete(new File(currentBloomFilter.fullPath));
}
} else {
// This scenario should not be possible.
- throw new HyracksDataException(
- "Found LSM files with overlapping but not contained
timetamp intervals.");
+ throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
}
}
@@ -198,53 +185,40 @@
public LSMComponentFileReferences getNewTransactionFileReference() throws
IOException {
String ts = getCurrentTimestamp();
// Create transaction lock file
- Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts));
-
- String baseName = baseDir + ts + DELIMITER + ts;
+ IoUtil.create(baseDir.getChild(TXN_PREFIX + ts));
+ String baseName = ts + DELIMITER + ts;
// Begin timestamp and end timestamp are identical since it is a
transaction
- return new LSMComponentFileReferences(createFlushFile(baseName +
DELIMITER + BTREE_SUFFIX), null,
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + BTREE_SUFFIX), null,
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit()
throws HyracksDataException {
FilenameFilter transactionFilter;
- File dir = new File(baseDir);
- String[] files = dir.list(txnFileNameFilter);
+ String[] files = baseDir.getFile().list(txnFileNameFilter);
if (files.length == 0) {
return null;
}
if (files.length != 1) {
- throw new HyracksDataException("More than one transaction lock
found:" + files.length);
+ throw
HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir);
} else {
transactionFilter = getTransactionFileFilter(true);
- String txnFileName = dir.getPath() + File.separator + files[0];
+ FileReference txnFile = baseDir.getChild(files[0]);
// get the actual transaction files
- files = dir.list(transactionFilter);
- if (files.length < 2) {
- throw new HyracksDataException("LSM Btree transaction has less
than 2 files :" + files.length);
- }
- try {
- Files.delete(Paths.get(txnFileName));
- } catch (IOException e) {
- throw new HyracksDataException("Failed to delete transaction
lock :" + txnFileName);
- }
+ files = baseDir.getFile().list(transactionFilter);
+ IoUtil.delete(txnFile);
}
- File bTreeFile = null;
- File bloomFilterFile = null;
-
+ FileReference bTreeFileRef = null;
+ FileReference bloomFilterFileRef = null;
for (String fileName : files) {
if (fileName.endsWith(BTREE_SUFFIX)) {
- bTreeFile = new File(dir.getPath() + File.separator +
fileName);
+ bTreeFileRef = baseDir.getChild(fileName);
} else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) {
- bloomFilterFile = new File(dir.getPath() + File.separator +
fileName);
+ bloomFilterFileRef = baseDir.getChild(fileName);
} else {
- throw new HyracksDataException("unrecognized file found = " +
fileName);
+ throw
HyracksDataException.create(ErrorCode.UNRECOGNIZED_INDEX_COMPONENT_FILE,
fileName);
}
}
- FileReference bTreeFileRef =
ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath());
- FileReference bloomFilterFileRef =
ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath());
-
return new LSMComponentFileReferences(bTreeFileRef, null,
bloomFilterFileRef);
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
index c866a45..f61d783 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -29,9 +29,11 @@
import java.util.Iterator;
import java.util.List;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -42,19 +44,10 @@
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
private final TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory;
- private static FilenameFilter btreeFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
- }
- };
+ private static FilenameFilter btreeFilter = (dir, name) ->
!name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
- private static FilenameFilter buddyBtreeFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(DELETE_TREE_SUFFIX);
- }
- };
+ private static FilenameFilter buddyBtreeFilter =
+ (dir, name) -> !name.startsWith(".") &&
name.endsWith(DELETE_TREE_SUFFIX);
public LSMBTreeWithBuddyFileManager(IIOManager ioManager, FileReference
file,
TreeIndexFactory<? extends ITreeIndex> btreeFactory,
@@ -67,11 +60,11 @@
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws
HyracksDataException {
String ts = getCurrentTimestamp();
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = ts + DELIMITER + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName +
DELIMITER + BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + DELETE_TREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
@@ -80,12 +73,12 @@
String[] firstTimestampRange = firstFileName.split(DELIMITER);
String[] lastTimestampRange = lastFileName.split(DELIMITER);
- String baseName = baseDir + firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1];
+ String baseName = firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
// timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName +
DELIMITER + BTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + DELETE_TREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
@@ -94,10 +87,8 @@
ArrayList<ComparableFileName> allBTreeFiles = new ArrayList<>();
ArrayList<ComparableFileName> allBuddyBTreeFiles = new ArrayList<>();
ArrayList<ComparableFileName> allBloomFilterFiles = new ArrayList<>();
-
// Create transaction file filter
FilenameFilter transactionFilefilter = getTransactionFileFilter(false);
-
// Gather files.
cleanupAndGetValidFilesInternal(getCompoundFilter(btreeFilter,
transactionFilefilter), btreeFactory,
allBTreeFiles);
@@ -110,18 +101,14 @@
buddyBtreeFactory);
validateFiles(btreeFilesSet, allBloomFilterFiles,
getCompoundFilter(bloomFilterFilter, transactionFilefilter),
null);
-
// Sanity check.
if (allBTreeFiles.size() != allBuddyBTreeFiles.size() ||
allBTreeFiles.size() != allBloomFilterFiles.size()) {
- throw new HyracksDataException(
- "Unequal number of valid BTree, Buddy BTree, and Bloom
Filter files found. Aborting cleanup.");
+ throw
HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
}
-
// Trivial cases.
if (allBTreeFiles.isEmpty() || allBuddyBTreeFiles.isEmpty() ||
allBloomFilterFiles.isEmpty()) {
return validFiles;
}
-
if (allBTreeFiles.size() == 1 && allBuddyBTreeFiles.size() == 1 &&
allBloomFilterFiles.size() == 1) {
validFiles.add(new
LSMComponentFileReferences(allBTreeFiles.get(0).fileRef,
allBuddyBTreeFiles.get(0).fileRef,
allBloomFilterFiles.get(0).fileRef));
@@ -166,16 +153,12 @@
&&
currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
// Invalid files are completely contained in last interval.
- File invalidBTreeFile = new File(currentBTree.fullPath);
- invalidBTreeFile.delete();
- File invalidBuddyBTreeFile = new
File(currentBuddyBTree.fullPath);
- invalidBuddyBTreeFile.delete();
- File invalidBloomFilterFile = new
File(currentBloomFilter.fullPath);
- invalidBloomFilterFile.delete();
+ IoUtil.delete(new File(currentBTree.fullPath));
+ IoUtil.delete(new File(currentBuddyBTree.fullPath));
+ IoUtil.delete(new File(currentBloomFilter.fullPath));
} else {
// This scenario should not be possible.
- throw new HyracksDataException(
- "Found LSM files with overlapping but not contained
timetamp intervals.");
+ throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
}
}
@@ -204,55 +187,45 @@
String ts = getCurrentTimestamp();
// Create transaction lock file
Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts));
-
- String baseName = baseDir + ts + DELIMITER + ts;
- return new LSMComponentFileReferences(createFlushFile(baseName +
DELIMITER + BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ String baseName = ts + DELIMITER + ts;
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + DELETE_TREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit()
throws HyracksDataException {
FilenameFilter transactionFilter;
- File dir = new File(baseDir);
- String[] files = dir.list(txnFileNameFilter);
+ String[] files = baseDir.getFile().list(txnFileNameFilter);
if (files.length == 0) {
return null;
}
if (files.length != 1) {
- throw new HyracksDataException("More than one transaction lock
found:" + files.length);
+ throw
HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir);
} else {
transactionFilter = getTransactionFileFilter(true);
- String txnFileName = dir.getPath() + File.separator + files[0];
// get the actual transaction files
- files = dir.list(transactionFilter);
+ files = baseDir.getFile().list(transactionFilter);
if (files.length < 3) {
- throw new HyracksDataException(
- "LSM Btree with buddy transaction has less than 3
files :" + files.length);
+ throw
HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
}
- try {
- Files.delete(Paths.get(txnFileName));
- } catch (IOException e) {
- throw new HyracksDataException("Failed to delete transaction
lock :" + txnFileName);
- }
+ IoUtil.delete(baseDir.getChild(files[0]));
}
- File bTreeFile = null;
- File buddyBTreeFile = null;
- File bloomFilterFile = null;
+ FileReference bTreeFileRef = null;
+ FileReference buddyBTreeFileRef = null;
+ FileReference bloomFilterFileRef = null;
for (String fileName : files) {
if (fileName.endsWith(BTREE_SUFFIX)) {
- bTreeFile = new File(dir.getPath() + File.separator +
fileName);
+ bTreeFileRef = baseDir.getChild(fileName);
} else if (fileName.endsWith(DELETE_TREE_SUFFIX)) {
- buddyBTreeFile = new File(dir.getPath() + File.separator +
fileName);
+ buddyBTreeFileRef = baseDir.getChild(fileName);
} else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) {
- bloomFilterFile = new File(dir.getPath() + File.separator +
fileName);
+ bloomFilterFileRef = baseDir.getChild(fileName);
} else {
- throw new HyracksDataException("unrecognized file found = " +
fileName);
+ throw
HyracksDataException.create(ErrorCode.UNRECOGNIZED_INDEX_COMPONENT_FILE,
fileName);
}
}
- FileReference bTreeFileRef =
ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath());
- FileReference buddyBTreeFileRef =
ioManager.resolveAbsolutePath(buddyBTreeFile.getAbsolutePath());
- FileReference bloomFilterFileRef =
ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath());
+
return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef,
bloomFilterFileRef);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
index 18d9a67..5580387 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
@@ -24,6 +24,7 @@
import java.util.List;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
/**
@@ -43,7 +44,7 @@
LSMComponentFileReferences getRelMergeFileReference(String firstFileName,
String lastFileName)
throws HyracksDataException;
- String getBaseDir();
+ FileReference getBaseDir();
// Deletes invalid files, and returns list of valid files from baseDir.
// The returned valid files are correctly sorted (based on the recency of
data).
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index ead9db4..d3c056d 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -22,8 +22,6 @@
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.text.Format;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -88,7 +86,7 @@
protected final IIOManager ioManager;
// baseDir should reflect dataset name and partition name and be absolute
- protected final String baseDir;
+ protected final FileReference baseDir;
protected final Format formatter = new
SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS");
protected final Comparator<ComparableFileName> recencyCmp = new
RecencyComparator();
protected final TreeIndexFactory<? extends ITreeIndex> treeFactory;
@@ -97,8 +95,7 @@
public AbstractLSMIndexFileManager(IIOManager ioManager, FileReference
file,
TreeIndexFactory<? extends ITreeIndex> treeFactory) {
this.ioManager = ioManager;
- this.baseDir =
file.getFile().getAbsolutePath().endsWith(File.separator) ?
file.getFile().getAbsolutePath()
- : file.getFile().getAbsolutePath() + File.separator;
+ this.baseDir = file;
this.treeFactory = treeFactory;
}
@@ -136,9 +133,8 @@
TreeIndexFactory<? extends ITreeIndex> treeFactory,
ArrayList<ComparableFileName> allFiles)
throws HyracksDataException {
String[] files = listDirFiles(baseDir, filter);
- File dir = new File(baseDir);
for (String fileName : files) {
- FileReference fileRef =
ioManager.resolveAbsolutePath(dir.getPath() + File.separator + fileName);
+ FileReference fileRef = baseDir.getChild(fileName);
if (treeFactory == null) {
allFiles.add(new ComparableFileName(fileRef));
continue;
@@ -152,21 +148,20 @@
}
}
- static String[] listDirFiles(String path, FilenameFilter filter) throws
HyracksDataException {
- File dir = new File(path);
+ static String[] listDirFiles(FileReference dir, FilenameFilter filter)
throws HyracksDataException {
/*
* Returns null if this abstract pathname does not denote a directory,
or if an I/O error occurs.
*/
- String[] files = dir.list(filter);
+ String[] files = dir.getFile().list(filter);
if (files == null) {
- if (!dir.canRead()) {
- throw HyracksDataException.create(ErrorCode.CANNOT_READ_FILE,
path);
- } else if (!dir.exists()) {
- throw
HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, path);
- } else if (!dir.isDirectory()) {
- throw
HyracksDataException.create(ErrorCode.FILE_IS_NOT_DIRECTORY, path);
+ if (!dir.getFile().canRead()) {
+ throw HyracksDataException.create(ErrorCode.CANNOT_READ_FILE,
dir);
+ } else if (!dir.getFile().exists()) {
+ throw
HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, dir);
+ } else if (!dir.getFile().isDirectory()) {
+ throw
HyracksDataException.create(ErrorCode.FILE_IS_NOT_DIRECTORY, dir);
}
- throw
HyracksDataException.create(ErrorCode.UNIDENTIFIED_IO_ERROR_READING_FILE, path);
+ throw
HyracksDataException.create(ErrorCode.UNIDENTIFIED_IO_ERROR_READING_FILE, dir);
}
return files;
}
@@ -189,31 +184,22 @@
@Override
public void createDirs() throws HyracksDataException {
- File f = new File(baseDir);
- if (f.exists()) {
+ if (baseDir.getFile().exists()) {
throw
HyracksDataException.create(ErrorCode.CANNOT_CREATE_EXISTING_INDEX);
}
- f.mkdirs();
+ baseDir.getFile().mkdirs();
}
@Override
public void deleteDirs() throws HyracksDataException {
- IoUtil.delete(new File(baseDir));
- }
-
- protected FileReference createFlushFile(String flushFileName) throws
HyracksDataException {
- return ioManager.resolveAbsolutePath(flushFileName);
- }
-
- protected FileReference createMergeFile(String mergeFileName) throws
HyracksDataException {
- return createFlushFile(mergeFileName);
+ IoUtil.delete(baseDir);
}
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws
HyracksDataException {
String ts = getCurrentTimestamp();
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseDir + ts +
DELIMITER + ts), null, null);
+ return new LSMComponentFileReferences(baseDir.getChild(ts + DELIMITER
+ ts), null, null);
}
@Override
@@ -223,7 +209,7 @@
String[] lastTimestampRange = lastFileName.split(DELIMITER);
// Get the range of timestamps by taking the earliest and the latest
timestamps
return new LSMComponentFileReferences(
- createMergeFile(baseDir + firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1]), null, null);
+ baseDir.getChild(firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1]), null, null);
}
@Override
@@ -267,8 +253,7 @@
current.fileRef.delete();
} else {
// This scenario should not be possible since timestamps are
monotonically increasing.
- throw new HyracksDataException("Found LSM files with
overlapping timestamp intervals, "
- + "but the intervals were not contained by another
file.");
+ throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
}
}
@@ -302,24 +287,19 @@
}
@Override
- public String getBaseDir() {
+ public FileReference getBaseDir() {
return baseDir;
}
@Override
public void recoverTransaction() throws HyracksDataException {
String[] files = listDirFiles(baseDir, txnFileNameFilter);
- File dir = new File(baseDir);
- try {
- if (files.length == 0) {
- // Do nothing
- } else if (files.length > 1) {
- throw new HyracksDataException("Found more than one
transaction");
- } else {
- Files.delete(Paths.get(dir.getPath() + File.separator +
files[0]));
- }
- } catch (IOException e) {
- throw new HyracksDataException("Failed to recover transaction", e);
+ if (files.length == 0) {
+ // Do nothing
+ } else if (files.length > 1) {
+ throw
HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir);
+ } else {
+ IoUtil.delete(baseDir.getChild(files[0]));
}
}
@@ -366,27 +346,17 @@
if (files.length == 0) {
// Do nothing
} else if (files.length > 1) {
- throw new HyracksDataException("Found more than one transaction");
+ throw
HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir);
} else {
- File dir = new File(baseDir);
//create transaction filter
FilenameFilter transactionFilter =
createTransactionFilter(files[0], true);
String[] componentsFiles = listDirFiles(baseDir,
transactionFilter);
for (String fileName : componentsFiles) {
- try {
- String absFileName = dir.getPath() + File.separator +
fileName;
- Files.delete(Paths.get(absFileName));
- } catch (IOException e) {
- throw new HyracksDataException("Failed to delete
transaction files", e);
- }
+ FileReference file = baseDir.getChild(fileName);
+ IoUtil.delete(file);
}
// delete the txn lock file
- String absFileName = dir.getPath() + File.separator + files[0];
- try {
- Files.delete(Paths.get(absFileName));
- } catch (IOException e) {
- throw new HyracksDataException("Failed to delete transaction
files", e);
- }
+ IoUtil.delete(baseDir.getChild(files[0]));
}
}
@@ -403,16 +373,7 @@
protected static FilenameFilter createTransactionFilter(String
transactionFileName, final boolean inclusive) {
final String timeStamp =
transactionFileName.substring(transactionFileName.indexOf(TXN_PREFIX) +
TXN_PREFIX.length());
- return new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- if (inclusive) {
- return name.startsWith(timeStamp);
- } else {
- return !name.startsWith(timeStamp);
- }
- }
- };
+ return (dir, name) -> inclusive ? name.startsWith(timeStamp) :
!name.startsWith(timeStamp);
}
protected FilenameFilter getTransactionFileFilter(boolean inclusive)
throws HyracksDataException {
@@ -425,12 +386,7 @@
}
protected FilenameFilter getCompoundFilter(final FilenameFilter filter1,
final FilenameFilter filter2) {
- return new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return (filter1.accept(dir, name) && filter2.accept(dir,
name));
- }
- };
+ return (dir, name) -> filter1.accept(dir, name) && filter2.accept(dir,
name);
}
/**
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 4a60db0..0f862ac 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -724,7 +724,7 @@
return new LSMInvertedIndexFlushOperation(new
LSMInvertedIndexAccessor(getLsmHarness(), opCtx),
flushingComponent,
componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getDeleteIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(),
- callback, fileManager.getBaseDir());
+ callback, fileManager.getBaseDir().getAbsolutePath());
}
@Override
@@ -735,7 +735,7 @@
IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(opCtx);
return new LSMInvertedIndexMergeOperation(accessor, mergingComponents,
cursor,
mergeFileRefs.getInsertIndexFileReference(),
mergeFileRefs.getDeleteIndexFileReference(),
- mergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir());
+ mergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir().getAbsolutePath());
}
@Override
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
index 9254c45..c4a2f73 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java
@@ -27,9 +27,11 @@
import java.util.Iterator;
import java.util.List;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.util.IoUtil;
import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -43,27 +45,12 @@
// We only need a BTree factory because the inverted indexes consistency
is validated against its dictionary BTree.
private final BTreeFactory btreeFactory;
-
- private static FilenameFilter dictBTreeFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(DICT_BTREE_SUFFIX);
- }
- };
-
- private static FilenameFilter invListFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(INVLISTS_SUFFIX);
- }
- };
-
- private static FilenameFilter deletedKeysBTreeFilter = new
FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") &&
name.endsWith(DELETED_KEYS_BTREE_SUFFIX);
- }
- };
+ private static final FilenameFilter dictBTreeFilter =
+ (dir, name) -> !name.startsWith(".") &&
name.endsWith(DICT_BTREE_SUFFIX);
+ private static final FilenameFilter invListFilter =
+ (dir, name) -> !name.startsWith(".") &&
name.endsWith(INVLISTS_SUFFIX);
+ private static final FilenameFilter deletedKeysBTreeFilter =
+ (dir, name) -> !name.startsWith(".") &&
name.endsWith(DELETED_KEYS_BTREE_SUFFIX);
public LSMInvertedIndexFileManager(IIOManager ioManager, FileReference
file, BTreeFactory btreeFactory) {
super(ioManager, file, null);
@@ -73,11 +60,11 @@
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws
HyracksDataException {
String ts = getCurrentTimestamp();
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = ts + DELIMITER + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName +
DELIMITER + DICT_BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER +
DELETED_KEYS_BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + DICT_BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER +
DELETED_KEYS_BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
@@ -85,12 +72,11 @@
throws HyracksDataException {
String[] firstTimestampRange = firstFileName.split(DELIMITER);
String[] lastTimestampRange = lastFileName.split(DELIMITER);
-
- String baseName = baseDir + firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1];
+ String baseName = firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName +
DELIMITER + DICT_BTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER +
DELETED_KEYS_BTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + DICT_BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER +
DELETED_KEYS_BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
@@ -118,8 +104,7 @@
if (allDictBTreeFiles.size() != allInvListsFiles.size()
|| allDictBTreeFiles.size() != allDeletedKeysBTreeFiles.size()
|| allDictBTreeFiles.size() != allBloomFilterFiles.size()) {
- throw new HyracksDataException(
- "Unequal number of valid Dictionary BTree, Inverted Lists,
Deleted BTree, and Bloom Filter files found. Aborting cleanup.");
+ throw
HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
}
// Trivial cases.
@@ -154,35 +139,31 @@
for (int i = 1; i < allDictBTreeFiles.size(); i++) {
ComparableFileName currentDeletedKeysBTree =
allDeletedKeysBTreeFiles.get(i);
- ComparableFileName CurrentDictBTree = allDictBTreeFiles.get(i);
+ ComparableFileName currentDictBTree = allDictBTreeFiles.get(i);
ComparableFileName currentBloomFilter = allBloomFilterFiles.get(i);
// Current start timestamp is greater than last stop timestamp.
if
(currentDeletedKeysBTree.interval[0].compareTo(lastDeletedKeysBTree.interval[1])
> 0
- &&
CurrentDictBTree.interval[0].compareTo(lastDictBTree.interval[1]) > 0
+ &&
currentDictBTree.interval[0].compareTo(lastDictBTree.interval[1]) > 0
&&
currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[1]) > 0) {
- validComparableDictBTreeFiles.add(CurrentDictBTree);
+ validComparableDictBTreeFiles.add(currentDictBTree);
validComparableDeletedKeysBTreeFiles.add(currentDeletedKeysBTree);
validComparableBloomFilterFiles.add(currentBloomFilter);
- lastDictBTree = CurrentDictBTree;
+ lastDictBTree = currentDictBTree;
lastDeletedKeysBTree = currentDeletedKeysBTree;
lastBloomFilter = currentBloomFilter;
} else if
(currentDeletedKeysBTree.interval[0].compareTo(lastDeletedKeysBTree.interval[0])
>= 0
&&
currentDeletedKeysBTree.interval[1].compareTo(lastDeletedKeysBTree.interval[1])
<= 0
- &&
CurrentDictBTree.interval[0].compareTo(lastDictBTree.interval[0]) >= 0
- &&
CurrentDictBTree.interval[1].compareTo(lastDictBTree.interval[1]) <= 0
+ &&
currentDictBTree.interval[0].compareTo(lastDictBTree.interval[0]) >= 0
+ &&
currentDictBTree.interval[1].compareTo(lastDictBTree.interval[1]) <= 0
&&
currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
// Invalid files are completely contained in last interval.
- File invalidDeletedBTreeFile = new
File(currentDeletedKeysBTree.fullPath);
- invalidDeletedBTreeFile.delete();
- File invalidDictBTreeFile = new
File(CurrentDictBTree.fullPath);
- invalidDictBTreeFile.delete();
- File invalidBloomFilterFile = new
File(currentBloomFilter.fullPath);
- invalidBloomFilterFile.delete();
+ IoUtil.delete(new File(currentDeletedKeysBTree.fullPath));
+ IoUtil.delete(new File(currentDictBTree.fullPath));
+ IoUtil.delete(new File(currentBloomFilter.fullPath));
} else {
// This scenario should not be possible.
- throw new HyracksDataException(
- "Found LSM files with overlapping but not contained
timetamp intervals.");
+ throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index 512e35c..e39c3f9 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -642,7 +642,7 @@
// create the merge operation.
LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor,
mergingComponents, cursor,
relMergeFileRefs.getInsertIndexFileReference(),
relMergeFileRefs.getDeleteIndexFileReference(),
- relMergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir());
+ relMergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir().getAbsolutePath());
ioScheduler.scheduleOperation(mergeOp);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
index 07d48de..0f291df 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java
@@ -431,7 +431,7 @@
LSMRTreeAccessor accessor = new LSMRTreeAccessor(getLsmHarness(),
opCtx, buddyBTreeFields);
return new LSMRTreeFlushOperation(accessor, flushingComponent,
componentFileRefs.getInsertIndexFileReference(),
componentFileRefs.getDeleteIndexFileReference(),
componentFileRefs.getBloomFilterFileReference(),
- callback, fileManager.getBaseDir());
+ callback, fileManager.getBaseDir().getAbsolutePath());
}
@Override
@@ -442,6 +442,6 @@
ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(),
opCtx, buddyBTreeFields);
return new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
mergeFileRefs.getInsertIndexFileReference(),
mergeFileRefs.getDeleteIndexFileReference(),
- mergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir());
+ mergeFileRefs.getBloomFilterFileReference(), callback,
fileManager.getBaseDir().getAbsolutePath());
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
index 4d5eb5b..001228c 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java
@@ -30,9 +30,11 @@
import java.util.Iterator;
import java.util.List;
+import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.storage.am.common.api.ITreeIndex;
import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
import
org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -43,19 +45,10 @@
private final TreeIndexFactory<? extends ITreeIndex> rtreeFactory;
private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
- private static FilenameFilter btreeFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
- }
- };
-
- private static FilenameFilter rtreeFilter = new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return !name.startsWith(".") && name.endsWith(RTREE_SUFFIX);
- }
- };
+ private static final FilenameFilter btreeFilter =
+ (dir, name) -> !name.startsWith(".") &&
name.endsWith(BTREE_SUFFIX);
+ private static final FilenameFilter rtreeFilter =
+ (dir, name) -> !name.startsWith(".") &&
name.endsWith(RTREE_SUFFIX);
public LSMRTreeFileManager(IIOManager ioManager, FileReference file,
TreeIndexFactory<? extends ITreeIndex> rtreeFactory,
TreeIndexFactory<? extends ITreeIndex> btreeFactory) {
@@ -67,11 +60,11 @@
@Override
public LSMComponentFileReferences getRelFlushFileReference() throws
HyracksDataException {
String ts = getCurrentTimestamp();
- String baseName = baseDir + ts + DELIMITER + ts;
+ String baseName = ts + DELIMITER + ts;
// Begin timestamp and end timestamp are identical since it is a flush
- return new LSMComponentFileReferences(createFlushFile(baseName +
DELIMITER + RTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + RTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
@@ -79,13 +72,12 @@
throws HyracksDataException {
String[] firstTimestampRange = firstFileName.split(DELIMITER);
String[] lastTimestampRange = lastFileName.split(DELIMITER);
-
- String baseName = baseDir + firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1];
+ String baseName = firstTimestampRange[0] + DELIMITER +
lastTimestampRange[1];
// Get the range of timestamps by taking the earliest and the latest
// timestamps
- return new LSMComponentFileReferences(createMergeFile(baseName +
DELIMITER + RTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + BTREE_SUFFIX),
- createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + RTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
@@ -111,8 +103,7 @@
// Sanity check.
if (allRTreeFiles.size() != allBTreeFiles.size() ||
allBTreeFiles.size() != allBloomFilterFiles.size()) {
- throw new HyracksDataException(
- "Unequal number of valid RTree, BTree, and Bloom Filter
files found. Aborting cleanup.");
+ throw
HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
}
// Trivial cases.
@@ -164,16 +155,12 @@
&&
currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0
&&
currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) {
// Invalid files are completely contained in last interval.
- File invalidRTreeFile = new File(currentRTree.fullPath);
- invalidRTreeFile.delete();
- File invalidBTreeFile = new File(currentBTree.fullPath);
- invalidBTreeFile.delete();
- File invalidBloomFilterFile = new
File(currentBloomFilter.fullPath);
- invalidBloomFilterFile.delete();
+ IoUtil.delete(new File(currentRTree.fullPath));
+ IoUtil.delete(new File(currentBTree.fullPath));
+ IoUtil.delete(new File(currentBloomFilter.fullPath));
} else {
// This scenario should not be possible.
- throw new HyracksDataException(
- "Found LSM files with overlapping but not contained
timetamp intervals.");
+ throw
HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir);
}
}
@@ -201,54 +188,44 @@
String ts = getCurrentTimestamp();
// Create transaction lock file
Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts));
-
- String baseName = baseDir + ts + DELIMITER + ts;
- return new LSMComponentFileReferences(createFlushFile(baseName +
DELIMITER + RTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BTREE_SUFFIX),
- createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+ String baseName = ts + DELIMITER + ts;
+ return new LSMComponentFileReferences(baseDir.getChild(baseName +
DELIMITER + RTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX),
+ baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
}
@Override
public LSMComponentFileReferences getTransactionFileReferenceForCommit()
throws HyracksDataException {
FilenameFilter transactionFilter;
- File dir = new File(baseDir);
- String[] files = dir.list(txnFileNameFilter);
+ String[] files = baseDir.getFile().list(txnFileNameFilter);
if (files.length == 0) {
return null;
}
if (files.length != 1) {
- throw new HyracksDataException("More than one transaction lock
found:" + files.length);
+ throw
HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir);
} else {
transactionFilter = getTransactionFileFilter(true);
- String txnFileName = dir.getPath() + File.separator + files[0];
// get the actual transaction files
- files = dir.list(transactionFilter);
+ files = baseDir.getFile().list(transactionFilter);
if (files.length < 3) {
- throw new HyracksDataException("LSM Rtree transaction has less
than 3 files :" + files.length);
+ throw
HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir);
}
- try {
- Files.delete(Paths.get(txnFileName));
- } catch (IOException e) {
- throw new HyracksDataException("Failed to delete transaction
lock :" + txnFileName);
- }
+ IoUtil.delete(baseDir.getChild(files[0]));
}
- File rTreeFile = null;
- File bTreeFile = null;
- File bloomFilterFile = null;
+ FileReference rTreeFileRef = null;
+ FileReference bTreeFileRef = null;
+ FileReference bloomFilterFileRef = null;
for (String fileName : files) {
if (fileName.endsWith(BTREE_SUFFIX)) {
- bTreeFile = new File(dir.getPath() + File.separator +
fileName);
+ bTreeFileRef = baseDir.getChild(fileName);
} else if (fileName.endsWith(RTREE_SUFFIX)) {
- rTreeFile = new File(dir.getPath() + File.separator +
fileName);
+ rTreeFileRef = baseDir.getChild(fileName);
} else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) {
- bloomFilterFile = new File(dir.getPath() + File.separator +
fileName);
+ bloomFilterFileRef = baseDir.getChild(fileName);
} else {
- throw new HyracksDataException("unrecognized file found = " +
fileName);
+ throw
HyracksDataException.create(ErrorCode.UNRECOGNIZED_INDEX_COMPONENT_FILE,
fileName);
}
}
- FileReference rTreeFileRef =
ioManager.resolveAbsolutePath(rTreeFile.getAbsolutePath());
- FileReference bTreeFileRef =
ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath());
- FileReference bloomFilterFileRef =
ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath());
return new LSMComponentFileReferences(rTreeFileRef, bTreeFileRef,
bloomFilterFileRef);
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
index 2b61f1e..58d4421 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java
@@ -339,7 +339,7 @@
ILSMIOOperationCallback callback) throws HyracksDataException {
ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(),
opCtx, cursorFactory);
return new LSMRTreeFlushOperation(accessor, flushingComponent,
componentFileRefs.getInsertIndexFileReference(),
- null, null, callback, fileManager.getBaseDir());
+ null, null, callback,
fileManager.getBaseDir().getAbsolutePath());
}
@Override
@@ -353,6 +353,6 @@
ITreeIndexCursor cursor = new
LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx, returnDeletedTuples);
ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(),
opCtx, cursorFactory);
return new MergeOperation(accessor,
mergeFileRefs.getInsertIndexFileReference(), callback,
- fileManager.getBaseDir(), mergingComponents, cursor);
+ fileManager.getBaseDir().getAbsolutePath(), mergingComponents,
cursor);
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
index 1c75387..87fb11a 100644
---
a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
@@ -19,7 +19,6 @@
package org.apache.hyracks.storage.am.lsm.common.component;
-import java.io.File;
import java.io.FilenameFilter;
import java.util.ArrayList;
@@ -40,11 +39,9 @@
protected void cleanupAndGetValidFilesInternal(FilenameFilter filter,
TreeIndexFactory<? extends ITreeIndex> treeFactory,
ArrayList<ComparableFileName> allFiles)
throws HyracksDataException {
- File dir = new File(baseDir);
- String[] files = dir.list(filter);
+ String[] files = baseDir.getFile().list(filter);
for (String fileName : files) {
- File file = new File(dir.getPath() + File.separator + fileName);
- FileReference fileRef =
ioManager.resolveAbsolutePath(file.getAbsolutePath());
+ FileReference fileRef = baseDir.getChild(fileName);
allFiles.add(new ComparableFileName(fileRef));
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/1849
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I97bab76888790ca282ad9508ce8416f7c7a52fb7
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <bamou...@gmail.com>
