xichen01 commented on code in PR #9042:
URL: https://github.com/apache/ozone/pull/9042#discussion_r2377642485
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java:
##########
@@ -366,175 +372,230 @@ private void evaluateFSOBucket(OmVolumeArgs volume,
OmBucketInfo bucket, String
// skip this rule if some directory doesn't exist for this rule's
prefix
continue;
}
- // use last directory's object ID to iterate the keys
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- StringBuffer directoryPath = new StringBuffer();
+ StringBuffer lastDirPath = new StringBuffer();
+ OmDirectoryInfo lastDir = null;
if (!dirList.isEmpty()) {
- OmDirectoryInfo lastDir = dirList.get(dirList.size() - 1);
- prefix += lastDir.getObjectID();
- for (OmDirectoryInfo dir : dirList) {
- directoryPath.append(dir.getName()).append(OM_KEY_PREFIX);
+ lastDir = dirList.get(dirList.size() - 1);
+ if (lastDir != null && !lastDir.getName().equals(TRASH_PREFIX)) {
+ for (int i = 0; i < dirList.size(); i++) {
+ lastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 1) {
+ lastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable,
lastDirPath.toString(), lastDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ } else {
+ LOG.info("Skip evaluate trash directory {}", TRASH_PREFIX);
}
- if ((directoryPath.toString().equals(rule.getEffectivePrefix()) ||
- directoryPath.toString().equals(rule.getEffectivePrefix() +
OM_KEY_PREFIX))
- && rule.match(lastDir, directoryPath.toString())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+
+ if (!rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) {
+ // if the prefix doesn't end with "/", then also search and
evaluate the directory itself
+ // for example, "dir1/dir2" matches both directory "dir1/dir2" and
"dir1/dir22"
+ // or "dir1" matches both directory "dir1" and "dir11"
+ long objID;
+ String objPrefix;
+ String objPath;
+ if (dirList.size() > 1) {
+ OmDirectoryInfo secondLastDir = dirList.get(dirList.size() - 2);
+ objID = secondLastDir.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + secondLastDir.getObjectID();
+ StringBuffer secondLastDirPath = new StringBuffer();
+ for (int i = 0; i < dirList.size() - 1; i++) {
+ secondLastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 2) {
+ secondLastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ objPath = secondLastDirPath.toString();
+ } else {
+ objID = bucket.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + bucket.getObjectID();
+ objPath = "";
+ }
+ try {
+ List<OmDirectoryInfo> subDirList = getSubDirectory(objID,
objPrefix, omMetadataManager);
+ for (OmDirectoryInfo subDir : subDirList) {
+ String subDirPath = objPath.isEmpty() ? subDir.getName() :
objPath + OM_KEY_PREFIX + subDir.getName();
+ if (!subDir.getName().equals(TRASH_PREFIX) &&
subDirPath.startsWith(rule.getEffectivePrefix()) &&
+ (lastDir == null || subDir.getObjectID() !=
lastDir.getObjectID())) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(),
keyTable, subDirPath, subDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ }
+ }
+ } catch (IOException e) {
+ // log failure and continue the process
+ LOG.warn("Failed to get sub directories of {} under {}/{}",
objPrefix,
+ bucket.getVolumeName(), bucket.getBucketName(), e);
+ return;
}
- expiredDirList.add(directoryPath.toString(), 0,
lastDir.getUpdateID());
}
+ } else {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
}
+ }
+
+ if (!noPrefixRuleList.isEmpty()) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ noPrefixRuleList, expiredKeyList, expiredDirList);
+ }
+ }
- LOG.info("Prefix {} for {}", prefix, bucketKey);
- evaluateKeyTable(keyTable, prefix, directoryPath.toString(), rule,
expiredKeyList, bucket);
- evaluateDirTable(directoryInfoTable, prefix, directoryPath.toString(),
rule,
- expiredDirList, bucket);
+ @SuppressWarnings("checkstyle:parameternumber")
+ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId,
Table<String, OmKeyInfo> keyTable,
+ String directoryPath, OmDirectoryInfo dir, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList keyList, LimitedExpiredObjectList dirList) {
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ Deque<PendingEvaluateDirectory> stack = new ArrayDeque<>();
Review Comment:
It might be better to check the deepest subdirectory first, but currently it
seems to be traversing from the top to the bottom.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java:
##########
@@ -366,175 +372,230 @@ private void evaluateFSOBucket(OmVolumeArgs volume,
OmBucketInfo bucket, String
// skip this rule if some directory doesn't exist for this rule's
prefix
continue;
}
- // use last directory's object ID to iterate the keys
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- StringBuffer directoryPath = new StringBuffer();
+ StringBuffer lastDirPath = new StringBuffer();
+ OmDirectoryInfo lastDir = null;
if (!dirList.isEmpty()) {
- OmDirectoryInfo lastDir = dirList.get(dirList.size() - 1);
- prefix += lastDir.getObjectID();
- for (OmDirectoryInfo dir : dirList) {
- directoryPath.append(dir.getName()).append(OM_KEY_PREFIX);
+ lastDir = dirList.get(dirList.size() - 1);
+ if (lastDir != null && !lastDir.getName().equals(TRASH_PREFIX)) {
+ for (int i = 0; i < dirList.size(); i++) {
+ lastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 1) {
+ lastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable,
lastDirPath.toString(), lastDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ } else {
+ LOG.info("Skip evaluate trash directory {}", TRASH_PREFIX);
}
- if ((directoryPath.toString().equals(rule.getEffectivePrefix()) ||
- directoryPath.toString().equals(rule.getEffectivePrefix() +
OM_KEY_PREFIX))
- && rule.match(lastDir, directoryPath.toString())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+
+ if (!rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) {
+ // if the prefix doesn't end with "/", then also search and
evaluate the directory itself
+ // for example, "dir1/dir2" matches both directory "dir1/dir2" and
"dir1/dir22"
+ // or "dir1" matches both directory "dir1" and "dir11"
+ long objID;
+ String objPrefix;
+ String objPath;
+ if (dirList.size() > 1) {
+ OmDirectoryInfo secondLastDir = dirList.get(dirList.size() - 2);
+ objID = secondLastDir.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + secondLastDir.getObjectID();
+ StringBuffer secondLastDirPath = new StringBuffer();
+ for (int i = 0; i < dirList.size() - 1; i++) {
+ secondLastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 2) {
+ secondLastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ objPath = secondLastDirPath.toString();
+ } else {
+ objID = bucket.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + bucket.getObjectID();
+ objPath = "";
+ }
+ try {
+ List<OmDirectoryInfo> subDirList = getSubDirectory(objID,
objPrefix, omMetadataManager);
+ for (OmDirectoryInfo subDir : subDirList) {
+ String subDirPath = objPath.isEmpty() ? subDir.getName() :
objPath + OM_KEY_PREFIX + subDir.getName();
+ if (!subDir.getName().equals(TRASH_PREFIX) &&
subDirPath.startsWith(rule.getEffectivePrefix()) &&
+ (lastDir == null || subDir.getObjectID() !=
lastDir.getObjectID())) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(),
keyTable, subDirPath, subDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ }
+ }
+ } catch (IOException e) {
+ // log failure and continue the process
+ LOG.warn("Failed to get sub directories of {} under {}/{}",
objPrefix,
+ bucket.getVolumeName(), bucket.getBucketName(), e);
+ return;
}
- expiredDirList.add(directoryPath.toString(), 0,
lastDir.getUpdateID());
}
+ } else {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
}
+ }
+
+ if (!noPrefixRuleList.isEmpty()) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ noPrefixRuleList, expiredKeyList, expiredDirList);
+ }
+ }
- LOG.info("Prefix {} for {}", prefix, bucketKey);
- evaluateKeyTable(keyTable, prefix, directoryPath.toString(), rule,
expiredKeyList, bucket);
- evaluateDirTable(directoryInfoTable, prefix, directoryPath.toString(),
rule,
- expiredDirList, bucket);
+ @SuppressWarnings("checkstyle:parameternumber")
+ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId,
Table<String, OmKeyInfo> keyTable,
+ String directoryPath, OmDirectoryInfo dir, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList keyList, LimitedExpiredObjectList dirList) {
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ Deque<PendingEvaluateDirectory> stack = new ArrayDeque<>();
+ if (dir != null) {
+ stack.push(new PendingEvaluateDirectory(dir, directoryPath));
+ } else {
+ // put a placeholder PendingEvaluateDirectory to stack for bucket
+ stack.push(new PendingEvaluateDirectory(null, ""));
}
- for (OmLCRule rule : nonDirectoryStylePrefixRuleList) {
- // find the directory for the prefix, it may not exist
- OmDirectoryInfo dirInfo = getDirectory(volume, bucket,
rule.getEffectivePrefix(), bucketKey);
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- if (dirInfo != null) {
- if (!dirInfo.getName().equals(TRASH_PREFIX)) {
- prefix += dirInfo.getObjectID();
- if (dirInfo.getName().equals(rule.getEffectivePrefix()) &&
rule.match(dirInfo, dirInfo.getName())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+ while (!stack.isEmpty()) {
+ PendingEvaluateDirectory item = stack.pop();
+ OmDirectoryInfo currentDir = item.getDirectoryInfo();
+ String currentDirPath = item.getDirPath();
+ long currentDirObjID = currentDir == null ? bucket.getObjectID() :
currentDir.getObjectID();
+
+ // use current directory's object ID to iterate the keys and
directories under it
+ String prefix =
+ OM_KEY_PREFIX + volumeObjId + OM_KEY_PREFIX + bucket.getObjectID()
+ OM_KEY_PREFIX + currentDirObjID;
+ LOG.info("Prefix {} for {}/{}", prefix, bucket.getVolumeName(),
bucket.getBucketName());
Review Comment:
Nit: Can set to debug level.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java:
##########
@@ -366,175 +372,230 @@ private void evaluateFSOBucket(OmVolumeArgs volume,
OmBucketInfo bucket, String
// skip this rule if some directory doesn't exist for this rule's
prefix
continue;
}
- // use last directory's object ID to iterate the keys
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- StringBuffer directoryPath = new StringBuffer();
+ StringBuffer lastDirPath = new StringBuffer();
+ OmDirectoryInfo lastDir = null;
if (!dirList.isEmpty()) {
- OmDirectoryInfo lastDir = dirList.get(dirList.size() - 1);
- prefix += lastDir.getObjectID();
- for (OmDirectoryInfo dir : dirList) {
- directoryPath.append(dir.getName()).append(OM_KEY_PREFIX);
+ lastDir = dirList.get(dirList.size() - 1);
+ if (lastDir != null && !lastDir.getName().equals(TRASH_PREFIX)) {
+ for (int i = 0; i < dirList.size(); i++) {
+ lastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 1) {
+ lastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable,
lastDirPath.toString(), lastDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ } else {
+ LOG.info("Skip evaluate trash directory {}", TRASH_PREFIX);
}
- if ((directoryPath.toString().equals(rule.getEffectivePrefix()) ||
- directoryPath.toString().equals(rule.getEffectivePrefix() +
OM_KEY_PREFIX))
- && rule.match(lastDir, directoryPath.toString())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+
+ if (!rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) {
+ // if the prefix doesn't end with "/", then also search and
evaluate the directory itself
+ // for example, "dir1/dir2" matches both directory "dir1/dir2" and
"dir1/dir22"
+ // or "dir1" matches both directory "dir1" and "dir11"
+ long objID;
+ String objPrefix;
+ String objPath;
+ if (dirList.size() > 1) {
+ OmDirectoryInfo secondLastDir = dirList.get(dirList.size() - 2);
+ objID = secondLastDir.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + secondLastDir.getObjectID();
+ StringBuffer secondLastDirPath = new StringBuffer();
+ for (int i = 0; i < dirList.size() - 1; i++) {
+ secondLastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 2) {
+ secondLastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ objPath = secondLastDirPath.toString();
+ } else {
+ objID = bucket.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + bucket.getObjectID();
+ objPath = "";
+ }
+ try {
+ List<OmDirectoryInfo> subDirList = getSubDirectory(objID,
objPrefix, omMetadataManager);
+ for (OmDirectoryInfo subDir : subDirList) {
+ String subDirPath = objPath.isEmpty() ? subDir.getName() :
objPath + OM_KEY_PREFIX + subDir.getName();
+ if (!subDir.getName().equals(TRASH_PREFIX) &&
subDirPath.startsWith(rule.getEffectivePrefix()) &&
+ (lastDir == null || subDir.getObjectID() !=
lastDir.getObjectID())) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(),
keyTable, subDirPath, subDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ }
+ }
+ } catch (IOException e) {
+ // log failure and continue the process
+ LOG.warn("Failed to get sub directories of {} under {}/{}",
objPrefix,
+ bucket.getVolumeName(), bucket.getBucketName(), e);
+ return;
}
- expiredDirList.add(directoryPath.toString(), 0,
lastDir.getUpdateID());
}
+ } else {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
}
+ }
+
+ if (!noPrefixRuleList.isEmpty()) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ noPrefixRuleList, expiredKeyList, expiredDirList);
+ }
+ }
- LOG.info("Prefix {} for {}", prefix, bucketKey);
- evaluateKeyTable(keyTable, prefix, directoryPath.toString(), rule,
expiredKeyList, bucket);
- evaluateDirTable(directoryInfoTable, prefix, directoryPath.toString(),
rule,
- expiredDirList, bucket);
+ @SuppressWarnings("checkstyle:parameternumber")
+ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId,
Table<String, OmKeyInfo> keyTable,
+ String directoryPath, OmDirectoryInfo dir, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList keyList, LimitedExpiredObjectList dirList) {
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ Deque<PendingEvaluateDirectory> stack = new ArrayDeque<>();
+ if (dir != null) {
+ stack.push(new PendingEvaluateDirectory(dir, directoryPath));
+ } else {
+ // put a placeholder PendingEvaluateDirectory to stack for bucket
+ stack.push(new PendingEvaluateDirectory(null, ""));
}
- for (OmLCRule rule : nonDirectoryStylePrefixRuleList) {
- // find the directory for the prefix, it may not exist
- OmDirectoryInfo dirInfo = getDirectory(volume, bucket,
rule.getEffectivePrefix(), bucketKey);
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- if (dirInfo != null) {
- if (!dirInfo.getName().equals(TRASH_PREFIX)) {
- prefix += dirInfo.getObjectID();
- if (dirInfo.getName().equals(rule.getEffectivePrefix()) &&
rule.match(dirInfo, dirInfo.getName())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+ while (!stack.isEmpty()) {
+ PendingEvaluateDirectory item = stack.pop();
+ OmDirectoryInfo currentDir = item.getDirectoryInfo();
+ String currentDirPath = item.getDirPath();
+ long currentDirObjID = currentDir == null ? bucket.getObjectID() :
currentDir.getObjectID();
+
+ // use current directory's object ID to iterate the keys and
directories under it
+ String prefix =
+ OM_KEY_PREFIX + volumeObjId + OM_KEY_PREFIX + bucket.getObjectID()
+ OM_KEY_PREFIX + currentDirObjID;
+ LOG.info("Prefix {} for {}/{}", prefix, bucket.getVolumeName(),
bucket.getBucketName());
+
+ // get direct sub directories
+ List<OmDirectoryInfo> subDirList;
+ try {
+ subDirList = getSubDirectory(currentDirObjID, prefix,
omMetadataManager);
+ } catch (IOException e) {
+ // log failure, continue to process other directories in stack
+ LOG.warn("Failed to get sub directories of {} under {}/{}",
currentDirPath, volumeName, bucketName, e);
+ continue;
+ }
+
+ // evaluate direct files, first check cache, then check table
+ long numKeysUnderDir = 0;
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter
= keyTable.cacheIterator();
+ while (cacheIter.hasNext()) {
+ Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry =
cacheIter.next();
+ OmKeyInfo key = entry.getValue().getCacheValue();
+ if (key == null || key.getParentObjectID() != currentDirObjID) {
+ continue;
+ }
+ numKeysUnderDir++;
+ String keyPath = currentDirPath.isEmpty() ? key.getKeyName() :
+ currentDirPath + OM_KEY_PREFIX + key.getKeyName();
+ for (OmLCRule rule : ruleList) {
+ if (key.getParentObjectID() == currentDirObjID && rule.match(key,
keyPath)) {
+ // mark key as expired, check next key
+ if (keyList.isFull()) {
+ // if keyList is full, send delete request for pending
deletion keys
+ handleAndClearFullList(bucket, keyList, false);
}
- expiredDirList.add(dirInfo.getName(), 0, dirInfo.getUpdateID());
+ keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID());
}
- } else {
- dirInfo = null;
- LOG.info("Skip evaluate trash directory {}", TRASH_PREFIX);
}
}
- LOG.info("Prefix {} for {}", prefix, bucketKey);
- evaluateKeyTable(keyTable, prefix, dirInfo == null ? "" :
dirInfo.getName() + OzoneConsts.OM_KEY_PREFIX,
- rule, expiredKeyList, bucket);
- evaluateDirTable(directoryInfoTable, prefix,
- dirInfo == null ? "" : dirInfo.getName() +
OzoneConsts.OM_KEY_PREFIX, rule, expiredDirList, bucket);
- }
- if (!noPrefixRuleList.isEmpty()) {
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- LOG.info("prefix {} for {}", prefix, bucketKey);
- // use bucket name as key iterator prefix
try (TableIterator<String, ? extends Table.KeyValue<String,
OmKeyInfo>> keyTblItr =
keyTable.iterator(prefix)) {
while (keyTblItr.hasNext()) {
Table.KeyValue<String, OmKeyInfo> keyValue = keyTblItr.next();
OmKeyInfo key = keyValue.getValue();
+ String keyPath = currentDirPath.isEmpty() ? key.getKeyName() :
+ currentDirPath + OM_KEY_PREFIX + key.getKeyName();
numKeyIterated++;
- for (OmLCRule rule : noPrefixRuleList) {
- if (rule.match(key)) {
+ numKeysUnderDir++;
+ for (OmLCRule rule : ruleList) {
+ if (key.getParentObjectID() == currentDirObjID &&
rule.match(key, keyPath)) {
// mark key as expired, check next key
- if (expiredKeyList.isFull()) {
- // if expiredKeyList is full, send delete/rename request for
expired keys
- handleAndClearFullList(bucket, expiredKeyList, false);
+ if (keyList.isFull()) {
+ // if keyList is full, send delete request for pending
deletion keys
+ handleAndClearFullList(bucket, keyList, false);
}
- expiredKeyList.add(key.getKeyName(), key.getReplicatedSize(),
key.getUpdateID());
- break;
+ keyList.add(keyPath, key.getReplicatedSize(),
key.getUpdateID());
}
}
}
} catch (IOException e) {
- // log failure and continue the process to delete/move files already
identified in this run
- LOG.warn("Failed to iterate keyTable for bucket {}", bucketKey, e);
+ // log failure and continue the process other directories in stack
+ LOG.warn("Failed to iterate keyTable for bucket {}/{}", volumeName,
bucketName, e);
+ continue;
}
- try (TableIterator<String, ? extends Table.KeyValue<String,
OmDirectoryInfo>> dirTblItr =
- directoryInfoTable.iterator(prefix)) {
- while (dirTblItr.hasNext()) {
- Table.KeyValue<String, OmDirectoryInfo> entry = dirTblItr.next();
- OmDirectoryInfo dir = entry.getValue();
- numDirIterated++;
- // skip TRASH_PREFIX directory
- if (dir.getName().equals(TRASH_PREFIX)) {
- continue;
- }
- for (OmLCRule rule : noPrefixRuleList) {
- if (rule.match(dir, dir.getPath())) {
- // mark directory as expired, check next directory
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
- }
- expiredDirList.add(dir.getPath(), 0, dir.getUpdateID());
- break;
+ // this directory is empty, evaluate itself
+ if (numKeysUnderDir == 0 && subDirList.isEmpty()) {
+ for (OmLCRule rule : ruleList) {
+ String path = rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX) ?
+ currentDirPath + OM_KEY_PREFIX : currentDirPath;
+ if (currentDir != null && rule.match(currentDir, path)) {
+ if (dirList.isFull()) {
+ // if expiredDirList is full, send delete request for pending
deletion directories
+ handleAndClearFullList(bucket, dirList, true);
}
+ dirList.add(currentDirPath, 0, currentDir.getUpdateID());
}
}
- } catch (IOException e) {
- // log failure and continue the process to delete/move directories
already identified in this run
- LOG.warn("Failed to iterate directoryTable for bucket {}",
bucketKey, e);
}
- }
- }
-
- private void evaluateKeyTable(Table<String, OmKeyInfo> keyTable, String
prefix, String directoryPath,
- OmLCRule rule, LimitedExpiredObjectList keyList, OmBucketInfo bucket) {
- String volumeName = bucket.getVolumeName();
- String bucketName = bucket.getBucketName();
- try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
keyTblItr =
- keyTable.iterator(prefix)) {
- while (keyTblItr.hasNext()) {
- Table.KeyValue<String, OmKeyInfo> keyValue = keyTblItr.next();
- OmKeyInfo key = keyValue.getValue();
- String keyPath = directoryPath + key.getKeyName();
- numKeyIterated++;
- if (rule.match(key, keyPath)) {
- // mark key as expired, check next key
- if (keyList.isFull()) {
- // if keyList is full, send delete/rename request for expired
keys
- handleAndClearFullList(bucket, keyList, false);
+ // evaluate sub directories recursively by push them to stack
+ for (OmDirectoryInfo subDir : subDirList) {
+ String subDirPath = currentDirPath.isEmpty() ? subDir.getName() :
+ currentDirPath + OM_KEY_PREFIX + subDir.getName();
+ if (subDir.getName().equals(TRASH_PREFIX)) {
+ continue;
+ }
+ for (OmLCRule rule : ruleList) {
+ if (subDirPath.startsWith(rule.getEffectivePrefix())) {
+ stack.push(new PendingEvaluateDirectory(subDir, subDirPath));
}
- keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID());
}
}
- } catch (IOException e) {
- // log failure and continue the process to delete/move files already
identified in this run
- LOG.warn("Failed to iterate keyTable for bucket {}/{}", volumeName,
bucketName, e);
}
}
- private void evaluateDirTable(Table<String, OmDirectoryInfo>
directoryInfoTable, String prefix,
- String directoryPath, OmLCRule rule, LimitedExpiredObjectList dirList,
OmBucketInfo bucket) {
- String volumeName = bucket.getVolumeName();
- String bucketName = bucket.getBucketName();
- try (TableIterator<String, ? extends Table.KeyValue<String,
OmDirectoryInfo>> dirTblItr =
- directoryInfoTable.iterator(prefix)) {
- while (dirTblItr.hasNext()) {
- Table.KeyValue<String, OmDirectoryInfo> entry = dirTblItr.next();
- OmDirectoryInfo dir = entry.getValue();
+ private List<OmDirectoryInfo> getSubDirectory(long dirObjID, String
prefix, OMMetadataManager metaMgr)
+ throws IOException {
+ List<OmDirectoryInfo> subDirList = new ArrayList<>();
+ // Check all dirTable cache for any sub paths.
+ Table dirTable = metaMgr.getDirectoryTable();
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>>>
+ cacheIter = dirTable.cacheIterator();
+
+ while (cacheIter.hasNext()) {
+ Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>> entry =
+ cacheIter.next();
Review Comment:
need `numDirIterated++` too.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java:
##########
@@ -366,175 +372,230 @@ private void evaluateFSOBucket(OmVolumeArgs volume,
OmBucketInfo bucket, String
// skip this rule if some directory doesn't exist for this rule's
prefix
continue;
}
- // use last directory's object ID to iterate the keys
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- StringBuffer directoryPath = new StringBuffer();
+ StringBuffer lastDirPath = new StringBuffer();
+ OmDirectoryInfo lastDir = null;
if (!dirList.isEmpty()) {
- OmDirectoryInfo lastDir = dirList.get(dirList.size() - 1);
- prefix += lastDir.getObjectID();
- for (OmDirectoryInfo dir : dirList) {
- directoryPath.append(dir.getName()).append(OM_KEY_PREFIX);
+ lastDir = dirList.get(dirList.size() - 1);
+ if (lastDir != null && !lastDir.getName().equals(TRASH_PREFIX)) {
+ for (int i = 0; i < dirList.size(); i++) {
+ lastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 1) {
+ lastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable,
lastDirPath.toString(), lastDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ } else {
+ LOG.info("Skip evaluate trash directory {}", TRASH_PREFIX);
}
- if ((directoryPath.toString().equals(rule.getEffectivePrefix()) ||
- directoryPath.toString().equals(rule.getEffectivePrefix() +
OM_KEY_PREFIX))
- && rule.match(lastDir, directoryPath.toString())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+
+ if (!rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) {
+ // if the prefix doesn't end with "/", then also search and
evaluate the directory itself
+ // for example, "dir1/dir2" matches both directory "dir1/dir2" and
"dir1/dir22"
+ // or "dir1" matches both directory "dir1" and "dir11"
+ long objID;
+ String objPrefix;
+ String objPath;
+ if (dirList.size() > 1) {
+ OmDirectoryInfo secondLastDir = dirList.get(dirList.size() - 2);
+ objID = secondLastDir.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + secondLastDir.getObjectID();
+ StringBuffer secondLastDirPath = new StringBuffer();
+ for (int i = 0; i < dirList.size() - 1; i++) {
+ secondLastDirPath.append(dirList.get(i).getName());
+ if (i != dirList.size() - 2) {
+ secondLastDirPath.append(OM_KEY_PREFIX);
+ }
+ }
+ objPath = secondLastDirPath.toString();
+ } else {
+ objID = bucket.getObjectID();
+ objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX
+ bucket.getObjectID() +
+ OM_KEY_PREFIX + bucket.getObjectID();
+ objPath = "";
+ }
+ try {
+ List<OmDirectoryInfo> subDirList = getSubDirectory(objID,
objPrefix, omMetadataManager);
+ for (OmDirectoryInfo subDir : subDirList) {
+ String subDirPath = objPath.isEmpty() ? subDir.getName() :
objPath + OM_KEY_PREFIX + subDir.getName();
+ if (!subDir.getName().equals(TRASH_PREFIX) &&
subDirPath.startsWith(rule.getEffectivePrefix()) &&
+ (lastDir == null || subDir.getObjectID() !=
lastDir.getObjectID())) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(),
keyTable, subDirPath, subDir,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
+ }
+ }
+ } catch (IOException e) {
+ // log failure and continue the process
+ LOG.warn("Failed to get sub directories of {} under {}/{}",
objPrefix,
+ bucket.getVolumeName(), bucket.getBucketName(), e);
+ return;
}
- expiredDirList.add(directoryPath.toString(), 0,
lastDir.getUpdateID());
}
+ } else {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ Arrays.asList(rule), expiredKeyList, expiredDirList);
}
+ }
+
+ if (!noPrefixRuleList.isEmpty()) {
+ evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "",
null,
+ noPrefixRuleList, expiredKeyList, expiredDirList);
+ }
+ }
- LOG.info("Prefix {} for {}", prefix, bucketKey);
- evaluateKeyTable(keyTable, prefix, directoryPath.toString(), rule,
expiredKeyList, bucket);
- evaluateDirTable(directoryInfoTable, prefix, directoryPath.toString(),
rule,
- expiredDirList, bucket);
+ @SuppressWarnings("checkstyle:parameternumber")
+ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId,
Table<String, OmKeyInfo> keyTable,
+ String directoryPath, OmDirectoryInfo dir, List<OmLCRule> ruleList,
+ LimitedExpiredObjectList keyList, LimitedExpiredObjectList dirList) {
+ String volumeName = bucket.getVolumeName();
+ String bucketName = bucket.getBucketName();
+ Deque<PendingEvaluateDirectory> stack = new ArrayDeque<>();
+ if (dir != null) {
+ stack.push(new PendingEvaluateDirectory(dir, directoryPath));
+ } else {
+ // put a placeholder PendingEvaluateDirectory to stack for bucket
+ stack.push(new PendingEvaluateDirectory(null, ""));
}
- for (OmLCRule rule : nonDirectoryStylePrefixRuleList) {
- // find the directory for the prefix, it may not exist
- OmDirectoryInfo dirInfo = getDirectory(volume, bucket,
rule.getEffectivePrefix(), bucketKey);
- String prefix = OM_KEY_PREFIX + volume.getObjectID() +
- OM_KEY_PREFIX + bucket.getObjectID() + OM_KEY_PREFIX;
- if (dirInfo != null) {
- if (!dirInfo.getName().equals(TRASH_PREFIX)) {
- prefix += dirInfo.getObjectID();
- if (dirInfo.getName().equals(rule.getEffectivePrefix()) &&
rule.match(dirInfo, dirInfo.getName())) {
- if (expiredDirList.isFull()) {
- // if expiredDirList is full, send delete/rename request for
expired directories
- handleAndClearFullList(bucket, expiredDirList, true);
+ while (!stack.isEmpty()) {
+ PendingEvaluateDirectory item = stack.pop();
+ OmDirectoryInfo currentDir = item.getDirectoryInfo();
+ String currentDirPath = item.getDirPath();
+ long currentDirObjID = currentDir == null ? bucket.getObjectID() :
currentDir.getObjectID();
+
+ // use current directory's object ID to iterate the keys and
directories under it
+ String prefix =
+ OM_KEY_PREFIX + volumeObjId + OM_KEY_PREFIX + bucket.getObjectID()
+ OM_KEY_PREFIX + currentDirObjID;
+ LOG.info("Prefix {} for {}/{}", prefix, bucket.getVolumeName(),
bucket.getBucketName());
+
+ // get direct sub directories
+ List<OmDirectoryInfo> subDirList;
+ try {
+ subDirList = getSubDirectory(currentDirObjID, prefix,
omMetadataManager);
+ } catch (IOException e) {
+ // log failure, continue to process other directories in stack
+ LOG.warn("Failed to get sub directories of {} under {}/{}",
currentDirPath, volumeName, bucketName, e);
+ continue;
+ }
+
+ // evaluate direct files, first check cache, then check table
+ long numKeysUnderDir = 0;
+ Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter
= keyTable.cacheIterator();
+ while (cacheIter.hasNext()) {
Review Comment:
This may cause some some Key be recored two times, and the keys marked as
deleted in the Cache traversal need to be excluded from the DB table.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
