jojochuang commented on code in PR #8774:
URL: https://github.com/apache/ozone/pull/8774#discussion_r2244054824
##########
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBBatchOperation.java:
##########
@@ -141,39 +339,99 @@ private class FamilyCache {
* When dbValue is a byte[]/{@link ByteBuffer}, it represents a put-op.
* Otherwise, it represents a delete-op (dbValue is {@link Op#DELETE}).
*/
- private final Map<Bytes, Object> ops = new HashMap<>();
+ private final Map<Bytes, Integer> opsKeys = new HashMap<>();
+ private final Map<Integer, Operation> batchOps = new HashMap<>();
private boolean isCommit;
private long batchSize;
private long discardedSize;
private int discardedCount;
private int putCount;
private int delCount;
+ private int delRangeCount;
+ private AtomicInteger opIndex;
FamilyCache(ColumnFamily family) {
this.family = family;
+ this.opIndex = new AtomicInteger(0);
}
- /** Prepare batch write for the entire family. */
+ /**
+ * Prepares a batch write operation for a RocksDB-backed system.
+ *
+ * This method ensures the orderly execution of operations accumulated
in the batch,
+ * respecting their respective types and order of insertion.
+ *
+ * Key functionalities:
+ * 1. Ensures that the batch is not already committed before proceeding.
+ * 2. Sorts all operations by their `opIndex` to maintain a consistent
execution order.
+ * 3. Filters and adapts operations to account for any delete range
operations that might
+ * affect other operations in the batch:
+ * - Operations with keys that fall within the range specified by a
delete range operation
+ * are discarded.
+ * - Delete range operations are executed in their correct order.
+ * 4. Applies remaining operations to the write batch, ensuring proper
filtering and execution.
+ * 5. Logs a summary of the batch execution for debugging purposes.
+ *
+ * Throws:
+ * - RocksDatabaseException if any error occurs while applying
operations to the write batch.
+ *
+ * Prerequisites:
+ * - The method assumes that the operations are represented by
`Operation` objects, each of which
+ * encapsulates the logic for its specific type.
+ * - Delete range operations must be represented by the
`DeleteRangeOperation` class.
+ */
void prepareBatchWrite() throws RocksDatabaseException {
Preconditions.checkState(!isCommit, "%s is already committed.", this);
isCommit = true;
- for (Map.Entry<Bytes, Object> op : ops.entrySet()) {
- final Bytes key = op.getKey();
- final Object value = op.getValue();
- if (value instanceof byte[]) {
- family.batchPut(writeBatch, key.array(), (byte[]) value);
- } else if (value instanceof CodecBuffer) {
- family.batchPut(writeBatch, key.asReadOnlyByteBuffer(),
- ((CodecBuffer) value).asReadOnlyByteBuffer());
- } else if (value == Op.DELETE) {
- family.batchDelete(writeBatch, key.array());
+ // Sort Entries based on opIndex and flush the operation to the batch
in the same order.
+ List<Operation> ops =
batchOps.entrySet().stream().sorted(Comparator.comparingInt(Map.Entry::getKey))
+ .map(Map.Entry::getValue).collect(Collectors.toList());
+ List<Integer> deleteRangeIndices = new ArrayList<>();
+ int index = 0;
+ for (Operation op : ops) {
+ if (Op.DELETE_RANGE == op.getOpType()) {
+ deleteRangeIndices.add(index);
+ }
+ index++;
+ }
+ // This is to apply the last batch of entries after the last
DeleteRangeOperation.
+ deleteRangeIndices.add(ops.size());
+ int startIndex = 0;
+
+ for (Integer deleteRangeIndex : deleteRangeIndices) {
+ DeleteRangeOperation deleteRangeOp = deleteRangeIndex < ops.size() ?
+ (DeleteRangeOperation) ops.get(deleteRangeIndex) : null;
+ Bytes startKey, endKey;
+ if (deleteRangeOp != null) {
+ startKey = new Bytes(deleteRangeOp.startKey);
+ endKey = new Bytes(deleteRangeOp.endKey);
} else {
- throw new IllegalStateException("Unexpected value: " + value
- + ", class=" + value.getClass().getSimpleName());
+ startKey = null;
+ endKey = null;
+ }
+ for (int i = startIndex; i < deleteRangeIndex; i++) {
+ Operation op = ops.get(i);
+ Bytes key = op.getKey();
Review Comment:
Suggest to add a check to verify key is not null pointer. And that is
expected because the key is between startIndex and deleteRangeIndices.
```suggestion
Bytes key = op.getKey();
Preconditions.checkArgument(key)
```
##########
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBBatchOperation.java:
##########
@@ -186,68 +444,65 @@ void clear() {
final boolean warn = !isCommit && batchSize > 0;
String details = warn ? summary() : null;
- for (Object value : ops.values()) {
- if (value instanceof CodecBuffer) {
- ((CodecBuffer) value).release(); // the key will also be released
- }
- }
- ops.clear();
+ IOUtils.close(LOG, batchOps.values());
+ batchOps.clear();
if (warn) {
LOG.warn("discarding changes {}", details);
}
}
- void putOrDelete(Bytes key, int keyLen, Object val, int valLen) {
- Preconditions.checkState(!isCommit, "%s is already committed.", this);
- batchSize += keyLen + valLen;
+ private void deleteIfExist(Bytes key, boolean removeFromIndexMap) {
// remove previous first in order to call release()
- final Object previous = ops.remove(key);
- if (previous != null) {
- final boolean isPut = previous != Op.DELETE;
- final int preLen;
- if (!isPut) {
- preLen = 0;
- } else if (previous instanceof CodecBuffer) {
- final CodecBuffer previousValue = (CodecBuffer) previous;
- preLen = previousValue.readableBytes();
- previousValue.release(); // key will also be released
- } else if (previous instanceof byte[]) {
- preLen = ((byte[]) previous).length;
- } else {
- throw new IllegalStateException("Unexpected previous: " + previous
- + ", class=" + previous.getClass().getSimpleName());
- }
- discardedSize += keyLen + preLen;
+ if (opsKeys.containsKey(key)) {
+ int previousIndex = removeFromIndexMap ? opsKeys.remove(key) :
opsKeys.get(key);
+ final Operation previous = batchOps.remove(previousIndex);
+ previous.close();
+ discardedSize += previous.totalLength();
discardedCount++;
- debug(() -> String.format("%s overwriting a previous %s", this,
- isPut ? "put (value: " + byteSize2String(preLen) + ")" : "del"));
+ debug(() -> String.format("%s overwriting a previous %s[valLen =>
%s]", this, previous.getOpType(),
+ previous.valLen()));
}
- final Object overwritten = ops.put(key, val);
- Preconditions.checkState(overwritten == null);
+ }
+ int overWriteOpIfExist(Bytes key, Operation operation) {
Review Comment:
return value is over used, and can be made a private method
##########
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBBatchOperation.java:
##########
@@ -186,68 +444,65 @@ void clear() {
final boolean warn = !isCommit && batchSize > 0;
String details = warn ? summary() : null;
- for (Object value : ops.values()) {
- if (value instanceof CodecBuffer) {
- ((CodecBuffer) value).release(); // the key will also be released
- }
- }
- ops.clear();
+ IOUtils.close(LOG, batchOps.values());
+ batchOps.clear();
if (warn) {
LOG.warn("discarding changes {}", details);
}
}
- void putOrDelete(Bytes key, int keyLen, Object val, int valLen) {
- Preconditions.checkState(!isCommit, "%s is already committed.", this);
- batchSize += keyLen + valLen;
+ private void deleteIfExist(Bytes key, boolean removeFromIndexMap) {
// remove previous first in order to call release()
- final Object previous = ops.remove(key);
- if (previous != null) {
- final boolean isPut = previous != Op.DELETE;
- final int preLen;
- if (!isPut) {
- preLen = 0;
- } else if (previous instanceof CodecBuffer) {
- final CodecBuffer previousValue = (CodecBuffer) previous;
- preLen = previousValue.readableBytes();
- previousValue.release(); // key will also be released
- } else if (previous instanceof byte[]) {
- preLen = ((byte[]) previous).length;
- } else {
- throw new IllegalStateException("Unexpected previous: " + previous
- + ", class=" + previous.getClass().getSimpleName());
- }
- discardedSize += keyLen + preLen;
+ if (opsKeys.containsKey(key)) {
+ int previousIndex = removeFromIndexMap ? opsKeys.remove(key) :
opsKeys.get(key);
+ final Operation previous = batchOps.remove(previousIndex);
+ previous.close();
+ discardedSize += previous.totalLength();
discardedCount++;
- debug(() -> String.format("%s overwriting a previous %s", this,
- isPut ? "put (value: " + byteSize2String(preLen) + ")" : "del"));
+ debug(() -> String.format("%s overwriting a previous %s[valLen =>
%s]", this, previous.getOpType(),
+ previous.valLen()));
}
- final Object overwritten = ops.put(key, val);
- Preconditions.checkState(overwritten == null);
+ }
+ int overWriteOpIfExist(Bytes key, Operation operation) {
Review Comment:
consider updating overWriteOpIfExist() so that DeleteRanger API usage is
included.
Otherwise it means they don't contribute to batchSize tracking. This
inconsistency could lead to inaccurate batch size reporting.
##########
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBBatchOperation.java:
##########
@@ -141,39 +339,99 @@ private class FamilyCache {
* When dbValue is a byte[]/{@link ByteBuffer}, it represents a put-op.
* Otherwise, it represents a delete-op (dbValue is {@link Op#DELETE}).
*/
- private final Map<Bytes, Object> ops = new HashMap<>();
+ private final Map<Bytes, Integer> opsKeys = new HashMap<>();
+ private final Map<Integer, Operation> batchOps = new HashMap<>();
Review Comment:
add a description for batchOps similar to opsKeys?
```suggestion
/**
* Cache of all batch operations to be applied for this family, keyed
by their insertion order index.
*
* Each entry maps a unique integer index (representing the order in
which the operation was added)
* to an {@link Operation} object (such as put, delete, or delete
range).
*
* This ensures that all database operations are executed in the same
order as they were added,
* which is crucial for correctly handling overlapping or dependent
operations, such as delete ranges.
*/
private final Map<Integer, Operation> batchOps = new HashMap<>();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]