ashishkumar50 commented on code in PR #8845:
URL: https://github.com/apache/ozone/pull/8845#discussion_r2230158359
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java:
##########
@@ -895,6 +909,102 @@ public void
testDeletedBlockTransactionsOfDeletedContainer() throws IOException
assertEquals(0, blocks.size());
}
+ @ParameterizedTest
+ @ValueSource(ints = {1, 10, 25, 50, 100})
+ public void testTransactionSerializedSize(int blockCount) {
+ long txID = 10000000;
+ long containerID = 1000000;
+ List<DeletedBlock> blocks = new ArrayList<>();
+ for (int i = 0; i < blockCount; i++) {
+ blocks.add(new DeletedBlock(new BlockID(containerID, 100000000 + i), 128
* 1024 * 1024, 128 * 1024 * 1024));
+ }
+ List<Long> localIdList = blocks.stream().map(b ->
b.getBlockID().getLocalID()).collect(Collectors.toList());
+ DeletedBlocksTransaction tx1 = DeletedBlocksTransaction.newBuilder()
+ .setTxID(txID)
+ .setContainerID(containerID)
+ .addAllLocalID(localIdList)
+ .setCount(0)
+
.setTotalBlockSize(blocks.stream().mapToLong(DeletedBlock::getSize).sum())
+
.setTotalBlockReplicatedSize(blocks.stream().mapToLong(DeletedBlock::getReplicatedSize).sum())
+ .build();
+ DeletedBlocksTransaction tx2 = DeletedBlocksTransaction.newBuilder()
+ .setTxID(txID)
+ .setContainerID(containerID)
+ .addAllLocalID(localIdList)
+ .setCount(0)
+ .build();
+ /*
+ * 1 blocks tx with totalBlockSize size is 26
+ * 1 blocks tx without totalBlockSize size is 16
+ * 10 blocks tx with totalBlockSize size is 73
+ * 10 blocks tx without totalBlockSize size is 61
+ * 25 blocks tx with totalBlockSize size is 148
+ * 25 blocks tx without totalBlockSize size is 136
+ * 50 blocks tx with totalBlockSize size is 273
+ * 50 blocks tx without totalBlockSize size is 261
+ * 100 blocks tx with totalBlockSize size is 523
+ * 100 blocks tx without totalBlockSize size is 511
+ */
+ System.out.println(blockCount + " blocks tx with totalBlockSize size is "
+ tx1.getSerializedSize());
+ System.out.println(blockCount + " blocks tx without totalBlockSize size is
" + tx2.getSerializedSize());
Review Comment:
Can you `assert` these instead of just printing like
```
@CsvSource({
"1, 26, 16",
....})
```
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java:
##########
@@ -895,6 +909,102 @@ public void
testDeletedBlockTransactionsOfDeletedContainer() throws IOException
assertEquals(0, blocks.size());
}
+ @ParameterizedTest
+ @ValueSource(ints = {1, 10, 25, 50, 100})
+ public void testTransactionSerializedSize(int blockCount) {
+ long txID = 10000000;
+ long containerID = 1000000;
+ List<DeletedBlock> blocks = new ArrayList<>();
+ for (int i = 0; i < blockCount; i++) {
+ blocks.add(new DeletedBlock(new BlockID(containerID, 100000000 + i), 128
* 1024 * 1024, 128 * 1024 * 1024));
+ }
+ List<Long> localIdList = blocks.stream().map(b ->
b.getBlockID().getLocalID()).collect(Collectors.toList());
+ DeletedBlocksTransaction tx1 = DeletedBlocksTransaction.newBuilder()
+ .setTxID(txID)
+ .setContainerID(containerID)
+ .addAllLocalID(localIdList)
+ .setCount(0)
+
.setTotalBlockSize(blocks.stream().mapToLong(DeletedBlock::getSize).sum())
+
.setTotalBlockReplicatedSize(blocks.stream().mapToLong(DeletedBlock::getReplicatedSize).sum())
+ .build();
+ DeletedBlocksTransaction tx2 = DeletedBlocksTransaction.newBuilder()
+ .setTxID(txID)
+ .setContainerID(containerID)
+ .addAllLocalID(localIdList)
+ .setCount(0)
+ .build();
+ /*
+ * 1 blocks tx with totalBlockSize size is 26
+ * 1 blocks tx without totalBlockSize size is 16
+ * 10 blocks tx with totalBlockSize size is 73
+ * 10 blocks tx without totalBlockSize size is 61
+ * 25 blocks tx with totalBlockSize size is 148
+ * 25 blocks tx without totalBlockSize size is 136
+ * 50 blocks tx with totalBlockSize size is 273
+ * 50 blocks tx without totalBlockSize size is 261
+ * 100 blocks tx with totalBlockSize size is 523
+ * 100 blocks tx without totalBlockSize size is 511
+ */
+ System.out.println(blockCount + " blocks tx with totalBlockSize size is "
+ tx1.getSerializedSize());
+ System.out.println(blockCount + " blocks tx without totalBlockSize size is
" + tx2.getSerializedSize());
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {100, 1000, 10000, 100000})
+ public void testAddRemoveTransactionPerformance(int txCount) throws
Exception {
+ Map<Long, List<DeletedBlock>> data = generateData(txCount);
+ SCMHAInvocationHandler handler =
+ (SCMHAInvocationHandler)
Proxy.getInvocationHandler(deletedBlockLog.getDeletedBlockLogStateManager());
+ DeletedBlockLogStateManagerImpl deletedBlockLogStateManager =
+ (DeletedBlockLogStateManagerImpl) handler.getLocalHandler();
+ HddsProtos.DeletedBlocksTransactionSummary summary =
deletedBlockLogStateManager.getTransactionSummary();
+ assertEquals(EMPTY_SUMMARY, summary);
+
+ long startTime = System.nanoTime();
+ deletedBlockLog.addTransactions(data);
+ scmHADBTransactionBuffer.flush();
+ /**
+ * Before DataDistribution is enabled
+ * - 979 ms to add 100 txs to DB
+ * - 275 ms to add 1000 txs to DB
+ * - 1106 ms to add 10000 txs to DB
+ * - 11103 ms to add 100000 txs to DB
+ * After DataDistribution is enabled
+ * - 908 ms to add 100 txs to DB
+ * - 351 ms to add 1000 txs to DB
+ * - 2875 ms to add 10000 txs to DB
+ * - 12446 ms to add 100000 txs to DB
+ */
+ System.out.println((System.nanoTime() - startTime) / 100000 + " ms to add
" + txCount + " txs to DB");
+ summary = deletedBlockLogStateManager.getTransactionSummary();
+ assertEquals(txCount, summary.getTotalTransactionCount());
+
+ ArrayList txIdList =
data.keySet().stream().collect(Collectors.toCollection(ArrayList::new));
+ startTime = System.nanoTime();
+ deletedBlockLogStateManager.removeTransactionsFromDB(txIdList);
+ scmHADBTransactionBuffer.flush();
+ /**
+ * Before DataDistribution is enabled
+ * - 19 ms to remove 100 txs from DB
+ * - 26 ms to remove 1000 txs from DB
+ * - 142 ms to remove 10000 txs from DB
+ * - 2571 ms to remove 100000 txs from DB
+ * After DataDistribution is enabled (all cache miss)
+ * - 62 ms to remove 100 txs from DB
+ * - 186 ms to remove 1000 txs from DB
+ * - 968 ms to remove 10000 txs from DB
+ * - 8635 ms to remove 100000 txs from DB
+ * After DataDistribution is enabled (all cache hit)
+ * - 40 ms to remove 100 txs from DB
+ * - 112 ms to remove 1000 txs from DB
+ * - 412 ms to remove 10000 txs from DB
+ * - 3499 ms to remove 100000 txs from DB
+ */
+ System.out.println((System.nanoTime() - startTime) / 100000 + " ms to
remove " + txCount + " txs from DB");
+ System.out.println("Cache hit: " +
metrics.getNumBlockDeletionTransactionSizeFromCache());
+ System.out.println("Cache miss: " +
metrics.getNumBlockDeletionTransactionSizeFromDB());
Review Comment:
Expected number of cache hit and miss can be asserted here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]