swamirishi commented on PR #8203: URL: https://github.com/apache/ozone/pull/8203#issuecomment-2808035270
@szetszwo @kerneltime I compared the changes with and without the buffer stack. Allocating direct byte buffer is still a significant portion of the Iteration.         ``` package org.apache.hadoop.hdds.utils.db; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES; import java.io.IOException; import java.util.LinkedList; import java.util.Queue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.ratis.util.function.CheckedFunction; import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Utility to benchmark splitIterator. */ public final class BenchmarkOmDBCodecBufferIter { private static final Logger LOG = LoggerFactory.getLogger(BenchmarkOmDBCodecBufferIter.class); private BenchmarkOmDBCodecBufferIter() { } public static void main(String[] args) throws IOException { OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ozoneConfiguration.setInt(OZONE_OM_SNAPSHOT_DB_MAX_OPEN_FILES, -1); ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, args[0]); RocksDBConfiguration rocksDBConfiguration = ozoneConfiguration.getObject(RocksDBConfiguration.class); Long timeToLive = Long.valueOf(args[2]); int numberOfBuffers = Integer.valueOf(args[3]); long loggingThreshold = Long.parseLong(args[1]); ozoneConfiguration.setFromObject(rocksDBConfiguration); OmMetadataManagerImpl omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, null); AtomicLong counter = new AtomicLong(); long startTime = System.currentTimeMillis(); AtomicBoolean finished = new AtomicBoolean(true); Queue<UncheckedAutoCloseableSupplier<RawKeyValue<CodecBuffer>>> q = new LinkedList<>(); CheckedFunction<UncheckedAutoCloseableSupplier<RawKeyValue<CodecBuffer>>, Boolean, IOException> func = kv -> { long id = counter.incrementAndGet(); q.add(kv); if (id % loggingThreshold == 0) { long time = System.currentTimeMillis(); LOG.info("Benchmark: " + id + "\t" + (time - startTime) + "\t" + StringCodec.get().fromCodecBuffer(kv.get().getKey())); finished.set(time - startTime <= timeToLive); } if (q.size() > numberOfBuffers) { q.poll().close(); } return finished.get(); }; RDBStore rdbStore = (RDBStore) omMetadataManager.getStore(); RDBTable rdbTable = rdbStore.getTable("fileTable"); RocksDatabase rocksDatabase = rdbStore.getDb(); try (TableIterator<CodecBuffer, UncheckedAutoCloseableSupplier<RawKeyValue<CodecBuffer>>> itr = new RDBStoreCodecBufferIterator(rocksDatabase.newIterator(rdbTable.getColumnFamily()), rdbTable, null, numberOfBuffers + 2)) { while (itr.hasNext()) { if (!func.apply(itr.next())) { break; } } while (!q.isEmpty()) { q.poll().close(); } } omMetadataManager.getStore().close(); } } ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
