merlimat closed pull request #1738: PIP-17: fix index entry offset error in OffloadIndexBlock URL: https://github.com/apache/incubator-pulsar/pull/1738
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockBuilderImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockBuilderImpl.java index ced3bf37c6..766083dcb3 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockBuilderImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockBuilderImpl.java @@ -35,6 +35,7 @@ private LedgerMetadata ledgerMetadata; private List<OffloadIndexEntryImpl> entries; + private int lastBlockSize; public OffloadIndexBlockBuilderImpl() { this.entries = Lists.newArrayList(); @@ -50,13 +51,14 @@ public OffloadIndexBlockBuilder withMetadata(LedgerMetadata metadata) { public OffloadIndexBlockBuilder addBlock(long firstEntryId, int partId, int blockSize) { // we should added one by one. long offset; - if(firstEntryId == 0) { + if (firstEntryId == 0) { checkState(entries.size() == 0); offset = 0; } else { checkState(entries.size() > 0); - offset = entries.get(entries.size() - 1).getOffset() + blockSize; + offset = entries.get(entries.size() - 1).getOffset() + lastBlockSize; } + lastBlockSize = blockSize; this.entries.add(OffloadIndexEntryImpl.of(firstEntryId, partId, offset)); return this; diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java index 31058b4f29..086d466879 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexBlockImpl.java @@ -96,7 +96,7 @@ public void recycle() { @Override public OffloadIndexEntry getIndexEntryForEntry(long messageEntryId) throws IOException { - if(messageEntryId > segmentMetadata.getLastEntryId()) { + if (messageEntryId > segmentMetadata.getLastEntryId()) { log.warn("Try to get entry: {}, which beyond lastEntryId {}, return null", messageEntryId, segmentMetadata.getLastEntryId()); throw new IndexOutOfBoundsException("Entry index: " + messageEntryId + @@ -302,7 +302,8 @@ private OffloadIndexBlock fromStream(InputStream stream) throws IOException { DataInputStream dis = new DataInputStream(stream); int magic = dis.readInt(); if (magic != this.INDEX_MAGIC_WORD) { - throw new IOException("Invalid MagicWord. read: " + magic + " expected: " + INDEX_MAGIC_WORD); + throw new IOException("Invalid MagicWord. read: " + Integer.toHexString(magic) + + " expected: 0x" + Integer.toHexString(INDEX_MAGIC_WORD)); } int indexBlockLength = dis.readInt(); int segmentMetadataLength = dis.readInt(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexTest.java index c0ae5ed993..3867899b57 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/s3offload/impl/OffloadIndexTest.java @@ -110,7 +110,7 @@ public void offloadIndexBlockImplTest() throws Exception { blockBuilder.withMetadata(metadata); - blockBuilder.addBlock(0, 2, 0); + blockBuilder.addBlock(0, 2, 64 * 1024 * 1024); blockBuilder.addBlock(1000, 3, 64 * 1024 * 1024); blockBuilder.addBlock(2000, 4, 64 * 1024 * 1024); OffloadIndexBlock indexBlock = blockBuilder.build(); ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services