This is an automated email from the ASF dual-hosted git repository. sodonnell pushed a commit to branch ozone-1.4 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit e453c61d0b214c1cfa4465520bc2dafb8064615a Author: Stephen O'Donnell <[email protected]> AuthorDate: Fri Apr 12 21:02:33 2024 +0100 HDDS-10682. EC Reconstruction creates empty chunks at the end of blocks with partial stripes (#6515) (cherry picked from commit a5fccbc1d6539cdccf5366edf26382a139897789) --- .../ec/reconstruction/ECReconstructionCoordinator.java | 10 +++++++--- .../hadoop/hdds/scm/storage/TestContainerCommandsEC.java | 14 ++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java index 3d9b5b77ee..62f96d8adf 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java @@ -320,9 +320,13 @@ public class ECReconstructionCoordinator implements Closeable { } // TODO: can be submitted in parallel for (int i = 0; i < bufs.length; i++) { - CompletableFuture<ContainerProtos.ContainerCommandResponseProto> - future = targetBlockStreams[i].write(bufs[i]); - checkFailures(targetBlockStreams[i], future); + if (bufs[i].remaining() != 0) { + // If the buffer is empty, we don't need to write it as it will cause + // an empty chunk to be added to the end of the block. + CompletableFuture<ContainerProtos.ContainerCommandResponseProto> + future = targetBlockStreams[i].write(bufs[i]); + checkFailures(targetBlockStreams[i], future); + } bufs[i].clear(); } length -= readLen; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java index 5cd4ce63f8..c5f1239353 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java @@ -135,7 +135,9 @@ public class TestContainerCommandsEC { private static final int EC_CHUNK_SIZE = 1024 * 1024; private static final int STRIPE_DATA_SIZE = EC_DATA * EC_CHUNK_SIZE; private static final int NUM_DN = EC_DATA + EC_PARITY + 3; - private static byte[][] inputChunks = new byte[EC_DATA][EC_CHUNK_SIZE]; + // Data slots are EC_DATA + 1 so we can generate enough data to have a full stripe + // plus one extra chunk. + private static byte[][] inputChunks = new byte[EC_DATA + 1][EC_CHUNK_SIZE]; // Each key size will be in range [min, max), min inclusive, max exclusive private static final int[][] KEY_SIZE_RANGES = @@ -621,13 +623,13 @@ public class TestContainerCommandsEC { testECReconstructionCoordinator(missingIndexes, 1); } - @Test - void testECReconstructParityWithPartialStripe() - throws Exception { - testECReconstructionCoordinator(ImmutableList.of(4, 5), 1); + @ParameterizedTest + @MethodSource("recoverableMissingIndexes") + void testECReconstructionCoordinatorWithFullAndPartialStripe(List<Integer> missingIndexes) + throws Exception { + testECReconstructionCoordinator(missingIndexes, 4); } - static Stream<List<Integer>> recoverableMissingIndexes() { return Stream .concat(IntStream.rangeClosed(1, 5).mapToObj(ImmutableList::of), Stream --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
