[
https://issues.apache.org/jira/browse/HDDS-15193?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Chu Cheng Li updated HDDS-15193:
--------------------------------
Status: Patch Available (was: Open)
> Remove old atomic key creation
> ------------------------------
>
> Key: HDDS-15193
> URL: https://issues.apache.org/jira/browse/HDDS-15193
> Project: Apache Ozone
> Issue Type: Sub-task
> Reporter: Chu Cheng Li
> Assignee: Chu Cheng Li
> Priority: Major
> Labels: pull-request-available
>
> [https://github.com/apache/ozone/pull/5524]
> {code:java}
> diff --git
> a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
>
> b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
> index ee5c754875..9f94384d4d 100644
> ---
> a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
> +++
> b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
> @@ -74,14 +74,6 @@ public final class ECKeyOutputStream extends
> KeyOutputStream
> private final Future<Boolean> flushFuture;
> private final AtomicLong flushCheckpoint;
>
> - /**
> - * Indicates if an atomic write is required. When set to true,
> - * the amount of data written must match the declared size during the
> commit.
> - * A mismatch will prevent the commit from succeeding.
> - * This is essential for operations like S3 put to ensure atomicity.
> - */
> - private boolean atomicKeyCreation;
> -
> private volatile boolean closed;
> private volatile boolean closing;
> // how much of data is actually written yet to underlying stream
> @@ -130,7 +122,6 @@ private ECKeyOutputStream(Builder builder) {
> return flushStripeFromQueue();
> });
> this.flushCheckpoint = new AtomicLong(0);
> - this.atomicKeyCreation = builder.getAtomicKeyCreation();
> }
>
> @Override
> @@ -489,12 +480,6 @@ public void close() throws IOException {
> Preconditions.checkArgument(writeOffset == offset,
> "Expected writeOffset= " + writeOffset
> + " Expected offset=" + offset);
> - if (atomicKeyCreation) {
> - long expectedSize = blockOutputStreamEntryPool.getDataSize();
> - Preconditions.checkState(expectedSize == offset, String.format(
> - "Expected: %d and actual %d write sizes do not match",
> - expectedSize, offset));
> - }
> for (CheckedRunnable<IOException> preCommit : preCommits) {
> preCommit.run();
> }
> diff --git
> a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
>
> b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
> index ceacd624e9..afdc206ee7 100644
> ---
> a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
> +++
> b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java
> @@ -77,14 +77,6 @@ public class KeyDataStreamOutput extends
> AbstractDataStreamOutput
>
> private long clientID;
>
> - /**
> - * Indicates if an atomic write is required. When set to true,
> - * the amount of data written must match the declared size during the
> commit.
> - * A mismatch will prevent the commit from succeeding.
> - * This is essential for operations like S3 put to ensure atomicity.
> - */
> - private boolean atomicKeyCreation;
> -
> private List<CheckedRunnable<IOException>> preCommits =
> Collections.emptyList();
>
> public void setPreCommits(@Nonnull List<CheckedRunnable<IOException>>
> preCommits) {
> @@ -129,7 +121,6 @@ public KeyDataStreamOutput() {
>
> this.writeOffset = 0;
> this.clientID = 0L;
> - this.atomicKeyCreation = false;
> }
>
> @SuppressWarnings({"parameternumber", "squid:S00107"})
> @@ -140,9 +131,7 @@ public KeyDataStreamOutput(
> OzoneManagerProtocol omClient, int chunkSize,
> String requestId, ReplicationConfig replicationConfig,
> String uploadID, int partNumber, boolean isMultipart,
> - boolean unsafeByteBufferConversion,
> - boolean atomicKeyCreation
> - ) {
> + boolean unsafeByteBufferConversion) {
> super(HddsClientUtils.getRetryPolicyByException(
> config.getMaxRetryCount(), config.getRetryInterval()));
> this.config = config;
> @@ -162,7 +151,6 @@ public KeyDataStreamOutput(
> // encrypted bucket.
> this.writeOffset = 0;
> this.clientID = handler.getId();
> - this.atomicKeyCreation = atomicKeyCreation;
> }
>
> /**
> @@ -457,12 +445,6 @@ public void close() throws IOException {
> if (!isException()) {
> Preconditions.checkArgument(writeOffset == offset);
> }
> - if (atomicKeyCreation) {
> - long expectedSize = blockDataStreamOutputEntryPool.getDataSize();
> - Preconditions.checkArgument(expectedSize == offset,
> - String.format("Expected: %d and actual %d write sizes do not
> match",
> - expectedSize, offset));
> - }
> for (CheckedRunnable<IOException> preCommit : preCommits) {
> preCommit.run();
> }
> @@ -501,7 +483,6 @@ public static class Builder {
> private boolean unsafeByteBufferConversion;
> private OzoneClientConfig clientConfig;
> private ReplicationConfig replicationConfig;
> - private boolean atomicKeyCreation = false;
>
> public Builder setMultipartUploadID(String uploadID) {
> this.multipartUploadID = uploadID;
> @@ -553,11 +534,6 @@ public Builder setReplicationConfig(ReplicationConfig
> replConfig) {
> return this;
> }
>
> - public Builder setAtomicKeyCreation(boolean atomicKey) {
> - this.atomicKeyCreation = atomicKey;
> - return this;
> - }
> -
> public KeyDataStreamOutput build() {
> return new KeyDataStreamOutput(
> clientConfig,
> @@ -570,8 +546,7 @@ public KeyDataStreamOutput build() {
> multipartUploadID,
> multipartNumber,
> isMultipartKey,
> - unsafeByteBufferConversion,
> - atomicKeyCreation);
> + unsafeByteBufferConversion);
> }
>
> }
> diff --git
> a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
>
> b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
> index 2f9edfa94e..791709eb52 100644
> ---
> a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
> +++
> b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java
> @@ -98,13 +98,6 @@ public class KeyOutputStream extends OutputStream
> private long clientID;
> private StreamBufferArgs streamBufferArgs;
>
> - /**
> - * Indicates if an atomic write is required. When set to true,
> - * the amount of data written must match the declared size during the
> commit.
> - * A mismatch will prevent the commit from succeeding.
> - * This is essential for operations like S3 put to ensure atomicity.
> - */
> - private boolean atomicKeyCreation;
> private ContainerClientMetrics clientMetrics;
> private OzoneManagerVersion ozoneManagerVersion;
> private final Lock writeLock = new ReentrantLock();
> @@ -186,7 +179,6 @@ public KeyOutputStream(Builder b) {
> this.isException = false;
> this.writeOffset = 0;
> this.clientID = b.getOpenHandler().getId();
> - this.atomicKeyCreation = b.getAtomicKeyCreation();
> this.streamBufferArgs = b.getStreamBufferArgs();
> this.clientMetrics = b.getClientMetrics();
> this.ozoneManagerVersion = b.ozoneManagerVersion;
> @@ -656,12 +648,6 @@ private void closeInternal() throws IOException {
> if (!isException) {
> Preconditions.checkArgument(writeOffset == offset);
> }
> - if (atomicKeyCreation) {
> - long expectedSize = blockOutputStreamEntryPool.getDataSize();
> - Preconditions.checkState(expectedSize == offset,
> - String.format("Expected: %d and actual %d write sizes do not
> match",
> - expectedSize, offset));
> - }
> for (CheckedRunnable<IOException> preCommit : preCommits) {
> preCommit.run();
> }
> @@ -701,7 +687,6 @@ public static class Builder {
> private OzoneClientConfig clientConfig;
> private ReplicationConfig replicationConfig;
> private ContainerClientMetrics clientMetrics;
> - private boolean atomicKeyCreation = false;
> private StreamBufferArgs streamBufferArgs;
> private Supplier<ExecutorService> executorServiceSupplier;
> private OzoneManagerVersion ozoneManagerVersion;
> @@ -800,11 +785,6 @@ public Builder setReplicationConfig(ReplicationConfig
> replConfig) {
> return this;
> }
>
> - public Builder setAtomicKeyCreation(boolean atomicKey) {
> - this.atomicKeyCreation = atomicKey;
> - return this;
> - }
> -
> public Builder setClientMetrics(ContainerClientMetrics clientMetrics) {
> this.clientMetrics = clientMetrics;
> return this;
> @@ -814,10 +794,6 @@ public ContainerClientMetrics getClientMetrics() {
> return clientMetrics;
> }
>
> - public boolean getAtomicKeyCreation() {
> - return atomicKeyCreation;
> - }
> -
> public Builder setExecutorServiceSupplier(Supplier<ExecutorService>
> executorServiceSupplier) {
> this.executorServiceSupplier = executorServiceSupplier;
> return this;
> diff --git
> a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
>
> b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
> index c52ac5d188..007f7fdc64 100644
> ---
> a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
> +++
> b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
> @@ -2579,7 +2579,6 @@ private OzoneDataStreamOutput
> createDataStreamOutput(OpenKeySession openKey,
> KeyDataStreamOutput keyOutputStream = newKeyOutputStreamBuilder()
> .setHandler(openKey)
> .setReplicationConfig(replicationConfig)
> - .setAtomicKeyCreation(isS3GRequest.get() && hasKnownLength)
> .build();
> keyOutputStream.addPreallocateBlocks(
> openKey.getKeyInfo().getLatestVersionLocations(),
> @@ -2594,15 +2593,12 @@ private OzoneDataStreamOutput
> createDataStreamOutput(OpenKeySession openKey,
> }
>
> private KeyDataStreamOutput.Builder newKeyOutputStreamBuilder() {
> - // Amazon S3 never adds partial objects, So for S3 requests we need to
> - // set atomicKeyCreation to true
> // refer:
> https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
> return new KeyDataStreamOutput.Builder()
> .setXceiverClientManager(xceiverClientManager)
> .setOmClient(ozoneManagerClient)
> .enableUnsafeByteBufferConversion(unsafeByteBufferConversion)
> - .setConfig(clientConfig)
> - .setAtomicKeyCreation(isS3GRequest.get());
> + .setConfig(clientConfig);
> }
>
> private OzoneOutputStream createOutputStream(OpenKeySession openKey)
> @@ -2687,7 +2683,6 @@ private KeyOutputStream.Builder createKeyOutputStream(
> .setOmClient(ozoneManagerClient)
> .enableUnsafeByteBufferConversion(unsafeByteBufferConversion)
> .setConfig(clientConfig)
> - .setAtomicKeyCreation(isS3GRequest.get() && hasKnownLength)
> .setClientMetrics(clientMetrics)
> .setExecutorServiceSupplier(writeExecutor)
> .setStreamBufferArgs(streamBufferArgs)
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
