rich7420 commented on code in PR #10955:
URL: https://github.com/apache/ozone/pull/10955#discussion_r4503633012
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java:
##########
@@ -1443,4 +1479,259 @@ protected KeyArgs
validateAndRewriteIfMatchAsExpectedGeneration(
.clearExpectedETag()
.build();
}
+
+ /**
+ * Supersedes the key's current version with a delete marker: a record with
+ * no data blocks that makes plain reads of the key return KEY_NOT_FOUND
+ * while every existing version stays readable by versionId. The superseded
+ * current version becomes a noncurrent version; a record written before
+ * versioning was enabled carries no versionId and becomes the key's null
+ * version. When the key does not exist the marker is still inserted, as S3
+ * does.
+ *
+ * <p>Applies everything to the table cache and to the bucket's quota, and
+ * returns what a response has to put into its WriteBatch. Shared so that a
+ * single delete and a batch delete insert the same marker.
+ *
+ * @param currentVersion the key's current version, or null if it has none
+ * @param proposedVersionId the id the request proposed for the marker
+ * @param modificationTime the time to stamp the marker with
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ protected DeleteMarkerInsertion insertDeleteMarker(OzoneManager ozoneManager,
+ OMMetadataManager omMetadataManager, OmBucketInfo omBucketInfo,
+ OmKeyInfo currentVersion, String objectKey, String keyName,
+ long proposedVersionId, long modificationTime, long trxnLogIndex,
+ long pendingNamespace)
+ throws IOException {
+
+ String volumeName = omBucketInfo.getVolumeName();
+ String bucketName = omBucketInfo.getBucketName();
+ // While versioning is suspended the marker is the key's null version, so
+ // it replaces whatever held that slot instead of superseding it.
+ final boolean suspended = omBucketInfo.isS3VersioningSuspended();
+ final boolean replacesCurrent = suspended && currentVersion != null
+ && currentVersion.isNullVersionRecord();
+
+ long markerVersionId = ozoneManager.getVersionIdAllocator().allocate(
+ proposedVersionId, currentVersion);
+
+ // Everything that can fail runs before the first cache entry is added: a
+ // request that throws here is answered with an OMKeyDeleteResponse, whose
+ // cleanup does not cover the versionedKeyTable, so a cache entry left
+ // behind would never be removed and would outlive the failed request.
+ // The null version the marker replaces is removed: the current one when
+ // the last write was also suspended, and a noncurrent one when versioning
+ // was enabled in between. Resolved before the quota check so that the
+ // check sees what the marker removes as well as what it adds.
+ final Pair<String, OmKeyInfo> noncurrentNullVersion =
+ suspended && !replacesCurrent
+ ? getNoncurrentNullVersion(ozoneManager,
+ omMetadataManager, volumeName, bucketName, keyName)
+ : null;
+ final OmKeyInfo replacedNullVersion = replacesCurrent ? currentVersion
+ : (noncurrentNullVersion == null
+ ? null : noncurrentNullVersion.getValue());
+ // The marker is a record of its own; it holds no blocks, so it consumes
+ // namespace but no space - unless it takes the null version slot, which
+ // replaces a record rather than adding one.
+ // The check also counts what earlier markers of the same request have
+ // claimed, which the bucket does not show yet.
+ final long addedNamespace = replacedNullVersion == null ? 1L : 0L;
+ checkBucketQuotaInNamespace(omBucketInfo,
+ pendingNamespace + addedNamespace);
+
+ // The marker is built from a stated set of fields rather than copied from
+ // the version it supersedes. It keeps the key's identity - objectID,
+ // owner, replication and ACLs - and holds nothing else: no blocks, no
+ // metadata or tags, no encryption info or checksum, which describe content
+ // a marker does not have. Listing the fields rather than copying and
+ // clearing also keeps a field added to OmKeyInfo off markers by default.
+ OmKeyInfo.Builder markerBuilder = new OmKeyInfo.Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setOmKeyLocationInfos(Collections.singletonList(
+ new OmKeyLocationInfoGroup(0, new ArrayList<>())))
+ .setDataSize(0L)
+ .setCreationTime(modificationTime)
+ .setModificationTime(modificationTime)
+ .setUpdateID(trxnLogIndex)
+ .setFile(true)
+ .setVersionId(markerVersionId)
+ .setDeleteMarker(true)
+ .setNullVersion(suspended);
+ if (currentVersion != null) {
+ markerBuilder
+ .setObjectID(currentVersion.getObjectID())
+ .setOwnerName(currentVersion.getOwnerName())
+ .setReplicationConfig(currentVersion.getReplicationConfig())
+ .setAcls(currentVersion.getAcls());
+ } else {
+ // S3 inserts a delete marker for a key that does not exist, so the
+ // marker has no identity to inherit and gets one of its own.
+ markerBuilder
+ .setObjectID(ozoneManager.getObjectIdFromTxId(trxnLogIndex))
+ .setOwnerName(omBucketInfo.getOwner())
+ .setReplicationConfig(RatisReplicationConfig.getInstance(
+ ReplicationFactor.ONE));
+ }
+ OmKeyInfo deleteMarker = markerBuilder.build();
+
+ String movedVersionedKeyName = null;
+ OmKeyInfo movedVersionedKeyInfo = null;
+ if (currentVersion != null && !replacesCurrent) {
+ movedVersionedKeyInfo = currentVersion.getVersionId() != null
+ ? currentVersion
+ : currentVersion.toBuilder()
+ .setVersionId(VersionIdGenerator.UNSET_VERSION_ID)
+ .setNullVersion(true)
+ .build();
+ movedVersionedKeyName = omMetadataManager.getVersionedOzoneKey(
+ volumeName, bucketName, keyName,
+ movedVersionedKeyInfo.getVersionId());
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ movedVersionedKeyName, movedVersionedKeyInfo, trxnLogIndex);
+ }
+
+ String replacedNullVersionKey = null;
+ if (noncurrentNullVersion != null) {
+ replacedNullVersionKey = noncurrentNullVersion.getKey();
+ omMetadataManager.getVersionedKeyTable().addCacheEntry(
+ new CacheKey<>(replacedNullVersionKey),
+ CacheValue.get(trxnLogIndex));
+ }
+ Map<String, RepeatedOmKeyInfo> keysToDelete = null;
+ if (replacedNullVersion != null) {
+ keysToDelete = addKeyInfoToDeleteMap(ozoneManager, trxnLogIndex,
+ objectKey, omBucketInfo.getObjectID(),
Review Comment:
Can null replacement reuse permanent deletion’s pending accounting and skip
blockless records? Repeated suspended DELETE currently queues an empty
committed marker without pending namespace; purge then makes the counter
negative.
##########
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java:
##########
@@ -303,29 +311,86 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
}
validateAtomicRewrite(keyToDelete, omKeyInfo, auditMap);
+ final boolean versioningEnabled = omBucketInfo.isS3VersioningEnabled();
+ final boolean keepsVersions = omBucketInfo.hasEverBeenVersioned();
+ // A write while versioning is suspended creates no version of its own:
it
+ // takes the key's null version slot, replacing whatever held it. The
+ // record still carries a generated versionId, so that it sorts among the
+ // key's versions by the time it was written.
+ final boolean suspendedWrite = omBucketInfo.isS3VersioningSuspended();
// Set the UpdateID to current transactionLogIndex
- omKeyInfo = omKeyInfo.toBuilder()
+ OmKeyInfo.Builder committedKeyBuilder = omKeyInfo.toBuilder()
.setExpectedDataGeneration(null)
.addAllMetadata(KeyValueUtil.getFromProtobuf(
commitKeyArgs.getMetadataList()))
.setUpdateID(trxnLogIndex)
- .setDataSize(commitKeyArgs.getDataSize())
- .build();
+ .setDataSize(commitKeyArgs.getDataSize());
+ if (keepsVersions) {
+ // The version identity is frozen when the version is created: an hsync
+ // re-commit keeps updating the same version, so it keeps its
versionId.
+ committedKeyBuilder.setVersionId(isSameHsyncKey
+ ? keyToDelete.getVersionId()
+ : ozoneManager.getVersionIdAllocator().allocate(
+ commitKeyArgs.getProposedVersionId(), keyToDelete));
+ committedKeyBuilder.setNullVersion(suspendedWrite);
Review Comment:
Can hsync preserve the null-version flag alongside its frozen version ID?
Suspending the bucket mid-write currently makes that numbered version
replaceable, so the next DELETE destroys it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]