This is an automated email from the ASF dual-hosted git repository.
hemant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 69f700bbcf HDDS-9236. Fix snapdiff output for key modification (#5258)
69f700bbcf is described below
commit 69f700bbcf5eb90afcf26dd59065e1f73ea29c1a
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Thu Sep 14 12:54:20 2023 -0700
HDDS-9236. Fix snapdiff output for key modification (#5258)
---
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 51 +++++++++++++++-------
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 43 ++++++++++++++++++
.../ozone/om/snapshot/SnapshotDiffManager.java | 16 ++++++-
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 13 +++++-
4 files changed, 105 insertions(+), 18 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index 57bcd39f6f..b3be2306ec 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -760,6 +760,41 @@ public final class OmKeyInfo extends WithParentObjectId
'}';
}
+
+ public boolean isKeyInfoSame(OmKeyInfo omKeyInfo, boolean checkPath,
+ boolean checkKeyLocationVersions,
+ boolean checkModificationTime,
+ boolean checkUpdateID) {
+ boolean isEqual = dataSize == omKeyInfo.dataSize &&
+ creationTime == omKeyInfo.creationTime &&
+ volumeName.equals(omKeyInfo.volumeName) &&
+ bucketName.equals(omKeyInfo.bucketName) &&
+ replicationConfig.equals(omKeyInfo.replicationConfig) &&
+ Objects.equals(metadata, omKeyInfo.metadata) &&
+ Objects.equals(acls, omKeyInfo.acls) &&
+ objectID == omKeyInfo.objectID;
+
+ if (isEqual && checkUpdateID) {
+ isEqual = updateID == omKeyInfo.updateID;
+ }
+
+ if (isEqual && checkModificationTime) {
+ isEqual = modificationTime == omKeyInfo.modificationTime;
+ }
+
+ if (isEqual && checkPath) {
+ isEqual = parentObjectID == omKeyInfo.parentObjectID &&
+ keyName.equals(omKeyInfo.keyName);
+ }
+
+ if (isEqual && checkKeyLocationVersions) {
+ isEqual = Objects
+ .equals(keyLocationVersions, omKeyInfo.keyLocationVersions);
+ }
+
+ return isEqual;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -768,21 +803,7 @@ public final class OmKeyInfo extends WithParentObjectId
if (o == null || getClass() != o.getClass()) {
return false;
}
- OmKeyInfo omKeyInfo = (OmKeyInfo) o;
- return dataSize == omKeyInfo.dataSize &&
- creationTime == omKeyInfo.creationTime &&
- modificationTime == omKeyInfo.modificationTime &&
- volumeName.equals(omKeyInfo.volumeName) &&
- bucketName.equals(omKeyInfo.bucketName) &&
- keyName.equals(omKeyInfo.keyName) &&
- Objects
- .equals(keyLocationVersions, omKeyInfo.keyLocationVersions) &&
- replicationConfig.equals(omKeyInfo.replicationConfig) &&
- Objects.equals(metadata, omKeyInfo.metadata) &&
- Objects.equals(acls, omKeyInfo.acls) &&
- objectID == omKeyInfo.objectID &&
- updateID == omKeyInfo.updateID &&
- parentObjectID == omKeyInfo.parentObjectID;
+ return isKeyInfoSame((OmKeyInfo) o, true, true, true, true);
}
@Override
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index dea28e03d8..566a75ec7c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.ObjectStore;
@@ -63,6 +64,8 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.service.SnapshotDiffCleanupService;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
+import org.apache.hadoop.ozone.security.acl.OzoneObj;
+import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
@@ -101,6 +104,7 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
+import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT;
import static
org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isDone;
import static
org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isStarting;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
@@ -114,6 +118,8 @@ import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION;
import static
org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
+import static
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType.USER;
+import static
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType.WRITE;
import static
org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_ALREADY_CANCELLED_JOB;
import static
org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_JOB_NOT_EXIST;
import static
org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage.CANCEL_SUCCEEDED;
@@ -1047,6 +1053,43 @@ public class TestOmSnapshot {
Assert.assertEquals(diff.getDiffList(), diffEntries);
}
+ private OzoneObj buildKeyObj(OzoneBucket bucket, String key) {
+ return OzoneObjInfo.Builder.newBuilder()
+ .setResType(OzoneObj.ResourceType.KEY)
+ .setStoreType(OzoneObj.StoreType.OZONE)
+ .setVolumeName(bucket.getVolumeName())
+ .setBucketName(bucket.getName())
+ .setKeyName(key).build();
+ }
+
+ @Test
+ public void testSnapdiffWithObjectMetaModification() throws Exception {
+ String testVolumeName = "vol" + counter.incrementAndGet();
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String snap1 = "snap1";
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ OzoneObj keyObj = buildKeyObj(bucket, key1);
+ OzoneAcl userAcl = new OzoneAcl(USER, "user",
+ WRITE, DEFAULT);
+ store.addAcl(keyObj, userAcl);
+
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ SnapshotDiffReport diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap2);
+ List<DiffReportEntry> diffEntries = Lists.newArrayList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.MODIFY,
+ key1));
+ Assert.assertEquals(diffEntries, diff.getDiffList());
+ }
+
@Test
public void testSnapdiffWithFilesystemCreate()
throws IOException, URISyntaxException, InterruptedException,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index 9067873ce9..9e82b11413 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -1467,6 +1467,19 @@ public class SnapshotDiffManager implements
AutoCloseable {
}
}
+ /**
+ * Checks if the key has been modified b/w snapshots.
+ * @param fromKey Key info in source snapshot.
+ * @param toKey Key info in target snapshot.
+ * @return true if key is modified otherwise false.
+ */
+ private boolean isKeyModified(OmKeyInfo fromKey, OmKeyInfo toKey) {
+ return !fromKey.isKeyInfoSame(toKey,
+ false, false, false, false)
+ || !SnapshotDeletingService.isBlockLocationInfoSame(
+ fromKey, toKey);
+ }
+
private boolean isObjectModified(String fromObjectName, String toObjectName,
final Table<String, ? extends WithObjectID> fromSnapshotTable,
final Table<String, ? extends WithObjectID> toSnapshotTable)
@@ -1477,8 +1490,7 @@ public class SnapshotDiffManager implements AutoCloseable
{
final WithObjectID fromObject = fromSnapshotTable.get(fromObjectName);
final WithObjectID toObject = toSnapshotTable.get(toObjectName);
if ((fromObject instanceof OmKeyInfo) && (toObject instanceof OmKeyInfo)) {
- return !SnapshotDeletingService.isBlockLocationInfoSame(
- (OmKeyInfo) fromObject, (OmKeyInfo) toObject);
+ return isKeyModified((OmKeyInfo) fromObject, (OmKeyInfo) toObject);
} else if ((fromObject instanceof OmDirectoryInfo)
&& (toObject instanceof OmDirectoryInfo)) {
return !areAclsSame((OmDirectoryInfo) fromObject,
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
index 7a07f97f56..f01b2bc9c4 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
@@ -811,14 +811,25 @@ public class TestSnapshotDiffManager {
.thenAnswer(i -> {
int keyVal = Integer.parseInt(((OmKeyInfo)i.getArgument(0))
.getKeyName().substring(3));
- return !(keyVal % 4 == 2 && keyVal >= 0 && keyVal <= 25);
+ return !(keyVal % 4 == 2 && keyVal >= 0 && keyVal <= 25
+ && keyVal % 8 == 0);
});
+
Table<String, OmKeyInfo> fromSnapTable = mock(Table.class);
Table<String, OmKeyInfo> toSnapTable = mock(Table.class);
when(fromSnapTable.get(anyString())).thenAnswer(i -> {
OmKeyInfo keyInfo = mock(OmKeyInfo.class);
Mockito.when(keyInfo.getKeyName()).thenReturn(i.getArgument(0));
+ Mockito.when(keyInfo.isKeyInfoSame(Mockito.any(OmKeyInfo.class),
+ Mockito.eq(false), Mockito.eq(false),
+ Mockito.eq(false), Mockito.eq(false)))
+ .thenAnswer(k -> {
+ int keyVal = Integer.parseInt(((String)i.getArgument(0))
+ .substring(3));
+ return !(keyVal % 4 == 2 && keyVal >= 0 && keyVal <= 25 &&
+ keyVal % 8 != 0);
+ });
return keyInfo;
});
when(toSnapTable.get(anyString())).thenAnswer(i -> {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
