This is an automated email from the ASF dual-hosted git repository.
hemant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e03e9f39f7 HDDS-8904. Fix Snapdiff output for key renames (#5177)
e03e9f39f7 is described below
commit e03e9f39f7cb08f81c4ccf4abab8e068a8e56e62
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Thu Aug 31 10:23:11 2023 -0700
HDDS-8904. Fix Snapdiff output for key renames (#5177)
---
.../apache/hadoop/hdds/utils/db/BooleanCodec.java | 74 ++++++++++
.../apache/hadoop/hdds/utils/db/CodecBuffer.java | 11 ++
.../apache/hadoop/hdds/utils/db/CodecRegistry.java | 3 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 2 +-
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 2 -
.../ozone/om/snapshot/SnapshotDiffManager.java | 99 +++++--------
.../ozone/om/snapshot/SnapshotDiffObject.java | 164 ---------------------
.../ozone/om/snapshot/TestSnapshotDiffManager.java | 154 ++++++++++---------
8 files changed, 209 insertions(+), 300 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/BooleanCodec.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/BooleanCodec.java
new file mode 100644
index 0000000000..dc1090e5b8
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/BooleanCodec.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.utils.db;
+
+import javax.annotation.Nonnull;
+import java.util.function.IntFunction;
+
+/**
+ * Codec to serialize/deserialize {@link Boolean}.
+ */
+public final class BooleanCodec implements Codec<Boolean> {
+
+ private static final byte TRUE = 1;
+ private static final byte FALSE = 0;
+ private static final BooleanCodec INSTANCE = new BooleanCodec();
+
+ public static BooleanCodec get() {
+ return INSTANCE;
+ }
+
+ private BooleanCodec() {
+ // singleton
+ }
+
+ @Override
+ public boolean supportCodecBuffer() {
+ return true;
+ }
+
+ @Override
+ public CodecBuffer toCodecBuffer(Boolean object,
+ IntFunction<CodecBuffer> allocator) {
+ return allocator.apply(1).put(TRUE);
+ }
+
+ @Override
+ public Boolean fromCodecBuffer(@Nonnull CodecBuffer buffer) {
+ return buffer.asReadOnlyByteBuffer().get() == 1;
+ }
+
+ @Override
+ public byte[] toPersistedFormat(Boolean object) {
+ return object ? new byte[]{TRUE} : new byte[]{FALSE};
+ }
+
+ @Override
+ public Boolean fromPersistedFormat(byte[] rawData) {
+ if (rawData.length != 1) {
+ throw new IllegalStateException("Byte Buffer for boolean should be of " +
+ "length 1 but provided byte array of length " + rawData.length);
+ }
+ return rawData[0] == 1;
+ }
+
+ @Override
+ public Boolean copyObject(Boolean object) {
+ return object;
+ }
+}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
index 8254022a30..6a404c3e75 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/CodecBuffer.java
@@ -289,6 +289,17 @@ public final class CodecBuffer implements AutoCloseable {
return this;
}
+ /**
+ * Similar to {@link ByteBuffer#put(byte)}.
+ *
+ * @return this object.
+ */
+ public CodecBuffer put(byte val) {
+ assertRefCnt(1);
+ buf.writeByte(val);
+ return this;
+ }
+
/**
* Similar to {@link ByteBuffer#put(byte[])}.
*
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/CodecRegistry.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/CodecRegistry.java
index 34e1d1c8fa..5e767537d4 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/CodecRegistry.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/CodecRegistry.java
@@ -52,7 +52,8 @@ public final class CodecRegistry {
.addCodec(String.class, StringCodec.get())
.addCodec(Long.class, LongCodec.get())
.addCodec(Integer.class, IntegerCodec.get())
- .addCodec(byte[].class, ByteArrayCodec.get());
+ .addCodec(byte[].class, ByteArrayCodec.get())
+ .addCodec(Boolean.class, BooleanCodec.get());
}
private static final class CodecMap {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index e592c17262..dea28e03d8 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -987,7 +987,7 @@ public class TestOmSnapshot {
key1);
Assert.assertEquals(diff.getDiffList(), Arrays.asList(
SnapshotDiffReportOzone.getDiffReportEntry(
- SnapshotDiffReport.DiffType.MODIFY, key1)));
+ SnapshotDiffReport.DiffType.RENAME, key1, key1)));
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 7465319888..1bcaefccfc 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.service.SnapshotDiffCleanupService;
-import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffObject;
import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted;
import org.apache.hadoop.ozone.om.snapshot.SnapshotCache;
@@ -392,7 +391,6 @@ public final class OmSnapshotManager implements
AutoCloseable {
registry.addCodec(SnapshotDiffReportOzone.DiffReportEntry.class,
SnapshotDiffReportOzone.getDiffReportEntryCodec());
registry.addCodec(SnapshotDiffJob.class, SnapshotDiffJob.getCodec());
- registry.addCodec(SnapshotDiffObject.class, SnapshotDiffObject.getCodec());
return registry.build();
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index 7ed8fa46e8..b8299541f0 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.WithObjectID;
import org.apache.hadoop.ozone.om.helpers.WithParentObjectId;
import org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
-import
org.apache.hadoop.ozone.om.snapshot.SnapshotDiffObject.SnapshotDiffObjectBuilder;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
@@ -894,9 +893,9 @@ public class SnapshotDiffManager implements AutoCloseable {
new RocksDbPersistentMap<>(db, toSnapshotColumnFamily, codecRegistry,
byte[].class, byte[].class);
// Set of unique objectId between fromSnapshot and toSnapshot.
- final PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject =
+ final PersistentMap<byte[], Boolean> objectIdToIsDirMap =
new RocksDbPersistentMap<>(db, objectIDsColumnFamily, codecRegistry,
- byte[].class, SnapshotDiffObject.class);
+ byte[].class, Boolean.class);
final BucketLayout bucketLayout = getBucketLayout(volumeName, bucketName,
fromSnapshot.getMetadataManager());
@@ -950,7 +949,7 @@ public class SnapshotDiffManager implements AutoCloseable {
fromSnapshot, toSnapshot, fsInfo, tsInfo, useFullDiff,
performNonNativeDiff, tablePrefixes,
objectIdToKeyNameMapForFromSnapshot,
- objectIdToKeyNameMapForToSnapshot, objectIdToDiffObject,
+ objectIdToKeyNameMapForToSnapshot, objectIdToIsDirMap,
oldParentIds, newParentIds, path.toString());
return null;
},
@@ -960,7 +959,7 @@ public class SnapshotDiffManager implements AutoCloseable {
fromSnapshot, toSnapshot, fsInfo, tsInfo, useFullDiff,
performNonNativeDiff, tablePrefixes,
objectIdToKeyNameMapForFromSnapshot,
- objectIdToKeyNameMapForToSnapshot, objectIdToDiffObject,
+ objectIdToKeyNameMapForToSnapshot, objectIdToIsDirMap,
oldParentIds, newParentIds, path.toString());
}
return null;
@@ -989,13 +988,13 @@ public class SnapshotDiffManager implements AutoCloseable
{
tsKeyTable,
fsDirTable,
tsDirTable,
- objectIdToDiffObject,
+ objectIdToIsDirMap,
objectIdToKeyNameMapForFromSnapshot,
objectIdToKeyNameMapForToSnapshot,
volumeName, bucketName,
fromSnapshotName, toSnapshotName,
bucketLayout.isFileSystemOptimized(), oldParentIdPathMap,
- newParentIdPathMap);
+ newParentIdPathMap, tablePrefixes);
// If job is cancelled, totalDiffEntries will be equal to -1.
if (totalDiffEntries >= 0 &&
areDiffJobAndSnapshotsActive(volumeName, bucketName,
@@ -1057,7 +1056,7 @@ public class SnapshotDiffManager implements AutoCloseable
{
final Map<String, String> tablePrefixes,
final PersistentMap<byte[], byte[]> oldObjIdToKeyMap,
final PersistentMap<byte[], byte[]> newObjIdToKeyMap,
- final PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject,
+ final PersistentMap<byte[], Boolean> objectIdToIsDirMap,
final Optional<Set<Long>> oldParentIds,
final Optional<Set<Long>> newParentIds,
final String diffDir) throws IOException, RocksDBException {
@@ -1082,7 +1081,7 @@ public class SnapshotDiffManager implements AutoCloseable
{
!skipNativeDiff && sstDumpTool.isPresent(),
oldObjIdToKeyMap,
newObjIdToKeyMap,
- objectIdToDiffObject,
+ objectIdToIsDirMap,
oldParentIds,
newParentIds,
tablePrefixes);
@@ -1100,7 +1099,7 @@ public class SnapshotDiffManager implements AutoCloseable
{
false,
oldObjIdToKeyMap,
newObjIdToKeyMap,
- objectIdToDiffObject,
+ objectIdToIsDirMap,
oldParentIds,
newParentIds,
tablePrefixes);
@@ -1117,7 +1116,7 @@ public class SnapshotDiffManager implements AutoCloseable
{
Set<String> deltaFiles, boolean nativeRocksToolsLoaded,
PersistentMap<byte[], byte[]> oldObjIdToKeyMap,
PersistentMap<byte[], byte[]> newObjIdToKeyMap,
- PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject,
+ PersistentMap<byte[], Boolean> objectIdToIsDirMap,
Optional<Set<Long>> oldParentIds,
Optional<Set<Long>> newParentIds,
Map<String, String> tablePrefixes) throws IOException,
@@ -1154,10 +1153,7 @@ public class SnapshotDiffManager implements
AutoCloseable {
byte[] rawValue = codecRegistry.asRawData(
key.substring(tablePrefix.length()));
oldObjIdToKeyMap.put(rawObjId, rawValue);
- SnapshotDiffObject diffObject =
- createDiffObjectWithOldName(fromObjectId.getObjectID(), key,
- objectIdToDiffObject.get(rawObjId), isDirectoryTable);
- objectIdToDiffObject.put(rawObjId, diffObject);
+ objectIdToIsDirMap.put(rawObjId, isDirectoryTable);
oldParentIds.ifPresent(set -> set.add(
fromObjectId.getParentObjectID()));
}
@@ -1166,10 +1162,7 @@ public class SnapshotDiffManager implements
AutoCloseable {
byte[] rawValue = codecRegistry.asRawData(
key.substring(tablePrefix.length()));
newObjIdToKeyMap.put(rawObjId, rawValue);
- SnapshotDiffObject diffObject =
- createDiffObjectWithNewName(toObjectId.getObjectID(), key,
- objectIdToDiffObject.get(rawObjId), isDirectoryTable);
- objectIdToDiffObject.put(rawObjId, diffObject);
+ objectIdToIsDirMap.put(rawObjId, isDirectoryTable);
newParentIds.ifPresent(set -> set.add(toObjectId
.getParentObjectID()));
}
@@ -1184,29 +1177,6 @@ public class SnapshotDiffManager implements
AutoCloseable {
}
}
- private SnapshotDiffObject createDiffObjectWithOldName(
- long objectId, String oldName, SnapshotDiffObject diffObject,
- boolean isDirectory) {
- SnapshotDiffObjectBuilder builder = new SnapshotDiffObjectBuilder(objectId)
- .withOldKeyName(oldName).setIsDirectory(isDirectory);
- if (diffObject != null) {
- builder.withNewKeyName(diffObject.getNewKeyName());
- }
- return builder.build();
- }
-
- private SnapshotDiffObject createDiffObjectWithNewName(
- long objectId, String newName, SnapshotDiffObject diffObject,
- boolean isDirectory) {
-
- SnapshotDiffObjectBuilder builder = new SnapshotDiffObjectBuilder(objectId)
- .withNewKeyName(newName).setIsDirectory(isDirectory);
- if (diffObject != null) {
- builder.withOldKeyName(diffObject.getOldKeyName());
- }
- return builder.build();
- }
-
@VisibleForTesting
@SuppressWarnings("checkstyle:ParameterNumber")
Set<String> getDeltaFiles(OmSnapshot fromSnapshot,
@@ -1311,7 +1281,7 @@ public class SnapshotDiffManager implements AutoCloseable
{
final Table<String, OmKeyInfo> tsTable,
final Table<String, OmDirectoryInfo> fsDirTable,
final Table<String, OmDirectoryInfo> tsDirTable,
- final PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject,
+ final PersistentMap<byte[], Boolean> objectIdToIsDirMap,
final PersistentMap<byte[], byte[]> oldObjIdToKeyMap,
final PersistentMap<byte[], byte[]> newObjIdToKeyMap,
final String volumeName,
@@ -1320,7 +1290,8 @@ public class SnapshotDiffManager implements AutoCloseable
{
final String toSnapshotName,
final boolean isFSOBucket,
final Optional<Map<Long, Path>> oldParentIdPathMap,
- final Optional<Map<Long, Path>> newParentIdPathMap) {
+ final Optional<Map<Long, Path>> newParentIdPathMap,
+ final Map<String, String> tablePrefix) {
LOG.info("Starting diff report generation for jobId: {}.", jobId);
ColumnFamilyHandle deleteDiffColumnFamily = null;
ColumnFamilyHandle renameDiffColumnFamily = null;
@@ -1349,8 +1320,8 @@ public class SnapshotDiffManager implements AutoCloseable
{
final PersistentList<byte[]> modifyDiffs =
createDiffReportPersistentList(modifyDiffColumnFamily);
- try (ClosableIterator<Map.Entry<byte[], SnapshotDiffObject>>
- iterator = objectIdToDiffObject.iterator()) {
+ try (ClosableIterator<Map.Entry<byte[], Boolean>>
+ iterator = objectIdToIsDirMap.iterator()) {
// This counter is used, so that we can check every 100 elements
// if the job is cancelled and snapshots are still active.
int counter = 0;
@@ -1361,9 +1332,9 @@ public class SnapshotDiffManager implements AutoCloseable
{
return -1L;
}
- Map.Entry<byte[], SnapshotDiffObject> nextEntry = iterator.next();
+ Map.Entry<byte[], Boolean> nextEntry = iterator.next();
byte[] id = nextEntry.getKey();
- SnapshotDiffObject snapshotDiffObject = nextEntry.getValue();
+ boolean isDirectoryObject = nextEntry.getValue();
/*
* This key can be
@@ -1397,32 +1368,38 @@ public class SnapshotDiffManager implements
AutoCloseable {
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(DELETE, key);
deleteDiffs.add(codecRegistry.asRawData(entry));
- } else if (Arrays.equals(oldKeyName, newKeyName)) { // Key modified.
+ } else if (isDirectoryObject &&
+ Arrays.equals(oldKeyName, newKeyName)) {
String key = resolveBucketRelativePath(isFSOBucket,
newParentIdPathMap, newKeyName);
DiffReportEntry entry =
SnapshotDiffReportOzone.getDiffReportEntry(MODIFY, key);
modifyDiffs.add(codecRegistry.asRawData(entry));
- } else { // Key Renamed.
+ } else {
+ String keyPrefix = getTablePrefix(tablePrefix,
+ (isDirectoryObject ? fsDirTable : fsTable).getName());
String oldKey = resolveBucketRelativePath(isFSOBucket,
oldParentIdPathMap, oldKeyName);
- String newKey = resolveBucketRelativePath(isFSOBucket,
- newParentIdPathMap, newKeyName);
- renameDiffs.add(codecRegistry.asRawData(
- SnapshotDiffReportOzone.getDiffReportEntry(RENAME, oldKey,
- newKey)));
-
// Check if block location is same or not. If it is not same,
// key must have been overridden as well.
- if (isObjectModified(snapshotDiffObject.getOldKeyName(),
- snapshotDiffObject.getNewKeyName(),
- snapshotDiffObject.isDirectory() ? fsDirTable : fsTable,
- snapshotDiffObject.isDirectory() ? tsDirTable : tsTable)) {
+ boolean isObjectModified = isObjectModified(
+ keyPrefix + codecRegistry.asObject(oldKeyName, String.class),
+ keyPrefix + codecRegistry.asObject(newKeyName, String.class),
+ isDirectoryObject ? fsDirTable : fsTable,
+ isDirectoryObject ? tsDirTable : tsTable);
+ if (isObjectModified) {
// Here, oldKey name is returned as modified. Modified key name
is
// based on base snapshot (from snapshot).
- renameDiffs.add(codecRegistry.asRawData(
+ modifyDiffs.add(codecRegistry.asRawData(
SnapshotDiffReportOzone.getDiffReportEntry(MODIFY, oldKey)));
}
+ if (!isObjectModified || !Arrays.equals(oldKeyName, newKeyName)) {
+ String newKey = resolveBucketRelativePath(isFSOBucket,
+ newParentIdPathMap, newKeyName);
+ renameDiffs.add(codecRegistry.asRawData(
+ SnapshotDiffReportOzone.getDiffReportEntry(RENAME, oldKey,
+ newKey)));
+ }
}
counter++;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffObject.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffObject.java
deleted file mode 100644
index c99757f364..0000000000
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffObject.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.om.snapshot;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.hadoop.hdds.utils.db.Codec;
-
-import java.io.IOException;
-
-/**
- * SnapshotDiffObject to keep objectId, key's oldName and newName.
- */
-public class SnapshotDiffObject {
-
- private static final Codec<SnapshotDiffObject> CODEC =
- new SnapshotDiffObjectCodec();
-
- public static Codec<SnapshotDiffObject> getCodec() {
- return CODEC;
- }
-
- private long objectId;
- private String oldKeyName;
- private String newKeyName;
-
- private boolean isDirectory;
-
- // Default constructor for Jackson Serializer.
- public SnapshotDiffObject() {
-
- }
-
- public SnapshotDiffObject(long objectId,
- String oldKeyName,
- String newKeyName,
- boolean isDirectory) {
- this.objectId = objectId;
- this.oldKeyName = oldKeyName;
- this.newKeyName = newKeyName;
- this.isDirectory = isDirectory;
- }
-
- public long getObjectId() {
- return objectId;
- }
-
- public String getOldKeyName() {
- return oldKeyName;
- }
-
- public String getNewKeyName() {
- return newKeyName;
- }
-
- // For Jackson Serializer.
- public void setObjectId(long objectId) {
- this.objectId = objectId;
- }
-
- // For Jackson Serializer.
- public void setOldKeyName(String oldKeyName) {
- this.oldKeyName = oldKeyName;
- }
-
- // For Jackson Serializer.
- public void setNewKeyName(String newKeyName) {
- this.newKeyName = newKeyName;
- }
-
- public void setDirectory(boolean directory) {
- isDirectory = directory;
- }
-
- public boolean isDirectory() {
- return isDirectory;
- }
-
- /**
- * Builder for SnapshotDiffObject.
- */
- public static class SnapshotDiffObjectBuilder {
- private final long objectId;
- private String oldKeyName;
- private String newKeyName;
-
- private boolean isDirectory;
-
- public SnapshotDiffObjectBuilder(long objectID) {
- this.objectId = objectID;
- }
-
- public static SnapshotDiffObjectBuilder newBuilder(long objectID) {
- return new SnapshotDiffObjectBuilder(objectID);
- }
-
- public SnapshotDiffObjectBuilder withOldKeyName(String keyName) {
- this.oldKeyName = keyName;
- return this;
- }
-
- public SnapshotDiffObjectBuilder withNewKeyName(String keyName) {
- this.newKeyName = keyName;
- return this;
- }
-
- public SnapshotDiffObjectBuilder setIsDirectory(boolean directory) {
- isDirectory = directory;
- return this;
- }
-
- public SnapshotDiffObject build() {
- return new SnapshotDiffObject(objectId, oldKeyName, newKeyName,
- isDirectory);
- }
- }
-
-
- /**
- * Codec to encode KeyObject as byte array.
- */
- private static final class SnapshotDiffObjectCodec
- implements Codec<SnapshotDiffObject> {
-
- private static final ObjectMapper MAPPER = new ObjectMapper()
- .setSerializationInclusion(JsonInclude.Include.NON_NULL)
- .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-
- @Override
- public byte[] toPersistedFormat(SnapshotDiffObject object)
- throws IOException {
- return MAPPER.writeValueAsBytes(object);
- }
-
- @Override
- public SnapshotDiffObject fromPersistedFormat(byte[] rawData)
- throws IOException {
- return MAPPER.readValue(rawData, SnapshotDiffObject.class);
- }
-
- @Override
- public SnapshotDiffObject copyObject(SnapshotDiffObject object) {
- // Note: Not really a "copy".
- return object;
- }
- }
-}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
index 3fa2e9d0a5..d2b9bdffe5 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java
@@ -54,7 +54,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.WithParentObjectId;
-import
org.apache.hadoop.ozone.om.snapshot.SnapshotDiffObject.SnapshotDiffObjectBuilder;
+import org.apache.hadoop.ozone.om.service.SnapshotDeletingService;
import
org.apache.hadoop.ozone.om.snapshot.SnapshotTestUtils.StubbedPersistentMap;
import org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse;
import
org.apache.hadoop.ozone.snapshot.CancelSnapshotDiffResponse.CancelMessage;
@@ -690,7 +690,7 @@ public class TestSnapshotDiffManager {
new StubbedPersistentMap<>();
PersistentMap<byte[], byte[]> newObjectIdKeyMap =
new SnapshotTestUtils.StubbedPersistentMap<>();
- PersistentMap<byte[], SnapshotDiffObject> objectIdsToCheck =
+ PersistentMap<byte[], Boolean> objectIdsToCheck =
new SnapshotTestUtils.StubbedPersistentMap<>();
Set<Long> oldParentIds = Sets.newHashSet();
@@ -733,11 +733,11 @@ public class TestSnapshotDiffManager {
assertEquals(12, newObjectIdCnt);
}
- try (ClosableIterator<Entry<byte[], SnapshotDiffObject>>
+ try (ClosableIterator<Entry<byte[], Boolean>>
objectIdsToCheckIter = objectIdsToCheck.iterator()) {
int objectIdCnt = 0;
while (objectIdsToCheckIter.hasNext()) {
- Entry<byte[], SnapshotDiffObject> entry =
objectIdsToCheckIter.next();
+ Entry<byte[], Boolean> entry = objectIdsToCheckIter.next();
byte[] v = entry.getKey();
long objectId = codecRegistry.asObject(v, Long.class);
assertEquals(0, objectId % 2);
@@ -756,40 +756,38 @@ public class TestSnapshotDiffManager {
new StubbedPersistentMap<>();
PersistentMap<byte[], byte[]> newObjectIdKeyMap =
new StubbedPersistentMap<>();
- PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject =
+ PersistentMap<byte[], Boolean> objectIdToIsDirMap =
new SnapshotTestUtils.StubbedPersistentMap<>();
Map<Long, SnapshotDiffReport.DiffType> diffMap = new HashMap<>();
LongStream.range(0, 100).forEach(objectId -> {
try {
- SnapshotDiffObjectBuilder builder =
- new SnapshotDiffObjectBuilder(objectId);
String key = "key" + objectId;
byte[] objectIdVal = codecRegistry.asRawData(objectId);
byte[] keyBytes = codecRegistry.asRawData(key);
if (objectId >= 0 && objectId <= 25 ||
objectId >= 50 && objectId <= 100) {
oldObjectIdKeyMap.put(objectIdVal, keyBytes);
- builder.withOldKeyName(key);
}
- if (objectId >= 0 && objectId <= 25 && objectId % 4 == 0 ||
+ if (objectId >= 0 && objectId <= 25 &&
+ (objectId % 4 == 0 || objectId % 4 == 2) ||
objectId > 25 && objectId < 50) {
newObjectIdKeyMap.put(objectIdVal, keyBytes);
- builder.withNewKeyName(key);
}
if (objectId >= 0 && objectId <= 25 && objectId % 4 == 1) {
String renamedKey = "renamed-key" + objectId;
byte[] renamedKeyBytes = codecRegistry.asRawData(renamedKey);
newObjectIdKeyMap.put(objectIdVal, renamedKeyBytes);
diffMap.put(objectId, SnapshotDiffReport.DiffType.RENAME);
- builder.withOldKeyName(key);
- builder.withNewKeyName(renamedKey);
}
- objectIdToDiffObject.put(objectIdVal, builder.build());
+ objectIdToIsDirMap.put(objectIdVal, false);
if (objectId >= 50 && objectId <= 100 ||
- objectId >= 0 && objectId <= 25 && objectId % 4 > 1) {
+ objectId >= 0 && objectId <= 25 && objectId % 4 > 2) {
diffMap.put(objectId, SnapshotDiffReport.DiffType.DELETE);
}
if (objectId >= 0 && objectId <= 25 && objectId % 4 == 0) {
+ diffMap.put(objectId, SnapshotDiffReport.DiffType.RENAME);
+ }
+ if (objectId >= 0 && objectId <= 25 && objectId % 4 == 2) {
diffMap.put(objectId, SnapshotDiffReport.DiffType.MODIFY);
}
if (objectId > 25 && objectId < 50) {
@@ -804,55 +802,68 @@ public class TestSnapshotDiffManager {
String bucketName = "buck";
String fromSnapName = "fs";
String toSnapName = "ts";
+ try (MockedStatic<SnapshotDeletingService>
+ mockedSnapshotDeletingService = Mockito.mockStatic(
+ SnapshotDeletingService.class)) {
+ mockedSnapshotDeletingService.when(() ->
+ SnapshotDeletingService.isBlockLocationInfoSame(any(OmKeyInfo.class),
+ any(OmKeyInfo.class)))
+ .thenAnswer(i -> {
+ int keyVal = Integer.parseInt(((OmKeyInfo)i.getArgument(0))
+ .getKeyName().substring(3));
+ return !(keyVal % 4 == 2 && keyVal >= 0 && keyVal <= 25);
+ });
- OmKeyInfo fromKeyInfo = mock(OmKeyInfo.class);
- OmKeyInfo toKeyInfo = mock(OmKeyInfo.class);
- // This is temporary to make sure that
- // SnapshotDeletingService#isBlockLocationInfoSame always return true.
- when(toKeyInfo.isHsync()).thenReturn(true);
- when(fromKeyInfo.isHsync()).thenReturn(true);
-
- Table<String, OmKeyInfo> fromSnapTable = mock(Table.class);
- Table<String, OmKeyInfo> toSnapTable = mock(Table.class);
- when(fromSnapTable.get(anyString())).thenReturn(fromKeyInfo);
- when(toSnapTable.get(anyString())).thenReturn(toKeyInfo);
-
-
- SnapshotDiffManager spy = spy(snapshotDiffManager);
- doReturn(true).when(spy)
- .areDiffJobAndSnapshotsActive(volumeName, bucketName, fromSnapName,
- toSnapName);
-
- long totalDiffEntries = spy.generateDiffReport("jobId",
- fromSnapTable, toSnapTable, null, null,
- objectIdToDiffObject, oldObjectIdKeyMap, newObjectIdKeyMap, volumeName,
- bucketName, fromSnapName, toSnapName, false, Optional.empty(),
- Optional.empty());
-
- assertEquals(100, totalDiffEntries);
- SnapshotDiffJob snapshotDiffJob = new SnapshotDiffJob(0, "jobId",
- JobStatus.DONE, "vol", "buck", "fs", "ts", false,
- true, diffMap.size());
- SnapshotDiffReportOzone snapshotDiffReportOzone =
- snapshotDiffManager.createPageResponse(snapshotDiffJob, "vol",
- "buck", "fs", "ts",
- 0, Integer.MAX_VALUE);
- Set<SnapshotDiffReport.DiffType> expectedOrder = new LinkedHashSet<>();
- expectedOrder.add(SnapshotDiffReport.DiffType.DELETE);
- expectedOrder.add(SnapshotDiffReport.DiffType.RENAME);
- expectedOrder.add(SnapshotDiffReport.DiffType.CREATE);
- expectedOrder.add(SnapshotDiffReport.DiffType.MODIFY);
-
- Set<SnapshotDiffReport.DiffType> actualOrder = new LinkedHashSet<>();
- for (DiffReportEntry entry :
- snapshotDiffReportOzone.getDiffList()) {
- actualOrder.add(entry.getType());
-
- long objectId = Long.parseLong(
- DFSUtilClient.bytes2String(entry.getSourcePath()).substring(3));
- assertEquals(diffMap.get(objectId), entry.getType());
+ Table<String, OmKeyInfo> fromSnapTable = mock(Table.class);
+ Table<String, OmKeyInfo> toSnapTable = mock(Table.class);
+ when(fromSnapTable.get(anyString())).thenAnswer(i -> {
+ OmKeyInfo keyInfo = mock(OmKeyInfo.class);
+ Mockito.when(keyInfo.getKeyName()).thenReturn(i.getArgument(0));
+ return keyInfo;
+ });
+ when(toSnapTable.get(anyString())).thenAnswer(i -> {
+ OmKeyInfo keyInfo = mock(OmKeyInfo.class);
+ Mockito.when(keyInfo.getKeyName()).thenReturn(i.getArgument(0));
+ return keyInfo;
+ });
+ when(fromSnapTable.getName()).thenReturn("table");
+ Map<String, String> tablePrefixes = Mockito.mock(Map.class);
+ Mockito.when(tablePrefixes.get(anyString())).thenReturn("");
+ SnapshotDiffManager spy = spy(snapshotDiffManager);
+ doReturn(true).when(spy)
+ .areDiffJobAndSnapshotsActive(volumeName, bucketName, fromSnapName,
+ toSnapName);
+
+ long totalDiffEntries = spy.generateDiffReport("jobId",
+ fromSnapTable, toSnapTable, null, null,
+ objectIdToIsDirMap, oldObjectIdKeyMap, newObjectIdKeyMap,
+ volumeName, bucketName, fromSnapName, toSnapName, false,
+ Optional.empty(), Optional.empty(), tablePrefixes);
+
+ assertEquals(100, totalDiffEntries);
+ SnapshotDiffJob snapshotDiffJob = new SnapshotDiffJob(0, "jobId",
+ JobStatus.DONE, "vol", "buck", "fs", "ts", false,
+ true, diffMap.size());
+ SnapshotDiffReportOzone snapshotDiffReportOzone =
+ snapshotDiffManager.createPageResponse(snapshotDiffJob, "vol",
+ "buck", "fs", "ts",
+ 0, Integer.MAX_VALUE);
+ Set<SnapshotDiffReport.DiffType> expectedOrder = new LinkedHashSet<>();
+ expectedOrder.add(SnapshotDiffReport.DiffType.DELETE);
+ expectedOrder.add(SnapshotDiffReport.DiffType.RENAME);
+ expectedOrder.add(SnapshotDiffReport.DiffType.CREATE);
+ expectedOrder.add(SnapshotDiffReport.DiffType.MODIFY);
+
+ Set<SnapshotDiffReport.DiffType> actualOrder = new LinkedHashSet<>();
+ for (DiffReportEntry entry :
+ snapshotDiffReportOzone.getDiffList()) {
+ actualOrder.add(entry.getType());
+ long objectId = Long.parseLong(
+ DFSUtilClient.bytes2String(entry.getSourcePath()).substring(3));
+ assertEquals(diffMap.get(objectId), entry.getType());
+ }
+ assertEquals(expectedOrder, actualOrder);
}
- assertEquals(expectedOrder, actualOrder);
}
private DiffReportEntry getTestDiffEntry(String jobId,
@@ -1185,19 +1196,18 @@ public class TestSnapshotDiffManager {
@Test
public void testGenerateDiffReportWhenThereInEntry() {
- PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject =
+ PersistentMap<byte[], Boolean> objectIdToIsDirectoryMap =
new StubbedPersistentMap<>();
PersistentMap<byte[], byte[]> oldObjIdToKeyMap =
new StubbedPersistentMap<>();
PersistentMap<byte[], byte[]> newObjIdToKeyMap =
new StubbedPersistentMap<>();
-
long totalDiffEntries = snapshotDiffManager.generateDiffReport("jobId",
keyInfoTable,
keyInfoTable,
null,
null,
- objectIdToDiffObject,
+ objectIdToIsDirectoryMap,
oldObjIdToKeyMap,
newObjIdToKeyMap,
"volume",
@@ -1206,7 +1216,8 @@ public class TestSnapshotDiffManager {
"toSnapshot",
false,
Optional.empty(),
- Optional.empty());
+ Optional.empty(),
+ Collections.emptyMap());
assertEquals(0, totalDiffEntries);
}
@@ -1218,14 +1229,14 @@ public class TestSnapshotDiffManager {
String fromSnapshotName = "snap-" + RandomStringUtils.randomNumeric(5);
String toSnapshotName = "snap-" + RandomStringUtils.randomNumeric(5);
- PersistentMap<byte[], SnapshotDiffObject> objectIdToDiffObject =
+ PersistentMap<byte[], Boolean> objectIdToIsDirectoryMap =
new SnapshotTestUtils.StubbedPersistentMap<>();
PersistentMap<byte[], byte[]> oldObjIdToKeyMap =
new StubbedPersistentMap<>();
PersistentMap<byte[], byte[]> newObjIdToKeyMap =
new StubbedPersistentMap<>();
- objectIdToDiffObject.put(codecRegistry.asRawData("randomKey"),
- new SnapshotDiffObjectBuilder(1L).build());
+ objectIdToIsDirectoryMap.put(codecRegistry.asRawData("randomKey"),
+ false);
SnapshotDiffManager spy = spy(snapshotDiffManager);
doReturn(true).when(spy)
@@ -1238,7 +1249,7 @@ public class TestSnapshotDiffManager {
keyInfoTable,
null,
null,
- objectIdToDiffObject,
+ objectIdToIsDirectoryMap,
oldObjIdToKeyMap,
newObjIdToKeyMap,
volumeName,
@@ -1247,7 +1258,8 @@ public class TestSnapshotDiffManager {
toSnapshotName,
false,
Optional.empty(),
- Optional.empty())
+ Optional.empty(),
+ Collections.emptyMap())
);
assertEquals("Old and new key name both are null",
exception.getMessage());
@@ -1511,7 +1523,7 @@ public class TestSnapshotDiffManager {
doReturn(10L).when(spy).generateDiffReport(anyString(),
any(), any(), any(), any(), any(), any(), any(),
anyString(), anyString(), anyString(), anyString(), anyBoolean(),
- any(), any());
+ any(), any(), anyMap());
doReturn(LEGACY).when(spy).getBucketLayout(VOLUME_NAME, BUCKET_NAME,
omMetadataManager);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
