Jackie-Jiang commented on code in PR #10639:
URL: https://github.com/apache/pinot/pull/10639#discussion_r1179596748


##########
pinot-segment-local/src/main/java/org/apache/pinot/segment/local/dedup/PartitionDedupMetadataManager.java:
##########
@@ -39,46 +44,73 @@ public class PartitionDedupMetadataManager {
   private final List<String> _primaryKeyColumns;
   private final int _partitionId;
   private final ServerMetrics _serverMetrics;
-  private final HashFunction _hashFunction;
-
-  @VisibleForTesting
-  final ConcurrentHashMap<Object, IndexSegment> _primaryKeyToSegmentMap = new 
ConcurrentHashMap<>();
+  private final DedupConfig _dedupConfig;
+  @VisibleForTesting final LocalKeyValueStore _keyValueStore;
 
   public PartitionDedupMetadataManager(String tableNameWithType, List<String> 
primaryKeyColumns, int partitionId,
-      ServerMetrics serverMetrics, HashFunction hashFunction) {
+      ServerMetrics serverMetrics, DedupConfig dedupConfig) {
     _tableNameWithType = tableNameWithType;
     _primaryKeyColumns = primaryKeyColumns;
     _partitionId = partitionId;
     _serverMetrics = serverMetrics;
-    _hashFunction = hashFunction;
+    _dedupConfig = dedupConfig;
+    try {
+      byte[] id = (tableNameWithType + "@" + partitionId).getBytes();
+      _keyValueStore = StringUtils.isEmpty(dedupConfig.getKeyStore())
+              ? new ConcurrentHashMapKeyValueStore()
+              : PluginManager.get().createInstance(
+                      dedupConfig.getKeyStore(),
+                      new Class[]{byte[].class},
+                      new Object[]{id}
+              );
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
   }
 
   public void addSegment(IndexSegment segment) {
-    // Add all PKs to _primaryKeyToSegmentMap
     Iterator<PrimaryKey> primaryKeyIterator = getPrimaryKeyIterator(segment);
+    byte[] serializedSegment = serializeSegment(segment);
+    List<Pair<byte[], byte[]>> keyValues = new ArrayList<>();
     while (primaryKeyIterator.hasNext()) {
       PrimaryKey pk = primaryKeyIterator.next();
-      _primaryKeyToSegmentMap.put(HashUtils.hashPrimaryKey(pk, _hashFunction), 
segment);
+      byte[] serializedPrimaryKey = 
serializePrimaryKey(HashUtils.hashPrimaryKey(pk, 
_dedupConfig.getHashFunction()));
+      keyValues.add(Pair.of(serializedPrimaryKey, serializedSegment));
     }
+    _keyValueStore.putBatch(keyValues);
     _serverMetrics.setValueOfPartitionGauge(_tableNameWithType, _partitionId, 
ServerGauge.DEDUP_PRIMARY_KEYS_COUNT,
-        _primaryKeyToSegmentMap.size());
+        _keyValueStore.getKeyCount());
+  }
+
+  @VisibleForTesting
+  static byte[] serializeSegment(IndexSegment segment) {
+    return segment.getSegmentName().getBytes();
+  }
+
+  @VisibleForTesting
+  static byte[] serializePrimaryKey(Object pk) {
+    if (pk instanceof PrimaryKey) {
+      return ((PrimaryKey) pk).asBytes();
+    }
+    if (pk instanceof ByteArray) {
+      return ((ByteArray) pk).getBytes();
+    }
+    throw new RuntimeException("Unsupported pk class: " + pk);
   }
 
   public void removeSegment(IndexSegment segment) {
     // TODO(saurabh): Explain reload scenario here
     Iterator<PrimaryKey> primaryKeyIterator = getPrimaryKeyIterator(segment);
+    byte[] segmentBytes = serializeSegment(segment);
     while (primaryKeyIterator.hasNext()) {
       PrimaryKey pk = primaryKeyIterator.next();
-      _primaryKeyToSegmentMap.compute(HashUtils.hashPrimaryKey(pk, 
_hashFunction), (primaryKey, currentSegment) -> {
-        if (currentSegment == segment) {
-          return null;
-        } else {
-          return currentSegment;
-        }
-      });
+      byte[] pkBytes = serializePrimaryKey(pk);
+      if (Objects.deepEquals(_keyValueStore.get(pkBytes), segmentBytes)) {

Review Comment:
   This can potentially cause race condition when the segment is added again 
during the segment removal, and that's why we perform the reference check before



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to