[GitHub] [hbase] ZhaoBQ commented on a change in pull request #528: HBASE-22890 Verify the files when RegionServer is starting and BucketCache is in file mode

[GitBox]

ZhaoBQ commented on a change in pull request #528: HBASE-22890 Verify the files 
when RegionServer is starting and BucketCache is in file mode
URL: https://github.com/apache/hbase/pull/528#discussion_r324593190
 
 

 ##########
 File path: 
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
 ##########
 @@ -19,46 +19,96 @@
 package org.apache.hadoop.hbase.io.hfile.bucket;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.ObjectInputStream;
 import java.io.RandomAccessFile;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedByInterruptException;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.FileChannel;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
 import java.util.concurrent.locks.ReentrantLock;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
 
 /**
  * IO engine that stores data to a file on the local file system.
  */
 @InterfaceAudience.Private
-public class FileIOEngine implements IOEngine {
+public class FileIOEngine implements PersistentIOEngine {
   private static final Log LOG = LogFactory.getLog(FileIOEngine.class);
   public static final String FILE_DELIMITER = ",";
+  private static final DuFileCommand DU = new DuFileCommand(new String[] 
{"du", ""});
+
   private final String[] filePaths;
   private final FileChannel[] fileChannels;
   private final RandomAccessFile[] rafs;
   private final ReentrantLock[] channelLocks;
 
   private final long sizePerFile;
   private final long capacity;
+  private final String algorithmName;
+  private boolean oldVersion;
 
   private FileReadAccessor readAccessor = new FileReadAccessor();
   private FileWriteAccessor writeAccessor = new FileWriteAccessor();
 
-  public FileIOEngine(long capacity, String... filePaths) throws IOException {
+  public FileIOEngine(String algorithmName, String persistentPath,
+    long capacity, String... filePaths) throws IOException {
     this.sizePerFile = capacity / filePaths.length;
     this.capacity = this.sizePerFile * filePaths.length;
     this.filePaths = filePaths;
     this.fileChannels = new FileChannel[filePaths.length];
     this.rafs = new RandomAccessFile[filePaths.length];
     this.channelLocks = new ReentrantLock[filePaths.length];
+    this.algorithmName = algorithmName;
+    verifyFileIntegrity(persistentPath);
+    init();
+  }
+
+  /**
+   * Verify cache files's integrity
+   * @param persistentPath the backingMap persistent path
+   */
+  @Override
+  public void verifyFileIntegrity(String persistentPath) {
+    if (persistentPath != null) {
+      byte[] persistentChecksum = readPersistentChecksum(persistentPath);
+      if (!oldVersion) {
+        try {
+          byte[] calculateChecksum = calculateChecksum();
+          if (!Bytes.equals(persistentChecksum, calculateChecksum)) {
+            LOG.warn("The persistent checksum is " + 
Bytes.toString(persistentChecksum) +
+              ", but the calculate checksum is " + 
Bytes.toString(calculateChecksum));
+            throw new IOException();
+          }
+        } catch (IOException ioex) {
+          LOG.error("File verification failed because of ", ioex);
+          // delete cache files and backingMap persistent file.
+          deleteCacheDataFile();
+          new File(persistentPath).delete();
+        } catch (NoSuchAlgorithmException nsae) {
+          LOG.error("No such algorithm " + algorithmName, nsae);
+          throw new RuntimeException(nsae);
+        }
+      }
+    } else {
+      // not configure persistent path
+      deleteCacheDataFile();
 
 Review comment:
   The cache files would be created in init() method. `rafs[i] = new 
RandomAccessFile(filePath, "rw");` will create new file if it not exist.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services