ZhaoBQ commented on a change in pull request #528: HBASE-22890 Verify the files when RegionServer is starting and BucketCache is in file mode URL: https://github.com/apache/hbase/pull/528#discussion_r319764613
########## File path: hbase-server/src/main/java/org/apache/hadoop/hbase/util/FileIOEngineUtils.java ########## @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.util; + +import org.apache.hadoop.util.Shell; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public final class FileIOEngineUtils { + + private FileIOEngineUtils() { + } + + /** + * When HBase is stoping, a key will be written to persistencePath + * @param persistencePath the backingMap persistence path + * @return the key + * @throws IOException something happened like file not exists + */ + public static String getPreFilesKey(String persistencePath) throws IOException { + try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(persistencePath))) { + ois.readLong(); + String result = ois.readUTF(); + return result; + } + } + + /** + * ReadLong from persistencePath + * @param persistencePath the backingMap persistence path + * @return -1 means has key + */ + public static long readLongFormPersistencePath(String persistencePath) { + if (persistencePath == null) { + return 0; + } + try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(persistencePath))) { + Long result = ois.readLong(); + return result; + } catch (IOException e) { + return 0; + } + } + /** + * delete bucketcache files + * @param filePaths the cache data files + */ + public static void deleteCacheDataFile(String[] filePaths) { + if (filePaths == null) { + return; + } + for (String file : filePaths) { + File file1 = new File(file); + file1.delete(); + } + } + + /** + * Using an encryption algorithm to get a key, the default encryption algorithm is MD5 + * @param filePaths the cache data files + * @param algorithmName the encryption algorithm + * @return the key which is convert to HexString + * @throws IOException something happened like file not exists + * @throws NoSuchAlgorithmException no such algorithm + */ + public static String getFilesKey(String[] filePaths, String algorithmName) + throws IOException, NoSuchAlgorithmException { + if (filePaths == null) { + return null; + } + StringBuffer sb = new StringBuffer(); + for (String filePath : filePaths){ + File file = new File(filePath); + if (file.exists()){ + sb.append((getFileSize(filePath)+""+file.lastModified())); + } else { + throw new IOException("Cache file " + filePath + " is not exists."); + } + } + MessageDigest messageDigest = MessageDigest.getInstance(algorithmName); + messageDigest.update(sb.toString().getBytes()); + return bytes2HexString(messageDigest.digest()); + } + + /** + * Using Linux command du to get file's real size + * @param filePath the file + * @return file's real size + * @throws IOException something happened like file not exists + */ + private static long getFileSize(String filePath) throws IOException { + String[] commands = {"du",filePath}; + Shell.ShellCommandExecutor shell = new Shell.ShellCommandExecutor(commands); Review comment: ShellCommandExecutor's constructor should have a “command” parameter, so every getFileSize is different. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
