Repository: hbase Updated Branches: refs/heads/master 97cb1d71b -> 3584537b0
HBASE-16463 Improve transparent table/CF encryption with Commons Crypto (Dapeng Sun) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3584537b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3584537b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3584537b Branch: refs/heads/master Commit: 3584537b0776b7393d24141dd624fa7551591964 Parents: 97cb1d7 Author: Ramkrishna <ramkrishna.s.vasude...@intel.com> Authored: Mon Oct 24 16:22:30 2016 +0530 Committer: Ramkrishna <ramkrishna.s.vasude...@intel.com> Committed: Mon Oct 24 16:22:30 2016 +0530 ---------------------------------------------------------------------- .../apache/hadoop/hbase/io/crypto/Cipher.java | 8 + .../hbase/io/crypto/CryptoCipherProvider.java | 76 +++++++++ .../apache/hadoop/hbase/io/crypto/aes/AES.java | 7 - .../hbase/io/crypto/aes/CommonsCryptoAES.java | 166 +++++++++++++++++++ .../crypto/aes/CommonsCryptoAESDecryptor.java | 84 ++++++++++ .../crypto/aes/CommonsCryptoAESEncryptor.java | 98 +++++++++++ .../hbase/io/crypto/aes/TestCommonsAES.java | 131 +++++++++++++++ .../hbase/HFilePerformanceEvaluation.java | 47 +++++- 8 files changed, 604 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java index beda267..e19a13d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/Cipher.java @@ -31,6 +31,14 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; @InterfaceStability.Evolving public abstract class Cipher { + public static final int KEY_LENGTH = 16; + public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8; + public static final int BLOCK_SIZE = 16; + public static final int IV_LENGTH = 16; + + public static final String RNG_ALGORITHM_KEY = "hbase.crypto.algorithm.rng"; + public static final String RNG_PROVIDER_KEY = "hbase.crypto.algorithm.rng.provider"; + private final CipherProvider provider; public Cipher(CipherProvider provider) { http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CryptoCipherProvider.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CryptoCipherProvider.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CryptoCipherProvider.java new file mode 100644 index 0000000..3f5cd2d --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/CryptoCipherProvider.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hbase.io.crypto; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.io.crypto.aes.CommonsCryptoAES; + +/** + * The default cipher provider. Supports AES via the Commons Crypto. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class CryptoCipherProvider implements CipherProvider { + + private static CryptoCipherProvider instance; + + public static CryptoCipherProvider getInstance() { + if (instance != null) { + return instance; + } + instance = new CryptoCipherProvider(); + return instance; + } + + private Configuration conf = HBaseConfiguration.create(); + + // Prevent instantiation + private CryptoCipherProvider() { } + + @Override + public Configuration getConf() { + return conf; + } + + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public String getName() { + return "commons"; + } + + @Override + public Cipher getCipher(String name) { + if (name.equalsIgnoreCase("AES")) { + return new CommonsCryptoAES(this); + } + throw new RuntimeException("Cipher '" + name + "' is not supported by provider '" + + getName() + "'"); + } + + @Override + public String[] getSupportedCiphers() { + return new String[] { "AES" }; + } + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java index 03e3161..302091f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/AES.java @@ -51,15 +51,8 @@ public class AES extends Cipher { private static final Log LOG = LogFactory.getLog(AES.class); - public static final int KEY_LENGTH = 16; - public static final int KEY_LENGTH_BITS = KEY_LENGTH * 8; - public static final int BLOCK_SIZE = 16; - public static final int IV_LENGTH = 16; - public static final String CIPHER_MODE_KEY = "hbase.crypto.algorithm.aes.mode"; public static final String CIPHER_PROVIDER_KEY = "hbase.crypto.algorithm.aes.provider"; - public static final String RNG_ALGORITHM_KEY = "hbase.crypto.algorithm.rng"; - public static final String RNG_PROVIDER_KEY = "hbase.crypto.algorithm.rng.provider"; private final String rngAlgorithm; private final String cipherMode; http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAES.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAES.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAES.java new file mode 100644 index 0000000..7ea4e63 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAES.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.crypto.aes; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.security.GeneralSecurityException; +import java.security.Key; +import java.security.SecureRandom; +import java.util.Properties; + +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.crypto.cipher.CryptoCipherFactory; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.io.crypto.Cipher; +import org.apache.hadoop.hbase.io.crypto.CipherProvider; +import org.apache.hadoop.hbase.io.crypto.Context; +import org.apache.hadoop.hbase.io.crypto.Decryptor; +import org.apache.hadoop.hbase.io.crypto.Encryptor; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class CommonsCryptoAES extends Cipher { + + private static final Log LOG = LogFactory.getLog(CommonsCryptoAES.class); + + public static final String CIPHER_MODE_KEY = "hbase.crypto.commons.mode"; + public static final String CIPHER_CLASSES_KEY = "hbase.crypto.commons.cipher.classes"; + public static final String CIPHER_JCE_PROVIDER_KEY = "hbase.crypto.commons.cipher.jce.provider"; + + private final String cipherMode; + private Properties props; + private final String rngAlgorithm; + private SecureRandom rng; + + public CommonsCryptoAES(CipherProvider provider) { + super(provider); + // The mode for Commons Crypto Ciphers + cipherMode = provider.getConf().get(CIPHER_MODE_KEY, "AES/CTR/NoPadding"); + // Reads Commons Crypto properties from HBase conf + props = readCryptoProps(provider.getConf()); + // RNG algorithm + rngAlgorithm = provider.getConf().get(RNG_ALGORITHM_KEY, "SHA1PRNG"); + // RNG provider, null if default + String rngProvider = provider.getConf().get(RNG_PROVIDER_KEY); + try { + if (rngProvider != null) { + rng = SecureRandom.getInstance(rngAlgorithm, rngProvider); + } else { + rng = SecureRandom.getInstance(rngAlgorithm); + } + } catch (GeneralSecurityException e) { + LOG.warn("Could not instantiate specified RNG, falling back to default", e); + rng = new SecureRandom(); + } + } + + private static Properties readCryptoProps(Configuration conf) { + Properties props = new Properties(); + + props.setProperty(CryptoCipherFactory.CLASSES_KEY, conf.get(CIPHER_CLASSES_KEY, "")); + props.setProperty(CryptoCipherFactory.JCE_PROVIDER_KEY, conf.get(CIPHER_JCE_PROVIDER_KEY, "")); + + return props; + } + + @Override + public String getName() { + return "AES"; + } + + @Override + public int getKeyLength() { + return KEY_LENGTH; + } + + @Override + public int getIvLength() { + return IV_LENGTH; + } + + @Override + public Key getRandomKey() { + byte[] keyBytes = new byte[getKeyLength()]; + rng.nextBytes(keyBytes); + return new SecretKeySpec(keyBytes, getName()); + } + + @Override + public Encryptor getEncryptor() { + return new CommonsCryptoAESEncryptor(cipherMode, props, rng); + } + + @Override + public Decryptor getDecryptor() { + return new CommonsCryptoAESDecryptor(cipherMode, props); + } + + @Override + public OutputStream createEncryptionStream(OutputStream out, Context context, + byte[] iv) throws IOException { + Preconditions.checkNotNull(context); + Preconditions.checkState(context.getKey() != null, "Context does not have a key"); + Preconditions.checkNotNull(iv); + Encryptor e = getEncryptor(); + e.setKey(context.getKey()); + e.setIv(iv); + return e.createEncryptionStream(out); + } + + @Override + public OutputStream createEncryptionStream(OutputStream out, + Encryptor encryptor) throws + IOException { + return encryptor.createEncryptionStream(out); + } + + @Override + public InputStream createDecryptionStream(InputStream in, Context context, + byte[] iv) throws IOException { + Preconditions.checkNotNull(context); + Preconditions.checkState(context.getKey() != null, "Context does not have a key"); + Preconditions.checkNotNull(iv); + Decryptor d = getDecryptor(); + d.setKey(context.getKey()); + d.setIv(iv); + return d.createDecryptionStream(in); + } + + @Override + public InputStream createDecryptionStream(InputStream in, + Decryptor decryptor) throws + IOException { + Preconditions.checkNotNull(decryptor); + return decryptor.createDecryptionStream(in); + } + + @VisibleForTesting + SecureRandom getRNG() { + return rng; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESDecryptor.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESDecryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESDecryptor.java new file mode 100644 index 0000000..e33e366 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESDecryptor.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.crypto.aes; + +import com.google.common.base.Preconditions; +import org.apache.commons.crypto.stream.CryptoInputStream; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.io.crypto.Decryptor; + +import javax.crypto.spec.IvParameterSpec; +import java.io.IOException; +import java.io.InputStream; +import java.security.Key; +import java.util.Properties; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class CommonsCryptoAESDecryptor implements Decryptor { + + private String cipherMode; + private Properties properties; + private Key key; + private byte[] iv; + + public CommonsCryptoAESDecryptor(String cipherMode, Properties properties) { + this.cipherMode = cipherMode; + this.properties = properties; + } + + @Override + public void setKey(Key key) { + Preconditions.checkNotNull(key, "Key cannot be null"); + this.key = key; + } + + @Override + public int getIvLength() { + return CommonsCryptoAES.IV_LENGTH; + } + + @Override + public int getBlockSize() { + return CommonsCryptoAES.BLOCK_SIZE; + } + + @Override + public void setIv(byte[] iv) { + Preconditions.checkNotNull(iv, "IV cannot be null"); + Preconditions.checkArgument(iv.length == CommonsCryptoAES.IV_LENGTH, "Invalid IV length"); + this.iv = iv; + } + + @Override + public InputStream createDecryptionStream(InputStream in) { + try { + return new CryptoInputStream(cipherMode, properties, in, key, new + IvParameterSpec(iv)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void reset() { + ; + } + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESEncryptor.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESEncryptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESEncryptor.java new file mode 100644 index 0000000..346cd79 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/aes/CommonsCryptoAESEncryptor.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.crypto.aes; + +import com.google.common.base.Preconditions; +import org.apache.commons.crypto.stream.CryptoOutputStream; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.io.crypto.Encryptor; + +import javax.crypto.spec.IvParameterSpec; +import java.io.IOException; +import java.io.OutputStream; +import java.security.Key; +import java.security.SecureRandom; +import java.util.Properties; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class CommonsCryptoAESEncryptor implements Encryptor { + + private String cipherMode; + private Properties properties; + private Key key; + private byte[] iv; + private boolean initialized = false; + private SecureRandom rng; + + public CommonsCryptoAESEncryptor(String cipherMode, Properties properties, SecureRandom rng) { + this.cipherMode = cipherMode; + this.properties = properties; + this.rng = rng; + } + + @Override + public void setKey(Key key) { + this.key = key; + } + + @Override + public int getIvLength() { + return CommonsCryptoAES.IV_LENGTH; + } + + @Override + public int getBlockSize() { + return CommonsCryptoAES.BLOCK_SIZE; + } + + @Override + public byte[] getIv() { + return iv; + } + + @Override + public void setIv(byte[] iv) { + Preconditions.checkNotNull(iv, "IV cannot be null"); + Preconditions.checkArgument(iv.length == CommonsCryptoAES.IV_LENGTH, "Invalid IV length"); + this.iv = iv; + } + + @Override + public OutputStream createEncryptionStream(OutputStream out) { + if (!initialized) { + reset(); + } + try { + return new CryptoOutputStream(cipherMode, properties, out, key, new + IvParameterSpec(iv)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void reset() { + if (iv == null) { + iv = new byte[getIvLength()]; + rng.nextBytes(iv); + } + initialized = true; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestCommonsAES.java ---------------------------------------------------------------------- diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestCommonsAES.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestCommonsAES.java new file mode 100644 index 0000000..dca62e5 --- /dev/null +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/aes/TestCommonsAES.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hbase.io.crypto.aes; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.io.crypto.Cipher; +import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider; +import org.apache.hadoop.hbase.io.crypto.Encryption; +import org.apache.hadoop.hbase.io.crypto.Encryptor; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.security.*; + +import static org.junit.Assert.*; + +@Category({MiscTests.class, SmallTests.class}) +public class TestCommonsAES { + + // Validation for AES in CTR mode with a 128 bit key + // From NIST Special Publication 800-38A + @Test + public void testAESAlgorithm() throws Exception { + Configuration conf = HBaseConfiguration.create(); + Cipher aes = Encryption.getCipher(conf, "AES"); + assertEquals(aes.getKeyLength(), CommonsCryptoAES.KEY_LENGTH); + assertEquals(aes.getIvLength(), CommonsCryptoAES.IV_LENGTH); + Encryptor e = aes.getEncryptor(); + e.setKey(new SecretKeySpec(Bytes.fromHex("2b7e151628aed2a6abf7158809cf4f3c"), "AES")); + e.setIv(Bytes.fromHex("f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff")); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + OutputStream cout = e.createEncryptionStream(out); + cout.write(Bytes.fromHex("6bc1bee22e409f96e93d7e117393172a")); + cout.write(Bytes.fromHex("ae2d8a571e03ac9c9eb76fac45af8e51")); + cout.write(Bytes.fromHex("30c81c46a35ce411e5fbc1191a0a52ef")); + cout.write(Bytes.fromHex("f69f2445df4f9b17ad2b417be66c3710")); + cout.close(); + + ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray()); + byte[] b = new byte[16]; + IOUtils.readFully(in, b); + assertTrue("Failed #1", Bytes.equals(b, Bytes.fromHex("874d6191b620e3261bef6864990db6ce"))); + IOUtils.readFully(in, b); + assertTrue("Failed #2", Bytes.equals(b, Bytes.fromHex("9806f66b7970fdff8617187bb9fffdff"))); + IOUtils.readFully(in, b); + assertTrue("Failed #3", Bytes.equals(b, Bytes.fromHex("5ae4df3edbd5d35e5b4f09020db03eab"))); + IOUtils.readFully(in, b); + assertTrue("Failed #4", Bytes.equals(b, Bytes.fromHex("1e031dda2fbe03d1792170a0f3009cee"))); + } + + @Test + public void testAlternateRNG() throws Exception { + Security.addProvider(new TestProvider()); + + Configuration conf = new Configuration(); + conf.set(AES.RNG_ALGORITHM_KEY, "TestRNG"); + conf.set(AES.RNG_PROVIDER_KEY, "TEST"); + DefaultCipherProvider.getInstance().setConf(conf); + + AES aes = new AES(DefaultCipherProvider.getInstance()); + assertEquals("AES did not find alternate RNG", aes.getRNG().getAlgorithm(), + "TestRNG"); + } + + static class TestProvider extends Provider { + private static final long serialVersionUID = 1L; + public TestProvider() { + super("TEST", 1.0, "Test provider"); + AccessController.doPrivileged(new PrivilegedAction<Object>() { + public Object run() { + put("SecureRandom.TestRNG", TestCommonsAES.class.getName() + "$TestRNG"); + return null; + } + }); + } + } + + // Must be public for instantiation by the SecureRandom SPI + public static class TestRNG extends SecureRandomSpi { + private static final long serialVersionUID = 1L; + private SecureRandom rng; + + public TestRNG() { + try { + rng = SecureRandom.getInstance("SHA1PRNG"); + } catch (NoSuchAlgorithmException e) { + fail("Unable to create SecureRandom instance"); + } + } + + @Override + protected void engineSetSeed(byte[] seed) { + rng.setSeed(seed); + } + + @Override + protected void engineNextBytes(byte[] bytes) { + rng.nextBytes(bytes); + } + + @Override + protected byte[] engineGenerateSeed(int numBytes) { + return rng.generateSeed(numBytes); + } + } + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/3584537b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java index e5aec57..562630a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java @@ -31,6 +31,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.hadoop.hbase.io.crypto.CryptoCipherProvider; +import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider; import org.apache.hadoop.hbase.io.crypto.Encryption; import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.io.crypto.aes.AES; @@ -130,6 +132,23 @@ public class HFilePerformanceEvaluation { runWriteBenchmark(aesconf, aesfs, aesmf, "gz", "aes"); runReadBenchmark(aesconf, aesfs, aesmf, "gz", "aes"); + // Add configuration for Commons cipher + final Configuration cryptoconf = new Configuration(); + cryptoconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName()); + cryptoconf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase"); + cryptoconf.setInt("hfile.format.version", 3); + cryptoconf.set(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY, CryptoCipherProvider.class.getName()); + final FileSystem cryptofs = FileSystem.get(cryptoconf); + final Path cryptof = cryptofs.makeQualified(new Path("performanceevaluation.aes.mapfile")); + + // codec=none cipher=aes + runWriteBenchmark(cryptoconf, cryptofs, aesmf, "none", "aes"); + runReadBenchmark(cryptoconf, cryptofs, aesmf, "none", "aes"); + + // codec=gz cipher=aes + runWriteBenchmark(cryptoconf, aesfs, aesmf, "gz", "aes"); + runReadBenchmark(cryptoconf, aesfs, aesmf, "gz", "aes"); + // cleanup test files if (fs.exists(mf)) { fs.delete(mf, true); @@ -137,7 +156,10 @@ public class HFilePerformanceEvaluation { if (aesfs.exists(aesmf)) { aesfs.delete(aesmf, true); } - + if (cryptofs.exists(aesmf)) { + cryptofs.delete(cryptof, true); + } + // Print Result Summary LOG.info("\n***************\n" + "Result Summary" + "\n***************\n"); LOG.info(testSummary.toString()); @@ -160,7 +182,7 @@ public class HFilePerformanceEvaluation { } runBenchmark(new SequentialWriteBenchmark(conf, fs, mf, ROW_COUNT, codec, cipher), - ROW_COUNT, codec, cipher); + ROW_COUNT, codec, getCipherName(conf, cipher)); } @@ -179,7 +201,7 @@ public class HFilePerformanceEvaluation { public void run() { try { runBenchmark(new UniformRandomSmallScan(conf, fs, mf, ROW_COUNT), - ROW_COUNT, codec, cipher); + ROW_COUNT, codec, getCipherName(conf, cipher)); } catch (Exception e) { testSummary.append("UniformRandomSmallScan failed " + e.getMessage()); e.printStackTrace(); @@ -192,7 +214,7 @@ public class HFilePerformanceEvaluation { public void run() { try { runBenchmark(new UniformRandomReadBenchmark(conf, fs, mf, ROW_COUNT), - ROW_COUNT, codec, cipher); + ROW_COUNT, codec, getCipherName(conf, cipher)); } catch (Exception e) { testSummary.append("UniformRandomReadBenchmark failed " + e.getMessage()); e.printStackTrace(); @@ -205,7 +227,7 @@ public class HFilePerformanceEvaluation { public void run() { try { runBenchmark(new GaussianRandomReadBenchmark(conf, fs, mf, ROW_COUNT), - ROW_COUNT, codec, cipher); + ROW_COUNT, codec, getCipherName(conf, cipher)); } catch (Exception e) { testSummary.append("GaussianRandomReadBenchmark failed " + e.getMessage()); e.printStackTrace(); @@ -218,7 +240,7 @@ public class HFilePerformanceEvaluation { public void run() { try { runBenchmark(new SequentialReadBenchmark(conf, fs, mf, ROW_COUNT), - ROW_COUNT, codec, cipher); + ROW_COUNT, codec, getCipherName(conf, cipher)); } catch (Exception e) { testSummary.append("SequentialReadBenchmark failed " + e.getMessage()); e.printStackTrace(); @@ -530,4 +552,17 @@ public class HFilePerformanceEvaluation { public static void main(String[] args) throws Exception { new HFilePerformanceEvaluation().runBenchmarks(); } + + private String getCipherName(Configuration conf, String cipherName) { + if (cipherName.equals("aes")) { + String provider = conf.get(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY); + if (provider == null || provider.equals("") + || provider.equals(DefaultCipherProvider.class.getName())) { + return "aes-default"; + } else if (provider.equals(CryptoCipherProvider.class.getName())) { + return "aes-commons"; + } + } + return cipherName; + } }