RANGER-2043 : Ranger KMS KeyProvider and HSM KeyProvider should have more debug logs
Signed-off-by: Mehul Parikh <me...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/bc2cd5e0 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/bc2cd5e0 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/bc2cd5e0 Branch: refs/heads/master Commit: bc2cd5e00290866eb48f5115edb0e33971e8fca3 Parents: 499650c Author: bpatel <bpa...@hortonworks.com> Authored: Wed Apr 18 11:32:34 2018 +0530 Committer: Mehul Parikh <me...@apache.org> Committed: Wed Apr 18 12:23:45 2018 +0530 ---------------------------------------------------------------------- kms/scripts/VerifyIsDBMasterkeyCorrect.sh | 26 + kms/scripts/VerifyIsHSMMasterkeyCorrect.sh | 26 + .../org/apache/hadoop/crypto/key/RangerHSM.java | 150 ++-- .../hadoop/crypto/key/RangerKeyStore.java | 857 ++++++++++--------- .../crypto/key/RangerKeyStoreProvider.java | 792 ++++++++--------- .../hadoop/crypto/key/RangerMasterKey.java | 736 +++++++++------- .../crypto/key/VerifyIsDBMasterkeyCorrect.java | 69 ++ .../crypto/key/VerifyIsHSMMasterkeyCorrect.java | 86 ++ src/main/assembly/kms.xml | 702 +++++++-------- 9 files changed, 1915 insertions(+), 1529 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/bc2cd5e0/kms/scripts/VerifyIsDBMasterkeyCorrect.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/VerifyIsDBMasterkeyCorrect.sh b/kms/scripts/VerifyIsDBMasterkeyCorrect.sh new file mode 100755 index 0000000..1c9a2e1 --- /dev/null +++ b/kms/scripts/VerifyIsDBMasterkeyCorrect.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +if [ "$JAVA_HOME" != "" ]; then + export PATH=$JAVA_HOME/bin:$PATH +else + exit ; +fi +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" + +#echo "${cp}" +java -cp "${cp}" org.apache.hadoop.crypto.key.VerifyIsDBMasterkeyCorrect ${1} http://git-wip-us.apache.org/repos/asf/ranger/blob/bc2cd5e0/kms/scripts/VerifyIsHSMMasterkeyCorrect.sh ---------------------------------------------------------------------- diff --git a/kms/scripts/VerifyIsHSMMasterkeyCorrect.sh b/kms/scripts/VerifyIsHSMMasterkeyCorrect.sh new file mode 100755 index 0000000..82eccd5 --- /dev/null +++ b/kms/scripts/VerifyIsHSMMasterkeyCorrect.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ------------------------------------------------------------------------------------- +if [ "$JAVA_HOME" != "" ]; then + export PATH=$JAVA_HOME/bin:$PATH +else + exit ; +fi +RANGER_KMS_HOME=`dirname $0` +cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF" + +#echo "${cp}" +java -cp "${cp}" org.apache.hadoop.crypto.key.VerifyIsHSMMasterkeyCorrect ${1} ${2} http://git-wip-us.apache.org/repos/asf/ranger/blob/bc2cd5e0/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java index 00dc069..a531f32 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java @@ -38,97 +38,109 @@ import java.security.cert.CertificateException; * This Class is for HSM Keystore */ public class RangerHSM implements RangerKMSMKI { - - static final Logger logger = Logger.getLogger(RangerHSM.class); - + + static final Logger logger = Logger.getLogger(RangerHSM.class); + // Configure these as required. private String passwd = null; private String alias = "RangerKMSKey"; private String partitionName = null; private KeyStore myStore = null; - private String hsm_keystore=null; + private String hsm_keystore = null; private static final String MK_CIPHER = "AES"; - private static final int MK_KeySize = 128; - private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; - private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name"; - private static final String HSM_TYPE = "ranger.ks.hsm.type"; - - public RangerHSM(){ - } + private static final int MK_KeySize = 128; + private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; + private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name"; + private static final String HSM_TYPE = "ranger.ks.hsm.type"; + + public RangerHSM() { + } public RangerHSM(Configuration conf) { - logger.info("RangerHSM provider"); + logger.info("RangerHSM provider"); /* * We will log in to the HSM */ - passwd = conf.get(PARTITION_PASSWORD); - partitionName = conf.get(PARTITION_NAME); - hsm_keystore = conf.get(HSM_TYPE); + passwd = conf.get(PARTITION_PASSWORD); + partitionName = conf.get(PARTITION_NAME); + hsm_keystore = conf.get(HSM_TYPE); try { ByteArrayInputStream is1 = new ByteArrayInputStream(("tokenlabel:" + partitionName).getBytes()); - logger.debug("Loading HSM tokenlabel : "+partitionName); + logger.debug("Loading HSM tokenlabel : " + partitionName); myStore = KeyStore.getInstance("Luna"); myStore.load(is1, passwd.toCharArray()); - if(myStore == null){ logger.error("Luna not found. Please verify the Ranger KMS HSM configuration setup."); } + if (myStore == null) { + logger.error("Luna not found. Please verify the Ranger KMS HSM configuration setup."); + } } catch (KeyStoreException kse) { - logger.error("Unable to create keystore object : "+kse.getMessage()); + logger.error("Unable to create keystore object : " + kse.getMessage()); } catch (NoSuchAlgorithmException nsae) { logger.error("Unexpected NoSuchAlgorithmException while loading keystore : " + nsae.getMessage()); } catch (CertificateException e) { - logger.error("Unexpected CertificateException while loading keystore : "+e.getMessage()); + logger.error("Unexpected CertificateException while loading keystore : " + e.getMessage()); } catch (IOException e) { - logger.error("Unexpected IOException while loading keystore : "+e.getMessage()); - } + logger.error("Unexpected IOException while loading keystore : " + e.getMessage()); + } } - @Override - public boolean generateMasterKey(String password) throws Throwable { - if(myStore != null && myStore.size() < 1){ - KeyGenerator keyGen = null; - SecretKey aesKey = null; - try { - logger.info("Generating AES Master Key for HSM Provider"); - keyGen = KeyGenerator.getInstance(MK_CIPHER, hsm_keystore); - keyGen.init(MK_KeySize); - aesKey = keyGen.generateKey(); + @Override + public boolean generateMasterKey(String password) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerHSM.generateMasterKey()"); + } + if (myStore != null && myStore.size() < 1) { + KeyGenerator keyGen = null; + SecretKey aesKey = null; + try { + logger.info("Generating AES Master Key for HSM Provider"); + keyGen = KeyGenerator.getInstance(MK_CIPHER, hsm_keystore); + keyGen.init(MK_KeySize); + aesKey = keyGen.generateKey(); myStore.setKeyEntry(alias, aesKey, password.toCharArray(), (java.security.cert.Certificate[]) null); - return true; - } catch (Exception e) { - logger.error("generateMasterKey : Exception during Ranger Master Key Generation - " + e.getMessage()); - return false; - } - } - return false; - } + return true; + } catch (Exception e) { + logger.error("generateMasterKey : Exception during Ranger Master Key Generation - " + e.getMessage()); + return false; + } + } + return false; + } - @Override - public String getMasterKey(String password) throws Throwable { - if(myStore != null){ - try { - logger.debug("Searching for Ranger Master Key in Luna Keystore"); - boolean result = myStore.containsAlias(alias); - if (result == true) { - logger.debug("Ranger Master Key is present in Keystore"); - SecretKey key = (SecretKey)myStore.getKey(alias, password.toCharArray()); - return Base64.encode(key.getEncoded()); - } - } catch (Exception e) { - logger.error("getMasterKey : Exception searching for Ranger Master Key - " + e.getMessage()); - } - } - return null; - } - - public boolean setMasterKey(String password, byte[] key){ - if(myStore != null){ - try { - Key aesKey = new SecretKeySpec(key, MK_CIPHER); - myStore.setKeyEntry(alias, aesKey, password.toCharArray(), (java.security.cert.Certificate[]) null); - return true; - } catch (KeyStoreException e) { - logger.error("setMasterKey : Exception while setting Master Key - " + e.getMessage()); - } - } - return false; - } + @Override + public String getMasterKey(String password) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerHSM.getMasterKey()"); + } + if (myStore != null) { + try { + if (logger.isDebugEnabled()) { + logger.debug("Searching for Ranger Master Key in Luna Keystore"); + } + boolean result = myStore.containsAlias(alias); + if (result == true) { + if (logger.isDebugEnabled()) { + logger.debug("Ranger Master Key is present in Keystore"); + } + SecretKey key = (SecretKey) myStore.getKey(alias, password.toCharArray()); + return Base64.encode(key.getEncoded()); + } + } catch (Exception e) { + logger.error("getMasterKey : Exception searching for Ranger Master Key - " + e.getMessage()); + } + } + return null; + } + + public boolean setMasterKey(String password, byte[] key) { + if (myStore != null) { + try { + Key aesKey = new SecretKeySpec(key, MK_CIPHER); + myStore.setKeyEntry(alias, aesKey, password.toCharArray(), (java.security.cert.Certificate[]) null); + return true; + } catch (KeyStoreException e) { + logger.error("setMasterKey : Exception while setting Master Key - " + e.getMessage()); + } + } + return false; + } } http://git-wip-us.apache.org/repos/asf/ranger/blob/bc2cd5e0/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java index e73b6d3..cd5a8c2 100644 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java @@ -72,21 +72,23 @@ import org.apache.ranger.kms.dao.RangerKMSDao; */ public class RangerKeyStore extends KeyStoreSpi { - - static final Logger logger = Logger.getLogger(RangerKeyStore.class); - private static final String KEY_NAME_VALIDATION = "[a-z,A-Z,0-9](?!.*--)(?!.*__)(?!.*-_)(?!.*_-)[\\w\\-\\_]*"; - private static final Pattern pattern = Pattern.compile(KEY_NAME_VALIDATION); - - private DaoManager daoManager; - + + static final Logger logger = Logger.getLogger(RangerKeyStore.class); + private static final String KEY_NAME_VALIDATION = "[a-z,A-Z,0-9](?!.*--)(?!.*__)(?!.*-_)(?!.*_-)[\\w\\-\\_]*"; + private static final Pattern pattern = Pattern.compile(KEY_NAME_VALIDATION); + + private DaoManager daoManager; + // keys private static class KeyEntry { - Date date=new Date(); // the creation date of this entry - }; + Date date = new Date(); // the creation date of this entry + } + + ; // Secret key private static final class SecretKeyEntry { - Date date=new Date(); // the creation date of this entry + Date date = new Date(); // the creation date of this entry SealedObject sealedKey; String cipher_field; int bit_length; @@ -102,17 +104,19 @@ public class RangerKeyStore extends KeyStoreSpi { } public RangerKeyStore(DaoManager daoManager) { - this.daoManager = daoManager; - } + this.daoManager = daoManager; + } - String convertAlias(String alias){ - return alias.toLowerCase(); + String convertAlias(String alias) { + return alias.toLowerCase(); } @Override - public Key engineGetKey(String alias, char[] password)throws NoSuchAlgorithmException, UnrecoverableKeyException - { - Key key = null; + public Key engineGetKey(String alias, char[] password) throws NoSuchAlgorithmException, UnrecoverableKeyException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.engineGetKey()"); + } + Key key = null; Object entry = keyEntries.get(convertAlias(alias)); @@ -120,34 +124,40 @@ public class RangerKeyStore extends KeyStoreSpi { return null; } - try { - key = unsealKey(((SecretKeyEntry)entry).sealedKey, password); - } catch (Exception e) { - logger.error(e.getMessage()); - } + try { + key = unsealKey(((SecretKeyEntry) entry).sealedKey, password); + } catch (Exception e) { + logger.error("==> RangerKeyStore.engineGetKey() error: ", e); + } + if (logger.isDebugEnabled()) { + logger.debug("<== RangerKeyStore.engineGetKey()"); + } return key; } @Override public Date engineGetCreationDate(String alias) { Object entry = keyEntries.get(convertAlias(alias)); - Date date=null; + Date date = null; if (entry != null) { - KeyEntry keyEntry=(KeyEntry)entry; - if(keyEntry.date!=null){ - date=new Date(keyEntry.date.getTime()); - } - } - return date; - } + KeyEntry keyEntry = (KeyEntry) entry; + if (keyEntry.date != null) { + date = new Date(keyEntry.date.getTime()); + } + } + return date; + } public void addKeyEntry(String alias, Key key, char[] password, String cipher, int bitLength, String description, int version, String attributes) - throws KeyStoreException - { - SecretKeyEntry entry = new SecretKeyEntry(); - synchronized(deltaEntries) { - try { + throws KeyStoreException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.addKeyEntry()"); + logger.debug("Adding entry for alias:" + alias); + } + SecretKeyEntry entry = new SecretKeyEntry(); + synchronized (deltaEntries) { + try { entry.date = new Date(); // seal and store the key entry.sealedKey = sealKey(key, password); @@ -159,21 +169,24 @@ public class RangerKeyStore extends KeyStoreSpi { entry.attributes = attributes; deltaEntries.put(convertAlias(alias), entry); } catch (Exception e) { - logger.error(e.getMessage()); - throw new KeyStoreException(e.getMessage()); + logger.error("==> RangerKeyStore.addKeyEntry() error: ", e); + throw new KeyStoreException(e.getMessage()); } } - synchronized(keyEntries) { - try { - keyEntries.put(convertAlias(alias), entry); - }catch (Exception e) { - logger.error(e.getMessage()); - throw new KeyStoreException(e.getMessage()); + synchronized (keyEntries) { + try { + keyEntries.put(convertAlias(alias), entry); + } catch (Exception e) { + logger.error("==> RangerKeyStore.addKeyEntry() error: ", e); + throw new KeyStoreException(e.getMessage()); } } } private SealedObject sealKey(Key key, char[] password) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.sealKey()"); + } // Create SecretKey SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndTripleDES"); PBEKeySpec pbeKeySpec = new PBEKeySpec(password); @@ -189,10 +202,16 @@ public class RangerKeyStore extends KeyStoreSpi { // Seal the Key Cipher cipher = Cipher.getInstance("PBEWithMD5AndTripleDES"); cipher.init(Cipher.ENCRYPT_MODE, secretKey, pbeSpec); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerKeyStore.sealKey()"); + } return new RangerSealedObject(key, cipher); } private Key unsealKey(SealedObject sealedKey, char[] password) throws Exception { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.unsealKey()"); + } // Create SecretKey SecretKeyFactory secretKeyFactory = SecretKeyFactory.getInstance("PBEWithMD5AndTripleDES"); PBEKeySpec pbeKeySpec = new PBEKeySpec(password); @@ -202,7 +221,7 @@ public class RangerKeyStore extends KeyStoreSpi { // Get the AlgorithmParameters from RangerSealedObject AlgorithmParameters algorithmParameters = null; if (sealedKey instanceof RangerSealedObject) { - algorithmParameters = ((RangerSealedObject)sealedKey).getParameters(); + algorithmParameters = ((RangerSealedObject) sealedKey).getParameters(); } else { algorithmParameters = new RangerSealedObject(sealedKey).getParameters(); } @@ -210,35 +229,38 @@ public class RangerKeyStore extends KeyStoreSpi { // Unseal the Key Cipher cipher = Cipher.getInstance("PBEWithMD5AndTripleDES"); cipher.init(Cipher.DECRYPT_MODE, secretKey, algorithmParameters); - - return (Key)sealedKey.getObject(cipher); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerKeyStore.unsealKey()"); + } + return (Key) sealedKey.getObject(cipher); } @Override public void engineDeleteEntry(String alias) - throws KeyStoreException - { - synchronized(keyEntries) { - dbOperationDelete(convertAlias(alias)); - keyEntries.remove(convertAlias(alias)); + throws KeyStoreException { + synchronized (keyEntries) { + dbOperationDelete(convertAlias(alias)); + keyEntries.remove(convertAlias(alias)); } - synchronized(deltaEntries) { - deltaEntries.remove(convertAlias(alias)); + synchronized (deltaEntries) { + deltaEntries.remove(convertAlias(alias)); } } private void dbOperationDelete(String alias) { - try{ - if(daoManager != null){ - RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager); - rangerKMSDao.deleteByAlias(alias); - } - }catch(Exception e){ - logger.error(e.getMessage()); - e.printStackTrace(); - } - } + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.dbOperationDelete(" + alias + ")"); + } + try { + if (daoManager != null) { + RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager); + rangerKMSDao.deleteByAlias(alias); + } + } catch (Exception e) { + logger.error("==> RangerKeyStore.dbOperationDelete() error : ", e); + } + } @Override @@ -258,9 +280,11 @@ public class RangerKeyStore extends KeyStoreSpi { @Override public void engineStore(OutputStream stream, char[] password) - throws IOException, NoSuchAlgorithmException, CertificateException - { - synchronized(deltaEntries) { + throws IOException, NoSuchAlgorithmException, CertificateException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.engineStore()"); + } + synchronized (deltaEntries) { // password is mandatory when storing if (password == null) { throw new IllegalArgumentException("Ranger Master Key can't be null"); @@ -268,27 +292,27 @@ public class RangerKeyStore extends KeyStoreSpi { MessageDigest md = getKeyedMessageDigest(password); - byte digest[] = md.digest(); - for (Enumeration<String> e = deltaEntries.keys(); e.hasMoreElements();) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte digest[] = md.digest(); + for (Enumeration<String> e = deltaEntries.keys(); e.hasMoreElements(); ) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(new DigestOutputStream(baos, md)); ObjectOutputStream oos = null; - try{ - - String alias = e.nextElement(); - Object entry = deltaEntries.get(alias); + try { + + String alias = e.nextElement(); + Object entry = deltaEntries.get(alias); oos = new ObjectOutputStream(dos); - oos.writeObject(((SecretKeyEntry)entry).sealedKey); + oos.writeObject(((SecretKeyEntry) entry).sealedKey); dos.write(digest); dos.flush(); - Long creationDate = ((SecretKeyEntry)entry).date.getTime(); - SecretKeyEntry secretKey = (SecretKeyEntry)entry; - XXRangerKeyStore xxRangerKeyStore = mapObjectToEntity(alias,creationDate,baos.toByteArray(), secretKey.cipher_field, secretKey.bit_length, secretKey.description, secretKey.version, secretKey.attributes); + Long creationDate = ((SecretKeyEntry) entry).date.getTime(); + SecretKeyEntry secretKey = (SecretKeyEntry) entry; + XXRangerKeyStore xxRangerKeyStore = mapObjectToEntity(alias, creationDate, baos.toByteArray(), secretKey.cipher_field, secretKey.bit_length, secretKey.description, secretKey.version, secretKey.attributes); dbOperationStore(xxRangerKeyStore); - }finally { + } finally { if (oos != null) { oos.close(); } else { @@ -296,165 +320,183 @@ public class RangerKeyStore extends KeyStoreSpi { } } } - clearDeltaEntires(); + clearDeltaEntires(); } } private XXRangerKeyStore mapObjectToEntity(String alias, Long creationDate, - byte[] byteArray, String cipher_field, int bit_length, - String description, int version, String attributes) { - XXRangerKeyStore xxRangerKeyStore = new XXRangerKeyStore(); - xxRangerKeyStore.setAlias(alias); - xxRangerKeyStore.setCreatedDate(creationDate); - xxRangerKeyStore.setEncoded(DatatypeConverter.printBase64Binary(byteArray)); - xxRangerKeyStore.setCipher(cipher_field); - xxRangerKeyStore.setBitLength(bit_length); - xxRangerKeyStore.setDescription(description); - xxRangerKeyStore.setVersion(version); - xxRangerKeyStore.setAttributes(attributes); - return xxRangerKeyStore; - } - - private void dbOperationStore(XXRangerKeyStore rangerKeyStore) { - try{ - if(daoManager != null){ - RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager); - XXRangerKeyStore xxRangerKeyStore = rangerKMSDao.findByAlias(rangerKeyStore.getAlias()); - boolean keyStoreExists = true; - if (xxRangerKeyStore == null) { - xxRangerKeyStore = new XXRangerKeyStore(); - keyStoreExists = false; - } - xxRangerKeyStore = mapToEntityBean(rangerKeyStore, xxRangerKeyStore); - if (keyStoreExists) { - xxRangerKeyStore = rangerKMSDao.update(xxRangerKeyStore); - } else { - xxRangerKeyStore = rangerKMSDao.create(xxRangerKeyStore); - } - } - }catch(Exception e){ - logger.error(e.getMessage()); - e.printStackTrace(); - } - } - - private XXRangerKeyStore mapToEntityBean(XXRangerKeyStore rangerKMSKeyStore, XXRangerKeyStore xxRangerKeyStore) { - xxRangerKeyStore.setAlias(rangerKMSKeyStore.getAlias()); - xxRangerKeyStore.setCreatedDate(rangerKMSKeyStore.getCreatedDate()); - xxRangerKeyStore.setEncoded(rangerKMSKeyStore.getEncoded()); - xxRangerKeyStore.setCipher(rangerKMSKeyStore.getCipher()); - xxRangerKeyStore.setBitLength(rangerKMSKeyStore.getBitLength()); - xxRangerKeyStore.setDescription(rangerKMSKeyStore.getDescription()); - xxRangerKeyStore.setVersion(rangerKMSKeyStore.getVersion()); - xxRangerKeyStore.setAttributes(rangerKMSKeyStore.getAttributes()); - return xxRangerKeyStore; - } - - - @Override - public void engineLoad(InputStream stream, char[] password) - throws IOException , NoSuchAlgorithmException, CertificateException - { - synchronized(keyEntries) { - List<XXRangerKeyStore> rangerKeyDetails = dbOperationLoad(); - + byte[] byteArray, String cipher_field, int bit_length, + String description, int version, String attributes) { + XXRangerKeyStore xxRangerKeyStore = new XXRangerKeyStore(); + xxRangerKeyStore.setAlias(alias); + xxRangerKeyStore.setCreatedDate(creationDate); + xxRangerKeyStore.setEncoded(DatatypeConverter.printBase64Binary(byteArray)); + xxRangerKeyStore.setCipher(cipher_field); + xxRangerKeyStore.setBitLength(bit_length); + xxRangerKeyStore.setDescription(description); + xxRangerKeyStore.setVersion(version); + xxRangerKeyStore.setAttributes(attributes); + return xxRangerKeyStore; + } + + private void dbOperationStore(XXRangerKeyStore rangerKeyStore) { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.dbOperationStore()"); + } + try { + if (daoManager != null) { + RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager); + XXRangerKeyStore xxRangerKeyStore = rangerKMSDao.findByAlias(rangerKeyStore.getAlias()); + boolean keyStoreExists = true; + if (xxRangerKeyStore == null) { + xxRangerKeyStore = new XXRangerKeyStore(); + keyStoreExists = false; + } + xxRangerKeyStore = mapToEntityBean(rangerKeyStore, xxRangerKeyStore); + if (keyStoreExists) { + xxRangerKeyStore = rangerKMSDao.update(xxRangerKeyStore); + } else { + xxRangerKeyStore = rangerKMSDao.create(xxRangerKeyStore); + } + } + } catch (Exception e) { + logger.error("==> RangerKeyStore.dbOperationStore() error : ", e); + } + } + + private XXRangerKeyStore mapToEntityBean(XXRangerKeyStore rangerKMSKeyStore, XXRangerKeyStore xxRangerKeyStore) { + xxRangerKeyStore.setAlias(rangerKMSKeyStore.getAlias()); + xxRangerKeyStore.setCreatedDate(rangerKMSKeyStore.getCreatedDate()); + xxRangerKeyStore.setEncoded(rangerKMSKeyStore.getEncoded()); + xxRangerKeyStore.setCipher(rangerKMSKeyStore.getCipher()); + xxRangerKeyStore.setBitLength(rangerKMSKeyStore.getBitLength()); + xxRangerKeyStore.setDescription(rangerKMSKeyStore.getDescription()); + xxRangerKeyStore.setVersion(rangerKMSKeyStore.getVersion()); + xxRangerKeyStore.setAttributes(rangerKMSKeyStore.getAttributes()); + return xxRangerKeyStore; + } + + + @Override + public void engineLoad(InputStream stream, char[] password) + throws IOException, NoSuchAlgorithmException, CertificateException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.engineLoad()"); + } + synchronized (keyEntries) { + List<XXRangerKeyStore> rangerKeyDetails = dbOperationLoad(); + DataInputStream dis; MessageDigest md = null; - if(rangerKeyDetails == null || rangerKeyDetails.size() < 1){ - return; - } - - keyEntries.clear(); - if(password!=null){ - md = getKeyedMessageDigest(password); - } - - byte computed[]={}; - if(md!=null){ - computed = md.digest(); - } - for(XXRangerKeyStore rangerKey : rangerKeyDetails){ - String encoded = rangerKey.getEncoded(); - byte[] data = DatatypeConverter.parseBase64Binary(encoded); - - if(data != null && data.length > 0){ - stream = new ByteArrayInputStream(data); - }else{ - logger.error("No Key found for alias "+rangerKey.getAlias()); - } - - if (computed != null) { - int counter = 0; - for (int i = computed.length-1; i >= 0; i--) { - if (computed[i] != data[data.length-(1+counter)]) { - Throwable t = new UnrecoverableKeyException - ("Password verification failed"); - throw (IOException)new IOException - ("Keystore was tampered with, or " - + "password was incorrect").initCause(t); - }else{ - counter++; - } - } - } - - if (password != null) { - dis = new DataInputStream(new DigestInputStream(stream, md)); - } else { - dis = new DataInputStream(stream); - } - - ObjectInputStream ois = null; - try{ - String alias; - - SecretKeyEntry entry = new SecretKeyEntry(); - - //read the alias - alias = rangerKey.getAlias(); - - //read the (entry creation) date - entry.date = new Date(rangerKey.getCreatedDate()); - entry.cipher_field = rangerKey.getCipher(); - entry.bit_length = rangerKey.getBitLength(); - entry.description = rangerKey.getDescription(); - entry.version = rangerKey.getVersion(); - entry.attributes = rangerKey.getAttributes(); - //read the sealed key - try { - ois = new ObjectInputStream(dis); - entry.sealedKey = (SealedObject)ois.readObject(); - } catch (ClassNotFoundException cnfe) { - throw new IOException(cnfe.getMessage()); - } - - //Add the entry to the list - keyEntries.put(alias, entry); - }finally { - if (ois != null) { - ois.close(); - } else { - dis.close(); - } - } + if (rangerKeyDetails == null || rangerKeyDetails.size() < 1) { + if (logger.isDebugEnabled()) { + logger.debug("RangerKeyStore might be null or key is not present in the database."); + } + return; + } + + keyEntries.clear(); + if (password != null) { + md = getKeyedMessageDigest(password); + } + + byte computed[] = {}; + if (md != null) { + computed = md.digest(); + } + for (XXRangerKeyStore rangerKey : rangerKeyDetails) { + + String encoded = rangerKey.getEncoded(); + byte[] data = DatatypeConverter.parseBase64Binary(encoded); + + if (data != null && data.length > 0) { + stream = new ByteArrayInputStream(data); + } else { + logger.error("No Key found for alias " + rangerKey.getAlias()); + } + + if (computed != null) { + int counter = 0; + for (int i = computed.length - 1; i >= 0; i--) { + if (computed[i] != data[data.length - (1 + counter)]) { + Throwable t = new UnrecoverableKeyException + ("Password verification failed"); + logger.error("Keystore was tampered with, or password was incorrect.", t); + throw (IOException) new IOException + ("Keystore was tampered with, or " + + "password was incorrect").initCause(t); + } else { + counter++; + } + } + } + + if (password != null) { + dis = new DataInputStream(new DigestInputStream(stream, md)); + } else { + dis = new DataInputStream(stream); + } + + ObjectInputStream ois = null; + try { + String alias; + + SecretKeyEntry entry = new SecretKeyEntry(); + + //read the alias + alias = rangerKey.getAlias(); + + //read the (entry creation) date + entry.date = new Date(rangerKey.getCreatedDate()); + entry.cipher_field = rangerKey.getCipher(); + entry.bit_length = rangerKey.getBitLength(); + entry.description = rangerKey.getDescription(); + entry.version = rangerKey.getVersion(); + entry.attributes = rangerKey.getAttributes(); + //read the sealed key + try { + ois = new ObjectInputStream(dis); + entry.sealedKey = (SealedObject) ois.readObject(); + } catch (ClassNotFoundException cnfe) { + throw new IOException(cnfe.getMessage()); + } + + //Add the entry to the list + keyEntries.put(alias, entry); + } finally { + if (ois != null) { + ois.close(); + } else { + dis.close(); + } + } } } } private List<XXRangerKeyStore> dbOperationLoad() throws IOException { - try{ - if(daoManager != null){ - RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager); - return rangerKMSDao.getAllKeys(); - } - }catch(Exception e){ - e.printStackTrace(); - } - return null; - } - - /** + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStore.dbOperationLoad()"); + } + try { + if (daoManager != null) { + RangerKMSDao rangerKMSDao = new RangerKMSDao(daoManager); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerKeyStore.dbOperationLoad()"); + } + return rangerKMSDao.getAllKeys(); + } + } catch (Exception e) { + logger.error("==> RangerKeyStore.dbOperationLoad() error:", e); + } + if (logger.isDebugEnabled()) { + logger.debug("<== RangerKeyStore.dbOperationLoad()"); + } + return null; + } + + /** * To guard against tampering with the keystore, we append a keyed * hash with a bit of whitener. */ @@ -462,205 +504,210 @@ public class RangerKeyStore extends KeyStoreSpi { private final String SECRET_KEY_HASH_WORD = "Apache Ranger"; private MessageDigest getKeyedMessageDigest(char[] aKeyPassword) - throws NoSuchAlgorithmException, UnsupportedEncodingException - { + throws NoSuchAlgorithmException, UnsupportedEncodingException { int i, j; MessageDigest md = MessageDigest.getInstance("SHA"); byte[] keyPasswordBytes = new byte[aKeyPassword.length * 2]; - for (i=0, j=0; i<aKeyPassword.length; i++) { - keyPasswordBytes[j++] = (byte)(aKeyPassword[i] >> 8); - keyPasswordBytes[j++] = (byte)aKeyPassword[i]; + for (i = 0, j = 0; i < aKeyPassword.length; i++) { + keyPasswordBytes[j++] = (byte) (aKeyPassword[i] >> 8); + keyPasswordBytes[j++] = (byte) aKeyPassword[i]; } md.update(keyPasswordBytes); - for (i=0; i<keyPasswordBytes.length; i++) + for (i = 0; i < keyPasswordBytes.length; i++) keyPasswordBytes[i] = 0; md.update(SECRET_KEY_HASH_WORD.getBytes("UTF8")); return md; } - @Override - public void engineSetKeyEntry(String arg0, byte[] arg1, Certificate[] arg2) - throws KeyStoreException { - } - - @Override - public Certificate engineGetCertificate(String alias) { - return null; - } - - @Override - public String engineGetCertificateAlias(Certificate cert) { - return null; - } - - @Override - public Certificate[] engineGetCertificateChain(String alias) { - return null; - } - - @Override - public boolean engineIsCertificateEntry(String alias) { - return false; - } - - @Override - public boolean engineIsKeyEntry(String alias) { - return false; - } - - @Override - public void engineSetCertificateEntry(String alias, Certificate cert) - throws KeyStoreException { - } - - @Override - public void engineSetKeyEntry(String alias, Key key, char[] password, - Certificate[] chain) throws KeyStoreException { - } - - // - // The method is created to support JKS migration (from hadoop-common KMS keystore to RangerKMS keystore) - // - - private static final String METADATA_FIELDNAME = "metadata"; - private static final int NUMBER_OF_BITS_PER_BYTE = 8; - - public void engineLoadKeyStoreFile(InputStream stream, char[] storePass, - char[] keyPass, char[] masterKey, String fileFormat) - throws IOException, NoSuchAlgorithmException, CertificateException { - synchronized (deltaEntries) { - KeyStore ks; - - try { - ks = KeyStore.getInstance(fileFormat); - ks.load(stream, storePass); - deltaEntries.clear(); - for (Enumeration<String> name = ks.aliases(); name - .hasMoreElements();) { - SecretKeyEntry entry = new SecretKeyEntry(); - String alias = (String) name.nextElement(); - Key k = ks.getKey(alias, keyPass); - - if (k instanceof JavaKeyStoreProvider.KeyMetadata) { - JavaKeyStoreProvider.KeyMetadata keyMetadata = (JavaKeyStoreProvider.KeyMetadata) k; - Field f = JavaKeyStoreProvider.KeyMetadata.class - .getDeclaredField(METADATA_FIELDNAME); - f.setAccessible(true); - Metadata metadata = (Metadata) f.get(keyMetadata); - entry.bit_length = metadata.getBitLength(); - entry.cipher_field = metadata.getAlgorithm(); - Constructor<RangerKeyStoreProvider.KeyMetadata> constructor = RangerKeyStoreProvider.KeyMetadata.class - .getDeclaredConstructor(Metadata.class); - constructor.setAccessible(true); - RangerKeyStoreProvider.KeyMetadata nk = constructor - .newInstance(metadata); - k = nk; - } else { - entry.bit_length = (k.getEncoded().length * NUMBER_OF_BITS_PER_BYTE); - entry.cipher_field = k.getAlgorithm(); - } - String keyName = alias.split("@")[0]; - validateKeyName(keyName); - entry.attributes = "{\"key.acl.name\":\"" + keyName + "\"}"; - Class<?> c = null; - Object o = null; - try { - c = Class - .forName("com.sun.crypto.provider.KeyProtector"); - Constructor<?> constructor = c - .getDeclaredConstructor(char[].class); - constructor.setAccessible(true); - o = constructor.newInstance(masterKey); - // seal and store the key - Method m = c.getDeclaredMethod("seal", Key.class); - m.setAccessible(true); - entry.sealedKey = (SealedObject) m.invoke(o, k); - } catch (ClassNotFoundException | NoSuchMethodException - | SecurityException | InstantiationException - | IllegalAccessException | IllegalArgumentException - | InvocationTargetException e) { - logger.error(e.getMessage()); - throw new IOException(e.getMessage()); - } - - entry.date = ks.getCreationDate(alias); - entry.version = (alias.split("@").length == 2) ? (Integer - .parseInt(alias.split("@")[1])) : 0; - entry.description = k.getFormat() + " - " + ks.getType(); - deltaEntries.put(alias, entry); - } - } catch (Throwable t) { - logger.error("Unable to load keystore file ", t); - throw new IOException(t); - } - } - } - - public void engineLoadToKeyStoreFile(OutputStream stream, char[] storePass, - char[] keyPass, char[] masterKey, String fileFormat) - throws IOException, NoSuchAlgorithmException, CertificateException { - synchronized (keyEntries) { - KeyStore ks; - try { - ks = KeyStore.getInstance(fileFormat); - if (ks != null) { - ks.load(null, storePass); - String alias = null; - engineLoad(null, masterKey); - Enumeration<String> e = engineAliases(); - Key key; - while (e.hasMoreElements()) { - alias = e.nextElement(); - key = engineGetKey(alias, masterKey); - ks.setKeyEntry(alias, key, keyPass, null); - } - ks.store(stream, storePass); - } - } catch (Throwable t) { - logger.error("Unable to load keystore file ", t); - throw new IOException(t); - } + @Override + public void engineSetKeyEntry(String arg0, byte[] arg1, Certificate[] arg2) + throws KeyStoreException { + } + + @Override + public Certificate engineGetCertificate(String alias) { + return null; + } + + @Override + public String engineGetCertificateAlias(Certificate cert) { + return null; + } + + @Override + public Certificate[] engineGetCertificateChain(String alias) { + return null; + } + + @Override + public boolean engineIsCertificateEntry(String alias) { + return false; + } + + @Override + public boolean engineIsKeyEntry(String alias) { + return false; + } + + @Override + public void engineSetCertificateEntry(String alias, Certificate cert) + throws KeyStoreException { + } + + @Override + public void engineSetKeyEntry(String alias, Key key, char[] password, + Certificate[] chain) throws KeyStoreException { + } + + // + // The method is created to support JKS migration (from hadoop-common KMS keystore to RangerKMS keystore) + // + + private static final String METADATA_FIELDNAME = "metadata"; + private static final int NUMBER_OF_BITS_PER_BYTE = 8; + + public void engineLoadKeyStoreFile(InputStream stream, char[] storePass, + char[] keyPass, char[] masterKey, String fileFormat) + throws IOException, NoSuchAlgorithmException, CertificateException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.engineLoadKeyStoreFile()"); + } + synchronized (deltaEntries) { + KeyStore ks; + try { + ks = KeyStore.getInstance(fileFormat); + ks.load(stream, storePass); + deltaEntries.clear(); + for (Enumeration<String> name = ks.aliases(); name + .hasMoreElements(); ) { + SecretKeyEntry entry = new SecretKeyEntry(); + String alias = (String) name.nextElement(); + Key k = ks.getKey(alias, keyPass); + + if (k instanceof JavaKeyStoreProvider.KeyMetadata) { + JavaKeyStoreProvider.KeyMetadata keyMetadata = (JavaKeyStoreProvider.KeyMetadata) k; + Field f = JavaKeyStoreProvider.KeyMetadata.class + .getDeclaredField(METADATA_FIELDNAME); + f.setAccessible(true); + Metadata metadata = (Metadata) f.get(keyMetadata); + entry.bit_length = metadata.getBitLength(); + entry.cipher_field = metadata.getAlgorithm(); + Constructor<RangerKeyStoreProvider.KeyMetadata> constructor = RangerKeyStoreProvider.KeyMetadata.class + .getDeclaredConstructor(Metadata.class); + constructor.setAccessible(true); + RangerKeyStoreProvider.KeyMetadata nk = constructor + .newInstance(metadata); + k = nk; + } else { + entry.bit_length = (k.getEncoded().length * NUMBER_OF_BITS_PER_BYTE); + entry.cipher_field = k.getAlgorithm(); + } + String keyName = alias.split("@")[0]; + validateKeyName(keyName); + entry.attributes = "{\"key.acl.name\":\"" + keyName + "\"}"; + Class<?> c = null; + Object o = null; + try { + c = Class + .forName("com.sun.crypto.provider.KeyProtector"); + Constructor<?> constructor = c + .getDeclaredConstructor(char[].class); + constructor.setAccessible(true); + o = constructor.newInstance(masterKey); + // seal and store the key + Method m = c.getDeclaredMethod("seal", Key.class); + m.setAccessible(true); + entry.sealedKey = (SealedObject) m.invoke(o, k); + } catch (ClassNotFoundException | NoSuchMethodException + | SecurityException | InstantiationException + | IllegalAccessException | IllegalArgumentException + | InvocationTargetException e) { + logger.error(e.getMessage()); + throw new IOException(e.getMessage()); + } + + entry.date = ks.getCreationDate(alias); + entry.version = (alias.split("@").length == 2) ? (Integer + .parseInt(alias.split("@")[1])) : 0; + entry.description = k.getFormat() + " - " + ks.getType(); + deltaEntries.put(alias, entry); } - } - - private void validateKeyName(String name) { - Matcher matcher = pattern.matcher(name); - if (!matcher.matches()) { - throw new IllegalArgumentException( - "Key Name : " - + name - + ", should start with alpha/numeric letters and can have special characters - (hypen) or _ (underscore)"); + } catch (Throwable t) { + logger.error("Unable to load keystore file ", t); + throw new IOException(t); + } + } + } + + public void engineLoadToKeyStoreFile(OutputStream stream, char[] storePass, + char[] keyPass, char[] masterKey, String fileFormat) + throws IOException, NoSuchAlgorithmException, CertificateException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.engineLoadToKeyStoreFile()"); + } + + synchronized (keyEntries) { + KeyStore ks; + try { + ks = KeyStore.getInstance(fileFormat); + if (ks != null) { + ks.load(null, storePass); + String alias = null; + engineLoad(null, masterKey); + Enumeration<String> e = engineAliases(); + Key key; + while (e.hasMoreElements()) { + alias = e.nextElement(); + key = engineGetKey(alias, masterKey); + ks.setKeyEntry(alias, key, keyPass, null); + } + ks.store(stream, storePass); } + } catch (Throwable t) { + logger.error("Unable to load keystore file ", t); + throw new IOException(t); + } + } + } + + private void validateKeyName(String name) { + Matcher matcher = pattern.matcher(name); + if (!matcher.matches()) { + throw new IllegalArgumentException( + "Key Name : " + + name + + ", should start with alpha/numeric letters and can have special characters - (hypen) or _ (underscore)"); } + } - public void clearDeltaEntires(){ - deltaEntries.clear(); - } + public void clearDeltaEntires() { + deltaEntries.clear(); + } - /** - * Encapsulate the encrypted key, so that we can retrieve the AlgorithmParameters object on the decryption side - */ - private static class RangerSealedObject extends SealedObject { + /** + * Encapsulate the encrypted key, so that we can retrieve the AlgorithmParameters object on the decryption side + */ + private static class RangerSealedObject extends SealedObject { - /** - * - */ - private static final long serialVersionUID = -7551578543434362070L; + /** + * + */ + private static final long serialVersionUID = -7551578543434362070L; - protected RangerSealedObject(SealedObject so) { - super(so); - } + protected RangerSealedObject(SealedObject so) { + super(so); + } - protected RangerSealedObject(Serializable object, Cipher cipher) throws IllegalBlockSizeException, IOException { - super(object, cipher); - } + protected RangerSealedObject(Serializable object, Cipher cipher) throws IllegalBlockSizeException, IOException { + super(object, cipher); + } - public AlgorithmParameters getParameters() throws NoSuchAlgorithmException, IOException { - AlgorithmParameters algorithmParameters = AlgorithmParameters.getInstance("PBEWithMD5AndTripleDES"); - algorithmParameters.init(super.encodedParams); - return algorithmParameters; - } + public AlgorithmParameters getParameters() throws NoSuchAlgorithmException, IOException { + AlgorithmParameters algorithmParameters = AlgorithmParameters.getInstance("PBEWithMD5AndTripleDES"); + algorithmParameters.init(super.encodedParams); + return algorithmParameters; + } - } + } }