http://git-wip-us.apache.org/repos/asf/ranger/blob/bc2cd5e0/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java index b9d948f..88a545e 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java @@ -52,397 +52,427 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @InterfaceAudience.Private -public class RangerKeyStoreProvider extends KeyProvider{ - - static final Logger logger = Logger.getLogger(RangerKeyStoreProvider.class); - - public static final String SCHEME_NAME = "dbks"; - public static final String KMS_CONFIG_DIR = "kms.config.dir"; - public static final String DBKS_SITE_XML = "dbks-site.xml"; - public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; - private static final String KEY_METADATA = "KeyMetadata"; - private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; - private static final String MK_CREDENTIAL_ALIAS = "ranger.ks.masterkey.credential.alias"; - private static final String DB_CREDENTIAL_ALIAS = "ranger.ks.jpa.jdbc.credential.alias"; - private static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password"; - private static final String HSM_ENABLED = "ranger.ks.hsm.enabled"; - private static final String HSM_PARTITION_PASSWORD_ALIAS = "ranger.ks.hsm.partition.password.alias"; - private static final String HSM_PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; - - private final RangerKeyStore dbStore; - private char[] masterKey; - private boolean changed = false; - private final Map<String, Metadata> cache = new HashMap<String, Metadata>(); - private DaoManager daoManager; - - private Lock readLock; - - public RangerKeyStoreProvider(Configuration conf) throws Throwable { - super(conf); - conf = getDBKSConf(); - getFromJceks(conf,CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); - getFromJceks(conf,CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); - getFromJceks(conf,CREDENTIAL_PATH, HSM_PARTITION_PASSWORD_ALIAS, HSM_PARTITION_PASSWORD); - RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); - daoManager = rangerKMSDB.getDaoManager(); - - RangerKMSMKI rangerMasterKey = null; - String password = conf.get(ENCRYPTION_KEY); - if(password == null || password.trim().equals("") || password.trim().equals("_") || password.trim().equals("crypted")){ - throw new IOException("The Ranger MasterKey Password is empty or not a valid Password"); - } - if(StringUtils.isEmpty(conf.get(HSM_ENABLED)) || conf.get(HSM_ENABLED).equalsIgnoreCase("false")){ - rangerMasterKey = new RangerMasterKey(daoManager); - }else{ - rangerMasterKey = new RangerHSM(conf); - String partitionPasswd = conf.get(HSM_PARTITION_PASSWORD); - if(partitionPasswd == null || partitionPasswd.trim().equals("") || partitionPasswd.trim().equals("_") || partitionPasswd.trim().equals("crypted")){ - throw new IOException("Partition Password doesn't exists"); - } - } - dbStore = new RangerKeyStore(daoManager); - rangerMasterKey.generateMasterKey(password); - //code to retrieve rangerMasterKey password - masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); - if(masterKey == null){ - // Master Key does not exists - throw new IOException("Ranger MasterKey does not exists"); - } +public class RangerKeyStoreProvider extends KeyProvider { + + static final Logger logger = Logger.getLogger(RangerKeyStoreProvider.class); + + public static final String SCHEME_NAME = "dbks"; + public static final String KMS_CONFIG_DIR = "kms.config.dir"; + public static final String DBKS_SITE_XML = "dbks-site.xml"; + public static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String KEY_METADATA = "KeyMetadata"; + private static final String CREDENTIAL_PATH = "ranger.ks.jpa.jdbc.credential.provider.path"; + private static final String MK_CREDENTIAL_ALIAS = "ranger.ks.masterkey.credential.alias"; + private static final String DB_CREDENTIAL_ALIAS = "ranger.ks.jpa.jdbc.credential.alias"; + private static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password"; + private static final String HSM_ENABLED = "ranger.ks.hsm.enabled"; + private static final String HSM_PARTITION_PASSWORD_ALIAS = "ranger.ks.hsm.partition.password.alias"; + private static final String HSM_PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; + + private final RangerKeyStore dbStore; + private char[] masterKey; + private boolean changed = false; + private final Map<String, Metadata> cache = new HashMap<String, Metadata>(); + private DaoManager daoManager; + + private Lock readLock; + + public RangerKeyStoreProvider(Configuration conf) throws Throwable { + super(conf); + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.Configuration(conf)"); + } + conf = getDBKSConf(); + getFromJceks(conf, CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); + getFromJceks(conf, CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); + getFromJceks(conf, CREDENTIAL_PATH, HSM_PARTITION_PASSWORD_ALIAS, HSM_PARTITION_PASSWORD); + RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); + daoManager = rangerKMSDB.getDaoManager(); + + RangerKMSMKI rangerMasterKey = null; + String password = conf.get(ENCRYPTION_KEY); + if (password == null || password.trim().equals("") || password.trim().equals("_") || password.trim().equals("crypted")) { + throw new IOException("The Ranger MasterKey Password is empty or not a valid Password"); + } + if (StringUtils.isEmpty(conf.get(HSM_ENABLED)) || conf.get(HSM_ENABLED).equalsIgnoreCase("false")) { + logger.info("Ranger KMS Database is enabled for storing master key."); + rangerMasterKey = new RangerMasterKey(daoManager); + } else { + logger.info("Ranger KMS HSM is enabled for storing master key."); + rangerMasterKey = new RangerHSM(conf); + String partitionPasswd = conf.get(HSM_PARTITION_PASSWORD); + if (partitionPasswd == null || partitionPasswd.trim().equals("") || partitionPasswd.trim().equals("_") || partitionPasswd.trim().equals("crypted")) { + throw new IOException("Partition Password doesn't exists"); + } + } + dbStore = new RangerKeyStore(daoManager); + rangerMasterKey.generateMasterKey(password); + //code to retrieve rangerMasterKey password + masterKey = rangerMasterKey.getMasterKey(password).toCharArray(); + if (masterKey == null) { + // Master Key does not exists + throw new IOException("Ranger MasterKey does not exists"); + } + reloadKeys(); + ReadWriteLock lock = new ReentrantReadWriteLock(true); + readLock = lock.readLock(); + } + + public static Configuration getDBKSConf() { + Configuration newConfig = getConfiguration(true, DBKS_SITE_XML); + getFromJceks(newConfig, CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); + getFromJceks(newConfig, CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); + return newConfig; + + } + + static Configuration getConfiguration(boolean loadHadoopDefaults, + String... resources) { + Configuration conf = new Configuration(loadHadoopDefaults); + String confDir = System.getProperty(KMS_CONFIG_DIR); + if (confDir != null) { + try { + Path confPath = new Path(confDir); + if (!confPath.isUriPathAbsolute()) { + throw new RuntimeException("System property '" + KMS_CONFIG_DIR + + "' must be an absolute path: " + confDir); + } + for (String resource : resources) { + conf.addResource(new URL("file://" + new Path(confDir, resource).toUri())); + } + } catch (MalformedURLException ex) { + logger.error("==> RangerKeyStoreProvider.getConfiguration() error : ", ex); + throw new RuntimeException(ex); + } + } else { + for (String resource : resources) { + conf.addResource(resource); + } + } + return conf; + } + + private void loadKeys(char[] masterKey) throws NoSuchAlgorithmException, CertificateException, IOException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.loadKeys()"); + } + dbStore.engineLoad(null, masterKey); + } + + @Override + public KeyVersion createKey(String name, byte[] material, Options options) + throws IOException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.createKey()"); + } + reloadKeys(); + if (dbStore.engineContainsAlias(name) || cache.containsKey(name)) { + throw new IOException("Key " + name + " already exists"); + } + Metadata meta = new Metadata(options.getCipher(), options.getBitLength(), + options.getDescription(), options.getAttributes(), new Date(), 1); + if (options.getBitLength() != 8 * material.length) { + throw new IOException("Wrong key length. Required " + + options.getBitLength() + ", but got " + (8 * material.length)); + } + cache.put(name, meta); + String versionName = buildVersionName(name, 0); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerKeyStoreProvider.createKey()"); + } + return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); + } + + KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material, String cipher, int bitLength, String description, int version, Map<String, String> attributes) throws IOException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.innerSetKeyVersion()"); + logger.debug("name : " + name + " and versionName : " + versionName); + } + try { + ObjectMapper om = new ObjectMapper(); + String attribute = om.writeValueAsString(attributes); + dbStore.addKeyEntry(versionName, new SecretKeySpec(material, cipher), masterKey, cipher, bitLength, description, version, attribute); + } catch (KeyStoreException e) { + throw new IOException("Can't store key " + versionName, e); + } + changed = true; + if (logger.isDebugEnabled()) { + logger.debug("<== RangerKeyStoreProvider.innerSetKeyVersion()"); + } + return new KeyVersion(name, versionName, material); + } + + @Override + public void deleteKey(String name) throws IOException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.deleteKey(" + name + ")"); + } + reloadKeys(); + Metadata meta = getMetadata(name); + if (meta == null) { + throw new IOException("Key " + name + " does not exist"); + } + for (int v = 0; v < meta.getVersions(); ++v) { + String versionName = buildVersionName(name, v); + try { + if (dbStore.engineContainsAlias(versionName)) { + dbStore.engineDeleteEntry(versionName); + } + } catch (KeyStoreException e) { + throw new IOException("Problem removing " + versionName, e); + } + } + try { + if (dbStore.engineContainsAlias(name)) { + dbStore.engineDeleteEntry(name); + } + } catch (KeyStoreException e) { + throw new IOException("Problem removing " + name + " from " + this, e); + } + cache.remove(name); + changed = true; + } + + @Override + public void flush() throws IOException { + try { + if (!changed) { + return; + } + // put all of the updates into the db + for (Map.Entry<String, Metadata> entry : cache.entrySet()) { + try { + Metadata metadata = entry.getValue(); + ObjectMapper om = new ObjectMapper(); + String attributes = om.writeValueAsString(metadata.getAttributes()); + dbStore.addKeyEntry(entry.getKey(), new KeyMetadata(metadata), masterKey, metadata.getAlgorithm(), metadata.getBitLength(), metadata.getDescription(), metadata.getVersions(), attributes); + } catch (KeyStoreException e) { + throw new IOException("Can't set metadata key " + entry.getKey(), e); + } + } + try { + dbStore.engineStore(null, masterKey); + reloadKeys(); + } catch (NoSuchAlgorithmException e) { + throw new IOException("No such algorithm storing key", e); + } catch (CertificateException e) { + throw new IOException("Certificate exception storing key", e); + } + changed = false; + } catch (IOException ioe) { + cache.clear(); + reloadKeys(); + throw ioe; + } + } + + @Override + public KeyVersion getKeyVersion(String versionName) throws IOException { + readLock.lock(); + try { + SecretKeySpec key = null; + try { + if (!dbStore.engineContainsAlias(versionName)) { + dbStore.engineLoad(null, masterKey); + if (!dbStore.engineContainsAlias(versionName)) { + return null; + } + } + key = (SecretKeySpec) dbStore.engineGetKey(versionName, masterKey); + } catch (NoSuchAlgorithmException e) { + throw new IOException("Can't get algorithm for key " + key, e); + } catch (UnrecoverableKeyException e) { + throw new IOException("Can't recover key " + key, e); + } catch (CertificateException e) { + throw new IOException("Certificate exception storing key", e); + } + if (key == null) { + return null; + } else { + return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded()); + } + } finally { + readLock.unlock(); + } + } + + @Override + public List<KeyVersion> getKeyVersions(String name) throws IOException { + List<KeyVersion> list = new ArrayList<KeyVersion>(); + Metadata km = getMetadata(name); + if (km != null) { + int latestVersion = km.getVersions(); + KeyVersion v = null; + String versionName = null; + for (int i = 0; i < latestVersion; i++) { + versionName = buildVersionName(name, i); + v = getKeyVersion(versionName); + if (v != null) { + list.add(v); + } + } + } + return list; + } + + @Override + public List<String> getKeys() throws IOException { + ArrayList<String> list = new ArrayList<String>(); + String alias = null; reloadKeys(); - ReadWriteLock lock = new ReentrantReadWriteLock(true); - readLock = lock.readLock(); - } - - public static Configuration getDBKSConf() { - Configuration newConfig = getConfiguration(true, DBKS_SITE_XML); - getFromJceks(newConfig,CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY); - getFromJceks(newConfig,CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD); - return newConfig; - - } - - static Configuration getConfiguration(boolean loadHadoopDefaults, - String ... resources) { - Configuration conf = new Configuration(loadHadoopDefaults); - String confDir = System.getProperty(KMS_CONFIG_DIR); - if (confDir != null) { - try { - Path confPath = new Path(confDir); - if (!confPath.isUriPathAbsolute()) { - throw new RuntimeException("System property '" + KMS_CONFIG_DIR + - "' must be an absolute path: " + confDir); - } - for (String resource : resources) { - conf.addResource(new URL("file://" + new Path(confDir, resource).toUri())); - } - } catch (MalformedURLException ex) { - ex.printStackTrace(); - throw new RuntimeException(ex); - } - } else { - for (String resource : resources) { - conf.addResource(resource); - } - } - return conf; - } - - private void loadKeys(char[] masterKey) throws NoSuchAlgorithmException, CertificateException, IOException { - dbStore.engineLoad(null, masterKey); - } - - @Override - public KeyVersion createKey(String name, byte[] material, Options options) - throws IOException { - reloadKeys(); - if (dbStore.engineContainsAlias(name) || cache.containsKey(name)) { - throw new IOException("Key " + name + " already exists"); - } - Metadata meta = new Metadata(options.getCipher(), options.getBitLength(), - options.getDescription(), options.getAttributes(), new Date(), 1); - if (options.getBitLength() != 8 * material.length) { - throw new IOException("Wrong key length. Required " + - options.getBitLength() + ", but got " + (8 * material.length)); - } - cache.put(name, meta); - String versionName = buildVersionName(name, 0); - return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); - } - - KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material, String cipher, int bitLength, String description, int version, Map<String, String> attributes) throws IOException { - try { - ObjectMapper om = new ObjectMapper(); - String attribute = om.writeValueAsString(attributes); - dbStore.addKeyEntry(versionName, new SecretKeySpec(material, cipher), masterKey, cipher, bitLength, description, version, attribute); - } catch (KeyStoreException e) { - throw new IOException("Can't store key " + versionName,e); - } - changed = true; - return new KeyVersion(name, versionName, material); - } - - @Override - public void deleteKey(String name) throws IOException { - reloadKeys(); - Metadata meta = getMetadata(name); - if (meta == null) { - throw new IOException("Key " + name + " does not exist"); - } - for(int v=0; v < meta.getVersions(); ++v) { - String versionName = buildVersionName(name, v); - try { - if (dbStore.engineContainsAlias(versionName)) { - dbStore.engineDeleteEntry(versionName); - } - } catch (KeyStoreException e) { - throw new IOException("Problem removing " + versionName, e); - } - } - try { - if (dbStore.engineContainsAlias(name)) { - dbStore.engineDeleteEntry(name); - } - } catch (KeyStoreException e) { - throw new IOException("Problem removing " + name + " from " + this, e); - } - cache.remove(name); - changed = true; - } - - @Override - public void flush() throws IOException { - try { - if (!changed) { - return; - } - // put all of the updates into the db - for(Map.Entry<String, Metadata> entry: cache.entrySet()) { - try { - Metadata metadata = entry.getValue(); - ObjectMapper om = new ObjectMapper(); - String attributes = om.writeValueAsString(metadata.getAttributes()); - dbStore.addKeyEntry(entry.getKey(), new KeyMetadata(metadata), masterKey, metadata.getAlgorithm(), metadata.getBitLength(), metadata.getDescription(), metadata.getVersions(), attributes); - } catch (KeyStoreException e) { - throw new IOException("Can't set metadata key " + entry.getKey(),e ); - } - } - try { - dbStore.engineStore(null, masterKey); - reloadKeys(); - } catch (NoSuchAlgorithmException e) { - throw new IOException("No such algorithm storing key", e); - } catch (CertificateException e) { - throw new IOException("Certificate exception storing key", e); - } - changed = false; - }catch (IOException ioe) { - cache.clear(); - reloadKeys(); - throw ioe; - } - } - - @Override - public KeyVersion getKeyVersion(String versionName) throws IOException { - readLock.lock(); - try { - SecretKeySpec key = null; - try { - if (!dbStore.engineContainsAlias(versionName)) { - dbStore.engineLoad(null, masterKey); - if (!dbStore.engineContainsAlias(versionName)) { - return null; - } - } - key = (SecretKeySpec) dbStore.engineGetKey(versionName, masterKey); - } catch (NoSuchAlgorithmException e) { - throw new IOException("Can't get algorithm for key " + key, e); - } catch (UnrecoverableKeyException e) { - throw new IOException("Can't recover key " + key, e); - } - catch (CertificateException e) { - throw new IOException("Certificate exception storing key", e); - } - if (key == null) { - return null; - } else { - return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded()); - } - } finally { - readLock.unlock(); - } - } - - @Override - public List<KeyVersion> getKeyVersions(String name) throws IOException { - List<KeyVersion> list = new ArrayList<KeyVersion>(); - Metadata km = getMetadata(name); - if (km != null) { - int latestVersion = km.getVersions(); - KeyVersion v = null; - String versionName = null; - for (int i = 0; i < latestVersion; i++) { - versionName = buildVersionName(name, i); - v = getKeyVersion(versionName); - if (v != null) { - list.add(v); - } - } - } - return list; - } - - @Override - public List<String> getKeys() throws IOException { - ArrayList<String> list = new ArrayList<String>(); - String alias = null; - reloadKeys(); - Enumeration<String> e = dbStore.engineAliases(); - while (e.hasMoreElements()) { - alias = e.nextElement(); - // only include the metadata key names in the list of names - if (!alias.contains("@")) { - list.add(alias); - } - } - return list; - } - - @Override - public Metadata getMetadata(String name) throws IOException { - try { - readLock.lock(); + Enumeration<String> e = dbStore.engineAliases(); + while (e.hasMoreElements()) { + alias = e.nextElement(); + // only include the metadata key names in the list of names + if (!alias.contains("@")) { + list.add(alias); + } + } + return list; + } + + @Override + public Metadata getMetadata(String name) throws IOException { + try { + readLock.lock(); if (cache.containsKey(name)) { - Metadata meta = cache.get(name); - return meta; - } - try { - if (!dbStore.engineContainsAlias(name)) { - dbStore.engineLoad(null, masterKey); - if (!dbStore.engineContainsAlias(name)) { - return null; - } - } - Key key = dbStore.engineGetKey(name, masterKey); - if(key != null){ - Metadata meta = ((KeyMetadata) key).metadata; - cache.put(name, meta); - return meta; - } - } catch (NoSuchAlgorithmException e) { - throw new IOException("Can't get algorithm for " + name, e); - } catch (UnrecoverableKeyException e) { - throw new IOException("Can't recover key for " + name, e); - } - return null; - } - catch(Exception e){ - throw new IOException("Please try again ", e); - } - finally { - readLock.unlock(); - } - } - - @Override - public KeyVersion rollNewVersion(String name, byte[] material)throws IOException { - reloadKeys(); - Metadata meta = getMetadata(name); + Metadata meta = cache.get(name); + return meta; + } + try { + if (!dbStore.engineContainsAlias(name)) { + dbStore.engineLoad(null, masterKey); + if (!dbStore.engineContainsAlias(name)) { + return null; + } + } + Key key = dbStore.engineGetKey(name, masterKey); + if (key != null) { + Metadata meta = ((KeyMetadata) key).metadata; + cache.put(name, meta); + return meta; + } + } catch (NoSuchAlgorithmException e) { + throw new IOException("Can't get algorithm for " + name, e); + } catch (UnrecoverableKeyException e) { + throw new IOException("Can't recover key for " + name, e); + } + return null; + } catch (Exception e) { + throw new IOException("Please try again ", e); + } finally { + readLock.unlock(); + } + } + + @Override + public KeyVersion rollNewVersion(String name, byte[] material) throws IOException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.rollNewVersion()"); + } + reloadKeys(); + Metadata meta = getMetadata(name); if (meta == null) { - throw new IOException("Key " + name + " not found"); - } - if (meta.getBitLength() != 8 * material.length) { - throw new IOException("Wrong key length. Required " + meta.getBitLength() + ", but got " + (8 * material.length)); - } - int nextVersion = meta.addVersion(); - String versionName = buildVersionName(name, nextVersion); - return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); - } - - private static void getFromJceks(Configuration conf, String path, String alias, String key){ - //update credential from keystore - if(conf!=null){ - String pathValue=conf.get(path); - String aliasValue=conf.get(alias); - if(pathValue!=null && aliasValue!=null){ - String xaDBPassword=CredentialReader.getDecryptedString(pathValue.trim(),aliasValue.trim()); - if(xaDBPassword!=null&& !xaDBPassword.trim().isEmpty() && - !xaDBPassword.trim().equalsIgnoreCase("none")){ - conf.set(key, xaDBPassword); - }else{ - logger.info("Credential keystore password not applied for KMS; clear text password shall be applicable"); - } - } - } - } + throw new IOException("Key " + name + " not found"); + } + if (meta.getBitLength() != 8 * material.length) { + throw new IOException("Wrong key length. Required " + meta.getBitLength() + ", but got " + (8 * material.length)); + } + int nextVersion = meta.addVersion(); + String versionName = buildVersionName(name, nextVersion); + return innerSetKeyVersion(name, versionName, material, meta.getCipher(), meta.getBitLength(), meta.getDescription(), meta.getVersions(), meta.getAttributes()); + } + + private static void getFromJceks(Configuration conf, String path, String alias, String key) { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.getFromJceks()"); + } + //update credential from keystore + if (conf != null) { + String pathValue = conf.get(path); + String aliasValue = conf.get(alias); + if (pathValue != null && aliasValue != null) { + String xaDBPassword = CredentialReader.getDecryptedString(pathValue.trim(), aliasValue.trim()); + if (xaDBPassword != null && !xaDBPassword.trim().isEmpty() && + !xaDBPassword.trim().equalsIgnoreCase("none")) { + conf.set(key, xaDBPassword); + } else { + logger.info("Credential keystore password not applied for KMS; clear text password shall be applicable"); + } + } + } + } private void reloadKeys() throws IOException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerKeyStoreProvider.reloadKeys()"); + } try { - cache.clear(); + cache.clear(); loadKeys(masterKey); } catch (NoSuchAlgorithmException e) { throw new IOException("Can't load Keys"); - }catch(CertificateException e){ + } catch (CertificateException e) { throw new IOException("Can't load Keys"); } } - - /** - * The factory to create JksProviders, which is used by the ServiceLoader. - */ - public static class Factory extends KeyProviderFactory { - @Override - public KeyProvider createProvider(URI providerName, - Configuration conf) throws IOException { - try { - if (SCHEME_NAME.equals(providerName.getScheme())) { - return new RangerKeyStoreProvider(conf); - } - } catch (Throwable e) { - e.printStackTrace(); - } - return null; - } - } - - /** - * An adapter between a KeyStore Key and our Metadata. This is used to store - * the metadata in a KeyStore even though isn't really a key. - */ - public static class KeyMetadata implements Key, Serializable { - private Metadata metadata; - private final static long serialVersionUID = 8405872419967874451L; - - private KeyMetadata(Metadata meta) { - this.metadata = meta; - } - - @Override - public String getAlgorithm() { - return metadata.getCipher(); - } - - @Override - public String getFormat() { - return KEY_METADATA; - } - - @Override - public byte[] getEncoded() { - return new byte[0]; - } - - private void writeObject(ObjectOutputStream out) throws IOException { - byte[] serialized = metadata.serialize(); - out.writeInt(serialized.length); - out.write(serialized); - } - - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { - byte[] buf = new byte[in.readInt()]; - in.readFully(buf); - metadata = new Metadata(buf); - } - - } + + /** + * The factory to create JksProviders, which is used by the ServiceLoader. + */ + public static class Factory extends KeyProviderFactory { + @Override + public KeyProvider createProvider(URI providerName, + Configuration conf) throws IOException { + try { + if (SCHEME_NAME.equals(providerName.getScheme())) { + return new RangerKeyStoreProvider(conf); + } + } catch (Throwable e) { + logger.error("==> RangerKeyStoreProvider.reloadKeys() error : " , e); + } + return null; + } + } + + /** + * An adapter between a KeyStore Key and our Metadata. This is used to store + * the metadata in a KeyStore even though isn't really a key. + */ + public static class KeyMetadata implements Key, Serializable { + private Metadata metadata; + private final static long serialVersionUID = 8405872419967874451L; + + private KeyMetadata(Metadata meta) { + this.metadata = meta; + } + + @Override + public String getAlgorithm() { + return metadata.getCipher(); + } + + @Override + public String getFormat() { + return KEY_METADATA; + } + + @Override + public byte[] getEncoded() { + return new byte[0]; + } + + private void writeObject(ObjectOutputStream out) throws IOException { + byte[] serialized = metadata.serialize(); + out.writeInt(serialized.length); + out.write(serialized); + } + + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + byte[] buf = new byte[in.readInt()]; + in.readFully(buf); + metadata = new Metadata(buf); + } + + } }
http://git-wip-us.apache.org/repos/asf/ranger/blob/bc2cd5e0/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java index 6cfd37e..eb4f75a 100755 --- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java @@ -47,340 +47,426 @@ import com.google.common.collect.Lists; import com.sun.org.apache.xml.internal.security.exceptions.Base64DecodingException; import com.sun.org.apache.xml.internal.security.utils.Base64; -public class RangerMasterKey implements RangerKMSMKI{ - - private static final Logger logger = Logger.getLogger(RangerMasterKey.class); - - private static final String DEFAULT_MK_CIPHER = "AES"; - private static final int DEFAULT_MK_KeySize = 256; - private static final int DEFAULT_SALT_SIZE = 8; - private static final String DEFAULT_SALT = "abcdefghijklmnopqrstuvwxyz01234567890"; - private static final String DEFAULT_CRYPT_ALGO = "PBEWithMD5AndTripleDES"; - private static final String DEFAULT_MD_ALGO = "MD5"; - private static final int DEFAULT_ITERATION_COUNT = 1000; - private static String password = null; - - public static final String DBKS_SITE_XML = "dbks-site.xml"; - private static Properties serverConfigProperties = new Properties(); - - public static String MK_CIPHER; - public static Integer MK_KeySize = 0; - public static Integer SALT_SIZE = 0; - public static String SALT; - public static String PBE_ALGO; - public static String MD_ALGO; - public static Integer ITERATION_COUNT = 0; - public static String paddingString; - - private DaoManager daoManager; - - public RangerMasterKey() { - } - - public RangerMasterKey(DaoManager daoManager) { - this.daoManager = daoManager; - } - - protected static String getConfig(String key, String defaultValue) { - String value = serverConfigProperties.getProperty(key); - if (value == null || value.trim().isEmpty()) { - // Value not found in properties file, let's try to get from - // System's property - value = System.getProperty(key); - } - if (value == null || value.trim().isEmpty()) { - value = defaultValue; - } - return value; +public class RangerMasterKey implements RangerKMSMKI { + + private static final Logger logger = Logger.getLogger(RangerMasterKey.class); + + private static final String DEFAULT_MK_CIPHER = "AES"; + private static final int DEFAULT_MK_KeySize = 256; + private static final int DEFAULT_SALT_SIZE = 8; + private static final String DEFAULT_SALT = "abcdefghijklmnopqrstuvwxyz01234567890"; + private static final String DEFAULT_CRYPT_ALGO = "PBEWithMD5AndTripleDES"; + private static final String DEFAULT_MD_ALGO = "MD5"; + private static final int DEFAULT_ITERATION_COUNT = 1000; + private static String password = null; + + public static final String DBKS_SITE_XML = "dbks-site.xml"; + private static Properties serverConfigProperties = new Properties(); + + public static String MK_CIPHER; + public static Integer MK_KeySize = 0; + public static Integer SALT_SIZE = 0; + public static String SALT; + public static String PBE_ALGO; + public static String MD_ALGO; + public static Integer ITERATION_COUNT = 0; + public static String paddingString; + + private DaoManager daoManager; + + public RangerMasterKey() { + } + + public RangerMasterKey(DaoManager daoManager) { + this.daoManager = daoManager; + } + + protected static String getConfig(String key, String defaultValue) { + String value = serverConfigProperties.getProperty(key); + if (value == null || value.trim().isEmpty()) { + // Value not found in properties file, let's try to get from + // System's property + value = System.getProperty(key); + } + if (value == null || value.trim().isEmpty()) { + value = defaultValue; } + return value; + } - protected static int getIntConfig(String key, int defaultValue) { - int ret = defaultValue; - String retStr = serverConfigProperties.getProperty(key); - try { - if (retStr != null) { - ret = Integer.parseInt(retStr); - } - } catch (Exception err) { - logger.warn(retStr + " can't be parsed to int. Reason: " + err.toString()); - } - return ret; - } - /** - * To get Master Key - * @param password password to be used for decryption - * @return Decrypted Master Key - * @throws Throwable - */ - @Override - public String getMasterKey(String password) throws Throwable { - logger.info("Getting Master Key"); - List result = getEncryptedMK(); - String encryptedPassString = null; - byte masterKeyByte[] = null; - if (CollectionUtils.isNotEmpty(result) && result.size() == 2) { - masterKeyByte = (byte[]) result.get(0); - encryptedPassString = (String) result.get(1); - } else if (CollectionUtils.isNotEmpty(result)) { - masterKeyByte = (byte[]) result.get(0); - } - if (masterKeyByte != null && masterKeyByte.length > 0) { - return decryptMasterKey(masterKeyByte, password, encryptedPassString); - } else { - throw new Exception("No Master Key Found"); - } - } - - public SecretKey getMasterSecretKey(String password) throws Throwable { - logger.info("Getting Master Key"); - List result = getEncryptedMK(); - String encryptedPassString = null; - byte masterKeyByte[] = null; - if (CollectionUtils.isNotEmpty(result) && result.size() == 2) { - masterKeyByte = (byte[]) result.get(0); - encryptedPassString = (String) result.get(1); - } else if (CollectionUtils.isNotEmpty(result)) { - masterKeyByte = (byte[]) result.get(0); - } - if (masterKeyByte != null && masterKeyByte.length > 0) { - return decryptMasterKeySK(masterKeyByte, password, encryptedPassString); + protected static int getIntConfig(String key, int defaultValue) { + int ret = defaultValue; + String retStr = serverConfigProperties.getProperty(key); + try { + if (retStr != null) { + ret = Integer.parseInt(retStr); + } + } catch (Exception err) { + logger.warn(retStr + " can't be parsed to int. Reason: " + err.toString()); + } + return ret; + } + + /** + * To get Master Key + * + * @param password password to be used for decryption + * @return Decrypted Master Key + * @throws Throwable + */ + @Override + public String getMasterKey(String password) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.getMasterKey()"); + } + logger.info("Getting Master Key"); + List result = getEncryptedMK(); + String encryptedPassString = null; + byte masterKeyByte[] = null; + if (CollectionUtils.isNotEmpty(result) && result.size() == 2) { + masterKeyByte = (byte[]) result.get(0); + encryptedPassString = (String) result.get(1); + } else if (CollectionUtils.isNotEmpty(result)) { + masterKeyByte = (byte[]) result.get(0); + } + if (masterKeyByte != null && masterKeyByte.length > 0) { + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.getMasterKey()"); + } + return decryptMasterKey(masterKeyByte, password, encryptedPassString); + } else { + throw new Exception("No Master Key Found"); + } + } + + public SecretKey getMasterSecretKey(String password) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.getMasterSecretKey()"); + } + logger.info("Getting Master Key"); + List result = getEncryptedMK(); + String encryptedPassString = null; + byte masterKeyByte[] = null; + if (CollectionUtils.isNotEmpty(result) && result.size() == 2) { + masterKeyByte = (byte[]) result.get(0); + encryptedPassString = (String) result.get(1); + } else if (CollectionUtils.isNotEmpty(result)) { + masterKeyByte = (byte[]) result.get(0); + } + if (masterKeyByte != null && masterKeyByte.length > 0) { + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.getMasterSecretKey()"); + } + return decryptMasterKeySK(masterKeyByte, password, encryptedPassString); + } else { + throw new Exception("No Master Key Found"); + } + } + + /** + * Generate the master key, encrypt it and save it in the database + * + * @param password password to be used for encryption + * @return true if the master key was successfully created false if master + * key generation was unsuccessful or the master key already exists + * @throws Throwable + */ + + public void init() { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.init()"); + } + XMLUtils.loadConfig(DBKS_SITE_XML, serverConfigProperties); + MK_CIPHER = getConfig("ranger.kms.service.masterkey.password.cipher", DEFAULT_MK_CIPHER); + MK_KeySize = getIntConfig("ranger.kms.service.masterkey.password.size", DEFAULT_MK_KeySize); + SALT_SIZE = getIntConfig("ranger.kms.service.masterkey.password.salt.size", DEFAULT_SALT_SIZE); + SALT = getConfig("ranger.kms.service.masterkey.password.salt", DEFAULT_SALT); + PBE_ALGO = getConfig("ranger.kms.service.masterkey.password.encryption.algorithm", DEFAULT_CRYPT_ALGO); + MD_ALGO = getConfig("ranger.kms.service.masterkey.password.md.algorithm", DEFAULT_MD_ALGO); + ITERATION_COUNT = getIntConfig("ranger.kms.service.masterkey.password.iteration.count", + DEFAULT_ITERATION_COUNT); + paddingString = Joiner.on(",").skipNulls().join(MK_CIPHER, MK_KeySize, SALT_SIZE, PBE_ALGO, MD_ALGO, + ITERATION_COUNT, SALT); + } + + @Override + public boolean generateMasterKey(String password) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.generateMasterKey()"); + } + logger.info("Generating Master Key..."); + init(); + String encryptedMasterKey = encryptMasterKey(password); + String savedKey = saveEncryptedMK(paddingString + "," + encryptedMasterKey, daoManager); + if (savedKey != null && !savedKey.trim().equals("")) { + if (logger.isDebugEnabled()) { + logger.debug("Master Key Created with id = " + savedKey); + logger.debug("<== RangerMasterKey.generateMasterKey()"); + } + return true; + } + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.generateMasterKey()"); + } + return false; + } + + public boolean generateMKFromHSMMK(String password, byte[] key) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.generateMKFromHSMMK()"); + } + init(); + String encryptedMasterKey = encryptMasterKey(password, key); + String savedKey = saveEncryptedMK(paddingString + "," + encryptedMasterKey, daoManager); + if (savedKey != null && !savedKey.trim().equals("")) { + if (logger.isDebugEnabled()) { + logger.debug("Master Key Created with id = " + savedKey); + logger.debug("<== RangerMasterKey.generateMKFromHSMMK()"); + } + return true; + } + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.generateMKFromHSMMK()"); + } + return false; + } + + private String decryptMasterKey(byte masterKey[], String password, String encryptedPassString) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.decryptMasterKey()"); + logger.debug("Decrypting Master Key..."); + } + if (encryptedPassString == null) { + getPasswordParam(password); + } + PBEKeySpec pbeKeyspec = getPBEParameterSpec(password); + byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec); + SecretKey masterKeyFromDB = getMasterKeyFromBytes(masterKeyFromDBDecrypted); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.decryptMasterKey()"); + } + return Base64.encode(masterKeyFromDB.getEncoded()); + } + + public static void getPasswordParam(String paddedEncryptedPwd) { + String[] encryptedPwd = null; + if (paddedEncryptedPwd != null && paddedEncryptedPwd.contains(",")) { + encryptedPwd = Lists.newArrayList(Splitter.on(",").split(paddedEncryptedPwd)).toArray(new String[0]); + } + if (encryptedPwd != null && encryptedPwd.length >= 7) { + int index = 0; + MK_CIPHER = encryptedPwd[index]; + MK_KeySize = Integer.parseInt(encryptedPwd[++index]); + SALT_SIZE = Integer.parseInt(encryptedPwd[++index]); + PBE_ALGO = encryptedPwd[++index]; + MD_ALGO = encryptedPwd[++index]; + ITERATION_COUNT = Integer.parseInt(encryptedPwd[++index]); + SALT = encryptedPwd[++index]; + password = encryptedPwd[++index]; + } else { + MK_CIPHER = DEFAULT_MK_CIPHER; + MK_KeySize = DEFAULT_MK_KeySize; + SALT_SIZE = DEFAULT_SALT_SIZE; + PBE_ALGO = DEFAULT_CRYPT_ALGO; + MD_ALGO = DEFAULT_MD_ALGO; + password = paddedEncryptedPwd; + SALT = password; + if (password != null) { + ITERATION_COUNT = password.toCharArray().length + 1; + } + } + } + + private SecretKey decryptMasterKeySK(byte masterKey[], String password, String encryptedPassString) + throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.decryptMasterKeySK()"); + } + if (encryptedPassString == null) { + getPasswordParam(password); + } + PBEKeySpec pbeKeyspec = getPBEParameterSpec(password); + byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.decryptMasterKeySK()"); + } + return getMasterKeyFromBytes(masterKeyFromDBDecrypted); + } + + private List getEncryptedMK() throws Base64DecodingException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.getEncryptedMK()"); + } + try { + if (daoManager != null) { + ArrayList ret = new ArrayList<>(); + RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager); + List<XXRangerMasterKey> lstRangerMasterKey = rangerKMSDao.getAll(); + if (lstRangerMasterKey.size() < 1) { + throw new Exception("No Master Key exists"); + } else if (lstRangerMasterKey.size() > 1) { + throw new Exception("More than one Master Key exists"); } else { - throw new Exception("No Master Key Found"); - } - } - - /** - * Generate the master key, encrypt it and save it in the database - * @param password password to be used for encryption - * @return true if the master key was successfully created - * false if master key generation was unsuccessful or the master key already exists - * @throws Throwable - */ - - public void init() { - XMLUtils.loadConfig(DBKS_SITE_XML, serverConfigProperties); - MK_CIPHER = getConfig("ranger.kms.service.masterkey.password.cipher", DEFAULT_MK_CIPHER); - MK_KeySize = getIntConfig("ranger.kms.service.masterkey.password.size", DEFAULT_MK_KeySize); - SALT_SIZE = getIntConfig("ranger.kms.service.masterkey.password.salt.size", DEFAULT_SALT_SIZE); - SALT = getConfig("ranger.kms.service.masterkey.password.salt", DEFAULT_SALT); - PBE_ALGO = getConfig("ranger.kms.service.masterkey.password.encryption.algorithm", DEFAULT_CRYPT_ALGO); - MD_ALGO = getConfig("ranger.kms.service.masterkey.password.md.algorithm", DEFAULT_MD_ALGO); - ITERATION_COUNT = getIntConfig("ranger.kms.service.masterkey.password.iteration.count", - DEFAULT_ITERATION_COUNT); - paddingString = Joiner.on(",").skipNulls().join(MK_CIPHER, MK_KeySize, SALT_SIZE, PBE_ALGO, MD_ALGO, - ITERATION_COUNT, SALT); - } - - @Override - public boolean generateMasterKey(String password) throws Throwable { - logger.info("Generating Master Key"); - init(); - String encryptedMasterKey = encryptMasterKey(password); - String savedKey = saveEncryptedMK(paddingString + "," + encryptedMasterKey, daoManager); - if (savedKey != null && !savedKey.trim().equals("")) { - logger.debug("Master Key Created with id = " + savedKey); - return true; - } - return false; - } - - public boolean generateMKFromHSMMK(String password, byte[] key) throws Throwable{ - logger.info("Generating Master Key"); - init(); - String encryptedMasterKey = encryptMasterKey(password, key); - String savedKey = saveEncryptedMK(paddingString + "," + encryptedMasterKey, daoManager); - if(savedKey != null && !savedKey.trim().equals("")){ - logger.debug("Master Key Created with id = "+savedKey); - return true; - } - return false; - } - - private String decryptMasterKey(byte masterKey[], String password, String encryptedPassString) throws Throwable { - logger.debug("Decrypting Master Key"); - if (encryptedPassString == null) { - getPasswordParam(password); - } - PBEKeySpec pbeKeyspec = getPBEParameterSpec(password); - byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec); - SecretKey masterKeyFromDB = getMasterKeyFromBytes(masterKeyFromDBDecrypted); - return Base64.encode(masterKeyFromDB.getEncoded()); - } - - public static void getPasswordParam(String paddedEncryptedPwd) { - String[] encryptedPwd = null; - if (paddedEncryptedPwd != null && paddedEncryptedPwd.contains(",")) { - encryptedPwd = Lists.newArrayList(Splitter.on(",").split(paddedEncryptedPwd)).toArray(new String[0]); - } - if (encryptedPwd != null && encryptedPwd.length >= 7) { - int index = 0; - MK_CIPHER = encryptedPwd[index]; - MK_KeySize = Integer.parseInt(encryptedPwd[++index]); - SALT_SIZE = Integer.parseInt(encryptedPwd[++index]); - PBE_ALGO = encryptedPwd[++index]; - MD_ALGO = encryptedPwd[++index]; - ITERATION_COUNT = Integer.parseInt(encryptedPwd[++index]); - SALT = encryptedPwd[++index]; - password = encryptedPwd[++index]; - - if (index > 7) { - for (int i = 8; i <= encryptedPwd.length; i++) { - password = password + "," + encryptedPwd[i]; - } + XXRangerMasterKey rangerMasterKey = rangerKMSDao.getById(lstRangerMasterKey.get(0).getId()); + String masterKeyStr = rangerMasterKey.getMasterKey(); + if (masterKeyStr.contains(",")) { + getPasswordParam(masterKeyStr); + ret.add(Base64.decode(password)); + ret.add(masterKeyStr); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.getEncryptedMK()"); } - } else { - MK_CIPHER = DEFAULT_MK_CIPHER; - MK_KeySize = DEFAULT_MK_KeySize; - SALT_SIZE = DEFAULT_SALT_SIZE; - PBE_ALGO = DEFAULT_CRYPT_ALGO; - MD_ALGO = DEFAULT_MD_ALGO; - password = paddedEncryptedPwd; - SALT = password; - ITERATION_COUNT = password.toCharArray().length + 1; + return ret; + } else { + ret.add(Base64.decode(masterKeyStr)); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.getEncryptedMK()"); + } + return ret; + } } + } + } catch (Exception e) { + logger.error("Unable to Retrieving Master Key from database!!! or ", e); + } + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.getEncryptedMK()"); } + return null; + } - private SecretKey decryptMasterKeySK(byte masterKey[], String password, String encryptedPassString) - throws Throwable { - logger.debug("Decrypting Master Key"); - if (encryptedPassString == null) { - getPasswordParam(password); + private String saveEncryptedMK(String encryptedMasterKey, DaoManager daoManager) { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.saveEncryptedMK()"); + } + XXRangerMasterKey xxRangerMasterKey = new XXRangerMasterKey(); + xxRangerMasterKey.setCipher(MK_CIPHER); + xxRangerMasterKey.setBitLength(MK_KeySize); + xxRangerMasterKey.setMasterKey(encryptedMasterKey); + try { + if (daoManager != null) { + RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager); + Long l = rangerKMSDao.getAllCount(); + if (l < 1) { + XXRangerMasterKey rangerMasterKey = rangerKMSDao.create(xxRangerMasterKey); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.saveEncryptedMK()"); + } + return rangerMasterKey.getId().toString(); } - PBEKeySpec pbeKeyspec = getPBEParameterSpec(password); - byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec); - return getMasterKeyFromBytes(masterKeyFromDBDecrypted); - } - - private List getEncryptedMK() throws Base64DecodingException { - logger.debug("Retrieving Encrypted Master Key from database"); - try { - if (daoManager != null) { - ArrayList ret = new ArrayList<>(); - RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager); - List<XXRangerMasterKey> lstRangerMasterKey = rangerKMSDao.getAll(); - if (lstRangerMasterKey.size() < 1) { - throw new Exception("No Master Key exists"); - } else if (lstRangerMasterKey.size() > 1) { - throw new Exception("More than one Master Key exists"); - } else { - XXRangerMasterKey rangerMasterKey = rangerKMSDao.getById(lstRangerMasterKey.get(0).getId()); - String masterKeyStr = rangerMasterKey.getMasterKey(); - if (masterKeyStr.contains(",")) { - getPasswordParam(masterKeyStr); - ret.add(Base64.decode(password)); - ret.add(masterKeyStr); - return ret; - } else { - ret.add(Base64.decode(masterKeyStr)); - return ret; - } - } - } - } catch (Exception e) { - e.printStackTrace(); + } + } catch (Exception e) { + logger.error("Error while saving master key in Database!!! ", e); + } + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.saveEncryptedMK()"); + } + return null; + } + + private String encryptMasterKey(String password) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.encryptMasterKey()"); + } + Key secretKey = generateMasterKey(); + PBEKeySpec pbeKeySpec = getPBEParameterSpec(password); + byte[] masterKeyToDB = encryptKey(secretKey.getEncoded(), pbeKeySpec); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.encryptMasterKey()"); + } + return Base64.encode(masterKeyToDB); + } + + private String encryptMasterKey(String password, byte[] secretKey) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.encryptMasterKey()"); + } + PBEKeySpec pbeKeySpec = getPBEParameterSpec(password); + byte[] masterKeyToDB = encryptKey(secretKey, pbeKeySpec); + if (logger.isDebugEnabled()) { + logger.debug("<== RangerMasterKey.encryptMasterKey()"); + } + return Base64.encode(masterKeyToDB); + } + + private Key generateMasterKey() throws NoSuchAlgorithmException { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.generateMasterKey()"); + } + KeyGenerator kg = KeyGenerator.getInstance(MK_CIPHER); + kg.init(MK_KeySize); + return kg.generateKey(); + } + + private PBEKeySpec getPBEParameterSpec(String password) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.getPBEParameterSpec()"); + } + MessageDigest md = MessageDigest.getInstance(MD_ALGO); + byte[] saltGen = md.digest(SALT.getBytes()); + byte[] salt = new byte[SALT_SIZE]; + System.arraycopy(saltGen, 0, salt, 0, SALT_SIZE); + return new PBEKeySpec(password.toCharArray(), salt, ITERATION_COUNT); + } + + private byte[] encryptKey(byte[] data, PBEKeySpec keyspec) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.encryptKey()"); + } + SecretKey key = getPasswordKey(keyspec); + if (keyspec.getSalt() != null) { + PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()); + Cipher c = Cipher.getInstance(key.getAlgorithm()); + c.init(Cipher.ENCRYPT_MODE, key, paramSpec); + return c.doFinal(data); + } + return null; + } + + private SecretKey getPasswordKey(PBEKeySpec keyspec) throws Throwable { + if (logger.isDebugEnabled()) { + logger.debug("==> RangerMasterKey.getPasswordKey()"); + } + SecretKeyFactory factory = SecretKeyFactory.getInstance(PBE_ALGO); + return factory.generateSecret(keyspec); + } + + private byte[] decryptKey(byte[] encrypted, PBEKeySpec keyspec) throws Throwable { + SecretKey key = getPasswordKey(keyspec); + if (keyspec.getSalt() != null) { + PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()); + Cipher c = Cipher.getInstance(key.getAlgorithm()); + c.init(Cipher.DECRYPT_MODE, key, paramSpec); + return c.doFinal(encrypted); + } + return null; + } + + private SecretKey getMasterKeyFromBytes(byte[] keyData) throws Throwable { + return new SecretKeySpec(keyData, MK_CIPHER); + } + + public Map<String, String> getPropertiesWithPrefix(Properties props, String prefix) { + Map<String, String> prefixedProperties = new HashMap<String, String>(); + + if (props != null && prefix != null) { + for (String key : props.stringPropertyNames()) { + if (key == null) { + continue; } - return null; - } - - private String saveEncryptedMK(String encryptedMasterKey, DaoManager daoManager) { - logger.debug("Saving Encrypted Master Key to database"); - XXRangerMasterKey xxRangerMasterKey = new XXRangerMasterKey(); - xxRangerMasterKey.setCipher(MK_CIPHER); - xxRangerMasterKey.setBitLength(MK_KeySize); - xxRangerMasterKey.setMasterKey(encryptedMasterKey); - try{ - if(daoManager != null){ - RangerMasterKeyDao rangerKMSDao = new RangerMasterKeyDao(daoManager); - Long l = rangerKMSDao.getAllCount(); - if(l < 1){ - XXRangerMasterKey rangerMasterKey = rangerKMSDao.create(xxRangerMasterKey); - return rangerMasterKey.getId().toString(); - } - } - }catch(Exception e){ - e.printStackTrace(); - } - return null; - } - - private String encryptMasterKey(String password) throws Throwable { - logger.debug("Encrypting Master Key"); - Key secretKey = generateMasterKey(); - PBEKeySpec pbeKeySpec = getPBEParameterSpec(password); - byte[] masterKeyToDB = encryptKey(secretKey.getEncoded(), pbeKeySpec); - return Base64.encode(masterKeyToDB); - } - - private String encryptMasterKey(String password, byte[] secretKey) throws Throwable { - logger.debug("Encrypting Master Key"); - PBEKeySpec pbeKeySpec = getPBEParameterSpec(password); - byte[] masterKeyToDB = encryptKey(secretKey, pbeKeySpec); - return Base64.encode(masterKeyToDB); - } - - private Key generateMasterKey() throws NoSuchAlgorithmException{ - KeyGenerator kg = KeyGenerator.getInstance(MK_CIPHER); - kg.init(MK_KeySize); - return kg.generateKey(); - } - - private PBEKeySpec getPBEParameterSpec(String password) throws Throwable { - MessageDigest md = MessageDigest.getInstance(MD_ALGO); - byte[] saltGen = md.digest(SALT.getBytes()); - byte[] salt = new byte[SALT_SIZE]; - System.arraycopy(saltGen, 0, salt, 0, SALT_SIZE); - return new PBEKeySpec(password.toCharArray(), salt, ITERATION_COUNT); - } - - private byte[] encryptKey(byte[] data, PBEKeySpec keyspec) throws Throwable { - SecretKey key = getPasswordKey(keyspec); - if(keyspec.getSalt() != null) { - PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()); - Cipher c = Cipher.getInstance(key.getAlgorithm()); - c.init(Cipher.ENCRYPT_MODE, key,paramSpec); - return c.doFinal(data); - } - return null; - } - private SecretKey getPasswordKey(PBEKeySpec keyspec) throws Throwable { - SecretKeyFactory factory = SecretKeyFactory.getInstance(PBE_ALGO); - return factory.generateSecret(keyspec); - } - private byte[] decryptKey(byte[] encrypted, PBEKeySpec keyspec) throws Throwable { - SecretKey key = getPasswordKey(keyspec); - if(keyspec.getSalt() != null) { - PBEParameterSpec paramSpec = new PBEParameterSpec(keyspec.getSalt(), keyspec.getIterationCount()); - Cipher c = Cipher.getInstance(key.getAlgorithm()); - c.init(Cipher.DECRYPT_MODE, key, paramSpec); - return c.doFinal(encrypted); - } - return null; - } - private SecretKey getMasterKeyFromBytes(byte[] keyData) throws Throwable { - return new SecretKeySpec(keyData, MK_CIPHER); - } - - public Map<String, String> getPropertiesWithPrefix(Properties props, String prefix) { - Map<String, String> prefixedProperties = new HashMap<String, String>(); - - if(props != null && prefix != null) { - for(String key : props.stringPropertyNames()) { - if(key == null) { - continue; - } - - String val = props.getProperty(key); - - if(key.startsWith(prefix)) { - key = key.substring(prefix.length()); - - if(key != null) { - prefixedProperties.put(key, val); + + String val = props.getProperty(key); + + if (key.startsWith(prefix)) { + key = key.substring(prefix.length()); + + if (key != null) { + prefixedProperties.put(key, val); } - } - } - } + } + } + } - return prefixedProperties; - } + return prefixedProperties; + } } http://git-wip-us.apache.org/repos/asf/ranger/blob/bc2cd5e0/kms/src/main/java/org/apache/hadoop/crypto/key/VerifyIsDBMasterkeyCorrect.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/VerifyIsDBMasterkeyCorrect.java b/kms/src/main/java/org/apache/hadoop/crypto/key/VerifyIsDBMasterkeyCorrect.java new file mode 100644 index 0000000..632e728 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/VerifyIsDBMasterkeyCorrect.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.kms.dao.DaoManager; + +public class VerifyIsDBMasterkeyCorrect { + RangerMasterKey rangerMasterKey; + RangerKeyStore dbStore; + DaoManager daoManager; + + public VerifyIsDBMasterkeyCorrect() throws Throwable { + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + RangerKMSDB rangerKMSDB = new RangerKMSDB(conf); + daoManager = rangerKMSDB.getDaoManager(); + dbStore = new RangerKeyStore(daoManager); + } + + public static void main(String[] args) throws Throwable { + if (args.length == 0) { + System.err.println("Invalid number of parameters found."); + System.exit(1); + } + try { + String password = args[0]; + if (password == null || password.trim().isEmpty()) { + System.err.println("KMS Masterkey Password not provided."); + System.exit(1); + } + new VerifyIsDBMasterkeyCorrect().verifyMasterkey(password); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void verifyMasterkey(String pass) { + try { + // Get Master Key from DB + rangerMasterKey = new RangerMasterKey(daoManager); + String masterKey = rangerMasterKey.getMasterKey(pass); + if(masterKey == null){ + // Master Key does not exists + throw new IOException("Ranger MasterKey does not exists"); + } + dbStore.engineLoad(null, masterKey.toCharArray()); + System.out.println("KMS keystore engine loaded successfully."); + } catch (Throwable e) { + throw new RuntimeException("Unable to load keystore engine with given password or Masterkey was tampered.", e); + } + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/bc2cd5e0/kms/src/main/java/org/apache/hadoop/crypto/key/VerifyIsHSMMasterkeyCorrect.java ---------------------------------------------------------------------- diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/VerifyIsHSMMasterkeyCorrect.java b/kms/src/main/java/org/apache/hadoop/crypto/key/VerifyIsHSMMasterkeyCorrect.java new file mode 100644 index 0000000..e5ebeb7 --- /dev/null +++ b/kms/src/main/java/org/apache/hadoop/crypto/key/VerifyIsHSMMasterkeyCorrect.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.crypto.key; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.ranger.kms.dao.DaoManager; + +public class VerifyIsHSMMasterkeyCorrect { + private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password"; + private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password"; + private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name"; + private static final String HSM_TYPE = "ranger.ks.hsm.type"; + + public static void main(String[] args) throws Throwable { + if (args.length < 2) { + System.err.println("Invalid number of parameters found."); + System.exit(1); + } + try { + String hsmType = args[0]; + if (hsmType == null || hsmType.trim().isEmpty()) { + System.err.println("HSM Type does not exists."); + System.exit(1); + } + + String partitionName = args[1]; + if (partitionName == null || partitionName.trim().isEmpty()) { + System.err.println("Partition name does not exists."); + System.exit(1); + } + new VerifyIsHSMMasterkeyCorrect().getHSMMasterkey(hsmType, partitionName); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void getHSMMasterkey(String hsmType, String partitionName) { + char[] partitionPassword = null; + try { + partitionPassword = ConsoleUtil + .getPasswordFromConsole("Enter Password for the Partition " + partitionName + " : "); + + Configuration conf = RangerKeyStoreProvider.getDBKSConf(); + conf.set(HSM_TYPE, hsmType); + conf.set(PARTITION_NAME, partitionName); + conf.set(PARTITION_PASSWORD, String.valueOf(partitionPassword)); + String password = conf.get(ENCRYPTION_KEY); + + RangerKMSDB rangerkmsDb = new RangerKMSDB(conf); + DaoManager daoManager = rangerkmsDb.getDaoManager(); + RangerKeyStore dbStore = new RangerKeyStore(daoManager); + + // Get Master Key from HSM + RangerHSM rangerHSM = new RangerHSM(conf); + String hsmMasterKey = rangerHSM.getMasterKey(password); + if(hsmMasterKey == null){ + // Master Key does not exists + throw new IOException("Ranger MasterKey does not exists in HSM!!!"); + } + + dbStore.engineLoad(null, hsmMasterKey.toCharArray()); + System.out.println("KMS keystore engine loaded successfully."); + } catch (Throwable t) { + throw new RuntimeException("Unable to load keystore engine with given password or Masterkey was tampered.", t); + } + + } + +}
