This is an automated email from the ASF dual-hosted git repository. srdo pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/storm.git
The following commit(s) were added to refs/heads/master by this push: new f5aecc1 STORM-3449: autocreds: fix all checkstyle warnings new 7ca1ffb Merge pull request #3065 from krichter722/checkstyle-autocreds f5aecc1 is described below commit f5aecc1b5d5e15c0c128f600c5a399eb0b0e8fc4 Author: Karl-Philipp Richter <krich...@posteo.de> AuthorDate: Mon Jul 1 21:54:37 2019 +0200 STORM-3449: autocreds: fix all checkstyle warnings --- external/storm-autocreds/pom.xml | 2 +- .../storm/common/AbstractHadoopAutoCreds.java | 29 +++---- .../AbstractHadoopNimbusPluginAutoCreds.java | 25 +++--- .../apache/storm/common/CredentialKeyProvider.java | 13 +-- .../apache/storm/common/HadoopCredentialUtil.java | 85 ++++++++++--------- .../org/apache/storm/hbase/security/AutoHBase.java | 4 +- .../storm/hbase/security/AutoHBaseCommand.java | 71 ++++++++-------- .../storm/hbase/security/AutoHBaseNimbus.java | 34 ++++---- .../storm/hbase/security/HBaseSecurityUtil.java | 14 ++-- .../org/apache/storm/hdfs/security/AutoHDFS.java | 5 +- .../storm/hdfs/security/AutoHDFSCommand.java | 58 ++++++------- .../apache/storm/hdfs/security/AutoHDFSNimbus.java | 57 +++++++------ .../storm/hdfs/security/HdfsSecurityUtil.java | 24 +++--- .../org/apache/storm/hive/security/AutoHive.java | 6 +- .../storm/hive/security/AutoHiveCommand.java | 62 +++++++------- .../apache/storm/hive/security/AutoHiveNimbus.java | 97 ++++++++++++---------- .../storm/hive/security/HiveSecurityUtil.java | 13 +-- 17 files changed, 312 insertions(+), 287 deletions(-) diff --git a/external/storm-autocreds/pom.xml b/external/storm-autocreds/pom.xml index a8ac75e..87b2a2f 100644 --- a/external/storm-autocreds/pom.xml +++ b/external/storm-autocreds/pom.xml @@ -207,7 +207,7 @@ <artifactId>maven-checkstyle-plugin</artifactId> <!--Note - the version would be inherited--> <configuration> - <maxAllowedViolations>249</maxAllowedViolations> + <maxAllowedViolations>0</maxAllowedViolations> </configuration> </plugin> <plugin> diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java index 569f5af..7800401 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopAutoCreds.java @@ -15,8 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.storm.common; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.security.auth.Subject; +import javax.xml.bind.DatatypeConverter; + import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.util.Pair; import org.apache.hadoop.security.Credentials; @@ -27,15 +37,6 @@ import org.apache.storm.security.auth.IAutoCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.security.auth.Subject; -import javax.xml.bind.DatatypeConverter; -import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - /** * The base class that for auto credential plugins that abstracts out some of the common functionality. */ @@ -62,7 +63,7 @@ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, Crede @Override public void populateSubject(Subject subject, Map<String, String> credentials) { addCredentialToSubject(subject, credentials); - addTokensToUGI(subject); + addTokensToUgi(subject); } /** @@ -71,7 +72,7 @@ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, Crede @Override public void updateSubject(Subject subject, Map<String, String> credentials) { addCredentialToSubject(subject, credentials); - addTokensToUGI(subject); + addTokensToUgi(subject); } public Set<Pair<String, Credentials>> getCredentials(Map<String, String> credentials) { @@ -79,14 +80,14 @@ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, Crede } /** - * Prepare the plugin + * Prepare the plugin. * * @param topoConf the topology conf */ protected abstract void doPrepare(Map<String, Object> topoConf); /** - * The lookup key for the config key string + * The lookup key for the config key string. * * @return the config key string */ @@ -104,7 +105,7 @@ public abstract class AbstractHadoopAutoCreds implements IAutoCredentials, Crede } } - private void addTokensToUGI(Subject subject) { + private void addTokensToUgi(Subject subject) { if (subject != null) { Set<Credentials> privateCredentials = subject.getPrivateCredentials(Credentials.class); if (privateCredentials != null) { diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java index 0ddf381..807d2ad 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/AbstractHadoopNimbusPluginAutoCreds.java @@ -15,8 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.storm.common; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.xml.bind.DatatypeConverter; + import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.util.Pair; import org.apache.hadoop.conf.Configuration; @@ -29,14 +38,6 @@ import org.apache.storm.utils.ConfigUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.xml.bind.DatatypeConverter; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - /** * The base class that for auto credential plugins that abstracts out some of the common functionality. */ @@ -51,7 +52,9 @@ public abstract class AbstractHadoopNimbusPluginAutoCreds } @Override - public void populateCredentials(Map<String, String> credentials, Map<String, Object> topologyConf, final String topologyOwnerPrincipal) { + public void populateCredentials(Map<String, String> credentials, + Map<String, Object> topologyConf, + final String topologyOwnerPrincipal) { try { List<String> configKeys = getConfigKeys(topologyConf); if (!configKeys.isEmpty()) { @@ -103,14 +106,14 @@ public abstract class AbstractHadoopNimbusPluginAutoCreds } /** - * Prepare the plugin + * Prepare the plugin. * * @param conf the storm cluster conf set via storm.yaml */ protected abstract void doPrepare(Map<String, Object> conf); /** - * The lookup key for the config key string + * The lookup key for the config key string. * * @return the config key string */ diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/CredentialKeyProvider.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/CredentialKeyProvider.java index 3826407..a3503a5 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/common/CredentialKeyProvider.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/CredentialKeyProvider.java @@ -15,16 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.storm.common; /** * Provider interface for credential key. */ public interface CredentialKeyProvider { - /** - * The lookup key for the config key string - * - * @return the config key string - */ - String getCredentialKey(String configKey); + /** + * The lookup key for the config key string. + * + * @return the config key string + */ + String getCredentialKey(String configKey); } diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java index f676abf..690c391 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/common/HadoopCredentialUtil.java @@ -15,66 +15,69 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.storm.common; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.math3.util.Pair; -import org.apache.hadoop.security.Credentials; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +package org.apache.storm.common; -import javax.xml.bind.DatatypeConverter; import java.io.ByteArrayInputStream; import java.io.ObjectInputStream; import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; +import javax.xml.bind.DatatypeConverter; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.math3.util.Pair; +import org.apache.hadoop.security.Credentials; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Utility class for getting credential for Hadoop. */ final class HadoopCredentialUtil { - private static final Logger LOG = LoggerFactory.getLogger(HadoopCredentialUtil.class); + private static final Logger LOG = LoggerFactory.getLogger(HadoopCredentialUtil.class); - private HadoopCredentialUtil() { - } + private HadoopCredentialUtil() { + } - static Set<Pair<String, Credentials>> getCredential(CredentialKeyProvider provider, - Map<String, String> credentials, Collection<String> configKeys) { - Set<Pair<String, Credentials>> res = new HashSet<>(); - if (!configKeys.isEmpty()) { - for (String configKey : configKeys) { - Credentials cred = doGetCredentials(provider, credentials, configKey); - if (cred != null) { - res.add(new Pair(configKey, cred)); + static Set<Pair<String, Credentials>> getCredential(CredentialKeyProvider provider, + Map<String, String> credentials, + Collection<String> configKeys) { + Set<Pair<String, Credentials>> res = new HashSet<>(); + if (!configKeys.isEmpty()) { + for (String configKey : configKeys) { + Credentials cred = doGetCredentials(provider, credentials, configKey); + if (cred != null) { + res.add(new Pair(configKey, cred)); + } + } + } else { + Credentials cred = doGetCredentials(provider, credentials, StringUtils.EMPTY); + if (cred != null) { + res.add(new Pair(StringUtils.EMPTY, cred)); + } } - } - } else { - Credentials cred = doGetCredentials(provider, credentials, StringUtils.EMPTY); - if (cred != null) { - res.add(new Pair(StringUtils.EMPTY, cred)); - } + return res; } - return res; - } - private static Credentials doGetCredentials(CredentialKeyProvider provider, - Map<String, String> credentials, String configKey) { - Credentials credential = null; - String credentialKey = provider.getCredentialKey(configKey); - if (credentials != null && credentials.containsKey(credentialKey)) { - try { - byte[] credBytes = DatatypeConverter.parseBase64Binary(credentialKey); - ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(credBytes)); + private static Credentials doGetCredentials(CredentialKeyProvider provider, + Map<String, String> credentials, + String configKey) { + Credentials credential = null; + String credentialKey = provider.getCredentialKey(configKey); + if (credentials != null && credentials.containsKey(credentialKey)) { + try { + byte[] credBytes = DatatypeConverter.parseBase64Binary(credentialKey); + ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(credBytes)); - credential = new Credentials(); - credential.readFields(in); - } catch (Exception e) { - LOG.error("Could not obtain credentials from credentials map.", e); - } + credential = new Credentials(); + credential.readFields(in); + } catch (Exception e) { + LOG.error("Could not obtain credentials from credentials map.", e); + } + } + return credential; } - return credential; - } } diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java index e38a54a..549f2d9 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBase.java @@ -18,11 +18,11 @@ package org.apache.storm.hbase.security; -import org.apache.storm.common.AbstractHadoopAutoCreds; +import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS; import java.util.Map; -import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS; +import org.apache.storm.common.AbstractHadoopAutoCreds; /** * Auto credentials plugin for HBase implementation. This class provides a way to automatically diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java index bd03ad9..189d32a 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseCommand.java @@ -15,49 +15,50 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.storm.hbase.security; -import org.apache.storm.Config; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_KEYTAB_FILE_KEY; +import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_PRINCIPAL_KEY; -import javax.security.auth.Subject; import java.util.HashMap; import java.util.Map; +import javax.security.auth.Subject; -import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_KEYTAB_FILE_KEY; -import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_PRINCIPAL_KEY; +import org.apache.storm.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Command tool of Hive credential renewer + * Command tool of Hive credential renewer. */ public final class AutoHBaseCommand { - private static final Logger LOG = LoggerFactory.getLogger(AutoHBaseCommand.class); - - private AutoHBaseCommand() { - } - - @SuppressWarnings("unchecked") - public static void main(String[] args) throws Exception { - Map<String, Object> conf = new HashMap<>(); - conf.put(HBASE_PRINCIPAL_KEY, args[1]); // hbase principal storm-hb...@witzen.com - conf.put(HBASE_KEYTAB_FILE_KEY, - args[2]); // storm hbase keytab /etc/security/keytabs/storm-hbase.keytab - - AutoHBase autoHBase = new AutoHBase(); - autoHBase.prepare(conf); - AutoHBaseNimbus autoHBaseNimbus = new AutoHBaseNimbus(); - autoHBaseNimbus.prepare(conf); - - Map<String, String> creds = new HashMap<>(); - autoHBaseNimbus.populateCredentials(creds, conf, args[0]); //with realm e.g. st...@witzend.com - LOG.info("Got HBase credentials" + autoHBase.getCredentials(creds)); - - Subject s = new Subject(); - autoHBase.populateSubject(s, creds); - LOG.info("Got a Subject " + s); - - autoHBaseNimbus.renew(creds, conf, args[0]); - LOG.info("renewed credentials" + autoHBase.getCredentials(creds)); - } + private static final Logger LOG = LoggerFactory.getLogger(AutoHBaseCommand.class); + + private AutoHBaseCommand() { + } + + @SuppressWarnings("unchecked") + public static void main(String[] args) throws Exception { + Map<String, Object> conf = new HashMap<>(); + conf.put(HBASE_PRINCIPAL_KEY, args[1]); // hbase principal storm-hb...@witzen.com + conf.put(HBASE_KEYTAB_FILE_KEY, + args[2]); // storm hbase keytab /etc/security/keytabs/storm-hbase.keytab + + AutoHBase autoHBase = new AutoHBase(); + autoHBase.prepare(conf); + AutoHBaseNimbus autoHBaseNimbus = new AutoHBaseNimbus(); + autoHBaseNimbus.prepare(conf); + + Map<String, String> creds = new HashMap<>(); + autoHBaseNimbus.populateCredentials(creds, conf, args[0]); //with realm e.g. st...@witzend.com + LOG.info("Got HBase credentials" + autoHBase.getCredentials(creds)); + + Subject s = new Subject(); + autoHBase.populateSubject(s, creds); + LOG.info("Got a Subject " + s); + + autoHBaseNimbus.renew(creds, conf, args[0]); + LOG.info("renewed credentials" + autoHBase.getCredentials(creds)); + } } diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java index c9361b1..2ad5819 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/AutoHBaseNimbus.java @@ -18,6 +18,15 @@ package org.apache.storm.hbase.security; +import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS; +import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_KEYTAB_FILE_KEY; +import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_PRINCIPAL_KEY; + +import java.io.ByteArrayOutputStream; +import java.io.ObjectOutputStream; +import java.net.InetAddress; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.Connection; @@ -34,15 +43,6 @@ import org.apache.storm.common.AbstractHadoopNimbusPluginAutoCreds; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.ObjectOutputStream; -import java.net.InetAddress; -import java.util.Map; - -import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_CREDENTIALS; -import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_KEYTAB_FILE_KEY; -import static org.apache.storm.hbase.security.HBaseSecurityUtil.HBASE_PRINCIPAL_KEY; - /** * Auto credentials nimbus plugin for HBase implementation. This class automatically * gets HBase delegation tokens and push it to user's topology. @@ -76,16 +76,10 @@ public class AutoHBaseNimbus extends AbstractHadoopNimbusPluginAutoCreds { return getHadoopCredentials(conf, HBaseConfiguration.create(), topologyOwnerPrincipal); } - private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) { - Configuration configuration = HBaseConfiguration.create(); - fillHadoopConfiguration(topoConf, configKey, configuration); - return configuration; - } - @SuppressWarnings("unchecked") protected byte[] getHadoopCredentials(Map<String, Object> conf, Configuration hbaseConf, final String topologySubmitterUser) { try { - if(UserGroupInformation.isSecurityEnabled()) { + if (UserGroupInformation.isSecurityEnabled()) { UserProvider provider = UserProvider.instantiate(hbaseConf); provider.login(HBASE_KEYTAB_FILE_KEY, HBASE_PRINCIPAL_KEY, InetAddress.getLocalHost().getCanonicalHostName()); @@ -97,7 +91,7 @@ public class AutoHBaseNimbus extends AbstractHadoopNimbusPluginAutoCreds { User user = User.create(proxyUser); - if(user.isHBaseSecurityEnabled(hbaseConf)) { + if (user.isHBaseSecurityEnabled(hbaseConf)) { final Connection connection = ConnectionFactory.createConnection(hbaseConf, user); TokenUtil.obtainAndCacheToken(connection, user); @@ -127,6 +121,12 @@ public class AutoHBaseNimbus extends AbstractHadoopNimbusPluginAutoCreds { } } + private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) { + Configuration configuration = HBaseConfiguration.create(); + fillHadoopConfiguration(topoConf, configKey, configuration); + return configuration; + } + @Override public void doRenew(Map<String, String> credentials, Map<String, Object> topologyConf, final String topologySubmitterUser) { //HBASE tokens are not renewable so we always have to get new ones. diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java index cb8329b..1d8e5b8 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hbase/security/HBaseSecurityUtil.java @@ -18,19 +18,19 @@ package org.apache.storm.hbase.security; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.security.UserProvider; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.storm.security.auth.kerberos.AutoTGT; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS; import java.io.IOException; import java.net.InetAddress; import java.util.List; import java.util.Map; -import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.security.UserProvider; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.storm.security.auth.kerberos.AutoTGT; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class provides util methods for storm-hbase connector communicating diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java index 6797db4..a5d1a03 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFS.java @@ -18,16 +18,17 @@ package org.apache.storm.hdfs.security; -import org.apache.storm.common.AbstractHadoopAutoCreds; +import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS; import java.util.Map; -import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS; +import org.apache.storm.common.AbstractHadoopAutoCreds; /** * Auto credentials plugin for HDFS implementation. This class provides a way to automatically * push credentials to a topology and to retrieve them in the worker. */ +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") public class AutoHDFS extends AbstractHadoopAutoCreds { @Override public void doPrepare(Map<String, Object> conf) { diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java index 4a6a9a4..1e9290a 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSCommand.java @@ -15,49 +15,51 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.storm.hdfs.security; -import org.apache.storm.Config; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY; +import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_USER_NAME_KEY; -import javax.security.auth.Subject; import java.util.HashMap; import java.util.Map; +import javax.security.auth.Subject; -import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY; -import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_USER_NAME_KEY; +import org.apache.storm.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Command tool of HDFS credential renewer + * Command tool of HDFS credential renewer. */ +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") public final class AutoHDFSCommand { - private static final Logger LOG = LoggerFactory.getLogger(AutoHDFSCommand.class); + private static final Logger LOG = LoggerFactory.getLogger(AutoHDFSCommand.class); - private AutoHDFSCommand() { - } + private AutoHDFSCommand() { + } - @SuppressWarnings("unchecked") - public static void main(String[] args) throws Exception { - Map<String, Object> conf = new HashMap<>(); - conf.put(STORM_USER_NAME_KEY, args[1]); //with realm e.g. h...@witzend.com - conf.put(STORM_KEYTAB_FILE_KEY, args[2]);// /etc/security/keytabs/storm.keytab + @SuppressWarnings("unchecked") + public static void main(String[] args) throws Exception { + Map<String, Object> conf = new HashMap<>(); + conf.put(STORM_USER_NAME_KEY, args[1]); //with realm e.g. h...@witzend.com + conf.put(STORM_KEYTAB_FILE_KEY, args[2]);// /etc/security/keytabs/storm.keytab - AutoHDFS autoHDFS = new AutoHDFS(); - autoHDFS.prepare(conf); - AutoHDFSNimbus autoHDFSNimbus = new AutoHDFSNimbus(); - autoHDFSNimbus.prepare(conf); + AutoHDFS autoHdfs = new AutoHDFS(); + autoHdfs.prepare(conf); + AutoHDFSNimbus autoHdfsNimbus = new AutoHDFSNimbus(); + autoHdfsNimbus.prepare(conf); - Map<String,String> creds = new HashMap<>(); - autoHDFSNimbus.populateCredentials(creds, conf, args[0]); - LOG.info("Got HDFS credentials", autoHDFS.getCredentials(creds)); + Map<String,String> creds = new HashMap<>(); + autoHdfsNimbus.populateCredentials(creds, conf, args[0]); + LOG.info("Got HDFS credentials", autoHdfs.getCredentials(creds)); - Subject s = new Subject(); - autoHDFS.populateSubject(s, creds); - LOG.info("Got a Subject "+ s); + Subject s = new Subject(); + autoHdfs.populateSubject(s, creds); + LOG.info("Got a Subject " + s); - autoHDFSNimbus.renew(creds, conf, args[0]); - LOG.info("renewed credentials", autoHDFS.getCredentials(creds)); - } + autoHdfsNimbus.renew(creds, conf, args[0]); + LOG.info("renewed credentials", autoHdfs.getCredentials(creds)); + } } diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java index aab16a9..87ab429 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/AutoHDFSNimbus.java @@ -18,6 +18,20 @@ package org.apache.storm.hdfs.security; +import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS; +import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY; +import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_USER_NAME_KEY; +import static org.apache.storm.hdfs.security.HdfsSecurityUtil.TOPOLOGY_HDFS_URI; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.net.URI; +import java.security.PrivilegedAction; +import java.util.Collection; +import java.util.List; +import java.util.Map; + import org.apache.commons.math3.util.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -31,24 +45,11 @@ import org.apache.storm.common.AbstractHadoopNimbusPluginAutoCreds; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.net.URI; -import java.security.PrivilegedAction; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import static org.apache.storm.hdfs.security.HdfsSecurityUtil.HDFS_CREDENTIALS; -import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_KEYTAB_FILE_KEY; -import static org.apache.storm.hdfs.security.HdfsSecurityUtil.STORM_USER_NAME_KEY; -import static org.apache.storm.hdfs.security.HdfsSecurityUtil.TOPOLOGY_HDFS_URI; - /** * Auto credentials nimbus plugin for HDFS implementation. This class automatically * gets HDFS delegation tokens and push it to user's topology. */ +@SuppressWarnings("checkstyle:AbbreviationAsWordInName") public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds { private static final Logger LOG = LoggerFactory.getLogger(AutoHDFSNimbus.class); @@ -57,7 +58,7 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds { @Override public void doPrepare(Map<String, Object> conf) { - if(conf.containsKey(STORM_KEYTAB_FILE_KEY) && conf.containsKey(STORM_USER_NAME_KEY)) { + if (conf.containsKey(STORM_KEYTAB_FILE_KEY) && conf.containsKey(STORM_USER_NAME_KEY)) { hdfsKeyTab = (String) conf.get(STORM_KEYTAB_FILE_KEY); hdfsPrincipal = (String) conf.get(STORM_USER_NAME_KEY); } @@ -84,19 +85,14 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds { return getHadoopCredentials(conf, new Configuration(), topologyOwnerPrincipal); } - private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) { - Configuration configuration = new Configuration(); - fillHadoopConfiguration(topoConf, configKey, configuration); - return configuration; - } - @SuppressWarnings("unchecked") private byte[] getHadoopCredentials(Map<String, Object> conf, final Configuration configuration, final String topologySubmitterUser) { try { - if(UserGroupInformation.isSecurityEnabled()) { + if (UserGroupInformation.isSecurityEnabled()) { login(configuration); - final URI nameNodeURI = conf.containsKey(TOPOLOGY_HDFS_URI) ? new URI(conf.get(TOPOLOGY_HDFS_URI).toString()) + final URI nameNodeUri = conf.containsKey(TOPOLOGY_HDFS_URI) + ? new URI(conf.get(TOPOLOGY_HDFS_URI).toString()) : FileSystem.getDefaultUri(configuration); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); @@ -107,8 +103,8 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds { @Override public Object run() { try { - FileSystem fileSystem = FileSystem.get(nameNodeURI, configuration); - Credentials credential= proxyUser.getCredentials(); + FileSystem fileSystem = FileSystem.get(nameNodeUri, configuration); + Credentials credential = proxyUser.getCredentials(); if (configuration.get(STORM_USER_NAME_KEY) == null) { configuration.set(STORM_USER_NAME_KEY, hdfsPrincipal); @@ -140,6 +136,12 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds { } } + private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) { + Configuration configuration = new Configuration(); + fillHadoopConfiguration(topoConf, configKey, configuration); + return configuration; + } + /** * {@inheritDoc} */ @@ -163,8 +165,9 @@ public class AutoHDFSNimbus extends AbstractHadoopNimbusPluginAutoCreds { LOG.debug("No tokens found for credentials, skipping renewal."); } } catch (Exception e) { - LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " + - "renewal period so attempting to get new tokens.", e); + LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " + + "renewal period so attempting to get new tokens.", + e); populateCredentials(credentials, topologyConf, topologyOwnerPrincipal); } } diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java index c5a3f0f..a0c8236 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hdfs/security/HdfsSecurityUtil.java @@ -15,22 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.storm.hdfs.security; -import org.apache.storm.security.auth.kerberos.AutoTGT; +package org.apache.storm.hdfs.security; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.SecurityUtil; -import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import static org.apache.storm.Config.TOPOLOGY_AUTO_CREDENTIALS; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; + +import org.apache.storm.security.auth.kerberos.AutoTGT; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class provides util methods for storm-hdfs connector communicating @@ -51,9 +53,9 @@ public final class HdfsSecurityUtil { public static void login(Map<String, Object> conf, Configuration hdfsConfig) throws IOException { //If AutoHDFS is specified, do not attempt to login using keytabs, only kept for backward compatibility. - if(conf.get(TOPOLOGY_AUTO_CREDENTIALS) == null || - (!(((List)conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class.getName())) && - !(((List)conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class.getName())))) { + if (conf.get(TOPOLOGY_AUTO_CREDENTIALS) == null + || (!(((List)conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoHDFS.class.getName())) + && !(((List)conf.get(TOPOLOGY_AUTO_CREDENTIALS)).contains(AutoTGT.class.getName())))) { if (UserGroupInformation.isSecurityEnabled()) { // compareAndSet added because of https://issues.apache.org/jira/browse/STORM-1535 if (isLoggedIn.compareAndSet(false, true)) { diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHive.java b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHive.java index 2160753..94b38fa 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHive.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHive.java @@ -18,12 +18,12 @@ package org.apache.storm.hive.security; -import org.apache.storm.common.AbstractHadoopAutoCreds; +import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS; +import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS_CONFIG_KEYS; import java.util.Map; -import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS; -import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS_CONFIG_KEYS; +import org.apache.storm.common.AbstractHadoopAutoCreds; /** * Auto credentials plugin for Hive implementation. This class provides a way to automatically diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveCommand.java b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveCommand.java index dcd99ae..f5c139b 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveCommand.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveCommand.java @@ -15,51 +15,53 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.storm.hive.security; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.storm.Config; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_KEYTAB_FILE_KEY; +import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_PRINCIPAL_KEY; -import javax.security.auth.Subject; import java.util.HashMap; import java.util.Map; +import javax.security.auth.Subject; -import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_KEYTAB_FILE_KEY; -import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_PRINCIPAL_KEY; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.storm.Config; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Command tool of Hive credential renewer + * Command tool of Hive credential renewer. */ public final class AutoHiveCommand { - private static final Logger LOG = LoggerFactory.getLogger(AutoHiveCommand.class); + private static final Logger LOG = LoggerFactory.getLogger(AutoHiveCommand.class); - private AutoHiveCommand() { - } + private AutoHiveCommand() { + } - @SuppressWarnings("unchecked") - public static void main(String[] args) throws Exception { - Map<String, Object> conf = new HashMap<>(); - conf.put(HIVE_PRINCIPAL_KEY, args[1]); // hive principal storm-h...@witzen.com - conf.put(HIVE_KEYTAB_FILE_KEY, args[2]); // storm hive keytab /etc/security/keytabs/storm-hive.keytab - conf.put(HiveConf.ConfVars.METASTOREURIS.varname, args[3]); // hive.metastore.uris : "thrift://pm-eng1-cluster1.field.hortonworks.com:9083" + @SuppressWarnings("unchecked") + public static void main(String[] args) throws Exception { + Map<String, Object> conf = new HashMap<>(); + conf.put(HIVE_PRINCIPAL_KEY, args[1]); // hive principal storm-h...@witzen.com + conf.put(HIVE_KEYTAB_FILE_KEY, args[2]); // storm hive keytab /etc/security/keytabs/storm-hive.keytab + // hive.metastore.uris : "thrift://pm-eng1-cluster1.field.hortonworks.com:9083" + conf.put(HiveConf.ConfVars.METASTOREURIS.varname, args[3]); - AutoHive autoHive = new AutoHive(); - autoHive.prepare(conf); - AutoHiveNimbus autoHiveNimbus = new AutoHiveNimbus(); - autoHiveNimbus.prepare(conf); + AutoHive autoHive = new AutoHive(); + autoHive.prepare(conf); + AutoHiveNimbus autoHiveNimbus = new AutoHiveNimbus(); + autoHiveNimbus.prepare(conf); - Map<String, String> creds = new HashMap<>(); - autoHiveNimbus.populateCredentials(creds, conf, args[0]); - LOG.info("Got Hive credentials" + autoHive.getCredentials(creds)); + Map<String, String> creds = new HashMap<>(); + autoHiveNimbus.populateCredentials(creds, conf, args[0]); + LOG.info("Got Hive credentials" + autoHive.getCredentials(creds)); - Subject subject = new Subject(); - autoHive.populateSubject(subject, creds); - LOG.info("Got a Subject " + subject); + Subject subject = new Subject(); + autoHive.populateSubject(subject, creds); + LOG.info("Got a Subject " + subject); - autoHiveNimbus.renew(creds, conf, args[0]); - LOG.info("Renewed credentials" + autoHive.getCredentials(creds)); - } + autoHiveNimbus.renew(creds, conf, args[0]); + LOG.info("Renewed credentials" + autoHive.getCredentials(creds)); + } } diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveNimbus.java b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveNimbus.java index 9854b69..e3b0ca7 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveNimbus.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/AutoHiveNimbus.java @@ -18,6 +18,20 @@ package org.apache.storm.hive.security; +import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS; +import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS_CONFIG_KEYS; +import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_KEYTAB_FILE_KEY; +import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_PRINCIPAL_KEY; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.Collection; +import java.util.List; +import java.util.Map; + import org.apache.commons.math3.util.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; @@ -29,25 +43,10 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hive.hcatalog.api.HCatClient; import org.apache.hive.hcatalog.common.HCatException; -import org.apache.storm.Config; import org.apache.storm.common.AbstractHadoopNimbusPluginAutoCreds; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS; -import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_CREDENTIALS_CONFIG_KEYS; -import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_KEYTAB_FILE_KEY; -import static org.apache.storm.hive.security.HiveSecurityUtil.HIVE_PRINCIPAL_KEY; - /** * Auto credentials nimbus plugin for Hive implementation. This class automatically * gets Hive delegation tokens and push it to user's topology. @@ -57,6 +56,7 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds { public String hiveKeytab; public String hivePrincipal; + @SuppressWarnings("checkstyle:AbbreviationAsWordInName") public String metaStoreURI; @Override @@ -90,29 +90,13 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds { return getHadoopCredentials(conf, configuration, topologyOwnerPrincipal); } - private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) { - Configuration configuration = new Configuration(); - fillHadoopConfiguration(topoConf, configKey, configuration); - return configuration; - } - - public HiveConf createHiveConf(String metaStoreURI, String hiveMetaStorePrincipal) throws IOException { - HiveConf hcatConf = new HiveConf(); - hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreURI); - hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); - hcatConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - hcatConf.setBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, true); - hcatConf.set(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, hiveMetaStorePrincipal); - return hcatConf; - } - @SuppressWarnings("unchecked") protected byte[] getHadoopCredentials(Map<String, Object> conf, final Configuration configuration, final String topologySubmitterUser) { try { if (UserGroupInformation.isSecurityEnabled()) { - String hiveMetaStoreURI = getMetaStoreURI(configuration); + String hiveMetaStoreUri = getMetaStoreUri(configuration); String hiveMetaStorePrincipal = getMetaStorePrincipal(configuration); - HiveConf hcatConf = createHiveConf(hiveMetaStoreURI, hiveMetaStorePrincipal); + HiveConf hcatConf = createHiveConf(hiveMetaStoreUri, hiveMetaStorePrincipal); login(configuration); UserGroupInformation currentUser = UserGroupInformation.getCurrentUser(); @@ -142,6 +126,22 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds { } } + private Configuration getHadoopConfiguration(Map<String, Object> topoConf, String configKey) { + Configuration configuration = new Configuration(); + fillHadoopConfiguration(topoConf, configKey, configuration); + return configuration; + } + + public HiveConf createHiveConf(String metaStoreUri, String hiveMetaStorePrincipal) throws IOException { + HiveConf hcatConf = new HiveConf(); + hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, metaStoreUri); + hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); + hcatConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + hcatConf.setBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, true); + hcatConf.set(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname, hiveMetaStorePrincipal); + return hcatConf; + } + private Token<DelegationTokenIdentifier> getDelegationToken(HiveConf hcatConf, String metaStoreServicePrincipal, String topologySubmitterUser) throws IOException { @@ -161,23 +161,26 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds { return delegationTokenId; } finally { - if (hcatClient != null) + if (hcatClient != null) { hcatClient.close(); + } } } - private String getMetaStoreURI(Configuration configuration) { - if (configuration.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) + private String getMetaStoreUri(Configuration configuration) { + if (configuration.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) { return metaStoreURI; - else + } else { return configuration.get(HiveConf.ConfVars.METASTOREURIS.varname); + } } private String getMetaStorePrincipal(Configuration configuration) { - if (configuration.get(HIVE_PRINCIPAL_KEY) == null) + if (configuration.get(HIVE_PRINCIPAL_KEY) == null) { return hivePrincipal; - else + } else { return configuration.get(HIVE_PRINCIPAL_KEY); + } } private void login(Configuration configuration) throws IOException { @@ -197,7 +200,7 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds { for (Pair<String, Credentials> cred : getCredentials(credentials, configKeys)) { try { Configuration configuration = getHadoopConfiguration(topologyConf, cred.getFirst()); - String hiveMetaStoreURI = getMetaStoreURI(configuration); + String hiveMetaStoreUri = getMetaStoreUri(configuration); String hiveMetaStorePrincipal = getMetaStorePrincipal(configuration); Collection<Token<? extends TokenIdentifier>> tokens = cred.getSecond().getAllTokens(); @@ -205,26 +208,27 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds { if (tokens != null && !tokens.isEmpty()) { for (Token<? extends TokenIdentifier> token : tokens) { - long expiration = renewToken(token, hiveMetaStoreURI, hiveMetaStorePrincipal); + long expiration = renewToken(token, hiveMetaStoreUri, hiveMetaStorePrincipal); LOG.info("Hive delegation token renewed, new expiration time {}", expiration); } } else { LOG.debug("No tokens found for credentials, skipping renewal."); } } catch (Exception e) { - LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " + - "renewal period so attempting to get new tokens.", e); + LOG.warn("could not renew the credentials, one of the possible reason is tokens are beyond " + + "renewal period so attempting to get new tokens.", + e); populateCredentials(credentials, topologyConf); } } } - private long renewToken(Token token, String metaStoreURI, String hiveMetaStorePrincipal) { + private long renewToken(Token token, String metaStoreUri, String hiveMetaStorePrincipal) { HCatClient hcatClient = null; if (UserGroupInformation.isSecurityEnabled()) { try { String tokenStr = token.encodeToUrlString(); - HiveConf hcatConf = createHiveConf(metaStoreURI, hiveMetaStorePrincipal); + HiveConf hcatConf = createHiveConf(metaStoreUri, hiveMetaStorePrincipal); LOG.debug("renewing delegation tokens for principal={}", hiveMetaStorePrincipal); hcatClient = HCatClient.create(hcatConf); Long expiryTime = hcatClient.renewDelegationToken(tokenStr); @@ -233,12 +237,13 @@ public class AutoHiveNimbus extends AbstractHadoopNimbusPluginAutoCreds { } catch (Exception ex) { throw new RuntimeException("Failed to renew delegation tokens.", ex); } finally { - if (hcatClient != null) + if (hcatClient != null) { try { hcatClient.close(); } catch (HCatException e) { LOG.error(" Exception", e); } + } } } else { throw new RuntimeException("Security is not enabled for Hadoop"); diff --git a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/HiveSecurityUtil.java b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/HiveSecurityUtil.java index 3e3186f..de67f2c 100644 --- a/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/HiveSecurityUtil.java +++ b/external/storm-autocreds/src/main/java/org/apache/storm/hive/security/HiveSecurityUtil.java @@ -15,6 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.storm.hive.security; /** @@ -22,11 +23,11 @@ package org.apache.storm.hive.security; * with secured Hive. */ public final class HiveSecurityUtil { - public static final String HIVE_KEYTAB_FILE_KEY = "hive.keytab.file"; - public static final String HIVE_PRINCIPAL_KEY = "hive.kerberos.principal"; - public static final String HIVE_CREDENTIALS_CONFIG_KEYS = "hiveCredentialsConfigKeys"; - public static final String HIVE_CREDENTIALS = "HIVE_CREDENTIALS"; + public static final String HIVE_KEYTAB_FILE_KEY = "hive.keytab.file"; + public static final String HIVE_PRINCIPAL_KEY = "hive.kerberos.principal"; + public static final String HIVE_CREDENTIALS_CONFIG_KEYS = "hiveCredentialsConfigKeys"; + public static final String HIVE_CREDENTIALS = "HIVE_CREDENTIALS"; - private HiveSecurityUtil() { - } + private HiveSecurityUtil() { + } }