Github user steveloughran commented on a diff in the pull request:
https://github.com/apache/spark/pull/9232#discussion_r43488794
--- Diff:
yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala ---
@@ -142,6 +145,81 @@ class YarnSparkHadoopUtil extends SparkHadoopUtil {
val containerIdString =
System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name())
ConverterUtils.toContainerId(containerIdString)
}
+
+ /**
+ * Obtains token for the Hive metastore, using the current user as the
principal.
+ * Some exceptions are caught and downgraded to a log message.
+ * @param conf hadoop configuration; the Hive configuration will be
based on this
+ * @return a token, or `None` if there's no need for a token (no
metastore URI or principal
+ * in the config), or if a binding exception was caught and
downgraded.
+ */
+ def obtainTokenForHiveMetastore(conf: Configuration):
Option[Token[DelegationTokenIdentifier]] = {
+ try {
+ obtainTokenForHiveMetastoreInner(conf,
UserGroupInformation.getCurrentUser().getUserName)
+ } catch {
+ case e: ClassNotFoundException =>
+ logInfo(s"Hive class not found $e")
+ logDebug("Hive class not found", e)
+ None
+ case t: Throwable =>
+ throw t
+ }
+ }
+
+ /**
+ * Inner routine to obtains token for the Hive metastore; exceptions are
raised on any problem.
+ * @param conf hadoop configuration; the Hive configuration will be
based on this.
+ * @param username the username of the principal requesting the
delegating token.
+ * @return a delegation token
+ */
+ private[yarn] def obtainTokenForHiveMetastoreInner(conf: Configuration,
+ username: String): Option[Token[DelegationTokenIdentifier]] = {
+ val mirror = universe.runtimeMirror(Utils.getContextOrSparkClassLoader)
+
+ // the hive configuration class is a subclass of Hadoop Configuration,
so can be cast down
+ // to a Configuration and used without reflection
+ val hiveConfClass =
mirror.classLoader.loadClass("org.apache.hadoop.hive.conf.HiveConf")
+ // using the (Configuration, Class) constructor allows the current
configuratin to be included
+ // in the hive config.
+ val ctor = hiveConfClass.getDeclaredConstructor(classOf[Configuration],
+ classOf[Object].getClass)
+ val hiveConf = ctor.newInstance(conf,
hiveConfClass).asInstanceOf[Configuration]
+ val metastoreUri = hiveConf.getTrimmed("hive.metastore.uris", "")
+
+ // Check for local metastore
+ if (metastoreUri.nonEmpty) {
+ require(username.nonEmpty, "Username undefined")
+ val principalKey = "hive.metastore.kerberos.principal"
+ val principal = hiveConf.getTrimmed(principalKey, "")
+ require(principal.nonEmpty, "Hive principal $principalKey undefined")
+ logDebug(s"Getting Hive delegation token for $username against
$principal at $metastoreUri")
+ val hiveClass =
mirror.classLoader.loadClass("org.apache.hadoop.hive.ql.metadata.Hive")
+ val closeCurrent = hiveClass.getMethod("closeCurrent")
+ try {
+ // get all the instance methods before invoking any
+ val getDelegationToken = hiveClass.getMethod("getDelegationToken",
+ classOf[String], classOf[String])
+ val getHive = hiveClass.getMethod("get", hiveConfClass)
+
+ // invoke
+ val hive = getHive.invoke(null, hiveConf)
+ val tokenStr = getDelegationToken.invoke(hive, username, principal)
+ .asInstanceOf[java.lang.String]
--- End diff --
copied from the original...cut that and joined the lines
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
