This is an automated email from the ASF dual-hosted git repository. daijy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 3e9614e HIVE-21247: Webhcat beeline in secure mode (Daniel Dai, reviewed by Thejas Nair) 3e9614e is described below commit 3e9614eebd19f9843b28d97aaee1c3dfb815fb3d Author: Daniel Dai <dai...@gmail.com> AuthorDate: Tue Feb 26 16:38:26 2019 -0800 HIVE-21247: Webhcat beeline in secure mode (Daniel Dai, reviewed by Thejas Nair) Signed-off-by: Thejas M Nair <the...@hortonworks.com> --- .../src/java/org/apache/hive/beeline/BeeLine.java | 4 ++ hcatalog/webhcat/svr/pom.xml | 16 +++++ .../apache/hive/hcatalog/templeton/AppConfig.java | 1 + .../hive/hcatalog/templeton/HiveDelegator.java | 6 ++ .../hcatalog/templeton/SecureProxySupport.java | 1 + .../hive/hcatalog/templeton/tool/LaunchMapper.java | 20 ++++++- .../templeton/tool/TempletonControllerJob.java | 68 ++++++++++++++++++---- .../java/org/apache/hive/jdbc/HiveConnection.java | 26 ++++++++- packaging/src/main/assembly/bin.xml | 3 + pom.xml | 1 + 10 files changed, 132 insertions(+), 14 deletions(-) diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 65eee2c..cded55f 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -1203,6 +1203,10 @@ public class BeeLine implements Closeable { if (password != null) { jdbcConnectionParams.getSessionVars().put(JdbcConnectionParams.AUTH_PASSWD, password); } + String auth = cl.getOptionValue("a"); + if (auth != null) { + jdbcConnectionParams.getSessionVars().put(JdbcConnectionParams.AUTH_TYPE, auth); + } mergedConnectionProperties = HS2ConnectionFileUtils.mergeUserConnectionPropertiesAndBeelineSite( userConnectionProperties, jdbcConnectionParams); diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml index 4dfade5..36d15cd 100644 --- a/hcatalog/webhcat/svr/pom.xml +++ b/hcatalog/webhcat/svr/pom.xml @@ -92,6 +92,12 @@ <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> <version>${jersey.version}</version> + <exclusions> + <exclusion> + <groupId>javax.ws.rs</groupId> + <artifactId>jsr311-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>com.sun.jersey</groupId> @@ -144,6 +150,11 @@ <version>${slf4j.version}</version> </dependency> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-jdbc</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> <version>${hadoop.version}</version> @@ -199,6 +210,11 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>javax.ws.rs</groupId> + <artifactId>javax.ws.rs-api</artifactId> + <version>${rs-api.version}</version> + </dependency> <!-- test inter-project --> <dependency> <groupId>org.apache.hive</groupId> diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java index 1fd9e47..b566cf8 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java @@ -163,6 +163,7 @@ public class AppConfig extends Configuration { * of escape/unescape methods in {@link org.apache.hadoop.util.StringUtils} in webhcat. */ public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; + public static final String HIVE_SERVER2_URL = "templeton.hive.hs2.url"; public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive"; public static final String SQOOP_PATH_NAME = "templeton.sqoop.path"; public static final String SQOOP_HOME_PATH = "templeton.sqoop.home"; diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java index 3f1968d..3f679ac 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java @@ -28,6 +28,7 @@ import java.util.Map; import org.apache.commons.exec.ExecuteException; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants; import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob; import org.apache.hive.hcatalog.templeton.tool.TempletonUtils; @@ -78,6 +79,11 @@ public class HiveDelegator extends LauncherDelegator { args.add("-p"); args.add("default"); + if (UserGroupInformation.isSecurityEnabled()) { + args.add("-a"); + args.add("delegationToken"); + } + //add mapreduce job tag placeholder args.add("--hiveconf"); args.add(TempletonControllerJob.HIVE_QUERY_TAG_ARG_PLACEHOLDER); diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java index c503a7a..bbe5947 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java @@ -50,6 +50,7 @@ import org.apache.thrift.TException; public class SecureProxySupport { private Path tokenPath; public static final String HCAT_SERVICE = "hcat"; + public static final String HIVE_SERVICE = "hive"; private final boolean isEnabled; private String user; diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java index b1f4a6a..a776a0b 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java @@ -18,6 +18,10 @@ */ package org.apache.hive.hcatalog.templeton.tool; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hive.hcatalog.templeton.SecureProxySupport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -153,10 +157,10 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text> env.put(pathVarName, paths); } } - protected Process startJob(Configuration conf, String jobId, String user, String overrideClasspath, + protected Process startJob(Context context, String jobId, String user, String overrideClasspath, LauncherDelegator.JobType jobType) throws IOException, InterruptedException { - + Configuration conf = context.getConfiguration(); copyLocal(COPY_NAME, conf); String[] jarArgs = TempletonUtils.decodeArray(conf.get(JAR_ARGS_NAME)); @@ -174,6 +178,16 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text> handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary"); handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path"); if (jobType == LauncherDelegator.JobType.HIVE) { + Credentials cred = new Credentials(); + Token<? extends TokenIdentifier> token = context.getCredentials().getToken(new + Text(SecureProxySupport.HIVE_SERVICE)); + cred.addToken(new + Text(SecureProxySupport.HIVE_SERVICE), token); + File t = File.createTempFile("templeton", null); + Path tokenPath = new Path(t.toURI()); + cred.writeTokenStorageFile(tokenPath, conf); + env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, + tokenPath.toUri().getPath()); replaceJobTag(jarArgsList, JobSubmissionConstants.HIVE_QUERY_TAG_ARG_PLACEHOLDER, JobSubmissionConstants.HIVE_QUERY_TAG, jobId); } else { @@ -405,7 +419,7 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text> killLauncherChildJobs(conf, context.getJobID().toString()); // Start the job - Process proc = startJob(conf, + Process proc = startJob(context, context.getJobID().toString(), conf.get("user.name"), conf.get(OVERRIDE_CLASSPATH), diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java index bbb33cc..834b54b 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java @@ -20,8 +20,12 @@ package org.apache.hive.hcatalog.templeton.tool; import java.io.IOException; import java.security.PrivilegedExceptionAction; +import java.sql.DriverManager; +import java.sql.SQLException; import java.util.Arrays; +import org.apache.hive.hcatalog.templeton.LauncherDelegator; +import org.apache.hive.jdbc.HiveConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -142,32 +146,43 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token")); job.getCredentials().addToken(new Text("mr token"), mrdt); } - String metastoreTokenStrForm = addHMSToken(job, user); + LauncherDelegator.JobType jobType = LauncherDelegator.JobType.valueOf(conf.get(JOB_TYPE)); + + String tokenStrForm = null; + if (jobType == LauncherDelegator.JobType.HIVE) { + tokenStrForm = addToken(job, user, SecureProxySupport.HIVE_SERVICE); + } else { + tokenStrForm = addToken(job, user, SecureProxySupport.HCAT_SERVICE); + } job.submit(); JobID submittedJobId = job.getJobID(); - if(metastoreTokenStrForm != null) { + if(tokenStrForm != null) { //so that it can be cancelled later from CompleteDelegator DelegationTokenCache.getStringFormTokenCache().storeDelegationToken( - submittedJobId.toString(), metastoreTokenStrForm); - LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() + + submittedJobId.toString(), tokenStrForm); + LOG.debug("Added delegation token for jobId=" + submittedJobId.toString() + " user=" + user); } return 0; } - private String addHMSToken(Job job, String user) throws IOException, InterruptedException, + private String addToken(Job job, String user, String type) throws IOException, InterruptedException, TException { if(!secureMetastoreAccess) { return null; } Token<org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier> hiveToken = new Token<org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier>(); - String metastoreTokenStrForm = buildHcatDelegationToken(user); - hiveToken.decodeFromUrlString(metastoreTokenStrForm); - job.getCredentials().addToken(new - Text(SecureProxySupport.HCAT_SERVICE), hiveToken); - return metastoreTokenStrForm; + String tokenStrForm; + if (type.equals(SecureProxySupport.HIVE_SERVICE)) { + tokenStrForm = buildHS2DelegationToken(user); + } else { + tokenStrForm = buildHcatDelegationToken(user); + } + hiveToken.decodeFromUrlString(tokenStrForm); + job.getCredentials().addToken(new Text(type), hiveToken); + return tokenStrForm; } private String buildHcatDelegationToken(String user) throws IOException, InterruptedException, TException { @@ -189,4 +204,37 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi } }); } + + private String buildHS2DelegationToken(String user) throws IOException, InterruptedException, + TException { + final HiveConf c = new HiveConf(); + LOG.debug("Creating hiveserver2 delegation token for user " + user); + final UserGroupInformation ugi = UgiFactory.getUgi(user); + UserGroupInformation real = ugi.getRealUser(); + return real.doAs(new PrivilegedExceptionAction<String>() { + @Override + public String run() throws IOException, TException, InterruptedException { + try { + Class.forName("org.apache.hive.jdbc.HiveDriver"); + } catch (ClassNotFoundException e) { + throw new IOException(e); + } + String hs2Url = appConf.get(AppConfig.HIVE_SERVER2_URL); + final HiveConnection con; + try { + con = (HiveConnection) DriverManager.getConnection(hs2Url); + } catch (SQLException e) { + throw new IOException(e); + } + String token = ugi.doAs(new PrivilegedExceptionAction<String>() { + @Override + public String run() throws SQLException { + String u = ugi.getUserName(); + return con.getDelegationToken(u,u); + } + }); + return token; + } + }); + } } diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index 8d5aa70..32a4761 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -18,6 +18,12 @@ package org.apache.hive.jdbc; +import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hive.service.rpc.thrift.TSetClientInfoResp; import org.apache.hive.service.rpc.thrift.TSetClientInfoReq; @@ -74,6 +80,7 @@ import javax.security.auth.Subject; import javax.security.sasl.Sasl; import javax.security.sasl.SaslException; import java.io.BufferedReader; +import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -759,7 +766,23 @@ public class HiveConnection implements java.sql.Connection { if (JdbcConnectionParams.AUTH_TOKEN.equalsIgnoreCase(jdbcConnConf.get(JdbcConnectionParams.AUTH_TYPE))) { // check delegation token in job conf if any try { - tokenStr = SessionUtils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN); + if (System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION) != null) { + try { + Credentials cred = new Credentials(); + DataInputStream dis = new DataInputStream(new FileInputStream(System.getenv(UserGroupInformation + .HADOOP_TOKEN_FILE_LOCATION))); + cred.readTokenStorageStream(dis); + dis.close(); + Token<? extends TokenIdentifier> token = cred.getToken(new Text("hive")); + tokenStr = token.encodeToUrlString(); + } catch (IOException e) { + LOG.warn("Cannot get token from environment variable $HADOOP_TOKEN_FILE_LOCATION=" + + System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION)); + } + } + if (tokenStr == null) { + tokenStr = SessionUtils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN); + } } catch (IOException e) { throw new SQLException("Error reading token ", e); } @@ -850,6 +873,7 @@ public class HiveConnection implements java.sql.Connection { private boolean isKerberosAuthMode() { return !JdbcConnectionParams.AUTH_SIMPLE.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE)) + && !JdbcConnectionParams.AUTH_TOKEN.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE)) && sessConfMap.containsKey(JdbcConnectionParams.AUTH_PRINCIPAL); } diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml index fceb1be..766161d 100644 --- a/packaging/src/main/assembly/bin.xml +++ b/packaging/src/main/assembly/bin.xml @@ -101,9 +101,12 @@ <useTransitiveFiltering>true</useTransitiveFiltering> <excludes> <exclude>org.apache.hadoop:*</exclude> + <exclude>org.apache.hive:hive-jdbc:jar:standalone</exclude> + <exclude>org.apache.httpcomponents:*</exclude> </excludes> <includes> <include>org.apache.hive.hcatalog:hive-webhcat:*</include> + <include>org.apache.hive:hive-jdbc:jar</include> </includes> </dependencySet> <dependencySet> diff --git a/pom.xml b/pom.xml index c2d8641..23c4413 100644 --- a/pom.xml +++ b/pom.xml @@ -217,6 +217,7 @@ <jsr305.version>3.0.0</jsr305.version> <tephra.version>0.6.0</tephra.version> <gson.version>2.2.4</gson.version> + <rs-api.version>2.0.1</rs-api.version> </properties> <repositories>