This is an automated email from the ASF dual-hosted git repository. daijy pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 77b9c3c Revert "HIVE-21247: Webhcat beeline in secure mode (Daniel Dai, reviewed by Thejas Nair)" 77b9c3c is described below commit 77b9c3c79953121c19554a7de6953a6e21de9db6 Author: Daniel Dai <dai...@gmail.com> AuthorDate: Tue Feb 26 19:16:07 2019 -0800 Revert "HIVE-21247: Webhcat beeline in secure mode (Daniel Dai, reviewed by Thejas Nair)" This reverts commit 3e9614eebd19f9843b28d97aaee1c3dfb815fb3d. --- .../src/java/org/apache/hive/beeline/BeeLine.java | 4 -- hcatalog/webhcat/svr/pom.xml | 16 ----- .../apache/hive/hcatalog/templeton/AppConfig.java | 1 - .../hive/hcatalog/templeton/HiveDelegator.java | 6 -- .../hcatalog/templeton/SecureProxySupport.java | 1 - .../hive/hcatalog/templeton/tool/LaunchMapper.java | 20 +------ .../templeton/tool/TempletonControllerJob.java | 68 ++++------------------ .../java/org/apache/hive/jdbc/HiveConnection.java | 26 +-------- packaging/src/main/assembly/bin.xml | 3 - pom.xml | 1 - 10 files changed, 14 insertions(+), 132 deletions(-) diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index cded55f..65eee2c 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -1203,10 +1203,6 @@ public class BeeLine implements Closeable { if (password != null) { jdbcConnectionParams.getSessionVars().put(JdbcConnectionParams.AUTH_PASSWD, password); } - String auth = cl.getOptionValue("a"); - if (auth != null) { - jdbcConnectionParams.getSessionVars().put(JdbcConnectionParams.AUTH_TYPE, auth); - } mergedConnectionProperties = HS2ConnectionFileUtils.mergeUserConnectionPropertiesAndBeelineSite( userConnectionProperties, jdbcConnectionParams); diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml index 36d15cd..4dfade5 100644 --- a/hcatalog/webhcat/svr/pom.xml +++ b/hcatalog/webhcat/svr/pom.xml @@ -92,12 +92,6 @@ <groupId>com.sun.jersey</groupId> <artifactId>jersey-core</artifactId> <version>${jersey.version}</version> - <exclusions> - <exclusion> - <groupId>javax.ws.rs</groupId> - <artifactId>jsr311-api</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>com.sun.jersey</groupId> @@ -150,11 +144,6 @@ <version>${slf4j.version}</version> </dependency> <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-jdbc</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> <version>${hadoop.version}</version> @@ -210,11 +199,6 @@ </exclusion> </exclusions> </dependency> - <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - <version>${rs-api.version}</version> - </dependency> <!-- test inter-project --> <dependency> <groupId>org.apache.hive</groupId> diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java index b566cf8..1fd9e47 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java @@ -163,7 +163,6 @@ public class AppConfig extends Configuration { * of escape/unescape methods in {@link org.apache.hadoop.util.StringUtils} in webhcat. */ public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; - public static final String HIVE_SERVER2_URL = "templeton.hive.hs2.url"; public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive"; public static final String SQOOP_PATH_NAME = "templeton.sqoop.path"; public static final String SQOOP_HOME_PATH = "templeton.sqoop.home"; diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java index 3f679ac..3f1968d 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java @@ -28,7 +28,6 @@ import java.util.Map; import org.apache.commons.exec.ExecuteException; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants; import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob; import org.apache.hive.hcatalog.templeton.tool.TempletonUtils; @@ -79,11 +78,6 @@ public class HiveDelegator extends LauncherDelegator { args.add("-p"); args.add("default"); - if (UserGroupInformation.isSecurityEnabled()) { - args.add("-a"); - args.add("delegationToken"); - } - //add mapreduce job tag placeholder args.add("--hiveconf"); args.add(TempletonControllerJob.HIVE_QUERY_TAG_ARG_PLACEHOLDER); diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java index bbe5947..c503a7a 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/SecureProxySupport.java @@ -50,7 +50,6 @@ import org.apache.thrift.TException; public class SecureProxySupport { private Path tokenPath; public static final String HCAT_SERVICE = "hcat"; - public static final String HIVE_SERVICE = "hive"; private final boolean isEnabled; private String user; diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java index a776a0b..b1f4a6a 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java @@ -18,10 +18,6 @@ */ package org.apache.hive.hcatalog.templeton.tool; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; -import org.apache.hive.hcatalog.templeton.SecureProxySupport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -157,10 +153,10 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text> env.put(pathVarName, paths); } } - protected Process startJob(Context context, String jobId, String user, String overrideClasspath, + protected Process startJob(Configuration conf, String jobId, String user, String overrideClasspath, LauncherDelegator.JobType jobType) throws IOException, InterruptedException { - Configuration conf = context.getConfiguration(); + copyLocal(COPY_NAME, conf); String[] jarArgs = TempletonUtils.decodeArray(conf.get(JAR_ARGS_NAME)); @@ -178,16 +174,6 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text> handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary"); handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path"); if (jobType == LauncherDelegator.JobType.HIVE) { - Credentials cred = new Credentials(); - Token<? extends TokenIdentifier> token = context.getCredentials().getToken(new - Text(SecureProxySupport.HIVE_SERVICE)); - cred.addToken(new - Text(SecureProxySupport.HIVE_SERVICE), token); - File t = File.createTempFile("templeton", null); - Path tokenPath = new Path(t.toURI()); - cred.writeTokenStorageFile(tokenPath, conf); - env.put(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, - tokenPath.toUri().getPath()); replaceJobTag(jarArgsList, JobSubmissionConstants.HIVE_QUERY_TAG_ARG_PLACEHOLDER, JobSubmissionConstants.HIVE_QUERY_TAG, jobId); } else { @@ -419,7 +405,7 @@ public class LaunchMapper extends Mapper<NullWritable, NullWritable, Text, Text> killLauncherChildJobs(conf, context.getJobID().toString()); // Start the job - Process proc = startJob(context, + Process proc = startJob(conf, context.getJobID().toString(), conf.get("user.name"), conf.get(OVERRIDE_CLASSPATH), diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java index 834b54b..bbb33cc 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java @@ -20,12 +20,8 @@ package org.apache.hive.hcatalog.templeton.tool; import java.io.IOException; import java.security.PrivilegedExceptionAction; -import java.sql.DriverManager; -import java.sql.SQLException; import java.util.Arrays; -import org.apache.hive.hcatalog.templeton.LauncherDelegator; -import org.apache.hive.jdbc.HiveConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -146,43 +142,32 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi Token<DelegationTokenIdentifier> mrdt = jc.getDelegationToken(new Text("mr token")); job.getCredentials().addToken(new Text("mr token"), mrdt); } - LauncherDelegator.JobType jobType = LauncherDelegator.JobType.valueOf(conf.get(JOB_TYPE)); - - String tokenStrForm = null; - if (jobType == LauncherDelegator.JobType.HIVE) { - tokenStrForm = addToken(job, user, SecureProxySupport.HIVE_SERVICE); - } else { - tokenStrForm = addToken(job, user, SecureProxySupport.HCAT_SERVICE); - } + String metastoreTokenStrForm = addHMSToken(job, user); job.submit(); JobID submittedJobId = job.getJobID(); - if(tokenStrForm != null) { + if(metastoreTokenStrForm != null) { //so that it can be cancelled later from CompleteDelegator DelegationTokenCache.getStringFormTokenCache().storeDelegationToken( - submittedJobId.toString(), tokenStrForm); - LOG.debug("Added delegation token for jobId=" + submittedJobId.toString() + + submittedJobId.toString(), metastoreTokenStrForm); + LOG.debug("Added metastore delegation token for jobId=" + submittedJobId.toString() + " user=" + user); } return 0; } - private String addToken(Job job, String user, String type) throws IOException, InterruptedException, + private String addHMSToken(Job job, String user) throws IOException, InterruptedException, TException { if(!secureMetastoreAccess) { return null; } Token<org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier> hiveToken = new Token<org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier>(); - String tokenStrForm; - if (type.equals(SecureProxySupport.HIVE_SERVICE)) { - tokenStrForm = buildHS2DelegationToken(user); - } else { - tokenStrForm = buildHcatDelegationToken(user); - } - hiveToken.decodeFromUrlString(tokenStrForm); - job.getCredentials().addToken(new Text(type), hiveToken); - return tokenStrForm; + String metastoreTokenStrForm = buildHcatDelegationToken(user); + hiveToken.decodeFromUrlString(metastoreTokenStrForm); + job.getCredentials().addToken(new + Text(SecureProxySupport.HCAT_SERVICE), hiveToken); + return metastoreTokenStrForm; } private String buildHcatDelegationToken(String user) throws IOException, InterruptedException, TException { @@ -204,37 +189,4 @@ public class TempletonControllerJob extends Configured implements Tool, JobSubmi } }); } - - private String buildHS2DelegationToken(String user) throws IOException, InterruptedException, - TException { - final HiveConf c = new HiveConf(); - LOG.debug("Creating hiveserver2 delegation token for user " + user); - final UserGroupInformation ugi = UgiFactory.getUgi(user); - UserGroupInformation real = ugi.getRealUser(); - return real.doAs(new PrivilegedExceptionAction<String>() { - @Override - public String run() throws IOException, TException, InterruptedException { - try { - Class.forName("org.apache.hive.jdbc.HiveDriver"); - } catch (ClassNotFoundException e) { - throw new IOException(e); - } - String hs2Url = appConf.get(AppConfig.HIVE_SERVER2_URL); - final HiveConnection con; - try { - con = (HiveConnection) DriverManager.getConnection(hs2Url); - } catch (SQLException e) { - throw new IOException(e); - } - String token = ugi.doAs(new PrivilegedExceptionAction<String>() { - @Override - public String run() throws SQLException { - String u = ugi.getUserName(); - return con.getDelegationToken(u,u); - } - }); - return token; - } - }); - } } diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index 32a4761..8d5aa70 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -18,12 +18,6 @@ package org.apache.hive.jdbc; -import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hive.service.rpc.thrift.TSetClientInfoResp; import org.apache.hive.service.rpc.thrift.TSetClientInfoReq; @@ -80,7 +74,6 @@ import javax.security.auth.Subject; import javax.security.sasl.Sasl; import javax.security.sasl.SaslException; import java.io.BufferedReader; -import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -766,23 +759,7 @@ public class HiveConnection implements java.sql.Connection { if (JdbcConnectionParams.AUTH_TOKEN.equalsIgnoreCase(jdbcConnConf.get(JdbcConnectionParams.AUTH_TYPE))) { // check delegation token in job conf if any try { - if (System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION) != null) { - try { - Credentials cred = new Credentials(); - DataInputStream dis = new DataInputStream(new FileInputStream(System.getenv(UserGroupInformation - .HADOOP_TOKEN_FILE_LOCATION))); - cred.readTokenStorageStream(dis); - dis.close(); - Token<? extends TokenIdentifier> token = cred.getToken(new Text("hive")); - tokenStr = token.encodeToUrlString(); - } catch (IOException e) { - LOG.warn("Cannot get token from environment variable $HADOOP_TOKEN_FILE_LOCATION=" + - System.getenv(UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION)); - } - } - if (tokenStr == null) { - tokenStr = SessionUtils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN); - } + tokenStr = SessionUtils.getTokenStrForm(HiveAuthConstants.HS2_CLIENT_TOKEN); } catch (IOException e) { throw new SQLException("Error reading token ", e); } @@ -873,7 +850,6 @@ public class HiveConnection implements java.sql.Connection { private boolean isKerberosAuthMode() { return !JdbcConnectionParams.AUTH_SIMPLE.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE)) - && !JdbcConnectionParams.AUTH_TOKEN.equals(sessConfMap.get(JdbcConnectionParams.AUTH_TYPE)) && sessConfMap.containsKey(JdbcConnectionParams.AUTH_PRINCIPAL); } diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml index 766161d..fceb1be 100644 --- a/packaging/src/main/assembly/bin.xml +++ b/packaging/src/main/assembly/bin.xml @@ -101,12 +101,9 @@ <useTransitiveFiltering>true</useTransitiveFiltering> <excludes> <exclude>org.apache.hadoop:*</exclude> - <exclude>org.apache.hive:hive-jdbc:jar:standalone</exclude> - <exclude>org.apache.httpcomponents:*</exclude> </excludes> <includes> <include>org.apache.hive.hcatalog:hive-webhcat:*</include> - <include>org.apache.hive:hive-jdbc:jar</include> </includes> </dependencySet> <dependencySet> diff --git a/pom.xml b/pom.xml index 23c4413..c2d8641 100644 --- a/pom.xml +++ b/pom.xml @@ -217,7 +217,6 @@ <jsr305.version>3.0.0</jsr305.version> <tephra.version>0.6.0</tephra.version> <gson.version>2.2.4</gson.version> - <rs-api.version>2.0.1</rs-api.version> </properties> <repositories>