Repository: hive Updated Branches: refs/heads/master 583a9511b -> a75b600c9
HIVE-17512 : Not use doAs if distcp privileged user same as user running hive (Anishek Agarwal, via Thejas Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a75b600c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a75b600c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a75b600c Branch: refs/heads/master Commit: a75b600c9e54c1b63ac64eab520805e268b838cb Parents: 583a951 Author: Anishek Agarwal <anis...@gmail.com> Authored: Wed Sep 20 15:37:29 2017 -0700 Committer: Thejas M Nair <the...@hortonworks.com> Committed: Wed Sep 20 15:38:03 2017 -0700 ---------------------------------------------------------------------- itests/hive-unit/pom.xml | 8 +- .../hadoop/hive/ql/parse/TestExportImport.java | 10 ++- .../hadoop/hive/ql/parse/WarehouseInstance.java | 93 ++++++++++++-------- .../hadoop/hive/metastore/ObjectStore.java | 3 +- .../hadoop/hive/ql/parse/repl/CopyUtils.java | 11 ++- .../ql/parse/repl/dump/io/FileOperations.java | 5 +- 6 files changed, 86 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a75b600c/itests/hive-unit/pom.xml ---------------------------------------------------------------------- diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml index bf600c2..eeb6e58 100644 --- a/itests/hive-unit/pom.xml +++ b/itests/hive-unit/pom.xml @@ -135,7 +135,13 @@ <version>${project.version}</version> <scope>test</scope> </dependency> - <dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-distcp</artifactId> + <version>${hadoop.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-cli</artifactId> <version>${project.version}</version> http://git-wip-us.apache.org/repos/asf/hive/blob/a75b600c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java index 1f19dfd..70a57f8 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.parse; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hive.conf.HiveConf; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -30,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.HashMap; public class TestExportImport { @@ -48,8 +50,12 @@ public class TestExportImport { conf.set("dfs.client.use.datanode.hostname", "true"); MiniDFSCluster miniDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); - srcHiveWarehouse = new WarehouseInstance(LOG, miniDFSCluster, false); - destHiveWarehouse = new WarehouseInstance(LOG, miniDFSCluster, false); + HashMap<String, String> overridesForHiveConf = new HashMap<String, String>() {{ + put(HiveConf.ConfVars.HIVE_IN_TEST.varname, "false"); + }}; + srcHiveWarehouse = + new WarehouseInstance(LOG, miniDFSCluster, overridesForHiveConf); + destHiveWarehouse = new WarehouseInstance(LOG, miniDFSCluster, overridesForHiveConf); } @AfterClass http://git-wip-us.apache.org/repos/asf/hive/blob/a75b600c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index c084d4d..19ad442 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -46,10 +46,13 @@ import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; class WarehouseInstance implements Closeable { @@ -64,7 +67,8 @@ class WarehouseInstance implements Closeable { private final static String LISTENER_CLASS = DbNotificationListener.class.getCanonicalName(); - WarehouseInstance(Logger logger, MiniDFSCluster cluster, boolean hiveInTests) throws Exception { + WarehouseInstance(Logger logger, MiniDFSCluster cluster, Map<String, String> overridesForHiveConf) + throws Exception { this.logger = logger; this.miniDFSCluster = cluster; assert miniDFSCluster.isClusterUp(); @@ -74,16 +78,22 @@ class WarehouseInstance implements Closeable { Path warehouseRoot = mkDir(fs, "/warehouse" + uniqueIdentifier); Path cmRootPath = mkDir(fs, "/cmroot" + uniqueIdentifier); this.functionsRoot = mkDir(fs, "/functions" + uniqueIdentifier).toString(); - initialize(cmRootPath.toString(), warehouseRoot.toString(), hiveInTests); + initialize(cmRootPath.toString(), warehouseRoot.toString(), overridesForHiveConf); } WarehouseInstance(Logger logger, MiniDFSCluster cluster) throws Exception { - this(logger, cluster, true); + this(logger, cluster, new HashMap<String, String>() {{ + put(HiveConf.ConfVars.HIVE_IN_TEST.varname, "true"); + }}); } - private void initialize(String cmRoot, String warehouseRoot, boolean hiveInTest) + private void initialize(String cmRoot, String warehouseRoot, + Map<String, String> overridesForHiveConf) throws Exception { hiveConf = new HiveConf(miniDFSCluster.getConfiguration(0), TestReplicationScenarios.class); + for (Map.Entry<String, String> entry : overridesForHiveConf.entrySet()) { + hiveConf.set(entry.getKey(), entry.getValue()); + } String metaStoreUri = System.getProperty("test." + HiveConf.ConfVars.METASTOREURIS.varname); String hiveWarehouseLocation = System.getProperty("test.warehouse.dir", "/tmp") + Path.SEPARATOR @@ -95,7 +105,7 @@ class WarehouseInstance implements Closeable { return; } - hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, hiveInTest); + // hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, hiveInTest); // turn on db notification listener on meta store hiveConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehouseRoot); hiveConf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, LISTENER_CLASS); @@ -107,7 +117,7 @@ class WarehouseInstance implements Closeable { hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:memory:${test.tmp.dir}/APP;create=true"); hiveConf.setVar(HiveConf.ConfVars.REPLDIR, - hiveWarehouseLocation + "/hrepl" + uniqueIdentifier + "/"); + hiveWarehouseLocation + "/hrepl" + uniqueIdentifier + "/"); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); @@ -184,39 +194,51 @@ class WarehouseInstance implements Closeable { return this; } - WarehouseInstance verifyResult (String data) throws IOException { - verifyResults(data == null ? new String[] {} : new String[] { data }); - return this; - } + WarehouseInstance verifyResult(String data) throws IOException { + verifyResults(data == null ? new String[] {} : new String[] { data }); + return this; + } - /** - * All the results that are read from the hive output will not preserve - * case sensitivity and will all be in lower case, hence we will check against - * only lower case data values. - * Unless for Null Values it actually returns in UpperCase and hence explicitly lowering case - * before assert. - */ - WarehouseInstance verifyResults(String[] data) throws IOException { - List<String> results = getOutput(); - logger.info("Expecting {}", StringUtils.join(data, ",")); - logger.info("Got {}", results); - assertEquals(data.length, results.size()); - for (int i = 0; i < data.length; i++) { - assertEquals(data[i].toLowerCase(), results.get(i).toLowerCase()); - } - return this; + /** + * All the results that are read from the hive output will not preserve + * case sensitivity and will all be in lower case, hence we will check against + * only lower case data values. + * Unless for Null Values it actually returns in UpperCase and hence explicitly lowering case + * before assert. + */ + WarehouseInstance verifyResults(String[] data) throws IOException { + List<String> results = getOutput(); + logger.info("Expecting {}", StringUtils.join(data, ",")); + logger.info("Got {}", results); + assertEquals(data.length, results.size()); + for (int i = 0; i < data.length; i++) { + assertEquals(data[i].toLowerCase(), results.get(i).toLowerCase()); } + return this; + } - List<String> getOutput() throws IOException { - List<String> results = new ArrayList<>(); - try { - driver.getResults(results); - } catch (CommandNeedRetryException e) { - logger.warn(e.getMessage(), e); - throw new RuntimeException(e); - } - return results; + /** + * verify's result without regard for ordering. + */ + WarehouseInstance verifyResults(List data) throws IOException { + List<String> results = getOutput(); + logger.info("Expecting {}", StringUtils.join(data, ",")); + logger.info("Got {}", results); + assertEquals(data.size(), results.size()); + assertTrue(results.containsAll(data)); + return this; + } + + List<String> getOutput() throws IOException { + List<String> results = new ArrayList<>(); + try { + driver.getResults(results); + } catch (CommandNeedRetryException e) { + logger.warn(e.getMessage(), e); + throw new RuntimeException(e); } + return results; + } private void printOutput() throws IOException { for (String s : getOutput()) { @@ -226,7 +248,6 @@ class WarehouseInstance implements Closeable { ReplicationV1CompatRule getReplivationV1CompatRule(List<String> testsToSkip) { return new ReplicationV1CompatRule(client, hiveConf, testsToSkip); - } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/a75b600c/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 1344e9c..b5e4bf0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -8569,7 +8569,8 @@ public class ObjectStore implements RawStore, Configurable { if (currentRetries >= maxRetries) { String message = "Couldn't acquire the DB log notification lock because we reached the maximum" - + " # of retries: {} retries. If this happens too often, then is recommended to " + + " # of retries: " + maxRetries + + " retries. If this happens too often, then is recommended to " + "increase the maximum number of retries on the" + " hive.notification.sequence.lock.max.retries configuration"; LOG.error(message, e); http://git-wip-us.apache.org/repos/asf/hive/blob/a75b600c/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java index 28e7bcb..a022b5d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/CopyUtils.java @@ -25,9 +25,12 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.hive.shims.Utils; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.security.auth.login.LoginException; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -52,10 +55,14 @@ public class CopyUtils { this.copyAsUser = distCpDoAsUser; } - public void doCopy(Path destination, List<Path> srcPaths) throws IOException { + public void doCopy(Path destination, List<Path> srcPaths) throws IOException, LoginException { Map<FileSystem, List<Path>> map = fsToFileMap(srcPaths); FileSystem destinationFs = destination.getFileSystem(hiveConf); + UserGroupInformation ugi = Utils.getUGI(); + String currentUser = ugi.getShortUserName(); + boolean usePrivilegedDistCp = copyAsUser != null && !currentUser.equals(copyAsUser); + for (Map.Entry<FileSystem, List<Path>> entry : map.entrySet()) { if (regularCopy(destinationFs, entry)) { Path[] paths = entry.getValue().toArray(new Path[] {}); @@ -66,7 +73,7 @@ public class CopyUtils { entry.getValue(), // list of source paths destination, false, - copyAsUser, + usePrivilegedDistCp ? copyAsUser : null, hiveConf, ShimLoader.getHadoopShims() ); http://git-wip-us.apache.org/repos/asf/hive/blob/a75b600c/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java index 3ae07f1..2f636b6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.parse.repl.CopyUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.security.auth.login.LoginException; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStreamWriter; @@ -54,7 +55,7 @@ public class FileOperations { exportFileSystem = exportRootDataDir.getFileSystem(hiveConf); } - public void export(ReplicationSpec forReplicationSpec) throws IOException, SemanticException { + public void export(ReplicationSpec forReplicationSpec) throws Exception { if (forReplicationSpec.isLazy()) { exportFilesAsList(); } else { @@ -65,7 +66,7 @@ public class FileOperations { /** * This writes the actual data in the exportRootDataDir from the source. */ - private void copyFiles() throws IOException { + private void copyFiles() throws IOException, LoginException { FileStatus[] fileStatuses = LoadSemanticAnalyzer.matchFilesOrDir(dataFileSystem, dataFileListPath); List<Path> srcPaths = new ArrayList<>();
