Repository: hive Updated Branches: refs/heads/master 1e048df35 -> 717a15b38
HIVE-20711: Race Condition when Multi-Threading in SessionState.createRootHDFSDir (Denys Kuzmenko, reviewed by Antal Skinkovics and Peter Vary) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/717a15b3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/717a15b3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/717a15b3 Branch: refs/heads/master Commit: 717a15b383e67c635870ec0369e77454541b994a Parents: 1e048df Author: denys kuzmenko <dkuzme...@cloudera.com> Authored: Tue Oct 9 09:47:58 2018 +0200 Committer: Peter Vary <pv...@cloudera.com> Committed: Tue Oct 9 09:47:58 2018 +0200 ---------------------------------------------------------------------- .../ql/exec/spark/TestSparkSessionTimeout.java | 27 +------------------- .../apache/hadoop/hive/ql/exec/Utilities.java | 14 +++++++--- 2 files changed, 11 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/717a15b3/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java index c887297..d8dd80a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/exec/spark/TestSparkSessionTimeout.java @@ -68,32 +68,7 @@ public class TestSparkSessionTimeout { HiveConf conf = new HiveConf(); conf.setBoolVar(HiveConf.ConfVars.SPARK_OPTIMIZE_SHUFFLE_SERDE, false); conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"), - "TestSparkSessionTimeout-testMultiSparkSessionTimeout-local-dir").toString()); - - SessionState.start(conf); - - runTestSparkSessionTimeout(conf); - return null; - })); - } - for (Future<Void> future : futures) { - future.get(); - } - } - - @Test - public void testMultiSparkSessionTimeout() throws ExecutionException, InterruptedException { - List<Future<Void>> futures = new ArrayList<>(); - ExecutorService es = Executors.newFixedThreadPool(10); - for (int i = 0; i < 10; i++) { - futures.add(es.submit(() -> { - String confDir = "../../data/conf/spark/local/hive-site.xml"; - HiveConf.setHiveSiteLocation(new File(confDir).toURI().toURL()); - - HiveConf conf = new HiveConf(); - conf.setBoolVar(HiveConf.ConfVars.SPARK_OPTIMIZE_SHUFFLE_SERDE, false); - conf.set("spark.local.dir", Paths.get(System.getProperty("test.tmp.dir"), - "TestSparkSessionTimeout-testMultiSparkSessionTimeout-local-dir").toString()); + "TestSparkSessionTimeout-testMultiSessionSparkSessionTimeout-local-dir").toString()); SessionState.start(conf); http://git-wip-us.apache.org/repos/asf/hive/blob/717a15b3/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 76a30eb..10aa94e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -255,6 +255,9 @@ public final class Utilities { public static Random randGen = new Random(); + private static final Object INPUT_SUMMARY_LOCK = new Object(); + private static final Object ROOT_HDFS_DIR_LOCK = new Object(); + /** * ReduceField: * KEY: record key @@ -2317,8 +2320,6 @@ public final class Utilities { } } - private static final Object INPUT_SUMMARY_LOCK = new Object(); - /** * Returns the maximum number of executors required to get file information from several input locations. * It checks whether HIVE_EXEC_INPUT_LISTING_MAX_THREADS or DEPRECATED_MAPRED_DFSCLIENT_PARALLELISM_MAX are > 1 @@ -4463,11 +4464,16 @@ public final class Utilities { public static void ensurePathIsWritable(Path rootHDFSDirPath, HiveConf conf) throws IOException { FsPermission writableHDFSDirPermission = new FsPermission((short)00733); FileSystem fs = rootHDFSDirPath.getFileSystem(conf); + if (!fs.exists(rootHDFSDirPath)) { - Utilities.createDirsWithPermission(conf, rootHDFSDirPath, writableHDFSDirPermission, true); + synchronized (ROOT_HDFS_DIR_LOCK) { + if (!fs.exists(rootHDFSDirPath)) { + Utilities.createDirsWithPermission(conf, rootHDFSDirPath, writableHDFSDirPermission, true); + } + } } FsPermission currentHDFSDirPermission = fs.getFileStatus(rootHDFSDirPath).getPermission(); - if (rootHDFSDirPath != null && rootHDFSDirPath.toUri() != null) { + if (rootHDFSDirPath.toUri() != null) { String schema = rootHDFSDirPath.toUri().getScheme(); LOG.debug("HDFS dir: " + rootHDFSDirPath + " with schema " + schema + ", permission: " + currentHDFSDirPermission);