This is an automated email from the ASF dual-hosted git repository. sankarh pushed a commit to branch branch-3 in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push: new 1316e66938a HIVE-27698: Backport of HIVE-22398: Remove legacy code that can cause issue with new Yarn releases (Slim Bouguerra via via Ashutosh Chauhan) 1316e66938a is described below commit 1316e66938a3eb3a29c4dd924bd1401f7783ead9 Author: Aman Raj <104416558+amanraj2...@users.noreply.github.com> AuthorDate: Wed Sep 20 10:12:50 2023 +0530 HIVE-27698: Backport of HIVE-22398: Remove legacy code that can cause issue with new Yarn releases (Slim Bouguerra via via Ashutosh Chauhan) Signed-off-by: Sankar Hariappan <sank...@apache.org> Closes (#4708) --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 - .../org/apache/hive/jdbc/TestSchedulerQueue.java | 175 --------------------- .../hive/service/cli/session/HiveSessionImpl.java | 10 -- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 4 +- .../apache/hadoop/hive/shims/SchedulerShim.java | 37 ----- .../org/apache/hadoop/hive/shims/ShimLoader.java | 9 -- .../hadoop/hive/schshim/FairSchedulerShim.java | 70 --------- 7 files changed, 1 insertion(+), 310 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 50a8d2c0977..3ec99315a27 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3430,12 +3430,6 @@ public class HiveConf extends Configuration { "SSL certificate keystore location."), HIVE_SERVER2_SSL_KEYSTORE_PASSWORD("hive.server2.keystore.password", "", "SSL certificate keystore password."), - HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE("hive.server2.map.fair.scheduler.queue", true, - "If the YARN fair scheduler is configured and HiveServer2 is running in non-impersonation mode,\n" + - "this setting determines the user for fair scheduler queue mapping.\n" + - "If set to true (default), the logged-in user determines the fair scheduler queue\n" + - "for submitted jobs, so that map reduce resource usage can be tracked by user.\n" + - "If set to false, all Hive jobs go to the 'hive' user's queue."), HIVE_SERVER2_BUILTIN_UDF_WHITELIST("hive.server2.builtin.udf.whitelist", "", "Comma separated list of builtin udf names allowed in queries.\n" + "An empty whitelist allows all builtin udfs to be executed. " + diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java deleted file mode 100644 index 6e57e811fe5..00000000000 --- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.jdbc; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.security.GroupMappingServiceProvider; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration; -import org.apache.hive.jdbc.miniHS2.MiniHS2; -import org.apache.hive.jdbc.miniHS2.MiniHS2.MiniClusterType; -import org.junit.After; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TestSchedulerQueue { - - // hadoop group mapping that maps user to same group - public static class HiveTestSimpleGroupMapping implements GroupMappingServiceProvider { - public static String primaryTag = ""; - @Override - public List<String> getGroups(String user) throws IOException { - List<String> results = new ArrayList<String>(); - results.add(user + primaryTag); - results.add(user + "-group"); - return results; - } - - @Override - public void cacheGroupsRefresh() throws IOException { - } - - @Override - public void cacheGroupsAdd(List<String> groups) throws IOException { - } - } - - private MiniHS2 miniHS2 = null; - private static HiveConf conf = new HiveConf(); - private Connection hs2Conn = null; - - @BeforeClass - public static void beforeTest() throws Exception { - Class.forName(MiniHS2.getJdbcDriverName()); - conf.set("hadoop.security.group.mapping", - HiveTestSimpleGroupMapping.class.getName()); - } - - @Before - public void setUp() throws Exception { - DriverManager.setLoginTimeout(0); - miniHS2 = new MiniHS2(conf, MiniClusterType.MR); - miniHS2.setConfProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false"); - miniHS2.setConfProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, - "true"); - miniHS2.setConfProperty(YarnConfiguration.RM_SCHEDULER, - "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); - miniHS2.start(new HashMap<String, String>()); - HiveTestSimpleGroupMapping.primaryTag = ""; - } - - @After - public void tearDown() throws Exception { - if (hs2Conn != null) { - hs2Conn.close(); - } - if (miniHS2 != null && miniHS2.isStarted()) { - miniHS2.stop(); - } - System.clearProperty("mapreduce.job.queuename"); - } - - /** - * Verify: - * Test is running with MR2 and queue mapping defaults are set. - * Queue mapping is set for the connected user. - * - * @throws Exception - */ - @Test - public void testFairSchedulerQueueMapping() throws Exception { - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1", "bar"); - verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false"); - verifyProperty("mapreduce.framework.name", "yarn"); - verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, - "true"); - verifyProperty(YarnConfiguration.RM_SCHEDULER, - "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); - verifyProperty("mapreduce.job.queuename", "root.user1"); - } - - /** - * Verify: - * Test is running with MR2 and queue mapping are set correctly for primary group rule. - * @throws Exception - */ - @Test - public void testFairSchedulerPrimaryQueueMapping() throws Exception { - miniHS2.setConfProperty(FairSchedulerConfiguration.ALLOCATION_FILE, "fair-scheduler-test.xml"); - HiveTestSimpleGroupMapping.primaryTag = "-test"; - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user2", "bar"); - verifyProperty("mapreduce.job.queuename", "root.user2" + HiveTestSimpleGroupMapping.primaryTag); - } - - /** - * Verify: - * Test is running with MR2 and queue mapping are set correctly for primary group rule. - * @throws Exception - */ - @Test - public void testFairSchedulerSecondaryQueueMapping() throws Exception { - miniHS2.setConfProperty(FairSchedulerConfiguration.ALLOCATION_FILE, "fair-scheduler-test.xml"); - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user3", "bar"); - verifyProperty("mapreduce.job.queuename", "root.user3-group"); - } - - /** - * Verify that the queue refresh doesn't happen when configured to be off. - * - * @throws Exception - */ - @Test - public void testQueueMappingCheckDisabled() throws Exception { - miniHS2.setConfProperty( - HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, "false"); - hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1", "bar"); - verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname, - "false"); - verifyProperty("mapreduce.job.queuename", YarnConfiguration.DEFAULT_QUEUE_NAME); - } - - /** - * Verify that the given property contains the expected value. - * - * @param propertyName - * @param expectedValue - * @throws Exception - */ - private void verifyProperty(String propertyName, String expectedValue) throws Exception { - Statement stmt = hs2Conn .createStatement(); - ResultSet res = stmt.executeQuery("set " + propertyName); - assertTrue(res.next()); - String results[] = res.getString(1).split("="); - assertEquals("Property should be set", results.length, 2); - assertEquals("Property should be set", expectedValue, results[1]); - } -} diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java index e5cdc7bb7c6..5ef71a69bb3 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java @@ -134,16 +134,6 @@ public class HiveSessionImpl implements HiveSession { this.forwardedAddresses = forwardedAddresses; this.operationLock = serverConf.getBoolVar( ConfVars.HIVE_SERVER2_PARALLEL_OPS_IN_SESSION) ? null : new Semaphore(1); - try { - // In non-impersonation mode, map scheduler queue to current user - // if fair scheduler is configured. - if (! sessionConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) && - sessionConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) { - ShimLoader.getHadoopShims().refreshDefaultQueue(sessionConf, username); - } - } catch (IOException e) { - LOG.warn("Error setting scheduler queue: " + e, e); - } // Set an explicit session name to control the download directory name sessionConf.set(ConfVars.HIVESESSIONID.varname, this.sessionHandle.getHandleIdentifier().toString()); diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index 79c69ee022b..f08154072cf 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -257,9 +257,7 @@ public class Hadoop23Shims extends HadoopShimsSecure { */ @Override public void refreshDefaultQueue(Configuration conf, String userName) throws IOException { - if (StringUtils.isNotBlank(userName) && isFairScheduler(conf)) { - ShimLoader.getSchedulerShims().refreshDefaultQueue(conf, userName); - } + //no op } private boolean isFairScheduler (Configuration conf) { diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/SchedulerShim.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/SchedulerShim.java deleted file mode 100644 index ec8d1a0ff68..00000000000 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/SchedulerShim.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.shims; - -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; - -/** - * Shim for Fair scheduler - * HiveServer2 uses fair scheduler API to resolve the queue mapping for non-impersonation - * mode. This shim is avoid direct dependency of yarn fair scheduler on Hive. - */ -public interface SchedulerShim { - /** - * Reset the default fair scheduler queue mapping to end user. - * @param conf - * @param userName end user name - */ - public void refreshDefaultQueue(Configuration conf, String userName) - throws IOException; -} diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java index 28ab7c4c605..bbeaa897ca2 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java @@ -36,7 +36,6 @@ public abstract class ShimLoader { private static volatile HadoopShims hadoopShims; private static JettyShims jettyShims; private static AppenderSkeleton eventCounter; - private static SchedulerShim schedulerShim; /** * The names of the classes for shimming Hadoop for each major version. @@ -100,14 +99,6 @@ public abstract class ShimLoader { } return eventCounter; } - - public static synchronized SchedulerShim getSchedulerShims() { - if (schedulerShim == null) { - schedulerShim = createShim(SCHEDULER_SHIM_CLASSE, SchedulerShim.class); - } - return schedulerShim; - } - private static <T> T loadShims(Map<String, String> classMap, Class<T> xface) { String vers = getMajorVersion(); String className = classMap.get(vers); diff --git a/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/FairSchedulerShim.java b/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/FairSchedulerShim.java deleted file mode 100644 index 6d12f05e696..00000000000 --- a/shims/scheduler/src/main/java/org/apache/hadoop/hive/schshim/FairSchedulerShim.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.schshim; - -import java.io.IOException; -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.shims.SchedulerShim; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy; - -public class FairSchedulerShim implements SchedulerShim { - private static final Logger LOG = LoggerFactory.getLogger(FairSchedulerShim.class); - private static final String MR2_JOB_QUEUE_PROPERTY = "mapreduce.job.queuename"; - - @Override - public void refreshDefaultQueue(Configuration conf, String userName) - throws IOException { - String requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME; - final AtomicReference<AllocationConfiguration> allocConf = new AtomicReference<AllocationConfiguration>(); - - AllocationFileLoaderService allocsLoader = new AllocationFileLoaderService(); - allocsLoader.init(conf); - allocsLoader.setReloadListener(new AllocationFileLoaderService.Listener() { - @Override - public void onReload(AllocationConfiguration allocs) { - allocConf.set(allocs); - } - }); - try { - allocsLoader.reloadAllocations(); - } catch (Exception ex) { - throw new IOException("Failed to load queue allocations", ex); - } - if (allocConf.get() == null) { - allocConf.set(new AllocationConfiguration(conf)); - } - QueuePlacementPolicy queuePolicy = allocConf.get().getPlacementPolicy(); - if (queuePolicy != null) { - requestedQueue = queuePolicy.assignAppToQueue(requestedQueue, userName); - if (StringUtils.isNotBlank(requestedQueue)) { - LOG.debug("Setting queue name to " + requestedQueue + " for user " - + userName); - conf.set(MR2_JOB_QUEUE_PROPERTY, requestedQueue); - } - } - } - -}