Author: prasadm
Date: Thu Oct 30 18:21:58 2014
New Revision: 1635574
URL: http://svn.apache.org/r1635574
Log:
HIVE-8634: HiveServer2 fair scheduler queue mapping doesn't handle the
secondary groups rules correctly (Prasad Mujumdar, reviewed by Brock Noland)
Modified:
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
Modified:
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java?rev=1635574&r1=1635573&r2=1635574&view=diff
==============================================================================
---
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java
(original)
+++
hive/trunk/itests/hive-unit-hadoop2/src/test/java/org/apache/hive/jdbc/TestSchedulerQueue.java
Thu Oct 30 18:21:58 2014
@@ -21,14 +21,19 @@ package org.apache.hive.jdbc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
import org.junit.After;
import org.junit.Before;
@@ -37,6 +42,26 @@ import org.junit.Test;
public class TestSchedulerQueue {
+ // hadoop group mapping that maps user to same group
+ public static class HiveTestSimpleGroupMapping implements
GroupMappingServiceProvider {
+ public static String primaryTag = "";
+ @Override
+ public List<String> getGroups(String user) throws IOException {
+ List<String> results = new ArrayList<String>();
+ results.add(user + primaryTag);
+ results.add(user + "-group");
+ return results;
+ }
+
+ @Override
+ public void cacheGroupsRefresh() throws IOException {
+ }
+
+ @Override
+ public void cacheGroupsAdd(List<String> groups) throws IOException {
+ }
+ }
+
private MiniHS2 miniHS2 = null;
private static HiveConf conf = new HiveConf();
private Connection hs2Conn = null;
@@ -44,6 +69,8 @@ public class TestSchedulerQueue {
@BeforeClass
public static void beforeTest() throws Exception {
Class.forName(MiniHS2.getJdbcDriverName());
+ conf.set("hadoop.security.group.mapping",
+ HiveTestSimpleGroupMapping.class.getName());
}
@Before
@@ -56,6 +83,7 @@ public class TestSchedulerQueue {
miniHS2.setConfProperty(YarnConfiguration.RM_SCHEDULER,
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
miniHS2.start(new HashMap<String, String>());
+ HiveTestSimpleGroupMapping.primaryTag = "";
}
@After
@@ -79,6 +107,7 @@ public class TestSchedulerQueue {
@Test
public void testFairSchedulerQueueMapping() throws Exception {
hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user1",
"bar");
+ verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname,
"false");
verifyProperty("mapreduce.framework.name", "yarn");
verifyProperty(HiveConf.ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE.varname,
"true");
@@ -88,6 +117,31 @@ public class TestSchedulerQueue {
}
/**
+ * Verify:
+ * Test is running with MR2 and queue mapping are set correctly for primary
group rule.
+ * @throws Exception
+ */
+ @Test
+ public void testFairSchedulerPrimaryQueueMapping() throws Exception {
+ miniHS2.setConfProperty(FairSchedulerConfiguration.ALLOCATION_FILE,
"fair-scheduler-test.xml");
+ HiveTestSimpleGroupMapping.primaryTag = "-test";
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user2",
"bar");
+ verifyProperty("mapreduce.job.queuename", "root.user2" +
HiveTestSimpleGroupMapping.primaryTag);
+ }
+
+ /**
+ * Verify:
+ * Test is running with MR2 and queue mapping are set correctly for primary
group rule.
+ * @throws Exception
+ */
+ @Test
+ public void testFairSchedulerSecondaryQueueMapping() throws Exception {
+ miniHS2.setConfProperty(FairSchedulerConfiguration.ALLOCATION_FILE,
"fair-scheduler-test.xml");
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "user3",
"bar");
+ verifyProperty("mapreduce.job.queuename", "root.user3-group");
+ }
+
+ /**
* Verify that the queue refresh doesn't happen when configured to be off.
*
* @throws Exception
Modified:
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL:
http://svn.apache.org/viewvc/hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1635574&r1=1635573&r2=1635574&view=diff
==============================================================================
---
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
(original)
+++
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
Thu Oct 30 18:21:58 2014
@@ -30,6 +30,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -77,6 +78,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationFileLoaderService;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementPolicy;
import org.apache.tez.test.MiniTezCluster;
@@ -232,8 +234,25 @@ public class Hadoop23Shims extends Hadoo
public void refreshDefaultQueue(Configuration conf, String userName) throws
IOException {
String requestedQueue = YarnConfiguration.DEFAULT_QUEUE_NAME;
if (StringUtils.isNotBlank(userName) && isFairScheduler(conf)) {
- AllocationConfiguration allocConf = new AllocationConfiguration(conf);
- QueuePlacementPolicy queuePolicy = allocConf.getPlacementPolicy();
+ final AtomicReference<AllocationConfiguration> allocConf = new
AtomicReference<AllocationConfiguration>();
+
+ AllocationFileLoaderService allocsLoader = new
AllocationFileLoaderService();
+ allocsLoader.init(conf);
+ allocsLoader.setReloadListener(new
AllocationFileLoaderService.Listener() {
+ @Override
+ public void onReload(AllocationConfiguration allocs) {
+ allocConf.set(allocs);
+ }
+ });
+ try {
+ allocsLoader.reloadAllocations();
+ } catch (Exception ex) {
+ throw new IOException("Failed to load queue allocations", ex);
+ }
+ if (allocConf.get() == null) {
+ allocConf.set(new AllocationConfiguration(conf));
+ }
+ QueuePlacementPolicy queuePolicy = allocConf.get().getPlacementPolicy();
if (queuePolicy != null) {
requestedQueue = queuePolicy.assignAppToQueue(requestedQueue,
userName);
if (StringUtils.isNotBlank(requestedQueue)) {
