This is an automated email from the ASF dual-hosted git repository.

abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 10efefbfbb3 HIVE-27884: LLAP: Reuse FileSystem objects from cache 
across different tasks in the same LLAP daemon (#4882) (Laszlo Bodor reviewed 
by Denys Kuzmenko, Eugene Ryan, Kokila N)
10efefbfbb3 is described below

commit 10efefbfbb3722e9fdc8b438838bd7c964bf5224
Author: Bodor Laszlo <[email protected]>
AuthorDate: Mon Sep 16 10:34:01 2024 +0200

    HIVE-27884: LLAP: Reuse FileSystem objects from cache across different 
tasks in the same LLAP daemon (#4882) (Laszlo Bodor reviewed by Denys Kuzmenko, 
Eugene Ryan, Kokila N)
---
 .../apache/hadoop/hive/llap/LlapUgiFactory.java    | 12 ++-
 .../apache/hadoop/hive/llap/LlapUgiManager.java    | 86 ++++++++++++++++++++++
 .../hive/llap/daemon/impl/ContainerRunnerImpl.java | 12 +--
 .../hadoop/hive/llap/daemon/impl/LlapDaemon.java   | 14 +---
 .../hive/llap/daemon/impl/TaskRunnerCallable.java  | 10 +--
 ...apUgiFactoryFactory.java => LlapUgiHelper.java} | 43 +++++------
 .../llap/daemon/impl/TaskExecutorTestHelpers.java  |  3 +-
 .../llap/daemon/impl/TestContainerRunnerImpl.java  |  6 +-
 8 files changed, 133 insertions(+), 53 deletions(-)

diff --git a/common/src/java/org/apache/hadoop/hive/common/UgiFactory.java 
b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapUgiFactory.java
similarity index 67%
rename from common/src/java/org/apache/hadoop/hive/common/UgiFactory.java
rename to llap-server/src/java/org/apache/hadoop/hive/llap/LlapUgiFactory.java
index 5b1ce60345c..5d3570d33a3 100644
--- a/common/src/java/org/apache/hadoop/hive/common/UgiFactory.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapUgiFactory.java
@@ -11,12 +11,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+package org.apache.hadoop.hive.llap;
 
-package org.apache.hadoop.hive.common;
+import org.apache.hadoop.security.UserGroupInformation;
 
 import java.io.IOException;
-import org.apache.hadoop.security.UserGroupInformation;
 
-public interface UgiFactory {
-  UserGroupInformation createUgi() throws IOException;
+public interface LlapUgiFactory {
+  /*
+   * Creates a UserGroupInformation instance for a user.
+   * Subclasses might implement a cache for taking care of reusing an existing 
ugi for the same query if possible.
+   */
+  UserGroupInformation createUgi(String user) throws IOException;
 }
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/LlapUgiManager.java 
b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapUgiManager.java
new file mode 100644
index 00000000000..34901729dbe
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/LlapUgiManager.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.llap.daemon.impl.QueryIdentifier;
+import org.apache.hadoop.hive.llap.security.LlapUgiHelper;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LlapUgiManager {
+  private static final Logger LOG = 
LoggerFactory.getLogger(LlapUgiManager.class);
+  private final ConcurrentMap<QueryIdentifier, UserGroupInformation> ugis = 
new ConcurrentHashMap<>();
+  private final LlapUgiFactory ugiFactory;
+
+  private LlapUgiManager(Configuration conf) {
+    try {
+      ugiFactory = LlapUgiHelper.createLlapUgiFactory(conf);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  public static LlapUgiManager getInstance(Configuration daemonConf) {
+    return new LlapUgiManager(daemonConf);
+  }
+
+  /**
+   * Creates (or returns) an ugi for tasks in the same query and merges the 
credentials.
+   * This is valid to be done once per query: no vertex-level ugi and 
credentials are needed, both of them
+   * are the same within the same query.
+   * Regarding vertex user: LlapTaskCommunicator has a single "user" field,
+   * which is passed into the SignableVertexSpec.
+   * Regarding credentials: LlapTaskCommunicator creates 
SubmitWorkRequestProto instances,
+   * into which dag-level credentials are passed.
+   * The most performant way would be to use a single UGI for the same user in 
the daemon, but that's not possible,
+   * because the credentials can theoretically change across queries.
+   */
+  public UserGroupInformation getOrCreateUgi(QueryIdentifier queryIdentifier, 
String user, Credentials credentials) {
+    return ugis.computeIfAbsent(queryIdentifier,
+        k -> {
+          try {
+            UserGroupInformation ugi = ugiFactory.createUgi(user);
+            ugi.addCredentials(credentials);
+            LOG.info("Created ugi {} for queryIdentifier '{}', current ugis #: 
{}", ugi, queryIdentifier, ugis.size());
+            return ugi;
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        });
+  }
+
+  /*
+   * Closes all filesystems for a specific query.
+   * In LLAP daemons for the same dag user and credentials, this call 
typically closes a single FileSystem instance.
+   */
+  public void closeAllForUgi(QueryIdentifier queryIdentifier) {
+    LOG.debug("Closing all FileSystems for queryIdentifier '{}'", 
queryIdentifier);
+    try {
+      FileSystem.closeAllForUGI(ugis.get(queryIdentifier));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    ugis.remove(queryIdentifier);
+  }
+}
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
index 336bf056915..a16ad881fbf 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/ContainerRunnerImpl.java
@@ -40,7 +40,7 @@ import io.jsonwebtoken.Jws;
 import io.jsonwebtoken.JwtException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.UgiFactory;
+import org.apache.hadoop.hive.llap.LlapUgiManager;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.llap.DaemonId;
@@ -132,7 +132,7 @@ public class ContainerRunnerImpl extends CompositeService 
implements ContainerRu
   private final LlapSignerImpl signer;
   private final String clusterId;
   private final DaemonId daemonId;
-  private final UgiFactory fsUgiFactory;
+  private final LlapUgiManager llapUgiManager;
   private final SocketFactory socketFactory;
   private final boolean execUseFQDN;
 
@@ -140,7 +140,7 @@ public class ContainerRunnerImpl extends CompositeService 
implements ContainerRu
       AtomicReference<InetSocketAddress> localAddress,
       long totalMemoryAvailableBytes, LlapDaemonExecutorMetrics metrics,
       AMReporter amReporter, QueryTracker queryTracker, 
Scheduler<TaskRunnerCallable> executorService,
-      DaemonId daemonId, UgiFactory fsUgiFactory,
+      DaemonId daemonId, LlapUgiManager llapUgiManager,
       SocketFactory socketFactory) {
     super("ContainerRunnerImpl");
     Preconditions.checkState(numExecutors > 0,
@@ -151,7 +151,7 @@ public class ContainerRunnerImpl extends CompositeService 
implements ContainerRu
     this.amReporter = amReporter;
     this.signer = UserGroupInformation.isSecurityEnabled()
         ? new LlapSignerImpl(conf, daemonId.getClusterString()) : null;
-    this.fsUgiFactory = fsUgiFactory;
+    this.llapUgiManager = llapUgiManager;
     this.socketFactory = socketFactory;
 
     this.clusterId = daemonId.getClusterString();
@@ -292,7 +292,8 @@ public class ContainerRunnerImpl extends CompositeService 
implements ContainerRu
 
       // Lazy create conf object, as it gets expensive in this codepath.
       Supplier<Configuration> callableConf = () -> new 
Configuration(getConfig());
-      UserGroupInformation fsTaskUgi = fsUgiFactory == null ? null : 
fsUgiFactory.createUgi();
+      UserGroupInformation fsTaskUgi =
+          llapUgiManager.getOrCreateUgi(queryIdentifier, vertex.getUser(), 
credentials);
       boolean isGuaranteed = request.hasIsGuaranteed() && 
request.getIsGuaranteed();
 
       // enable the printing of (per daemon) LLAP task queue/run times via 
LLAP_TASK_TIME_SUMMARY
@@ -488,6 +489,7 @@ public class ContainerRunnerImpl extends CompositeService 
implements ContainerRu
           fragmentInfo.getFragmentIdentifierString());
         
executorService.killFragment(fragmentInfo.getFragmentIdentifierString());
       }
+      llapUgiManager.closeAllForUgi(queryIdentifier);
       amReporter.queryComplete(queryIdentifier);
     }
     return QueryCompleteResponseProto.getDefaultInstance();
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java 
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index 3c24d4d3e35..106f1826f95 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -38,13 +38,13 @@ import javax.net.SocketFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.common.JvmPauseMonitor;
 import org.apache.hadoop.hive.common.LogUtils;
-import org.apache.hadoop.hive.common.UgiFactory;
 import org.apache.hadoop.hive.conf.Constants;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.llap.DaemonId;
 import org.apache.hadoop.hive.llap.LlapDaemonInfo;
 import org.apache.hadoop.hive.llap.LlapOutputFormatService;
+import org.apache.hadoop.hive.llap.LlapUgiManager;
 import org.apache.hadoop.hive.llap.LlapUtil;
 import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
 import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
@@ -70,8 +70,6 @@ import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem;
 import org.apache.hadoop.hive.llap.metrics.MetricsUtils;
 import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
 import org.apache.hadoop.hive.llap.security.LlapExtClientJwtHelper;
-import org.apache.hadoop.hive.llap.security.LlapUgiFactoryFactory;
-import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
 import org.apache.hadoop.hive.llap.security.SecretManager;
 import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
 import org.apache.hadoop.hive.ql.ServiceContext;
@@ -83,7 +81,6 @@ import 
org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge.UdfWhitelistChecke
 import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.StringUtils;
@@ -336,12 +333,7 @@ public class LlapDaemon extends CompositeService 
implements ContainerRunner, Lla
     this.server = new LlapProtocolServerImpl(secretManager, numHandlers, this, 
srvAddress, mngAddress, srvPort,
         externalClientsRpcPort, mngPort, daemonId, 
metrics).withTokenManager(this.llapTokenManager);
 
-    UgiFactory fsUgiFactory = null;
-    try {
-      fsUgiFactory = LlapUgiFactoryFactory.createFsUgiFactory(daemonConf);
-    } catch (IOException e) {
-      throw new RuntimeException(e);
-    }
+    LlapUgiManager llapUgiManager = LlapUgiManager.getInstance(daemonConf);
 
     QueryTracker queryTracker = new QueryTracker(daemonConf, localDirs,
         daemonId.getClusterString());
@@ -357,7 +349,7 @@ public class LlapDaemon extends CompositeService implements 
ContainerRunner, Lla
 
     this.containerRunner = new ContainerRunnerImpl(daemonConf, numExecutors,
         this.shufflePort, srvAddress, executorMemoryPerInstance, metrics,
-        amReporter, queryTracker, executorService, daemonId, fsUgiFactory, 
socketFactory);
+        amReporter, queryTracker, executorService, daemonId, llapUgiManager, 
socketFactory);
     addIfService(containerRunner);
 
     // Not adding the registry as a service, since we need to control when it 
is initialized - conf used to pickup properties.
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
 
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
index f299c58c5b8..b0fe7ea5ded 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/TaskRunnerCallable.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.llap.daemon.impl;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
@@ -25,8 +26,6 @@ import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hive.llap.counters.LlapWmCounters;
 import org.apache.hadoop.hive.llap.counters.WmFragmentCounters;
 import org.apache.hadoop.hive.llap.daemon.FragmentCompletionHandler;
 import org.apache.hadoop.hive.llap.daemon.HistoryLogger;
@@ -178,6 +177,8 @@ public class TaskRunnerCallable extends 
CallableWithNdc<TaskRunner2Result> {
     this.tezHadoopShim = tezHadoopShim;
     this.initialEvent = initialEvent;
     this.fsTaskUgi = fsTaskUgi;
+    Preconditions.checkArgument(fsTaskUgi != null,
+        "fsTaskUgi must not be null, this implies a problem somewhere in 
LlapUgiManager");
     this.completionListener = completionListener;
     this.socketFactory = socketFactory;
     this.isGuaranteed = isGuaranteed;
@@ -226,10 +227,6 @@ public class TaskRunnerCallable extends 
CallableWithNdc<TaskRunner2Result> {
 
       // TODO Consolidate this code with TezChild.
       runtimeWatch.start();
-      if (fsTaskUgi == null) {
-        fsTaskUgi = UserGroupInformation.createRemoteUser(vertex.getUser());
-      }
-      fsTaskUgi.addCredentials(credentials);
 
       Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<>();
       serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID,
@@ -299,7 +296,6 @@ public class TaskRunnerCallable extends 
CallableWithNdc<TaskRunner2Result> {
           isCompleted.set(true);
           return result;
         } finally {
-          FileSystem.closeAllForUGI(fsTaskUgi);
           fragmentInfo.getQueryInfo().returnUmbilicalUgi(taskOwner);
           LOG.info("ExecutionTime for Container: " + 
request.getContainerIdString() + "=" +
                   runtimeWatch.stop().elapsed(TimeUnit.MILLISECONDS));
diff --git 
a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapUgiFactoryFactory.java
 b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapUgiHelper.java
similarity index 68%
rename from 
llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapUgiFactoryFactory.java
rename to 
llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapUgiHelper.java
index f6c216f3423..b946454627e 100644
--- 
a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapUgiFactoryFactory.java
+++ 
b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapUgiHelper.java
@@ -17,7 +17,7 @@ package org.apache.hadoop.hive.llap.security;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.UgiFactory;
+import org.apache.hadoop.hive.llap.LlapUgiFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.llap.LlapUtil;
@@ -25,43 +25,44 @@ import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.security.UserGroupInformation;
 
-/** No Java application is complete until it has a FactoryFactory. */
-public class LlapUgiFactoryFactory {
+public class LlapUgiHelper {
   private static final HadoopShims SHIMS = ShimLoader.getHadoopShims();
 
-  private static class KerberosUgiFactory implements UgiFactory {
+  public static LlapUgiFactory createLlapUgiFactory(Configuration conf) throws 
IOException {
+    String fsKeytab = HiveConf.getVar(conf, 
ConfVars.LLAP_FS_KERBEROS_KEYTAB_FILE),
+        fsPrincipal = HiveConf.getVar(conf, 
ConfVars.LLAP_FS_KERBEROS_PRINCIPAL);
+    boolean hasFsKeytab = fsKeytab != null && !fsKeytab.isEmpty(),
+        hasFsPrincipal = fsPrincipal != null && !fsPrincipal.isEmpty();
+    if (hasFsKeytab != hasFsPrincipal) {
+      throw new IOException("Inconsistent FS keytab settings " + fsKeytab + "; 
" + fsPrincipal);
+    }
+    return hasFsKeytab ? new KerberosLlapUgiFactory(fsKeytab, fsPrincipal) : 
new NoopLlapUgiFactory();
+  }
+
+  private static class KerberosLlapUgiFactory implements LlapUgiFactory {
     private final UserGroupInformation baseUgi;
 
-    public KerberosUgiFactory(String keytab, String principal) throws 
IOException {
+    public KerberosLlapUgiFactory(String keytab, String principal) throws 
IOException {
       baseUgi = LlapUtil.loginWithKerberos(principal, keytab);
     }
 
     @Override
-    public UserGroupInformation createUgi() throws IOException {
+    public UserGroupInformation createUgi(String user) throws IOException {
       // Make sure the UGI is current.
       baseUgi.checkTGTAndReloginFromKeytab();
       // TODO: the only reason this is done this way is because we want unique 
Subject-s so that
       //       the FS.get gives different FS objects to different fragments.
-      // TODO: could we log in from ticket cache instead? no good method on 
UGI right now.
+      // TODO: could we log in from ticket cache instead? no good method on 
UGI right now
       return SHIMS.cloneUgi(baseUgi);
     }
   }
 
-  private static class NoopUgiFactory implements UgiFactory {
+  private static class NoopLlapUgiFactory implements LlapUgiFactory {
     @Override
-    public UserGroupInformation createUgi() throws IOException {
-      return null;
-    }
-  }
-
-  public static UgiFactory createFsUgiFactory(Configuration conf) throws 
IOException {
-    String fsKeytab = HiveConf.getVar(conf, 
ConfVars.LLAP_FS_KERBEROS_KEYTAB_FILE),
-        fsPrincipal = HiveConf.getVar(conf, 
ConfVars.LLAP_FS_KERBEROS_PRINCIPAL);
-    boolean hasFsKeytab = fsKeytab != null && !fsKeytab.isEmpty(),
-        hasFsPrincipal = fsPrincipal != null && !fsPrincipal.isEmpty();
-    if (hasFsKeytab != hasFsPrincipal) {
-      throw new IOException("Inconsistent FS keytab settings " + fsKeytab + "; 
" + fsPrincipal);
+    public UserGroupInformation createUgi(String user) throws IOException {
+      // create clone in order to have a unique subject (== unique ugi) per 
query,
+      // otherwise closeFileSystemsForQuery will close a FileSystem used by 
another query
+      return SHIMS.cloneUgi(UserGroupInformation.createRemoteUser(user));
     }
-    return hasFsKeytab ? new KerberosUgiFactory(fsKeytab, fsPrincipal) : new 
NoopUgiFactory();
   }
 }
diff --git 
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
 
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
index 6934e5d65aa..d8b5849a169 100644
--- 
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
+++ 
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
@@ -35,6 +35,7 @@ import 
org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWor
 import 
org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.VertexOrBinary;
 import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics;
 import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.tez.dag.records.TezDAGID;
 import org.apache.tez.dag.records.TezVertexID;
@@ -231,7 +232,7 @@ public class TaskExecutorTestHelpers {
           null, new Credentials(), 0, mock(AMReporter.class), null, mock(
           LlapDaemonExecutorMetrics.class), mock(KilledTaskHandler.class), 
mock(
           FragmentCompletionHandler.class), new DefaultHadoopShim(), null,
-          requestProto.getWorkSpec().getVertex(), initialEvent, null, mock(
+          requestProto.getWorkSpec().getVertex(), initialEvent, 
mock(UserGroupInformation.class), mock(
           SchedulerFragmentCompletingListener.class), 
mock(SocketFactory.class), isGuaranteed, null);
       this.workTime = workTime;
       this.canFinish = canFinish;
diff --git 
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestContainerRunnerImpl.java
 
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestContainerRunnerImpl.java
index e1c069016b5..cdc660407af 100644
--- 
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestContainerRunnerImpl.java
+++ 
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestContainerRunnerImpl.java
@@ -18,6 +18,7 @@ import com.google.common.primitives.Ints;
 import com.google.protobuf.ByteString;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.llap.DaemonId;
+import org.apache.hadoop.hive.llap.LlapUgiManager;
 import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
 import org.apache.hadoop.hive.llap.daemon.LlapDaemonTestUtils;
 import 
org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.RegisterDagRequestProto;
@@ -27,7 +28,6 @@ import 
org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics;
 import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem;
 import org.apache.hadoop.hive.llap.metrics.MetricsUtils;
 import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier;
-import org.apache.hadoop.hive.llap.security.LlapUgiFactoryFactory;
 import org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler;
 import org.apache.hadoop.hive.llap.tezplugins.LlapTezUtils;
 import org.apache.hadoop.io.Text;
@@ -45,7 +45,6 @@ import java.io.File;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.mockito.Mockito.mock;
@@ -118,8 +117,7 @@ public class TestContainerRunnerImpl {
         daemonConf, HiveConf.ConfVars.LLAP_DAEMON_RPC_PORT));
     containerRunner = new ContainerRunnerImpl(daemonConf, numExecutors,
         this.shufflePort, srvAddress, executorMemoryPerInstance, metrics,
-        amReporter, queryTracker, executorService, daemonId, 
LlapUgiFactoryFactory
-        .createFsUgiFactory(daemonConf), socketFactory);
+        amReporter, queryTracker, executorService, daemonId, 
LlapUgiManager.getInstance(daemonConf), socketFactory);
 
     ShuffleHandler.initializeAndStart(daemonConf);
 

Reply via email to