HIVE-15935: ACL is not set in ATS data (Daniel Dai, reviewed by Siddharth Seth)
Change-Id: I95734893c18cefe58731323ad33e6097118c3378 Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44e49982 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44e49982 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44e49982 Branch: refs/heads/branch-2.2 Commit: 44e49982a4049990fbfff9b3ad53e104e8499414 Parents: badea3f Author: Daniel Dai <da...@hortonworks.com> Authored: Tue Feb 28 15:40:50 2017 -0800 Committer: Owen O'Malley <omal...@apache.org> Committed: Tue Mar 28 15:27:53 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/hooks/ATSHook.java | 88 +++++++++++++++++++- .../hadoop/hive/ql/session/SessionState.java | 10 +++ 2 files changed, 95 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/44e49982/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java index 2dd03f5..218e0b4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java @@ -33,6 +33,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; import org.apache.hadoop.hive.ql.QueryPlan; import org.apache.hadoop.hive.ql.QueryState; @@ -43,12 +44,16 @@ import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.log.PerfLogger; import org.apache.hadoop.hive.ql.parse.ExplainConfiguration; import org.apache.hadoop.hive.ql.plan.ExplainWork; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hive.common.util.ShutdownHookManager; +import org.apache.tez.dag.api.TezConfiguration; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,6 +74,9 @@ public class ATSHook implements ExecuteWithHookContext { private static TimelineClient timelineClient; private enum EntityTypes { HIVE_QUERY_ID }; private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED }; + private static final String ATS_DOMAIN_PREFIX = "hive_"; + private static boolean defaultATSDomainCreated = false; + private static final String DEFAULT_ATS_DOMAIN = "hive_default_ats"; private enum OtherInfoTypes { QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, SESSION_ID, THREAD_NAME, VERSION, @@ -136,6 +144,78 @@ public class ATSHook implements ExecuteWithHookContext { LOG.info("Created ATS Hook"); } + private void createTimelineDomain(String domainId, String readers, String writers) throws Exception { + TimelineDomain timelineDomain = new TimelineDomain(); + timelineDomain.setId(domainId); + timelineDomain.setReaders(readers); + timelineDomain.setWriters(writers); + timelineClient.putDomain(timelineDomain); + LOG.info("ATS domain created:" + domainId + "(" + readers + "," + writers + ")"); + } + + private String createOrGetDomain(final HookContext hookContext) throws Exception { + final String domainId; + String domainReaders = null; + String domainWriters = null; + boolean create = false; + if (SessionState.get() != null) { + if (SessionState.get().getATSDomainId() == null) { + domainId = ATS_DOMAIN_PREFIX + hookContext.getSessionId(); + // Create session domain if not present + if (SessionState.get().getATSDomainId() == null) { + String requestuser = hookContext.getUserName(); + if (hookContext.getUserName() == null ){ + requestuser = hookContext.getUgi().getShortUserName() ; + } + boolean addHs2User = + HiveConf.getBoolVar(hookContext.getConf(), ConfVars.HIVETEZHS2USERACCESS); + + UserGroupInformation loginUserUgi = UserGroupInformation.getLoginUser(); + String loginUser = + loginUserUgi == null ? null : loginUserUgi.getShortUserName(); + + // In Tez, TEZ_AM_VIEW_ACLS/TEZ_AM_MODIFY_ACLS is used as the base for Tez ATS ACLS, + // so if exists, honor it. So we get the same ACLS for Tez ATS entries and + // Hive entries + domainReaders = Utilities.getAclStringWithHiveModification(hookContext.getConf(), + TezConfiguration.TEZ_AM_VIEW_ACLS, addHs2User, requestuser, loginUser); + + domainWriters = Utilities.getAclStringWithHiveModification(hookContext.getConf(), + TezConfiguration.TEZ_AM_MODIFY_ACLS, addHs2User, requestuser, loginUser); + SessionState.get().setATSDomainId(domainId); + create = true; + } + } else { + domainId = SessionState.get().getATSDomainId(); + } + } else { + // SessionState is null, this is unlikely to happen, just in case + if (!defaultATSDomainCreated) { + domainReaders = domainWriters = UserGroupInformation.getCurrentUser().getShortUserName(); + defaultATSDomainCreated = true; + create = true; + } + domainId = DEFAULT_ATS_DOMAIN; + } + if (create) { + final String readers = domainReaders; + final String writers = domainWriters; + // executor is single thread, so we can guarantee + // domain created before any ATS entries + executor.submit(new Runnable() { + @Override + public void run() { + try { + createTimelineDomain(domainId, readers, writers); + } catch (Exception e) { + LOG.warn("Failed to create ATS domain " + domainId, e); + } + } + }); + } + return domainId; + } + @Override public void run(final HookContext hookContext) throws Exception { final long currentTime = System.currentTimeMillis(); @@ -151,6 +231,7 @@ public class ATSHook implements ExecuteWithHookContext { try { setupAtsExecutor(conf); + final String domainId = createOrGetDomain(hookContext); executor.submit(new Runnable() { @Override public void run() { @@ -223,12 +304,12 @@ public class ATSHook implements ExecuteWithHookContext { case POST_EXEC_HOOK: fireAndForget(createPostHookEvent(queryId, currentTime, user, requestuser, true, opId, - durations)); + durations, domainId)); break; case ON_FAILURE_HOOK: fireAndForget(createPostHookEvent(queryId, currentTime, user, requestuser, false, - opId, durations)); + opId, durations, domainId)); break; default: // ignore @@ -342,7 +423,7 @@ public class ATSHook implements ExecuteWithHookContext { } TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser, boolean success, - String opId, Map<String, Long> durations) throws Exception { + String opId, Map<String, Long> durations, String domainId) throws Exception { LOG.info("Received post-hook notification for :" + queryId); TimelineEntity atsEntity = new TimelineEntity(); @@ -367,6 +448,7 @@ public class ATSHook implements ExecuteWithHookContext { perfObj.put(entry.getKey(), entry.getValue()); } atsEntity.addOtherInfo(OtherInfoTypes.PERF.name(), perfObj.toString()); + atsEntity.setDomainId(domainId); return atsEntity; } http://git-wip-us.apache.org/repos/asf/hive/blob/44e49982/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 847081b..6fc9055 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -283,6 +283,8 @@ public class SessionState { private List<String> forwardedAddresses; + private String atsDomainId; + /** * Get the lineage state stored in this session. * @@ -1251,6 +1253,14 @@ public class SessionState { } } + public String getATSDomainId() { + return atsDomainId; + } + + public void setATSDomainId(String domainId) { + this.atsDomainId = domainId; + } + /** * ResourceType. *
