hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [33/50] [abbrv] hive git commit: HIVE-15935: ACL is not set in ATS data (Daniel Dai, reviewed by Siddharth Seth)
Date Tue, 28 Mar 2017 21:09:50 GMT
HIVE-15935: ACL is not set in ATS data (Daniel Dai, reviewed by Siddharth Seth)

Change-Id: I95734893c18cefe58731323ad33e6097118c3378


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fa53585a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fa53585a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fa53585a

Branch: refs/heads/branch-2.2
Commit: fa53585a891d88b30d88413c96d9881fdda73e7b
Parents: 38c9ac7
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Tue Feb 28 15:40:50 2017 -0800
Committer: Owen O'Malley <omalley@apache.org>
Committed: Tue Mar 28 14:02:44 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/hooks/ATSHook.java    | 88 +++++++++++++++++++-
 .../hadoop/hive/ql/session/SessionState.java    | 10 +++
 2 files changed, 95 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/fa53585a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
index 2dd03f5..218e0b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java
@@ -33,6 +33,7 @@ import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
 import org.apache.hadoop.hive.ql.QueryPlan;
 import org.apache.hadoop.hive.ql.QueryState;
@@ -43,12 +44,16 @@ import org.apache.hadoop.hive.ql.exec.tez.TezTask;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.parse.ExplainConfiguration;
 import org.apache.hadoop.hive.ql.plan.ExplainWork;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
 import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hive.common.util.ShutdownHookManager;
+import org.apache.tez.dag.api.TezConfiguration;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -69,6 +74,9 @@ public class ATSHook implements ExecuteWithHookContext {
   private static TimelineClient timelineClient;
   private enum EntityTypes { HIVE_QUERY_ID };
   private enum EventTypes { QUERY_SUBMITTED, QUERY_COMPLETED };
+  private static final String ATS_DOMAIN_PREFIX = "hive_";
+  private static boolean defaultATSDomainCreated = false;
+  private static final String DEFAULT_ATS_DOMAIN = "hive_default_ats";
 
   private enum OtherInfoTypes {
     QUERY, STATUS, TEZ, MAPRED, INVOKER_INFO, SESSION_ID, THREAD_NAME, VERSION,
@@ -136,6 +144,78 @@ public class ATSHook implements ExecuteWithHookContext {
     LOG.info("Created ATS Hook");
   }
 
+  private void createTimelineDomain(String domainId, String readers, String writers) throws
Exception {
+    TimelineDomain timelineDomain = new TimelineDomain();
+    timelineDomain.setId(domainId);
+    timelineDomain.setReaders(readers);
+    timelineDomain.setWriters(writers);
+    timelineClient.putDomain(timelineDomain);
+    LOG.info("ATS domain created:" + domainId + "(" + readers + "," + writers + ")");
+  }
+ 
+  private String createOrGetDomain(final HookContext hookContext) throws Exception {
+    final String domainId;
+    String domainReaders = null;
+    String domainWriters = null;
+    boolean create = false;
+    if (SessionState.get() != null) {
+      if (SessionState.get().getATSDomainId() == null) {
+        domainId = ATS_DOMAIN_PREFIX + hookContext.getSessionId();
+        // Create session domain if not present
+        if (SessionState.get().getATSDomainId() == null) {
+          String requestuser = hookContext.getUserName();
+          if (hookContext.getUserName() == null ){
+            requestuser = hookContext.getUgi().getShortUserName() ;
+          }
+          boolean addHs2User =
+              HiveConf.getBoolVar(hookContext.getConf(), ConfVars.HIVETEZHS2USERACCESS);
+
+          UserGroupInformation loginUserUgi = UserGroupInformation.getLoginUser();
+          String loginUser =
+              loginUserUgi == null ? null : loginUserUgi.getShortUserName();
+
+          // In Tez, TEZ_AM_VIEW_ACLS/TEZ_AM_MODIFY_ACLS is used as the base for Tez ATS
ACLS,
+          // so if exists, honor it. So we get the same ACLS for Tez ATS entries and
+          // Hive entries
+          domainReaders = Utilities.getAclStringWithHiveModification(hookContext.getConf(),
+              TezConfiguration.TEZ_AM_VIEW_ACLS, addHs2User, requestuser, loginUser);
+
+          domainWriters = Utilities.getAclStringWithHiveModification(hookContext.getConf(),
+              TezConfiguration.TEZ_AM_MODIFY_ACLS, addHs2User, requestuser, loginUser);
+          SessionState.get().setATSDomainId(domainId);
+          create = true;
+        }
+      } else {
+        domainId = SessionState.get().getATSDomainId();
+      }
+    } else {
+      // SessionState is null, this is unlikely to happen, just in case
+      if (!defaultATSDomainCreated) {
+        domainReaders = domainWriters = UserGroupInformation.getCurrentUser().getShortUserName();
+        defaultATSDomainCreated = true;
+        create = true;
+      }
+      domainId = DEFAULT_ATS_DOMAIN;
+    }
+    if (create) {
+      final String readers = domainReaders;
+      final String writers = domainWriters;
+      // executor is single thread, so we can guarantee
+      // domain created before any ATS entries
+      executor.submit(new Runnable() {
+        @Override
+        public void run() {
+          try {
+            createTimelineDomain(domainId, readers, writers);
+          } catch (Exception e) {
+            LOG.warn("Failed to create ATS domain " + domainId, e);
+          }
+        }
+      });
+    }
+    return domainId;
+  }
+
   @Override
   public void run(final HookContext hookContext) throws Exception {
           final long currentTime = System.currentTimeMillis();
@@ -151,6 +231,7 @@ public class ATSHook implements ExecuteWithHookContext {
           try {
                   setupAtsExecutor(conf);
 
+                  final String domainId = createOrGetDomain(hookContext);
                   executor.submit(new Runnable() {
                           @Override
                           public void run() {
@@ -223,12 +304,12 @@ public class ATSHook implements ExecuteWithHookContext {
                                           case POST_EXEC_HOOK:
                                                   fireAndForget(createPostHookEvent(queryId,
                                                                   currentTime, user, requestuser,
true, opId,
-                                                                  durations));
+                                                                  durations, domainId));
                                                   break;
                                           case ON_FAILURE_HOOK:
                                                   fireAndForget(createPostHookEvent(queryId,
                                                                   currentTime, user, requestuser,
false,
-                                                                  opId, durations));
+                                                                  opId, durations, domainId));
                                                   break;
                                           default:
                                                   // ignore
@@ -342,7 +423,7 @@ public class ATSHook implements ExecuteWithHookContext {
   }
 
   TimelineEntity createPostHookEvent(String queryId, long stopTime, String user, String requestuser,
boolean success,
-      String opId, Map<String, Long> durations) throws Exception {
+      String opId, Map<String, Long> durations, String domainId) throws Exception {
     LOG.info("Received post-hook notification for :" + queryId);
 
     TimelineEntity atsEntity = new TimelineEntity();
@@ -367,6 +448,7 @@ public class ATSHook implements ExecuteWithHookContext {
       perfObj.put(entry.getKey(), entry.getValue());
     }
     atsEntity.addOtherInfo(OtherInfoTypes.PERF.name(), perfObj.toString());
+    atsEntity.setDomainId(domainId);
 
     return atsEntity;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/fa53585a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 847081b..6fc9055 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -283,6 +283,8 @@ public class SessionState {
 
   private List<String> forwardedAddresses;
 
+  private String atsDomainId;
+
   /**
    * Get the lineage state stored in this session.
    *
@@ -1251,6 +1253,14 @@ public class SessionState {
     }
   }
 
+  public String getATSDomainId() {
+    return atsDomainId;
+  }
+
+  public void setATSDomainId(String domainId) {
+    this.atsDomainId = domainId;
+  }
+
   /**
    * ResourceType.
    *


Mime
View raw message