hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject hive git commit: HIVE-13128. NullScan fails on a secure setup. (Siddharth Seth, reviewed by Sergey Shelukhin)
Date Wed, 24 Feb 2016 18:32:20 GMT
Repository: hive
Updated Branches:
  refs/heads/master 2bf0d92d1 -> 2f7323396


HIVE-13128. NullScan fails on a secure setup. (Siddharth Seth, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2f732339
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2f732339
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2f732339

Branch: refs/heads/master
Commit: 2f73233961e6f527f5460c124cc021a4a9628b01
Parents: 2bf0d92
Author: Siddharth Seth <sseth@apache.org>
Authored: Wed Feb 24 10:31:41 2016 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Wed Feb 24 10:31:41 2016 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/ql/exec/tez/DagUtils.java |  5 ++---
 .../hadoop/hive/ql/exec/tez/TezSessionState.java     |  2 ++
 .../org/apache/hadoop/hive/ql/exec/tez/TezTask.java  |  7 ++-----
 .../apache/hadoop/hive/ql/io/NullScanFileSystem.java | 15 +++++++++++++++
 4 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/2f732339/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
index 39866a3..473dbd6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
@@ -754,7 +754,7 @@ public class DagUtils {
     URL resourceURL = ConverterUtils.getYarnUrlFromPath(file);
     long resourceSize = fstat.getLen();
     long resourceModificationTime = fstat.getModificationTime();
-    LOG.info("Resource modification time: " + resourceModificationTime);
+    LOG.info("Resource modification time: " + resourceModificationTime + " for " + file);
 
     LocalResource lr = Records.newRecord(LocalResource.class);
     lr.setResource(resourceURL);
@@ -912,7 +912,7 @@ public class DagUtils {
 
     if (fstatus == null) {
       Path destDir = getDefaultDestDir(conf);
-      LOG.info("Jar dir is null/directory doesn't exist. Choosing HIVE_INSTALL_DIR - " +
destDir);
+      LOG.info("Jar dir is null / directory doesn't exist. Choosing HIVE_INSTALL_DIR - "
+ destDir);
       fstatus = validateTargetDir(destDir, conf);
     }
 
@@ -1275,5 +1275,4 @@ public class DagUtils {
   private DagUtils() {
     // don't instantiate
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/2f732339/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 6675f0d..3ea5ef9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -548,6 +548,8 @@ public class TezSessionState {
   private LocalResource createJarLocalResource(String localJarPath)
       throws IOException, LoginException, IllegalArgumentException,
       FileNotFoundException {
+    // TODO Reduce the number of lookups that happen here. This shouldn't go to HDFS for
each call.
+    // The hiveJarDir can be determined once per client.
     FileStatus destDirStatus = utils.getHiveJarDirectory(conf);
     assert destDirStatus != null;
     Path destDirPath = destDirStatus.getPath();

http://git-wip-us.apache.org/repos/asf/hive/blob/2f732339/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
index 83defea..3789ce9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
@@ -173,9 +173,6 @@ public class TezTask extends Task<TezWork> {
 
       // next we translate the TezWork to a Tez DAG
       DAG dag = build(jobConf, work, scratchDir, appJarLr, additionalLr, ctx);
-      if (driverContext.getCtx() == null) {
-        boolean a = false;
-      }
       CallerContext callerContext = CallerContext.create(
           "HIVE", queryPlan.getQueryId(),
           "HIVE_QUERY_ID", queryPlan.getQueryStr());
@@ -460,10 +457,10 @@ public class TezTask extends Task<TezWork> {
     } catch (Exception e) {
       // In case of any other exception, retry. If this also fails, report original error
and exit.
       try {
-        TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars,
-            true);
         console.printInfo("Dag submit failed due to " + e.getMessage() + " stack trace: "
             + Arrays.toString(e.getStackTrace()) + " retrying...");
+        TezSessionPoolManager.getInstance().closeAndOpen(sessionState, this.conf, inputOutputJars,
+            true);
         dagClient = sessionState.getSession().submitDAG(dag);
       } catch (Exception retryException) {
         // we failed to submit after retrying. Destroy session and bail.

http://git-wip-us.apache.org/repos/asf/hive/blob/2f732339/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java b/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java
index 88e9445..5bfde18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/NullScanFileSystem.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Progressable;
 
 /**
@@ -45,6 +47,8 @@ public class NullScanFileSystem extends FileSystem {
     return "nullscan";
   }
 
+  private final Token<?>[] DEFAULT_EMPTY_TOKEN_ARRAY = new Token<?>[0];
+
   public NullScanFileSystem() {
   }
 
@@ -113,4 +117,15 @@ public class NullScanFileSystem extends FileSystem {
   public FileStatus getFileStatus(Path f) throws IOException {
     return new FileStatus(0, false, 0, 0, 0, f);
   }
+
+  @Override
+  public Token<?>[] addDelegationTokens(String renewer, Credentials credentials) throws
+      IOException {
+    return DEFAULT_EMPTY_TOKEN_ARRAY;
+  }
+
+  @Override
+  public Token<?> getDelegationToken(String renewer) throws IOException {
+    return null;
+  }
 }
\ No newline at end of file


Mime
View raw message