hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ser...@apache.org
Subject svn commit: r1586454 - in /hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez: DagUtils.java TezSessionState.java
Date Thu, 10 Apr 2014 19:58:50 GMT
Author: sershe
Date: Thu Apr 10 19:58:49 2014
New Revision: 1586454

URL: http://svn.apache.org/r1586454
Log:
HIVE-6816 : jar upload path w/o schema is not handled correctly (Sergey Shelukhin, reviewed
by Vikram Dixit K)

Modified:
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1586454&r1=1586453&r2=1586454&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Thu Apr 10 19:58:49 2014
@@ -676,6 +676,8 @@ public class DagUtils {
 
     String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS);
 
+    // need to localize the additional jars and files
+    // we need the directory on hdfs to which we shall put all these files
     String allFiles = auxJars + "," + addedJars + "," + addedFiles + "," + addedArchives;
     addTempFiles(conf, tmpResources, hdfsDirPathStr, allFiles.split(","));
     return tmpResources;
@@ -712,27 +714,38 @@ public class DagUtils {
     }
   }
 
-  public String getHiveJarDirectory(Configuration conf) throws IOException, LoginException
{
+  public FileStatus getHiveJarDirectory(Configuration conf) throws IOException, LoginException
{
     FileStatus fstatus = null;
     String hdfsDirPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_JAR_DIRECTORY, null);
     if (hdfsDirPathStr != null) {
-      Path hdfsDirPath = new Path(hdfsDirPathStr);
-      FileSystem fs = hdfsDirPath.getFileSystem(conf);
-      if (!(fs instanceof DistributedFileSystem)) {
-        throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(hdfsDirPathStr));
-      }
-      try {
-        fstatus = fs.getFileStatus(hdfsDirPath);
-      } catch (FileNotFoundException fe) {
-        // do nothing
-      }
+      LOG.info("Hive jar directory is " + hdfsDirPathStr);
+      fstatus = validateTargetDir(new Path(hdfsDirPathStr), conf);
     }
 
-    if ((fstatus == null) || (!fstatus.isDir())) {
+    if (fstatus == null) {
       Path destDir = getDefaultDestDir(conf);
-      hdfsDirPathStr = destDir.toString();
+      LOG.info("Jar dir is null/directory doesn't exist. Choosing HIVE_INSTALL_DIR - " +
destDir);
+      fstatus = validateTargetDir(destDir, conf);
+    }
+
+    if (fstatus == null) {
+      throw new IOException(ErrorMsg.NO_VALID_LOCATIONS.getMsg());
+    }
+    return fstatus;
+  }
+
+  public static FileStatus validateTargetDir(Path path, Configuration conf) throws IOException
{
+    FileSystem fs = path.getFileSystem(conf);
+    if (!(fs instanceof DistributedFileSystem)) {
+      throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(path.toString()));
+    }
+    FileStatus fstatus = null;
+    try {
+      fstatus = fs.getFileStatus(path);
+    } catch (FileNotFoundException fe) {
+      // do nothing
     }
-    return hdfsDirPathStr;
+    return (fstatus != null && fstatus.isDir()) ? fstatus : null;
   }
 
   // the api that finds the jar being used by this class on disk

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java?rev=1586454&r1=1586453&r2=1586454&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
Thu Apr 10 19:58:49 2014
@@ -286,13 +286,13 @@ public class TezSessionState {
     throws IOException {
 
     // tez needs its own scratch dir (per session)
-    Path tezDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR),
-        TEZ_DIR);
+    Path tezDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), TEZ_DIR);
     tezDir = new Path(tezDir, sessionId);
     FileSystem fs = tezDir.getFileSystem(conf);
     FsPermission fsPermission = new FsPermission((short)00777);
     Utilities.createDirsWithPermission(conf, tezDir, fsPermission, true);
-
+    // Make sure the path is normalized (we expect validation to pass since we just created
it).
+    tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath();
     // don't keep the directory around on non-clean exit
     fs.deleteOnExit(tezDir);
 
@@ -311,39 +311,9 @@ public class TezSessionState {
   private LocalResource createJarLocalResource(String localJarPath)
       throws IOException, LoginException, IllegalArgumentException,
       FileNotFoundException {
-    Path destDirPath = null;
-    FileSystem destFs = null;
-    FileStatus destDirStatus = null;
-
-    {
-      String hiveJarDir = utils.getHiveJarDirectory(conf);
-      if (hiveJarDir != null) {
-        LOG.info("Hive jar directory is " + hiveJarDir);
-        // check if it is a valid directory in HDFS
-        destDirPath = new Path(hiveJarDir);
-        destFs = destDirPath.getFileSystem(conf);
-        destDirStatus = validateTargetDir(destDirPath, destFs);
-      }
-    }
-
-    /*
-     * Specified location does not exist or is not a directory.
-     * Try to push the jar to the hdfs location pointed by config variable HIVE_INSTALL_DIR.
-     * Path will be HIVE_INSTALL_DIR/{username}/.hiveJars/
-     * This will probably never ever happen.
-     */
-    if (destDirStatus == null || !destDirStatus.isDir()) {
-      destDirPath = utils.getDefaultDestDir(conf);
-      LOG.info("Jar dir is null/directory doesn't exist. Choosing HIVE_INSTALL_DIR - "
-          + destDirPath);
-      destFs = destDirPath.getFileSystem(conf);
-      destDirStatus = validateTargetDir(destDirPath, destFs);
-    }
-
-    // we couldn't find any valid locations. Throw exception
-    if (destDirStatus == null || !destDirStatus.isDir()) {
-      throw new IOException(ErrorMsg.NO_VALID_LOCATIONS.getMsg());
-    }
+    FileStatus destDirStatus = utils.getHiveJarDirectory(conf);
+    assert destDirStatus != null;
+    Path destDirPath = destDirStatus.getPath();
 
     Path localFile = new Path(localJarPath);
     String sha = getSha(localFile);
@@ -365,17 +335,6 @@ public class TezSessionState {
     return utils.localizeResource(localFile, destFile, conf);
   }
 
-  private FileStatus validateTargetDir(Path hiveJarDirPath, FileSystem fs) throws IOException
{
-    if (!(fs instanceof DistributedFileSystem)) {
-      throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(hiveJarDirPath.toString()));
-    }
-    try {
-      return fs.getFileStatus(hiveJarDirPath);
-    } catch (FileNotFoundException fe) {
-      // do nothing
-    }
-    return null;
-  }
 
   private String getSha(Path localFile) throws IOException, IllegalArgumentException {
     InputStream is = null;



Mime
View raw message