hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1671903 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/ hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ ql/src/j...
Date Tue, 07 Apr 2015 18:04:40 GMT
Author: gunther
Date: Tue Apr  7 18:04:40 2015
New Revision: 1671903

URL: http://svn.apache.org/r1671903
Log:
HIVE-10223: Consolidate several redundant FileSystem API calls. (Chris Nauroth via Gunther
Hagleitner)

Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Tue Apr  7 18:04:40
2015
@@ -335,16 +335,16 @@ public final class FileUtils {
    * @param fs
    *          file system
    * @param path
-   * @return the argument path if it exists or a parent path exists. Returns
-   *         NULL root is only parent that exists
+   * @return FileStatus for argument path if it exists or the first ancestor in the path
that exists
    * @throws IOException
    */
-  public static Path getPathOrParentThatExists(FileSystem fs, Path path) throws IOException
{
-    if (!fs.exists(path)) {
-      Path parentPath = path.getParent();
-      return getPathOrParentThatExists(fs, parentPath);
+  public static FileStatus getPathOrParentThatExists(FileSystem fs, Path path) throws IOException
{
+    FileStatus stat = FileUtils.getFileStatusOrNull(fs, path);
+    if (stat != null) {
+      return stat;
     }
-    return path;
+    Path parentPath = path.getParent();
+    return getPathOrParentThatExists(fs, parentPath);
   }
 
   /**
@@ -743,4 +743,20 @@ public final class FileUtils {
 
   }
 
+  /**
+   * Attempts to get file status.  This method differs from the FileSystem API in that it
returns
+   * null instead of throwing FileNotFoundException if the path does not exist.
+   *
+   * @param fs file system to check
+   * @param path file system path to check
+   * @return FileStatus for path or null if path does not exist
+   * @throws IOException if there is an I/O error
+   */
+  public static FileStatus getFileStatusOrNull(FileSystem fs, Path path) throws IOException
{
+    try {
+      return fs.getFileStatus(path);
+    } catch (FileNotFoundException e) {
+      return null;
+    }
+  }
 }

Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
(original)
+++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java
Tue Apr  7 18:04:40 2015
@@ -510,8 +510,9 @@ class FileOutputCommitterContainer exten
     }
 
     final Path finalOutputPath = getFinalPath(fs, file, srcDir, destDir, immutable);
+    FileStatus fileStatus = fs.getFileStatus(file);
 
-    if (fs.isFile(file)) {
+    if (fileStatus.isFile()) {
       if (dryRun){
         if (immutable){
           // Dryrun checks are meaningless for mutable table - we should always succeed
@@ -541,7 +542,7 @@ class FileOutputCommitterContainer exten
           }
         }
       }
-    } else if(fs.getFileStatus(file).isDir()) {
+    } else if (fileStatus.isDirectory()) {
 
       FileStatus[] children = fs.listStatus(file);
       FileStatus firstChild = null;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Tue Apr  7 18:04:40
2015
@@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.Context;
@@ -926,8 +927,9 @@ public class DagUtils {
     throws IOException {
     FileSystem destFS = dest.getFileSystem(conf);
     FileSystem sourceFS = src.getFileSystem(conf);
-    if (destFS.exists(dest)) {
-      return (sourceFS.getFileStatus(src).getLen() == destFS.getFileStatus(dest).getLen());
+    FileStatus destStatus = FileUtils.getFileStatusOrNull(destFS, dest);
+    if (destStatus != null) {
+      return (sourceFS.getFileStatus(src).getLen() == destStatus.getLen());
     }
     return false;
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java Tue Apr
 7 18:04:40 2015
@@ -460,7 +460,7 @@ public class OrcRawRecordMerger implemen
         Path deltaFile = AcidUtils.createBucketFile(delta, bucket);
         FileSystem fs = deltaFile.getFileSystem(conf);
         long length = getLastFlushLength(fs, deltaFile);
-        if (fs.exists(deltaFile) && length != -1) {
+        if (length != -1 && fs.exists(deltaFile)) {
           Reader deltaReader = OrcFile.createReader(deltaFile,
               OrcFile.readerOptions(conf).maxLength(length));
           ReaderPair deltaPair = new ReaderPair(key, deltaReader, bucket, minKey,

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Apr  7 18:04:40
2015
@@ -2416,7 +2416,7 @@ private void constructOneLBLocationMap(F
 
     List<List<Path[]>> result = new ArrayList<List<Path[]>>();
     try {
-      FileStatus destStatus = !replace && fs.exists(destf) ? fs.getFileStatus(destf)
: null;
+      FileStatus destStatus = !replace ? FileUtils.getFileStatusOrNull(fs, destf) : null;
       if (destStatus != null && !destStatus.isDir()) {
         throw new HiveException("checkPaths: destination " + destf
             + " should be a directory");

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
Tue Apr  7 18:04:40 2015
@@ -361,19 +361,22 @@ public class StorageBasedAuthorizationPr
 
     final FileSystem fs = path.getFileSystem(conf);
 
-    if (fs.exists(path)) {
-      checkPermissions(fs, path, actions, authenticator.getUserName());
+    FileStatus pathStatus = FileUtils.getFileStatusOrNull(fs, path);
+    if (pathStatus != null) {
+      checkPermissions(fs, pathStatus, actions, authenticator.getUserName());
     } else if (path.getParent() != null) {
       // find the ancestor which exists to check its permissions
       Path par = path.getParent();
+      FileStatus parStatus = null;
       while (par != null) {
-        if (fs.exists(par)) {
+        parStatus = FileUtils.getFileStatusOrNull(fs, par);
+        if (parStatus != null) {
           break;
         }
         par = par.getParent();
       }
 
-      checkPermissions(fs, par, actions, authenticator.getUserName());
+      checkPermissions(fs, parStatus, actions, authenticator.getUserName());
     }
   }
 
@@ -382,18 +385,20 @@ public class StorageBasedAuthorizationPr
    * does not exists, it returns.
    */
   @SuppressWarnings("deprecation")
-  protected static void checkPermissions(final FileSystem fs, final Path path,
+  protected static void checkPermissions(final FileSystem fs, final FileStatus stat,
       final EnumSet<FsAction> actions, String user) throws IOException,
       AccessControlException, HiveException {
 
-    try {
-      FileStatus stat = fs.getFileStatus(path);
-      for (FsAction action : actions) {
-        FileUtils.checkFileAccessWithImpersonation(fs, stat, action, user);
-      }
-    } catch (FileNotFoundException fnfe) {
+    if (stat == null) {
       // File named by path doesn't exist; nothing to validate.
       return;
+    }
+    FsAction checkActions = FsAction.NONE;
+    for (FsAction action : actions) {
+      checkActions = checkActions.or(action);
+    }
+    try {
+      FileUtils.checkFileAccessWithImpersonation(fs, stat, checkActions, user);
     } catch (org.apache.hadoop.fs.permission.AccessControlException ace) {
       // Older hadoop version will throw this @deprecated Exception.
       throw accessControlException(ace);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
Tue Apr  7 18:04:40 2015
@@ -385,8 +385,7 @@ public class SQLAuthorizationUtils {
     FileSystem fs;
     try {
       fs = FileSystem.get(filePath.toUri(), conf);
-      Path path = FileUtils.getPathOrParentThatExists(fs, filePath);
-      FileStatus fileStatus = fs.getFileStatus(path);
+      FileStatus fileStatus = FileUtils.getPathOrParentThatExists(fs, filePath);
       if (FileUtils.isOwnerOfFileHierarchy(fs, fileStatus, userName)) {
         availPrivs.addPrivilege(SQLPrivTypeGrant.OWNER_PRIV);
       }



Mime
View raw message