Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D4BD81797E for ; Tue, 7 Apr 2015 18:04:41 +0000 (UTC) Received: (qmail 69588 invoked by uid 500); 7 Apr 2015 18:04:41 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 69545 invoked by uid 500); 7 Apr 2015 18:04:41 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 69532 invoked by uid 99); 7 Apr 2015 18:04:41 -0000 Received: from eris.apache.org (HELO hades.apache.org) (140.211.11.105) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Apr 2015 18:04:41 +0000 Received: from hades.apache.org (localhost [127.0.0.1]) by hades.apache.org (ASF Mail Server at hades.apache.org) with ESMTP id 19568AC00B4 for ; Tue, 7 Apr 2015 18:04:41 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1671903 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/ hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ ql/src/java/org/apache/hadoop/hive/ql/io/orc/ ql/src/j... Date: Tue, 07 Apr 2015 18:04:40 -0000 To: commits@hive.apache.org From: gunther@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20150407180441.19568AC00B4@hades.apache.org> Author: gunther Date: Tue Apr 7 18:04:40 2015 New Revision: 1671903 URL: http://svn.apache.org/r1671903 Log: HIVE-10223: Consolidate several redundant FileSystem API calls. (Chris Nauroth via Gunther Hagleitner) Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff ============================================================================== --- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original) +++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Tue Apr 7 18:04:40 2015 @@ -335,16 +335,16 @@ public final class FileUtils { * @param fs * file system * @param path - * @return the argument path if it exists or a parent path exists. Returns - * NULL root is only parent that exists + * @return FileStatus for argument path if it exists or the first ancestor in the path that exists * @throws IOException */ - public static Path getPathOrParentThatExists(FileSystem fs, Path path) throws IOException { - if (!fs.exists(path)) { - Path parentPath = path.getParent(); - return getPathOrParentThatExists(fs, parentPath); + public static FileStatus getPathOrParentThatExists(FileSystem fs, Path path) throws IOException { + FileStatus stat = FileUtils.getFileStatusOrNull(fs, path); + if (stat != null) { + return stat; } - return path; + Path parentPath = path.getParent(); + return getPathOrParentThatExists(fs, parentPath); } /** @@ -743,4 +743,20 @@ public final class FileUtils { } + /** + * Attempts to get file status. This method differs from the FileSystem API in that it returns + * null instead of throwing FileNotFoundException if the path does not exist. + * + * @param fs file system to check + * @param path file system path to check + * @return FileStatus for path or null if path does not exist + * @throws IOException if there is an I/O error + */ + public static FileStatus getFileStatusOrNull(FileSystem fs, Path path) throws IOException { + try { + return fs.getFileStatus(path); + } catch (FileNotFoundException e) { + return null; + } + } } Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1671903&r1=1671902&r2=1671903&view=diff ============================================================================== --- hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java (original) +++ hive/trunk/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java Tue Apr 7 18:04:40 2015 @@ -510,8 +510,9 @@ class FileOutputCommitterContainer exten } final Path finalOutputPath = getFinalPath(fs, file, srcDir, destDir, immutable); + FileStatus fileStatus = fs.getFileStatus(file); - if (fs.isFile(file)) { + if (fileStatus.isFile()) { if (dryRun){ if (immutable){ // Dryrun checks are meaningless for mutable table - we should always succeed @@ -541,7 +542,7 @@ class FileOutputCommitterContainer exten } } } - } else if(fs.getFileStatus(file).isDir()) { + } else if (fileStatus.isDirectory()) { FileStatus[] children = fs.listStatus(file); FileStatus firstChild = null; Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java Tue Apr 7 18:04:40 2015 @@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.Context; @@ -926,8 +927,9 @@ public class DagUtils { throws IOException { FileSystem destFS = dest.getFileSystem(conf); FileSystem sourceFS = src.getFileSystem(conf); - if (destFS.exists(dest)) { - return (sourceFS.getFileStatus(src).getLen() == destFS.getFileStatus(dest).getLen()); + FileStatus destStatus = FileUtils.getFileStatusOrNull(destFS, dest); + if (destStatus != null) { + return (sourceFS.getFileStatus(src).getLen() == destStatus.getLen()); } return false; } Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java?rev=1671903&r1=1671902&r2=1671903&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java Tue Apr 7 18:04:40 2015 @@ -460,7 +460,7 @@ public class OrcRawRecordMerger implemen Path deltaFile = AcidUtils.createBucketFile(delta, bucket); FileSystem fs = deltaFile.getFileSystem(conf); long length = getLastFlushLength(fs, deltaFile); - if (fs.exists(deltaFile) && length != -1) { + if (length != -1 && fs.exists(deltaFile)) { Reader deltaReader = OrcFile.createReader(deltaFile, OrcFile.readerOptions(conf).maxLength(length)); ReaderPair deltaPair = new ReaderPair(key, deltaReader, bucket, minKey, Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1671903&r1=1671902&r2=1671903&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Tue Apr 7 18:04:40 2015 @@ -2416,7 +2416,7 @@ private void constructOneLBLocationMap(F List> result = new ArrayList>(); try { - FileStatus destStatus = !replace && fs.exists(destf) ? fs.getFileStatus(destf) : null; + FileStatus destStatus = !replace ? FileUtils.getFileStatusOrNull(fs, destf) : null; if (destStatus != null && !destStatus.isDir()) { throw new HiveException("checkPaths: destination " + destf + " should be a directory"); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1671903&r1=1671902&r2=1671903&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java Tue Apr 7 18:04:40 2015 @@ -361,19 +361,22 @@ public class StorageBasedAuthorizationPr final FileSystem fs = path.getFileSystem(conf); - if (fs.exists(path)) { - checkPermissions(fs, path, actions, authenticator.getUserName()); + FileStatus pathStatus = FileUtils.getFileStatusOrNull(fs, path); + if (pathStatus != null) { + checkPermissions(fs, pathStatus, actions, authenticator.getUserName()); } else if (path.getParent() != null) { // find the ancestor which exists to check its permissions Path par = path.getParent(); + FileStatus parStatus = null; while (par != null) { - if (fs.exists(par)) { + parStatus = FileUtils.getFileStatusOrNull(fs, par); + if (parStatus != null) { break; } par = par.getParent(); } - checkPermissions(fs, par, actions, authenticator.getUserName()); + checkPermissions(fs, parStatus, actions, authenticator.getUserName()); } } @@ -382,18 +385,20 @@ public class StorageBasedAuthorizationPr * does not exists, it returns. */ @SuppressWarnings("deprecation") - protected static void checkPermissions(final FileSystem fs, final Path path, + protected static void checkPermissions(final FileSystem fs, final FileStatus stat, final EnumSet actions, String user) throws IOException, AccessControlException, HiveException { - try { - FileStatus stat = fs.getFileStatus(path); - for (FsAction action : actions) { - FileUtils.checkFileAccessWithImpersonation(fs, stat, action, user); - } - } catch (FileNotFoundException fnfe) { + if (stat == null) { // File named by path doesn't exist; nothing to validate. return; + } + FsAction checkActions = FsAction.NONE; + for (FsAction action : actions) { + checkActions = checkActions.or(action); + } + try { + FileUtils.checkFileAccessWithImpersonation(fs, stat, checkActions, user); } catch (org.apache.hadoop.fs.permission.AccessControlException ace) { // Older hadoop version will throw this @deprecated Exception. throw accessControlException(ace); Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java?rev=1671903&r1=1671902&r2=1671903&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java Tue Apr 7 18:04:40 2015 @@ -385,8 +385,7 @@ public class SQLAuthorizationUtils { FileSystem fs; try { fs = FileSystem.get(filePath.toUri(), conf); - Path path = FileUtils.getPathOrParentThatExists(fs, filePath); - FileStatus fileStatus = fs.getFileStatus(path); + FileStatus fileStatus = FileUtils.getPathOrParentThatExists(fs, filePath); if (FileUtils.isOwnerOfFileHierarchy(fs, fileStatus, userName)) { availPrivs.addPrivilege(SQLPrivTypeGrant.OWNER_PRIV); }