hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1588947 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/common/FileUtils.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
Date Mon, 21 Apr 2014 18:06:41 GMT
Author: xuefu
Date: Mon Apr 21 18:06:41 2014
New Revision: 1588947

URL: http://svn.apache.org/r1588947
Log:
HIVE-6916: Export/import inherit permissions from parent directory (Szehon via Xuefu)

Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java?rev=1588947&r1=1588946&r2=1588947&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/FileUtils.java Mon Apr 21 18:06:41
2014
@@ -31,6 +31,8 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
@@ -489,4 +491,32 @@ public final class FileUtils {
       }
     }
   }
+
+  /**
+   * Copies files between filesystems.
+   */
+  public static boolean copy(FileSystem srcFS, Path src,
+    FileSystem dstFS, Path dst,
+    boolean deleteSource,
+    boolean overwrite,
+    HiveConf conf) throws IOException {
+    boolean copied = FileUtil.copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf);
+    boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
+    if (copied && inheritPerms) {
+      FileStatus destFileStatus = dstFS.getFileStatus(dst);
+      FsPermission perm = destFileStatus.getPermission();
+      String permString = Integer.toString(perm.toShort(), 8);
+      String group = destFileStatus.getGroup();
+      //use FsShell to change group and permissions recursively
+      try {
+        FsShell fshell = new FsShell();
+        fshell.setConf(conf);
+        fshell.run(new String[]{"-chgrp", "-R", group, dst.toString()});
+        fshell.run(new String[]{"-chmod", "-R", permString, dst.toString()});
+      } catch (Exception e) {
+        throw new IOException("Unable to set permissions of " + dst, e);
+      }
+    }
+    return copied;
+  }
 }

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java?rev=1588947&r1=1588946&r2=1588947&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java
(original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestFolderPermissions.java
Mon Apr 21 18:06:41 2014
@@ -223,6 +223,76 @@ public class TestFolderPermissions {
     }
   }
 
+  @Test
+  public void testEximPermissionInheritance() throws Exception {
+
+    //export the table to external file.
+    String myLocation = testDir + "/exim";
+    FileSystem fs = FileSystem.get(new URI(myLocation), conf);
+    fs.mkdirs(new Path(myLocation), FsPermission.createImmutable((short) 0777));
+
+    myLocation = myLocation + "/temp";
+
+    CommandProcessorResponse ret = driver.run("export table mysrc to '" + myLocation + "'");
+    Assert.assertEquals(0,ret.getResponseCode());
+
+    //check if exported data has inherited the permissions.
+    assertExistence(myLocation);
+    Assert.assertEquals(getPermissions(myLocation).toString(), "rwxrwxrwx");
+
+    assertExistence(myLocation + "/part1=1/part2=1");
+    Assert.assertEquals(getPermissions(myLocation + "/part1=1/part2=1").toString(), "rwxrwxrwx");
+    Assert.assertTrue(listChildrenPerms(myLocation + "/part1=1/part2=1").size() > 0);
+    for (FsPermission perm : listChildrenPerms(myLocation + "/part1=1/part2=1")) {
+      Assert.assertEquals("rwxrwxrwx", perm.toString());
+    }
+
+    assertExistence(myLocation + "/part1=2/part2=2");
+    Assert.assertEquals(getPermissions(myLocation + "/part1=2/part2=2").toString(), "rwxrwxrwx");
+    Assert.assertTrue(listChildrenPerms(myLocation + "/part1=2/part2=2").size() > 0);
+    for (FsPermission perm : listChildrenPerms(myLocation + "/part1=2/part2=2")) {
+      Assert.assertEquals("rwxrwxrwx", perm.toString());
+    }
+
+    //import the table back into another database
+    String testDb = "eximdb";
+    ret = driver.run("CREATE DATABASE " + testDb);
+    Assert.assertEquals(0,ret.getResponseCode());
+
+    //use another permission for this import location, to verify that it is really set.
+    assertExistence(testDir + "/" + testDb + ".db");
+    setPermissions(testDir + "/" + testDb + ".db", FsPermission.createImmutable((short) 0766));
+
+    ret = driver.run("USE " + testDb);
+    Assert.assertEquals(0,ret.getResponseCode());
+
+    ret = driver.run("import from '" + myLocation + "'");
+    Assert.assertEquals(0,ret.getResponseCode());
+
+    //check permissions of imported, from the exported table
+    assertExistence(testDir + "/" + testDb + ".db/mysrc");
+    Assert.assertEquals("rwxrw-rw-", getPermissions(testDir + "/" + testDb + ".db/mysrc").toString());
+
+    myLocation = testDir + "/" + testDb + ".db/mysrc";
+    assertExistence(myLocation);
+    Assert.assertEquals(getPermissions(myLocation).toString(), "rwxrw-rw-");
+
+    assertExistence(myLocation + "/part1=1/part2=1");
+    Assert.assertEquals(getPermissions(myLocation + "/part1=1/part2=1").toString(), "rwxrw-rw-");
+    Assert.assertTrue(listChildrenPerms(myLocation + "/part1=1/part2=1").size() > 0);
+    for (FsPermission perm : listChildrenPerms(myLocation + "/part1=1/part2=1")) {
+      Assert.assertEquals("rwxrw-rw-", perm.toString());
+    }
+
+    assertExistence(myLocation + "/part1=2/part2=2");
+    Assert.assertEquals(getPermissions(myLocation + "/part1=2/part2=2").toString(), "rwxrw-rw-");
+    Assert.assertTrue(listChildrenPerms(myLocation + "/part1=2/part2=2").size() > 0);
+    for (FsPermission perm : listChildrenPerms(myLocation + "/part1=2/part2=2")) {
+      Assert.assertEquals("rwxrw-rw-", perm.toString());
+    }
+  }
+
+
   private void setPermissions(String locn, FsPermission permissions) throws Exception {
     fs.setPermission(new Path(locn), permissions);
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java?rev=1588947&r1=1588946&r2=1588947&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CopyTask.java Mon Apr 21 18:06:41
2014
@@ -24,8 +24,9 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.DriverContext;
 import org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer;
 import org.apache.hadoop.hive.ql.plan.CopyWork;
@@ -69,7 +70,8 @@ public class CopyTask extends Task<CopyW
         }
       }
 
-      if (!dstFs.mkdirs(toPath)) {
+      boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
+      if (!FileUtils.mkdir(dstFs, toPath, inheritPerms)) {
         console.printError("Cannot make target directory: " + toPath.toString());
         return 2;
       }
@@ -77,8 +79,8 @@ public class CopyTask extends Task<CopyW
       for (FileStatus oneSrc : srcs) {
         console.printInfo("Copying file: " + oneSrc.getPath().toString());
         LOG.debug("Copying file: " + oneSrc.getPath().toString());
-        if (!FileUtil.copy(srcFs, oneSrc.getPath(), dstFs, toPath, false, // delete
-            // source
+        if (!FileUtils.copy(srcFs, oneSrc.getPath(), dstFs, toPath,
+            false, // delete source
             true, // overwrite destination
             conf)) {
           console.printError("Failed to copy: '" + oneSrc.getPath().toString()



Mime
View raw message