hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject hadoop git commit: HDFS-10918. Add a tool to get FileEncryptionInfo from CLI. Contributed by Xiao Chen.
Date Mon, 03 Oct 2016 23:09:51 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 b0b29de8a -> bf043e63f


HDFS-10918. Add a tool to get FileEncryptionInfo from CLI. Contributed by Xiao Chen.

(cherry picked from commit 853d65a157362661ccab10379c2d82e780382f83)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf043e63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf043e63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf043e63

Branch: refs/heads/branch-2.8
Commit: bf043e63f9a2ee215e0ca10a61f258a00cde2062
Parents: b0b29de
Author: Xiao Chen <xiao@apache.org>
Authored: Mon Oct 3 16:01:54 2016 -0700
Committer: Xiao Chen <xiao@apache.org>
Committed: Mon Oct 3 16:02:11 2016 -0700

----------------------------------------------------------------------
 .../apache/hadoop/fs/FileEncryptionInfo.java    | 21 +++++
 .../hadoop/hdfs/DistributedFileSystem.java      | 30 +++++++
 .../apache/hadoop/hdfs/client/HdfsAdmin.java    | 14 +++
 .../apache/hadoop/hdfs/tools/CryptoAdmin.java   | 51 ++++++++++-
 .../src/site/markdown/TransparentEncryption.md  | 16 ++++
 .../src/test/resources/testCryptoConf.xml       | 90 ++++++++++++++++++++
 6 files changed, 221 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf043e63/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
index 00ddfe8..1129e07 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileEncryptionInfo.java
@@ -121,4 +121,25 @@ public class FileEncryptionInfo {
     builder.append("}");
     return builder.toString();
   }
+
+  /**
+   * A frozen version of {@link #toString()} to be backward compatible.
+   * When backward compatibility is not needed, use {@link #toString()}, which
+   * provides more info and is supposed to evolve.
+   * Don't change this method except for major revisions.
+   *
+   * NOTE:
+   * Currently this method is used by CLI for backward compatibility.
+   */
+  public String toStringStable() {
+    StringBuilder builder = new StringBuilder("{");
+    builder.append("cipherSuite: " + cipherSuite);
+    builder.append(", cryptoProtocolVersion: " + version);
+    builder.append(", edek: " + Hex.encodeHexString(edek));
+    builder.append(", iv: " + Hex.encodeHexString(iv));
+    builder.append(", keyName: " + keyName);
+    builder.append(", ezKeyVersionName: " + ezKeyVersionName);
+    builder.append("}");
+    return builder.toString();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf043e63/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 737afdb..a608ba0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FSLinkResolver;
 import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileSystemLinkResolver;
@@ -2273,6 +2274,35 @@ public class DistributedFileSystem extends FileSystem {
     return dfs.listEncryptionZones();
   }
 
+  /* HDFS only */
+  public FileEncryptionInfo getFileEncryptionInfo(final Path path)
+      throws IOException {
+    Path absF = fixRelativePart(path);
+    return new FileSystemLinkResolver<FileEncryptionInfo>() {
+      @Override
+      public FileEncryptionInfo doCall(final Path p) throws IOException {
+        final HdfsFileStatus fi = dfs.getFileInfo(getPathName(p));
+        if (fi == null) {
+          throw new FileNotFoundException("File does not exist: " + p);
+        }
+        return fi.getFileEncryptionInfo();
+      }
+
+      @Override
+      public FileEncryptionInfo next(final FileSystem fs, final Path p)
+          throws IOException {
+        if (fs instanceof DistributedFileSystem) {
+          DistributedFileSystem myDfs = (DistributedFileSystem)fs;
+          return myDfs.getFileEncryptionInfo(p);
+        }
+        throw new UnsupportedOperationException(
+            "Cannot call getFileEncryptionInfo"
+                + " on a symlink to a non-DistributedFileSystem: " + path
+                + " -> " + p);
+      }
+    }.resolve(this, absF);
+  }
+
   @Override
   public void setXAttr(Path path, final String name, final byte[] value,
       final EnumSet<XAttrSetFlag> flag) throws IOException {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf043e63/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index fc62914..3cee861 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockStoragePolicySpi;
 import org.apache.hadoop.fs.CacheFlag;
+import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -353,6 +354,19 @@ public class HdfsAdmin {
   }
 
   /**
+   * Returns the FileEncryptionInfo on the HdfsFileStatus for the given path.
+   * The return value can be null if the path points to a directory, or a file
+   * that is not in an encryption zone.
+   *
+   * @throws FileNotFoundException if the path does not exist.
+   * @throws AccessControlException if no execute permission on parent path.
+   */
+  public FileEncryptionInfo getFileEncryptionInfo(final Path path)
+      throws IOException {
+    return dfs.getFileEncryptionInfo(path);
+  }
+
+  /**
    * Exposes a stream of namesystem events. Only events occurring after the
    * stream is created are available.
    * See {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf043e63/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
index 06389a1..b78da31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
@@ -25,6 +25,7 @@ import java.util.List;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileEncryptionInfo;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
@@ -193,6 +194,53 @@ public class CryptoAdmin extends Configured implements Tool {
     }
   }
 
+  private static class GetFileEncryptionInfoCommand
+      implements AdminHelper.Command {
+    @Override
+    public String getName() {
+      return "-getFileEncryptionInfo";
+    }
+
+    @Override
+    public String getShortUsage() {
+      return "[" + getName() + " -path <path>]\n";
+    }
+
+    @Override
+    public String getLongUsage() {
+      final TableListing listing = AdminHelper.getOptionDescriptionListing();
+      listing.addRow("<path>", "The path to the file to show encryption info.");
+      return getShortUsage() + "\n" + "Get encryption info of a file.\n\n" +
+          listing.toString();
+    }
+
+    @Override
+    public int run(Configuration conf, List<String> args) throws IOException {
+      final String path = StringUtils.popOptionWithArgument("-path", args);
+
+      if (!args.isEmpty()) {
+        System.err.println("Can't understand argument: " + args.get(0));
+        return 1;
+      }
+
+      final HdfsAdmin admin =
+          new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+      try {
+        final FileEncryptionInfo fei =
+            admin.getFileEncryptionInfo(new Path(path));
+        if (fei == null) {
+          System.out.println("No FileEncryptionInfo found for path " + path);
+          return 2;
+        }
+        System.out.println(fei.toStringStable());
+      } catch (IOException e) {
+        System.err.println(prettifyException(e));
+        return 3;
+      }
+      return 0;
+    }
+  }
+
   private static class ProvisionTrashCommand implements AdminHelper.Command {
     @Override
     public String getName() {
@@ -237,6 +285,7 @@ public class CryptoAdmin extends Configured implements Tool {
   private static final AdminHelper.Command[] COMMANDS = {
       new CreateZoneCommand(),
       new ListZonesCommand(),
-      new ProvisionTrashCommand()
+      new ProvisionTrashCommand(),
+      new GetFileEncryptionInfoCommand()
   };
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf043e63/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
index ee98df8..e7d9f1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
@@ -29,6 +29,8 @@ Transparent Encryption in HDFS
 * [crypto command-line interface](#crypto_command-line_interface)
     * [createZone](#createZone)
     * [listZones](#listZones)
+    * [provisionTrash](#provisionTrash)
+    * [getFileEncryptionInfo](#getFileEncryptionInfo)
 * [Example usage](#Example_usage)
 * [Distcp considerations](#Distcp_considerations)
     * [Running as the superuser](#Running_as_the_superuser)
@@ -189,6 +191,16 @@ Provision a trash directory for an encryption zone.
 |:---- |:---- |
 | *path* | The path to the root of the encryption zone. |
 
+### <a name="getFileEncryptionInfo"></a>getFileEncryptionInfo
+
+Usage: `[-getFileEncryptionInfo -path <path>]`
+
+Get encryption information from a file. This can be used to find out whether a file is being
encrypted, and the key name / key version used to encrypt it.
+
+| | |
+|:---- |:---- |
+| *path* | The path of the file to get encryption information. |
+
 <a name="Example_usage"></a>Example usage
 -------------
 
@@ -208,6 +220,10 @@ These instructions assume that you are running as the normal user or
HDFS superu
     hadoop fs -put helloWorld /zone
     hadoop fs -cat /zone/helloWorld
 
+    # As the normal user, get encryption information from the file
+    hdfs crypto -getFileEncryptionInfo -path /zone/helloWorld
+    # console output: {cipherSuite: {name: AES/CTR/NoPadding, algorithmBlockSize: 16}, cryptoProtocolVersion:
CryptoProtocolVersion{description='Encryption zones', version=1, unknownValue=null}, edek:
2010d301afbd43b58f10737ce4e93b39, iv: ade2293db2bab1a2e337f91361304cb3, keyName: mykey, ezKeyVersionName:
mykey@0}
+
 <a name="Distcp_considerations"></a>Distcp considerations
 ---------------------
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf043e63/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
index ddd4adc..0294368 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
@@ -388,5 +388,95 @@
         </comparator>
       </comparators>
     </test>
+
+    <test>
+      <description>Test success of getFileEncryptionInfo on a EZ file</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /src</command>
+        <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+        <command>-fs NAMENODE -touchz /src/file</command>
+        <crypto-admin-command>-getFileEncryptionInfo -path /src/file</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /src</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>keyName: myKey, ezKeyVersionName: myKey@0</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test>
+      <description>Test failure of getFileEncryptionInfo on a non-EZ file</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /src</command>
+        <command>-fs NAMENODE -touchz /src/cleartext</command>
+        <crypto-admin-command>-getFileEncryptionInfo -path /src/cleartext</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /src</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>No FileEncryptionInfo found for path</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test>
+      <description>Test failure of getFileEncryptionInfo on a non-exist file</description>
+      <test-commands>
+        <crypto-admin-command>-getFileEncryptionInfo -path /src/file</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /src</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>FileNotFoundException:</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test>
+      <description>Test failure of getFileEncryptionInfo on a EZ dir</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /src</command>
+        <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+        <crypto-admin-command>-getFileEncryptionInfo -path /src</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /src</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>No FileEncryptionInfo found for path </expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
+    <test>
+      <description>Test failure of getFileEncryptionInfo on a EZ subdir</description>
+      <test-commands>
+        <command>-fs NAMENODE -mkdir /src</command>
+        <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+        <command>-fs NAMENODE -mkdir /src/dir</command>
+        <crypto-admin-command>-getFileEncryptionInfo -path /src/dir</crypto-admin-command>
+      </test-commands>
+      <cleanup-commands>
+        <command>-fs NAMENODE -rm -r /src</command>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>No FileEncryptionInfo found for path </expected-output>
+        </comparator>
+      </comparators>
+    </test>
   </tests>
 </configuration>


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message