Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F2D0D1155E for ; Thu, 21 Aug 2014 05:22:44 +0000 (UTC) Received: (qmail 54518 invoked by uid 500); 21 Aug 2014 05:22:44 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 54471 invoked by uid 500); 21 Aug 2014 05:22:44 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 54460 invoked by uid 99); 21 Aug 2014 05:22:44 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 21 Aug 2014 05:22:44 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 21 Aug 2014 05:22:39 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id AC4B22388831; Thu, 21 Aug 2014 05:22:19 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1619293 [1/4] - in /hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/bin/ src/main/java/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/client/ sr... Date: Thu, 21 Aug 2014 05:22:16 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140821052219.AC4B22388831@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Thu Aug 21 05:22:10 2014 New Revision: 1619293 URL: http://svn.apache.org/r1619293 Log: Merge r1609845 through r1619277 from trunk. Added: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/UnknownCipherSuiteException.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/UnknownCipherSuiteException.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZone.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithId.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithIdIterator.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneWithIdIterator.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/encryption.proto hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/TransparentEncryption.apt.vm - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/TransparentEncryption.apt.vm hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandCryptoAdmin.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandCryptoAdmin.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CryptoAdminCmdExecutor.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CryptoAdminCmdExecutor.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteBlockGetsBlockLengthHint.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/crypto/ - copied from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/crypto/ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml - copied unchanged from r1619277, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/pom.xml hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataInputStream.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/sasl/DataTransferSaslUtil.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed) hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSRename.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddBlockRetry.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml Propchange: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs:r1594376-1619194 Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1618764-1619277 Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Aug 21 05:22:10 2014 @@ -278,6 +278,97 @@ Trunk (Unreleased) HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI. (Vinayakumar B via wheat 9) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS + + HDFS-6387. HDFS CLI admin tool for creating & deleting an + encryption zone. (clamb) + + HDFS-6386. HDFS Encryption Zones (clamb) + + HDFS-6388. HDFS integration with KeyProvider. (clamb) + + HDFS-6473. Protocol and API for Encryption Zones (clamb) + + HDFS-6392. Wire crypto streams for encrypted files in + DFSClient. (clamb and yliu) + + HDFS-6476. Print out the KeyProvider after finding KP successfully on + startup. (Juan Yu via wang) + + HDFS-6391. Get the Key/IV from the NameNode for encrypted files in + DFSClient. (Charles Lamb and wang) + + HDFS-6389. Rename restrictions for encryption zones. (clamb) + + HDFS-6605. Client server negotiation of cipher suite. (wang) + + HDFS-6625. Remove the Delete Encryption Zone function (clamb) + + HDFS-6516. List of Encryption Zones should be based on inodes (clamb) + + HDFS-6629. Not able to create symlinks after HDFS-6516 (umamaheswararao) + + HDFS-6635. Refactor encryption zone functionality into new + EncryptionZoneManager class. (wang) + + HDFS-6474. Namenode needs to get the actual keys and iv from the + KeyProvider. (wang) + + HDFS-6619. Clean up encryption-related tests. (wang) + + HDFS-6405. Test Crypto streams in HDFS. (yliu via wang) + + HDFS-6490. Fix the keyid format for generated keys in + FSNamesystem.createEncryptionZone (clamb) + + HDFS-6716. Update usage of KeyProviderCryptoExtension APIs on NameNode. + (wang) + + HDFS-6718. Remove EncryptionZoneManager lock. (wang) + + HDFS-6720. Remove KeyProvider in EncryptionZoneManager. (wang) + + HDFS-6738. Remove unnecessary getEncryptionZoneForPath call in + EZManager#createEncryptionZone. (clamb) + + HDFS-6724. Decrypt EDEK before creating + CryptoInputStream/CryptoOutputStream. (wang) + + HDFS-6509. Create a special /.reserved/raw directory for raw access to + encrypted data. (clamb via wang) + + HDFS-6771. Require specification of an encryption key when creating + an encryption zone. (wang) + + HDFS-6730. Create a .RAW extended attribute namespace. (clamb) + + HDFS-6692. Add more HDFS encryption tests. (wang) + + HDFS-6780. Batch the encryption zones listing API. (wang) + + HDFS-6394. HDFS encryption documentation. (wang) + + HDFS-6834. Improve the configuration guidance in DFSClient when there + are no Codec classes found in configs. (umamahesh) + + HDFS-6546. Add non-superuser capability to get the encryption zone + for a specific path. (clamb) + + HDFS-6733. Creating encryption zone results in NPE when + KeyProvider is null. (clamb) + + HDFS-6785. Should not be able to create encryption zone using path + to a non-directory file. (clamb) + + HDFS-6807. Fix TestReservedRawPaths. (clamb) + + HDFS-6814. Mistakenly dfs.namenode.list.encryption.zones.num.responses configured + as boolean. (umamahesh) + + HDFS-6817. Fix findbugs and other warnings. (yliu) + + HDFS-6839. Fix TestCLI to expect new output. (clamb) + Release 2.6.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -428,6 +519,15 @@ Release 2.6.0 - UNRELEASED HDFS-6188. An ip whitelist based implementation of TrustedChannelResolver. (Benoy Antony via Arpit Agarwal) + HDFS-6858. Allow dfs.data.transfer.saslproperties.resolver.class default to + hadoop.security.saslproperties.resolver.class. (Benoy Antony via cnauroth) + + HDFS-6878. Change MiniDFSCluster to support StorageType configuration + for individual directories. (Arpit Agarwal) + + HDFS-6758. block writer should pass the expected block size to + DataXceiverServer. (Arpit Agarwal) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) @@ -543,7 +643,13 @@ Release 2.6.0 - UNRELEASED HDFS-6569. OOB message can't be sent to the client when DataNode shuts down for upgrade (brandonli) -Release 2.5.0 - UNRELEASED + HDFS-6868. portmap and nfs3 are documented as hadoop commands instead of hdfs + (brandonli) + + HDFS-6870. Blocks and INodes could leak for Rename with overwrite flag. (Yi + Liu via jing9) + +Release 2.5.0 - 2014-08-11 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/pom.xml Thu Aug 21 05:22:10 2014 @@ -304,6 +304,7 @@ http://maven.apache.org/xsd/maven-4.0.0. datatransfer.proto fsimage.proto hdfs.proto + encryption.proto ${project.build.directory}/generated-sources/java Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/distribute-exclude.sh Thu Aug 21 05:22:10 2014 @@ -57,9 +57,9 @@ excludeFilenameRemote=$("$HADOOP_PREFIX/ if [ "$excludeFilenameRemote" = '' ] ; then echo \ - "Error: hdfs getconf -excludeFile returned empty string, " \ - "please setup dfs.hosts.exclude in hdfs-site.xml in local cluster " \ - "configuration and on all namenodes" + "Error: hdfs getconf -excludeFile returned empty string, " \ + "please setup dfs.hosts.exclude in hdfs-site.xml in local cluster " \ + "configuration and on all namenodes" exit 1 fi Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs Thu Aug 21 05:22:10 2014 @@ -15,250 +15,241 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Environment Variables -# -# JSVC_HOME home directory of jsvc binary. Required for starting secure -# datanode. -# -# JSVC_OUTFILE path to jsvc output file. Defaults to -# $HADOOP_LOG_DIR/jsvc.out. -# -# JSVC_ERRFILE path to jsvc error file. Defaults to $HADOOP_LOG_DIR/jsvc.err. - -bin=`which $0` -bin=`dirname ${bin}` -bin=`cd "$bin" > /dev/null; pwd` - -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh - -function print_usage(){ - echo "Usage: hdfs [--config confdir] COMMAND" +function hadoop_usage +{ + echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)] COMMAND" echo " where COMMAND is one of:" - echo " dfs run a filesystem command on the file systems supported in Hadoop." - echo " namenode -format format the DFS filesystem" - echo " secondarynamenode run the DFS secondary namenode" - echo " namenode run the DFS namenode" - echo " journalnode run the DFS journalnode" - echo " zkfc run the ZK Failover Controller daemon" + echo " balancer run a cluster balancing utility" + echo " cacheadmin configure the HDFS cache" + echo " classpath prints the class path needed to get the" + echo " Hadoop jar and the required libraries" echo " datanode run a DFS datanode" + echo " dfs run a filesystem command on the file system" echo " dfsadmin run a DFS admin client" - echo " haadmin run a DFS HA admin client" - echo " fsck run a DFS filesystem checking utility" - echo " balancer run a cluster balancing utility" - echo " jmxget get JMX exported values from NameNode or DataNode." - echo " oiv apply the offline fsimage viewer to an fsimage" - echo " oiv_legacy apply the offline fsimage viewer to an legacy fsimage" - echo " oev apply the offline edits viewer to an edits file" echo " fetchdt fetch a delegation token from the NameNode" + echo " fsck run a DFS filesystem checking utility" echo " getconf get config values from configuration" echo " groups get the groups which users belong to" - echo " snapshotDiff diff two snapshots of a directory or diff the" - echo " current directory contents with a snapshot" + echo " haadmin run a DFS HA admin client" + echo " jmxget get JMX exported values from NameNode or DataNode." + echo " journalnode run the DFS journalnode" echo " lsSnapshottableDir list all snapshottable dirs owned by the current user" - echo " Use -help to see options" - echo " portmap run a portmap service" + echo " Use -help to see options" + echo " namenode run the DFS namenode" + echo " Use -format to initialize the DFS filesystem" echo " nfs3 run an NFS version 3 gateway" - echo " cacheadmin configure the HDFS cache" + echo " oev apply the offline edits viewer to an edits file" + echo " oiv apply the offline fsimage viewer to an fsimage" + echo " oiv_legacy apply the offline fsimage viewer to a legacy fsimage" + echo " portmap run a portmap service" + echo " secondarynamenode run the DFS secondary namenode" + echo " snapshotDiff diff two snapshots of a directory or diff the" + echo " current directory contents with a snapshot" + echo " zkfc run the ZK Failover Controller daemon" + echo " crypto configure HDFS encryption zones" echo "" echo "Most commands print help when invoked w/o parameters." } -if [ $# = 0 ]; then - print_usage - exit +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi + +if [[ $# = 0 ]]; then + hadoop_exit_with_usage 1 fi COMMAND=$1 shift -case $COMMAND in - # usage flags - --help|-help|-h) - print_usage +case ${COMMAND} in + balancer) + CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}" + ;; + cacheadmin) + CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin + ;; + classpath) + hadoop_finalize + echo "${CLASSPATH}" exit - ;; -esac - -# Determine if we're starting a secure datanode, and if so, redefine appropriate variables -if [ "$COMMAND" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then - if [ -n "$JSVC_HOME" ]; then - if [ -n "$HADOOP_SECURE_DN_PID_DIR" ]; then - HADOOP_PID_DIR=$HADOOP_SECURE_DN_PID_DIR - fi - - if [ -n "$HADOOP_SECURE_DN_LOG_DIR" ]; then - HADOOP_LOG_DIR=$HADOOP_SECURE_DN_LOG_DIR - HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" - fi - - HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER - HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING" - starting_secure_dn="true" - else - echo "It looks like you're trying to start a secure DN, but \$JSVC_HOME"\ - "isn't set. Falling back to starting insecure DN." - fi -fi - -# Determine if we're starting a privileged NFS daemon, and if so, redefine appropriate variables -if [ "$COMMAND" == "nfs3" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_PRIVILEGED_NFS_USER" ]; then - if [ -n "$JSVC_HOME" ]; then - if [ -n "$HADOOP_PRIVILEGED_NFS_PID_DIR" ]; then - HADOOP_PID_DIR=$HADOOP_PRIVILEGED_NFS_PID_DIR + ;; + crypto) + CLASS=org.apache.hadoop.hdfs.tools.CryptoAdmin + ;; + datanode) + daemon="true" + # Determine if we're starting a secure datanode, and + # if so, redefine appropriate variables + if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then + secure_service="true" + secure_user="${HADOOP_SECURE_DN_USER}" + + # backward compatiblity + HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}" + HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}" + + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS} ${HADOOP_DATANODE_OPTS}" + CLASS="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter" + else + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}" + CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode' fi - - if [ -n "$HADOOP_PRIVILEGED_NFS_LOG_DIR" ]; then - HADOOP_LOG_DIR=$HADOOP_PRIVILEGED_NFS_LOG_DIR - HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" + ;; + dfs) + CLASS=org.apache.hadoop.fs.FsShell + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + dfsadmin) + CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + fetchdt) + CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher + ;; + fsck) + CLASS=org.apache.hadoop.hdfs.tools.DFSck + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + getconf) + CLASS=org.apache.hadoop.hdfs.tools.GetConf + ;; + groups) + CLASS=org.apache.hadoop.hdfs.tools.GetGroups + ;; + haadmin) + CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin + CLASSPATH="${CLASSPATH}:${TOOL_PATH}" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" + ;; + journalnode) + daemon="true" + CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode' + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}" + ;; + jmxget) + CLASS=org.apache.hadoop.hdfs.tools.JMXGet + ;; + lsSnapshottableDir) + CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir + ;; + namenode) + daemon="true" + CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}" + ;; + nfs3) + daemon="true" + if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then + secure_service="true" + secure_user="${HADOOP_PRIVILEGED_NFS_USER}" + + # backward compatiblity + HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}" + HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}" + + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS} ${HADOOP_NFS3_OPTS}" + CLASS=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter + else + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}" + CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 fi - - HADOOP_IDENT_STRING=$HADOOP_PRIVILEGED_NFS_USER - HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING" - starting_privileged_nfs="true" - else - echo "It looks like you're trying to start a privileged NFS server, but"\ - "\$JSVC_HOME isn't set. Falling back to starting unprivileged NFS server." - fi -fi + ;; + oev) + CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer + ;; + oiv) + CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB + ;; + oiv_legacy) + CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer + ;; + portmap) + daemon="true" + CLASS=org.apache.hadoop.portmap.Portmap + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}" + ;; + secondarynamenode) + daemon="true" + CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode' + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}" + ;; + snapshotDiff) + CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff + ;; + zkfc) + daemon="true" + CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController' + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}" + ;; + -*) + hadoop_exit_with_usage 1 + ;; + *) + CLASS="${COMMAND}" + ;; +esac -if [ "$COMMAND" = "namenode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS" -elif [ "$COMMAND" = "zkfc" ] ; then - CLASS='org.apache.hadoop.hdfs.tools.DFSZKFailoverController' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_ZKFC_OPTS" -elif [ "$COMMAND" = "secondarynamenode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_SECONDARYNAMENODE_OPTS" -elif [ "$COMMAND" = "datanode" ] ; then - CLASS='org.apache.hadoop.hdfs.server.datanode.DataNode' - if [ "$starting_secure_dn" = "true" ]; then - HADOOP_OPTS="$HADOOP_OPTS -jvm server $HADOOP_DATANODE_OPTS" - else - HADOOP_OPTS="$HADOOP_OPTS -server $HADOOP_DATANODE_OPTS" +if [[ -n "${secure_service}" ]]; then + HADOOP_SECURE_USER="${secure_user}" + if hadoop_verify_secure_prereq; then + hadoop_setup_secure_service + priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.out" + priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.err" + priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${COMMAND-$HOSTNAME}.pid" + daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" + daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}.pid" fi -elif [ "$COMMAND" = "journalnode" ] ; then - CLASS='org.apache.hadoop.hdfs.qjournal.server.JournalNode' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOURNALNODE_OPTS" -elif [ "$COMMAND" = "dfs" ] ; then - CLASS=org.apache.hadoop.fs.FsShell - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "dfsadmin" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "haadmin" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin - CLASSPATH=${CLASSPATH}:${TOOL_PATH} - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "fsck" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DFSck - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" -elif [ "$COMMAND" = "balancer" ] ; then - CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_BALANCER_OPTS" -elif [ "$COMMAND" = "jmxget" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.JMXGet -elif [ "$COMMAND" = "oiv" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewerPB -elif [ "$COMMAND" = "oiv_legacy" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer -elif [ "$COMMAND" = "oev" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer -elif [ "$COMMAND" = "fetchdt" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher -elif [ "$COMMAND" = "getconf" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.GetConf -elif [ "$COMMAND" = "groups" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.GetGroups -elif [ "$COMMAND" = "snapshotDiff" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff -elif [ "$COMMAND" = "lsSnapshottableDir" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir -elif [ "$COMMAND" = "portmap" ] ; then - CLASS=org.apache.hadoop.portmap.Portmap - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_PORTMAP_OPTS" -elif [ "$COMMAND" = "nfs3" ] ; then - CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NFS3_OPTS" -elif [ "$COMMAND" = "cacheadmin" ] ; then - CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin else - CLASS="$COMMAND" + daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.out" + daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${COMMAND}.pid" fi -export CLASSPATH=$CLASSPATH - -HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}" - -# Check to see if we should start a secure datanode -if [ "$starting_secure_dn" = "true" ]; then - if [ "$HADOOP_PID_DIR" = "" ]; then - HADOOP_SECURE_DN_PID="/tmp/hadoop_secure_dn.pid" +if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then + # shellcheck disable=SC2034 + HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}" + if [[ -n "${secure_service}" ]]; then + # shellcheck disable=SC2034 + HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log" else - HADOOP_SECURE_DN_PID="$HADOOP_PID_DIR/hadoop_secure_dn.pid" - fi - - JSVC=$JSVC_HOME/jsvc - if [ ! -f $JSVC ]; then - echo "JSVC_HOME is not set correctly so jsvc cannot be found. jsvc is required to run secure datanodes. " - echo "Please download and install jsvc from http://archive.apache.org/dist/commons/daemon/binaries/ "\ - "and set JSVC_HOME to the directory containing the jsvc binary." - exit + # shellcheck disable=SC2034 + HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${COMMAND}-${HOSTNAME}.log" fi +fi - if [[ ! $JSVC_OUTFILE ]]; then - JSVC_OUTFILE="$HADOOP_LOG_DIR/jsvc.out" - fi +hadoop_add_param HADOOP_OPTS Xmx "${JAVA_HEAP_MAX}" +hadoop_finalize - if [[ ! $JSVC_ERRFILE ]]; then - JSVC_ERRFILE="$HADOOP_LOG_DIR/jsvc.err" - fi +export CLASSPATH - exec "$JSVC" \ - -Dproc_$COMMAND -outfile "$JSVC_OUTFILE" \ - -errfile "$JSVC_ERRFILE" \ - -pidfile "$HADOOP_SECURE_DN_PID" \ - -nodetach \ - -user "$HADOOP_SECURE_DN_USER" \ - -cp "$CLASSPATH" \ - $JAVA_HEAP_MAX $HADOOP_OPTS \ - org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter "$@" -elif [ "$starting_privileged_nfs" = "true" ] ; then - if [ "$HADOOP_PID_DIR" = "" ]; then - HADOOP_PRIVILEGED_NFS_PID="/tmp/hadoop_privileged_nfs3.pid" +if [[ -n "${daemon}" ]]; then + if [[ -n "${secure_service}" ]]; then + hadoop_secure_daemon_handler \ + "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\ + "${daemon_pidfile}" "${daemon_outfile}" \ + "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}" "$@" else - HADOOP_PRIVILEGED_NFS_PID="$HADOOP_PID_DIR/hadoop_privileged_nfs3.pid" - fi - - JSVC=$JSVC_HOME/jsvc - if [ ! -f $JSVC ]; then - echo "JSVC_HOME is not set correctly so jsvc cannot be found. jsvc is required to run privileged NFS gateways. " - echo "Please download and install jsvc from http://archive.apache.org/dist/commons/daemon/binaries/ "\ - "and set JSVC_HOME to the directory containing the jsvc binary." - exit - fi - - if [[ ! $JSVC_OUTFILE ]]; then - JSVC_OUTFILE="$HADOOP_LOG_DIR/nfs3_jsvc.out" + hadoop_daemon_handler "${HADOOP_DAEMON_MODE}" "${COMMAND}" "${CLASS}"\ + "${daemon_pidfile}" "${daemon_outfile}" "$@" fi - - if [[ ! $JSVC_ERRFILE ]]; then - JSVC_ERRFILE="$HADOOP_LOG_DIR/nfs3_jsvc.err" - fi - - exec "$JSVC" \ - -Dproc_$COMMAND -outfile "$JSVC_OUTFILE" \ - -errfile "$JSVC_ERRFILE" \ - -pidfile "$HADOOP_PRIVILEGED_NFS_PID" \ - -nodetach \ - -user "$HADOOP_PRIVILEGED_NFS_USER" \ - -cp "$CLASSPATH" \ - $JAVA_HEAP_MAX $HADOOP_OPTS \ - org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter "$@" + exit $? else - # run it - exec "$JAVA" -Dproc_$COMMAND $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@" + # shellcheck disable=SC2086 + hadoop_java_exec "${COMMAND}" "${CLASS}" "$@" fi - Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh Thu Aug 21 05:22:10 2014 @@ -18,19 +18,67 @@ # included in all the hdfs scripts with source command # should not be executed directly -bin=`which "$0"` -bin=`dirname "${bin}"` -bin=`cd "$bin"; pwd` +function hadoop_subproject_init +{ + if [ -e "${HADOOP_CONF_DIR}/hdfs-env.sh" ]; then + . "${HADOOP_CONF_DIR}/hdfs-env.sh" + fi + + # at some point in time, someone thought it would be a good idea to + # create separate vars for every subproject. *sigh* + # let's perform some overrides and setup some defaults for bw compat + # this way the common hadoop var's == subproject vars and can be + # used interchangeable from here on out + # ... + # this should get deprecated at some point. + HADOOP_LOG_DIR="${HADOOP_HDFS_LOG_DIR:-$HADOOP_LOG_DIR}" + HADOOP_HDFS_LOG_DIR="${HADOOP_LOG_DIR}" + + HADOOP_LOGFILE="${HADOOP_HDFS_LOGFILE:-$HADOOP_LOGFILE}" + HADOOP_HDFS_LOGFILE="${HADOOP_LOGFILE}" + + HADOOP_NICENESS=${HADOOP_HDFS_NICENESS:-$HADOOP_NICENESS} + HADOOP_HDFS_NICENESS="${HADOOP_NICENESS}" + + HADOOP_STOP_TIMEOUT=${HADOOP_HDFS_STOP_TIMEOUT:-$HADOOP_STOP_TIMEOUT} + HADOOP_HDFS_STOP_TIMEOUT="${HADOOP_STOP_TIMEOUT}" + + HADOOP_PID_DIR="${HADOOP_HDFS_PID_DIR:-$HADOOP_PID_DIR}" + HADOOP_HDFS_PID_DIR="${HADOOP_PID_DIR}" + + HADOOP_ROOT_LOGGER=${HADOOP_HDFS_ROOT_LOGGER:-$HADOOP_ROOT_LOGGER} + HADOOP_HDFS_ROOT_LOGGER="${HADOOP_ROOT_LOGGER}" + + HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME_DIR}" + + HADOOP_IDENT_STRING="${HADOOP_HDFS_IDENT_STRING:-$HADOOP_IDENT_STRING}" + HADOOP_HDFS_IDENT_STRING="${HADOOP_IDENT_STRING}" + + # turn on the defaults + + export HADOOP_NAMENODE_OPTS=${HADOOP_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"} + export HADOOP_SECONDARYNAMENODE_OPTS=${HADOOP_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS -Dhdfs.audit.logger=INFO,NullAppender"} + export HADOOP_DATANODE_OPTS=${HADOOP_DATANODE_OPTS:-"-Dhadoop.security.logger=ERROR,RFAS"} + export HADOOP_DN_SECURE_EXTRA_OPTS=${HADOOP_DN_SECURE_EXTRA_OPTS:-"-jvm server"} + export HADOOP_NFS3_SECURE_EXTRA_OPTS=${HADOOP_NFS3_SECURE_EXTRA_OPTS:-"-jvm server"} + export HADOOP_PORTMAP_OPTS=${HADOOP_PORTMAP_OPTS:-"-Xmx512m"} + + +} + +if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then + _hd_this="${BASH_SOURCE-$0}" + HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hd_this}")" >/dev/null && pwd -P) +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} if [ -e "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]; then - . ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh + . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" elif [ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" ]; then - . "$HADOOP_COMMON_HOME"/libexec/hadoop-config.sh + . "${HADOOP_COMMON_HOME}/libexec/hadoop-config.sh" elif [ -e "${HADOOP_HOME}/libexec/hadoop-config.sh" ]; then - . "$HADOOP_HOME"/libexec/hadoop-config.sh + . "${HADOOP_HOME}/libexec/hadoop-config.sh" else - echo "Hadoop common not found." - exit + echo "ERROR: Hadoop common not found." 2>&1 + exit 1 fi + Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/refresh-namenodes.sh Thu Aug 21 05:22:10 2014 @@ -20,24 +20,40 @@ # This script refreshes all namenodes, it's a simple wrapper # for dfsadmin to support multiple namenodes. -bin=`dirname "$0"` -bin=`cd "$bin"; pwd` +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + this="${BASH_SOURCE-$0}" + bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi -namenodes=$("$HADOOP_PREFIX/bin/hdfs" getconf -nnRpcAddresses) -if [ "$?" != '0' ] ; then errorFlag='1' ; +namenodes=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -nnRpcAddresses) +if [[ "$?" != '0' ]] ; then + errorFlag='1' ; else - for namenode in $namenodes ; do - echo "Refreshing namenode [$namenode]" - "$HADOOP_PREFIX/bin/hdfs" dfsadmin -fs hdfs://$namenode -refreshNodes - if [ "$?" != '0' ] ; then errorFlag='1' ; fi + for namenode in ${namenodes} ; do + echo "Refreshing namenode [${namenode}]" + "${HADOOP_HDFS_HOME}/bin/hdfs" dfsadmin \ + -fs hdfs://${namenode} -refreshNodes + if [[ "$?" != '0' ]]; then + errorFlag='1' + fi done fi -if [ "$errorFlag" = '1' ] ; then +if [[ "${errorFlag}" = '1' ]] ; then echo "Error: refresh of namenodes failed, see error messages above." exit 1 else Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh Thu Aug 21 05:22:10 2014 @@ -15,13 +15,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function usage +{ + echo "Usage: start-balancer.sh [--config confdir] [-policy ] [-threshold ]" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi # Start balancer daemon. -"$HADOOP_PREFIX"/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start balancer $@ +exec "${bin}/hadoop-daemon.sh" --config "${HADOOP_CONF_DIR}" start balancer "$@" Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh Thu Aug 21 05:22:10 2014 @@ -20,98 +20,128 @@ # Optinally upgrade or rollback dfs state. # Run this on master node. -usage="Usage: start-dfs.sh [-upgrade|-rollback] [other options such as -clusterId]" +function hadoop_usage +{ + echo "Usage: start-dfs.sh [-upgrade|-rollback] [-clusterId]" +} + +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh # get arguments -if [ $# -ge 1 ]; then - nameStartOpt="$1" - shift - case "$nameStartOpt" in - (-upgrade) - ;; - (-rollback) - dataStartOpt="$nameStartOpt" - ;; - (*) - echo $usage - exit 1 - ;; - esac +if [[ $# -ge 1 ]]; then + nameStartOpt="$1" + shift + case "$nameStartOpt" in + -upgrade) + ;; + -rollback) + dataStartOpt="$nameStartOpt" + ;; + *) + hadoop_exit_with_usage 1 + ;; + esac fi + #Add other possible options nameStartOpt="$nameStartOpt $@" #--------------------------------------------------------- # namenodes -NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes) +NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes 2>/dev/null) + +if [[ -z "${NAMENODES}" ]]; then + NAMENODES=$(hostname) +fi echo "Starting namenodes on [$NAMENODES]" -"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$NAMENODES" \ - --script "$bin/hdfs" start namenode $nameStartOpt +"${bin}/hadoop-daemons.sh" \ +--config "${HADOOP_CONF_DIR}" \ +--hostnames "${NAMENODES}" \ +start namenode ${nameStartOpt} #--------------------------------------------------------- # datanodes (using default slaves file) -if [ -n "$HADOOP_SECURE_DN_USER" ]; then - echo \ - "Attempting to start secure cluster, skipping datanodes. " \ - "Run start-secure-dns.sh as root to complete startup." +if [[ -n "${HADOOP_SECURE_DN_USER}" ]] && +[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then + echo "ERROR: Attempting to start secure cluster, skipping datanodes. " + echo "Run start-secure-dns.sh as root or configure " + echo "\${HADOOP_SECURE_COMMAND} to complete startup." else - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --script "$bin/hdfs" start datanode $dataStartOpt + + echo "Starting datanodes" + + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + start datanode ${dataStartOpt} fi #--------------------------------------------------------- # secondary namenodes (if any) -SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes 2>/dev/null) +SECONDARY_NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -secondarynamenodes 2>/dev/null) -if [ -n "$SECONDARY_NAMENODES" ]; then - echo "Starting secondary namenodes [$SECONDARY_NAMENODES]" +if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then + SECONDARY_NAMENODES=$(hostname) +fi - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$SECONDARY_NAMENODES" \ - --script "$bin/hdfs" start secondarynamenode +if [[ -n "${SECONDARY_NAMENODES}" ]]; then + echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]" + + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${SECONDARY_NAMENODES}" \ + start secondarynamenode fi #--------------------------------------------------------- # quorumjournal nodes (if any) -SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-) +SHARED_EDITS_DIR=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.namenode.shared.edits.dir 2>&-) -case "$SHARED_EDITS_DIR" in -qjournal://*) - JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') - echo "Starting journal nodes [$JOURNAL_NODES]" - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$JOURNAL_NODES" \ - --script "$bin/hdfs" start journalnode ;; +case "${SHARED_EDITS_DIR}" in + qjournal://*) + JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') + echo "Starting journal nodes [${JOURNAL_NODES}]" + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${JOURNAL_NODES}" \ + start journalnode + ;; esac #--------------------------------------------------------- # ZK Failover controllers, if auto-HA is enabled -AUTOHA_ENABLED=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.ha.automatic-failover.enabled) -if [ "$(echo "$AUTOHA_ENABLED" | tr A-Z a-z)" = "true" ]; then - echo "Starting ZK Failover Controllers on NN hosts [$NAMENODES]" - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$NAMENODES" \ - --script "$bin/hdfs" start zkfc +AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]') +if [[ "${AUTOHA_ENABLED}" = "true" ]]; then + echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]" + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${NAMENODES}" \ + start zkfc fi # eof Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh Thu Aug 21 05:22:10 2014 @@ -17,17 +17,33 @@ # Run as root to start secure datanodes in a security-enabled cluster. -usage="Usage (run as root in order to start secure datanodes): start-secure-dns.sh" -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function hadoop_usage { + echo "Usage: start-secure-dns.sh" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then - "$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs start datanode $dataStartOpt +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else - echo $usage + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi + +if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then + exec "${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" start datanode "${dataStartOpt}" +else + echo hadoop_usage_and_exit 1 fi Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh Thu Aug 21 05:22:10 2014 @@ -15,14 +15,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function hadoop_usage +{ + echo "Usage: stop-balancer.sh [--config confdir]" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi # Stop balancer daemon. # Run this on the machine where the balancer is running -"$HADOOP_PREFIX"/sbin/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop balancer +"${bin}/hadoop-daemon.sh" --config "${HADOOP_CONF_DIR}" stop balancer Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh Thu Aug 21 05:22:10 2014 @@ -15,75 +15,100 @@ # See the License for the specific language governing permissions and # limitations under the License. -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function hadoop_usage +{ + echo "Usage: start-balancer.sh [--config confdir] [-policy ] [-threshold ]" +} + +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" +else + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi #--------------------------------------------------------- # namenodes -NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -namenodes) +NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -namenodes) echo "Stopping namenodes on [$NAMENODES]" -"$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$NAMENODES" \ - --script "$bin/hdfs" stop namenode +"${bin}/hadoop-daemons.sh" \ +--config "${HADOOP_CONF_DIR}" \ +--hostnames "${NAMENODES}" \ +stop namenode #--------------------------------------------------------- # datanodes (using default slaves file) -if [ -n "$HADOOP_SECURE_DN_USER" ]; then +if [[ -n "${HADOOP_SECURE_DN_USER}" ]] && +[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then echo \ - "Attempting to stop secure cluster, skipping datanodes. " \ - "Run stop-secure-dns.sh as root to complete shutdown." + "ERROR: Attempting to stop secure cluster, skipping datanodes. " \ + "Run stop-secure-dns.sh as root to complete shutdown." else - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --script "$bin/hdfs" stop datanode + + echo "Stopping datanodes" + + "${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" stop datanode fi #--------------------------------------------------------- # secondary namenodes (if any) -SECONDARY_NAMENODES=$($HADOOP_PREFIX/bin/hdfs getconf -secondarynamenodes 2>/dev/null) +SECONDARY_NAMENODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -secondarynamenodes 2>/dev/null) -if [ -n "$SECONDARY_NAMENODES" ]; then - echo "Stopping secondary namenodes [$SECONDARY_NAMENODES]" +if [[ "${SECONDARY_NAMENODES}" == "0.0.0.0" ]]; then + SECONDARY_NAMENODES=$(hostname) +fi - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$SECONDARY_NAMENODES" \ - --script "$bin/hdfs" stop secondarynamenode +if [[ -n "${SECONDARY_NAMENODES}" ]]; then + echo "Stopping secondary namenodes [${SECONDARY_NAMENODES}]" + + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${SECONDARY_NAMENODES}" \ + stop secondarynamenode fi #--------------------------------------------------------- # quorumjournal nodes (if any) -SHARED_EDITS_DIR=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.namenode.shared.edits.dir 2>&-) +SHARED_EDITS_DIR=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.namenode.shared.edits.dir 2>&-) -case "$SHARED_EDITS_DIR" in -qjournal://*) - JOURNAL_NODES=$(echo "$SHARED_EDITS_DIR" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') - echo "Stopping journal nodes [$JOURNAL_NODES]" - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$JOURNAL_NODES" \ - --script "$bin/hdfs" stop journalnode ;; +case "${SHARED_EDITS_DIR}" in + qjournal://*) + JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g') + echo "Stopping journal nodes [${JOURNAL_NODES}]" + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${JOURNAL_NODES}" \ + stop journalnode + ;; esac #--------------------------------------------------------- # ZK Failover controllers, if auto-HA is enabled -AUTOHA_ENABLED=$($HADOOP_PREFIX/bin/hdfs getconf -confKey dfs.ha.automatic-failover.enabled) -if [ "$(echo "$AUTOHA_ENABLED" | tr A-Z a-z)" = "true" ]; then - echo "Stopping ZK Failover Controllers on NN hosts [$NAMENODES]" - "$HADOOP_PREFIX/sbin/hadoop-daemons.sh" \ - --config "$HADOOP_CONF_DIR" \ - --hostnames "$NAMENODES" \ - --script "$bin/hdfs" stop zkfc +AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automatic-failover.enabled | tr '[:upper:]' '[:lower:]') +if [[ "${AUTOHA_ENABLED}" = "true" ]]; then + echo "Stopping ZK Failover Controllers on NN hosts [${NAMENODES}]" + "${bin}/hadoop-daemons.sh" \ + --config "${HADOOP_CONF_DIR}" \ + --hostnames "${NAMENODES}" \ + stop zkfc fi # eof Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh Thu Aug 21 05:22:10 2014 @@ -17,17 +17,33 @@ # Run as root to start secure datanodes in a security-enabled cluster. -usage="Usage (run as root in order to stop secure datanodes): stop-secure-dns.sh" -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin"; pwd` +function hadoop_usage { + echo "Usage (run as root in order to stop secure datanodes): stop-secure-dns.sh" +} -DEFAULT_LIBEXEC_DIR="$bin"/../libexec -HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR} -. $HADOOP_LIBEXEC_DIR/hdfs-config.sh +this="${BASH_SOURCE-$0}" +bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) -if [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_USER" ]; then - "$HADOOP_PREFIX"/sbin/hadoop-daemons.sh --config $HADOOP_CONF_DIR --script "$bin"/hdfs stop datanode +# let's locate libexec... +if [[ -n "${HADOOP_PREFIX}" ]]; then + DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else - echo $usage + DEFAULT_LIBEXEC_DIR="${bin}/../libexec" +fi + +HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}" +# shellcheck disable=SC2034 +HADOOP_NEW_CONFIG=true +if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then + . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" +else + echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1 + exit 1 +fi + +if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then + "${bin}/hadoop-daemons.sh" --config "${HADOOP_CONF_DIR}" stop datanode +else + hadoop_exit_with_usage 1 fi Propchange: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1594376-1619194 Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1618764-1619277 Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java Thu Aug 21 05:22:10 2014 @@ -17,7 +17,6 @@ */ package org.apache.hadoop.fs; - import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; @@ -31,6 +30,7 @@ import java.util.NoSuchElementException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.CryptoCodec; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; @@ -38,6 +38,8 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.hdfs.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DFSInputStream; +import org.apache.hadoop.hdfs.DFSOutputStream; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; @@ -59,6 +61,7 @@ import org.apache.hadoop.util.Progressab public class Hdfs extends AbstractFileSystem { DFSClient dfs; + final CryptoCodec factory; private boolean verifyChecksum = true; static { @@ -85,6 +88,7 @@ public class Hdfs extends AbstractFileSy } this.dfs = new DFSClient(theUri, conf, getStatistics()); + this.factory = CryptoCodec.getInstance(conf); } @Override @@ -97,9 +101,12 @@ public class Hdfs extends AbstractFileSy EnumSet createFlag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt, boolean createParent) throws IOException { - return new HdfsDataOutputStream(dfs.primitiveCreate(getUriPath(f), - absolutePermission, createFlag, createParent, replication, blockSize, - progress, bufferSize, checksumOpt), getStatistics()); + + final DFSOutputStream dfsos = dfs.primitiveCreate(getUriPath(f), + absolutePermission, createFlag, createParent, replication, blockSize, + progress, bufferSize, checksumOpt); + return dfs.createWrappedOutputStream(dfsos, statistics, + dfsos.getInitialLen()); } @Override @@ -308,8 +315,9 @@ public class Hdfs extends AbstractFileSy @Override public HdfsDataInputStream open(Path f, int bufferSize) throws IOException, UnresolvedLinkException { - return new DFSClient.DFSDataInputStream(dfs.open(getUriPath(f), - bufferSize, verifyChecksum)); + final DFSInputStream dfsis = dfs.open(getUriPath(f), + bufferSize, verifyChecksum); + return dfs.createWrappedInputStream(dfsis); } @Override Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java Thu Aug 21 05:22:10 2014 @@ -26,8 +26,8 @@ import org.apache.hadoop.classification. /** * XAttr is the POSIX Extended Attribute model similar to that found in * traditional Operating Systems. Extended Attributes consist of one - * or more name/value pairs associated with a file or directory. Four - * namespaces are defined: user, trusted, security and system. + * or more name/value pairs associated with a file or directory. Five + * namespaces are defined: user, trusted, security, system and raw. * 1) USER namespace attributes may be used by any user to store * arbitrary information. Access permissions in this namespace are * defined by a file directory's permission bits. For sticky directories, @@ -43,6 +43,12 @@ import org.apache.hadoop.classification. *
* 4) SECURITY namespace attributes are used by the fs kernel for * security features. It is not visible to users. + *
+ * 5) RAW namespace attributes are used for internal system attributes that + * sometimes need to be exposed. Like SYSTEM namespace attributes they are + * not visible to the user except when getXAttr/getXAttrs is called on a file + * or directory in the /.reserved/raw HDFS directory hierarchy. These + * attributes can only be accessed by the superuser. *

* @see * http://en.wikipedia.org/wiki/Extended_file_attributes @@ -55,7 +61,8 @@ public class XAttr { USER, TRUSTED, SECURITY, - SYSTEM; + SYSTEM, + RAW; } private final NameSpace ns; Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Thu Aug 21 05:22:10 2014 @@ -17,6 +17,11 @@ */ package org.apache.hadoop.hdfs; +import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; +import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension + .EncryptedKeyVersion; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; @@ -76,6 +81,7 @@ import java.net.Socket; import java.net.SocketAddress; import java.net.URI; import java.net.UnknownHostException; +import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; @@ -95,6 +101,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.CipherSuite; +import org.apache.hadoop.crypto.CryptoCodec; +import org.apache.hadoop.crypto.CryptoInputStream; +import org.apache.hadoop.crypto.CryptoOutputStream; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStorageLocation; import org.apache.hadoop.fs.CacheFlag; @@ -102,6 +113,7 @@ import org.apache.hadoop.fs.CommonConfig import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsStatus; @@ -140,6 +152,9 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; +import org.apache.hadoop.hdfs.protocol.EncryptionZone; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator; +import org.apache.hadoop.hdfs.protocol.EncryptionZoneWithId; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -249,7 +264,11 @@ public class DFSClient implements java.i private static final DFSHedgedReadMetrics HEDGED_READ_METRIC = new DFSHedgedReadMetrics(); private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL; - + private final CryptoCodec codec; + @VisibleForTesting + List cipherSuites; + @VisibleForTesting + KeyProviderCryptoExtension provider; /** * DFSClient configuration */ @@ -581,7 +600,17 @@ public class DFSClient implements java.i this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority(); this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" + DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId(); - + this.codec = CryptoCodec.getInstance(conf); + this.cipherSuites = Lists.newArrayListWithCapacity(1); + if (codec != null) { + cipherSuites.add(codec.getCipherSuite()); + } + provider = DFSUtil.createKeyProviderCryptoExtension(conf); + if (provider == null) { + LOG.info("No KeyProvider found."); + } else { + LOG.info("Found KeyProvider: " + provider.toString()); + } int numResponseToDrop = conf.getInt( DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY, DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT); @@ -1280,7 +1309,93 @@ public class DFSClient implements java.i return volumeBlockLocations; } - + + /** + * Decrypts a EDEK by consulting the KeyProvider. + */ + private KeyVersion decryptEncryptedDataEncryptionKey(FileEncryptionInfo + feInfo) throws IOException { + if (provider == null) { + throw new IOException("No KeyProvider is configured, cannot access" + + " an encrypted file"); + } + EncryptedKeyVersion ekv = EncryptedKeyVersion.createForDecryption( + feInfo.getEzKeyVersionName(), feInfo.getIV(), + feInfo.getEncryptedDataEncryptionKey()); + try { + return provider.decryptEncryptedKey(ekv); + } catch (GeneralSecurityException e) { + throw new IOException(e); + } + } + + /** + * Wraps the stream in a CryptoInputStream if the underlying file is + * encrypted. + */ + public HdfsDataInputStream createWrappedInputStream(DFSInputStream dfsis) + throws IOException { + final FileEncryptionInfo feInfo = dfsis.getFileEncryptionInfo(); + if (feInfo != null) { + // File is encrypted, wrap the stream in a crypto stream. + KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); + CryptoCodec codec = CryptoCodec + .getInstance(conf, feInfo.getCipherSuite()); + if (codec == null) { + throw new IOException("No configuration found for the cipher suite " + + feInfo.getCipherSuite().getConfigSuffix() + " prefixed with " + + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX + + ". Please see the example configuration " + + "hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE " + + "at core-default.xml for details."); + } + final CryptoInputStream cryptoIn = + new CryptoInputStream(dfsis, codec, decrypted.getMaterial(), + feInfo.getIV()); + return new HdfsDataInputStream(cryptoIn); + } else { + // No FileEncryptionInfo so no encryption. + return new HdfsDataInputStream(dfsis); + } + } + + /** + * Wraps the stream in a CryptoOutputStream if the underlying file is + * encrypted. + */ + public HdfsDataOutputStream createWrappedOutputStream(DFSOutputStream dfsos, + FileSystem.Statistics statistics) throws IOException { + return createWrappedOutputStream(dfsos, statistics, 0); + } + + /** + * Wraps the stream in a CryptoOutputStream if the underlying file is + * encrypted. + */ + public HdfsDataOutputStream createWrappedOutputStream(DFSOutputStream dfsos, + FileSystem.Statistics statistics, long startPos) throws IOException { + final FileEncryptionInfo feInfo = dfsos.getFileEncryptionInfo(); + if (feInfo != null) { + if (codec == null) { + throw new IOException("No configuration found for the cipher suite " + + HADOOP_SECURITY_CRYPTO_CIPHER_SUITE_KEY + " value prefixed with " + + HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_KEY_PREFIX + + ". Please see the example configuration " + + "hadoop.security.crypto.codec.classes.EXAMPLECIPHERSUITE " + + "at core-default.xml for details."); + } + // File is encrypted, wrap the stream in a crypto stream. + KeyVersion decrypted = decryptEncryptedDataEncryptionKey(feInfo); + final CryptoOutputStream cryptoOut = + new CryptoOutputStream(dfsos, codec, + decrypted.getMaterial(), feInfo.getIV(), startPos); + return new HdfsDataOutputStream(cryptoOut, statistics, startPos); + } else { + // No FileEncryptionInfo present so no encryption. + return new HdfsDataOutputStream(dfsos, statistics, startPos); + } + } + public DFSInputStream open(String src) throws IOException, UnresolvedLinkException { return open(src, dfsClientConf.ioBufferSize, true, null); @@ -1483,7 +1598,8 @@ public class DFSClient implements java.i } final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this, src, masked, flag, createParent, replication, blockSize, progress, - buffersize, dfsClientConf.createChecksum(checksumOpt), favoredNodeStrs); + buffersize, dfsClientConf.createChecksum(checksumOpt), + favoredNodeStrs, cipherSuites); beginFileLease(result.getFileId(), result); return result; } @@ -1530,7 +1646,7 @@ public class DFSClient implements java.i DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt); result = DFSOutputStream.newStreamForCreate(this, src, absPermission, flag, createParent, replication, blockSize, progress, buffersize, - checksum); + checksum, null, cipherSuites); } beginFileLease(result.getFileId(), result); return result; @@ -1608,7 +1724,7 @@ public class DFSClient implements java.i final Progressable progress, final FileSystem.Statistics statistics ) throws IOException { final DFSOutputStream out = append(src, buffersize, progress); - return new HdfsDataOutputStream(out, statistics, out.getInitialLen()); + return createWrappedOutputStream(out, statistics, out.getInitialLen()); } private DFSOutputStream append(String src, int buffersize, Progressable progress) @@ -2772,6 +2888,36 @@ public class DFSClient implements java.i } } + public void createEncryptionZone(String src, String keyName) + throws IOException { + checkOpen(); + try { + namenode.createEncryptionZone(src, keyName); + } catch (RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + SafeModeException.class, + UnresolvedPathException.class); + } + } + + public EncryptionZone getEZForPath(String src) + throws IOException { + checkOpen(); + try { + final EncryptionZoneWithId ezi = namenode.getEZForPath(src); + return (ezi.getId() < 0) ? null : ezi; + } catch (RemoteException re) { + throw re.unwrapRemoteException(AccessControlException.class, + UnresolvedPathException.class); + } + } + + public RemoteIterator listEncryptionZones() + throws IOException { + checkOpen(); + return new EncryptionZoneIterator(namenode); + } + public void setXAttr(String src, String name, byte[] value, EnumSet flag) throws IOException { checkOpen(); Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Aug 21 05:22:10 2014 @@ -582,7 +582,9 @@ public class DFSConfigKeys extends Commo public static final String DFS_TRUSTEDCHANNEL_RESOLVER_CLASS = "dfs.trustedchannel.resolver.class"; public static final String DFS_DATA_TRANSFER_PROTECTION_KEY = "dfs.data.transfer.protection"; public static final String DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY = "dfs.data.transfer.saslproperties.resolver.class"; - + public static final int DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT = 100; + public static final String DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES = "dfs.namenode.list.encryption.zones.num.responses"; + // Journal-node related configs. These are read on the JN side. public static final String DFS_JOURNALNODE_EDITS_DIR_KEY = "dfs.journalnode.edits.dir"; public static final String DFS_JOURNALNODE_EDITS_DIR_DEFAULT = "/tmp/hadoop/dfs/journalnode/"; Modified: hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1619293&r1=1619292&r2=1619293&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Thu Aug 21 05:22:10 2014 @@ -56,6 +56,7 @@ import org.apache.hadoop.fs.UnresolvedLi import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException; @@ -92,6 +93,7 @@ implements ByteBufferReadable, CanSetDro private final boolean verifyChecksum; private LocatedBlocks locatedBlocks = null; private long lastBlockBeingWrittenLength = 0; + private FileEncryptionInfo fileEncryptionInfo = null; private DatanodeInfo currentNode = null; private LocatedBlock currentLocatedBlock = null; private long pos = 0; @@ -301,6 +303,8 @@ implements ByteBufferReadable, CanSetDro } } + fileEncryptionInfo = locatedBlocks.getFileEncryptionInfo(); + currentNode = null; return lastBlockBeingWrittenLength; } @@ -1525,6 +1529,10 @@ implements ByteBufferReadable, CanSetDro return new ReadStatistics(readStatistics); } + public synchronized FileEncryptionInfo getFileEncryptionInfo() { + return fileEncryptionInfo; + } + private synchronized void closeCurrentBlockReader() { if (blockReader == null) return; // Close the current block reader so that the new caching settings can