Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2F63E10F93 for ; Wed, 22 Jan 2014 21:43:43 +0000 (UTC) Received: (qmail 86135 invoked by uid 500); 22 Jan 2014 21:43:39 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 86075 invoked by uid 500); 22 Jan 2014 21:43:38 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 86052 invoked by uid 99); 22 Jan 2014 21:43:37 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jan 2014 21:43:37 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Jan 2014 21:43:29 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id ED31E23889D5; Wed, 22 Jan 2014 21:43:08 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1560522 [1/9] - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./ dev-support/ src/main/bin/ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/client/ src/m... Date: Wed, 22 Jan 2014 21:43:04 -0000 To: hdfs-commits@hadoop.apache.org From: wang@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140122214308.ED31E23889D5@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: wang Date: Wed Jan 22 21:43:00 2014 New Revision: 1560522 URL: http://svn.apache.org/r1560522 Log: Merge HDFS-4949 changes from trunk to branch-2. Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/CacheFlag.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveEntry.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveStats.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolEntry.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolStats.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetCache.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/MappableBlock.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachedBlock.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockIdCommand.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/TableListing.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/CentralizedCacheManagement.apt.vm (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/site/resources/images/caching.png (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCacheAdminCLI.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandCacheAdmin.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CacheAdminCmdExecutor.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCachedBlocksList.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java (with props) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml (with props) Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ClientMmap.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/startupprogress/StepType.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/LogVerificationAppender.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Jan 22 21:43:00 2014 @@ -36,6 +36,8 @@ Release 2.4.0 - UNRELEASED HDFS-5703. Add support for HTTPS and swebhdfs to HttpFS. (tucu) + HDFS-4949. Centralized cache management in HDFS (wang and cmccabe) + IMPROVEMENTS HDFS-5267. Remove volatile from LightWeightHashSet. (Junping Du via llu) @@ -426,6 +428,211 @@ Release 2.4.0 - UNRELEASED HDFS-5667. Include DatanodeStorage in StorageReport. (Arpit Agarwal) + BREAKDOWN OF HDFS-4949 SUBTASKS AND RELATED JIRAS + + HDFS-5049. Add JNI mlock support. (Andrew Wang via Colin Patrick McCabe) + + HDFS-5051. Propagate cache status information from the DataNode to the + NameNode (Andrew Wang via Colin Patrick McCabe) + + HDFS-5052. Add cacheRequest/uncacheRequest support to NameNode. + (Contributed by Colin Patrick McCabe.) + + HDFS-5050. Add DataNode support for mlock and munlock (contributed by + Andrew Wang) + + HDFS-5141. Add cache status information to datanode heartbeat. (Contributed + by Andrew Wang) + + HDFS-5121. Add RPCs for creating and manipulating cache pools. + (Contributed by Colin Patrick McCabe) + + HDFS-5163. Miscellaneous cache pool RPC fixes (Contributed by Colin Patrick + McCabe) + + HDFS-5169. hdfs.c: translateZCRException: null pointer deref when + translating some exceptions (Contributed by Colin Patrick McCabe) + + HDFS-5120. Add command-line support for manipulating cache pools. (cmccabe) + + HDFS-5158. Add command-line support for manipulating cache directives. + (cmccabe) + + HDFS-5198. NameNodeRpcServer must not send back DNA_FINALIZE in reply to a + cache report. (cmccabe) + + HDFS-5195. Prevent passing null pointer to mlock and munlock. Contributed + by Chris Nauroth. + + HDFS-5053. NameNode should invoke DataNode APIs to coordinate caching. + (Andrew Wang) + + HDFS-5201. NativeIO: consolidate getrlimit into NativeIO#getMemlockLimit. + (Contributed by Colin Patrick McCabe) + + HDFS-5197. Document dfs.cachereport.intervalMsec in hdfs-default.xml. + Contributed by Chris Nauroth. + + HDFS-5210. Fix some failing unit tests on HDFS-4949 branch. (Contributed by + Andrew Wang) + + HDFS-5213. Separate PathBasedCacheEntry and PathBasedCacheDirectiveWithId. + Contributed by Colin Patrick McCabe. + + HDFS-5236. Change PathBasedCacheDirective APIs to be a single value rather + than batch. (Contributed by Andrew Wang) + + HDFS-5119. Persist CacheManager state in the edit log. (Contributed by + Andrew Wang) + + HDFS-5190. Move cache pool related CLI commands to CacheAdmin. (Contributed + by Andrew Wang) + + HDFS-5309. Fix failing caching unit tests. (Andrew Wang) + + HDFS-5314. Do not expose CachePool type in AddCachePoolOp (Colin Patrick + McCabe) + + HDFS-5304. Expose if a block replica is cached in getFileBlockLocations. + (Contributed by Andrew Wang) + + HDFS-5224. Refactor PathBasedCache* methods to use a Path rather than a + String. Contributed by Chris Nauroth. + + HDFS-5348. Fix error message when dfs.datanode.max.locked.memory is + improperly configured. (Contributed by Colin Patrick McCabe) + + HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only (cmccabe) + + HDFS-5358. Add replication field to PathBasedCacheDirective. (Contributed + by Colin Patrick McCabe) + + HDFS-5359. Allow LightWeightGSet#Iterator to remove elements. (Contributed + by Colin Patrick McCabe) + + HDFS-5373. hdfs cacheadmin -addDirective short usage does not mention + -replication parameter. Contributed by Chris Nauroth. + + HDFS-5096. Automatically cache new data added to a cached path (contributed + by Colin Patrick McCabe) + + HDFS-5383. fix broken caching unit tests (Andrew Wang) + + HDFS-5388. Loading fsimage fails to find cache pools during namenode + startup (Chris Nauroth via Colin Patrick McCabe) + + HDFS-5203. Concurrent clients that add a cache directive on the same path + may prematurely uncache each other. (Chris Nauroth via Colin Patrick McCabe) + + HDFS-5378. In CacheReport, don't send genstamp and length on the wire + (Contributed by Colin Patrick McCabe) + + HDFS-5385. Caching RPCs are AtMostOnce, but do not persist client ID and + call ID to edit log. (Chris Nauroth via Colin Patrick McCabe) + + HDFS-5404 Resolve regressions in Windows compatibility on HDFS-4949 branch. + Contributed by Chris Nauroth. + + HDFS-5405. Fix possible RetryCache hang for caching RPC handlers in + FSNamesystem. (Contributed by Andrew Wang) + + HDFS-5419. Fixup test-patch.sh warnings on HDFS-4949 branch. (wang) + + HDFS-5386. Add feature documentation for datanode caching. Contributed by + Colin Patrick McCabe. + + HDFS-5468. CacheAdmin help command does not recognize commands (Stephen + Chu via Colin Patrick McCabe) + + HDFS-5326. add modifyDirective to cacheAdmin (cmccabe) + + HDFS-5394: Fix race conditions in DN caching and uncaching (cmccabe) + + HDFS-5320. Add datanode caching metrics. Contributed by Andrew Wang. + + HDFS-5482. DistributedFileSystem#listPathBasedCacheDirectives must support + relative paths. Contributed by Colin Patrick McCabe. + + HDFS-5471. CacheAdmin -listPools fails when user lacks permissions to view + all pools (Andrew Wang via Colin Patrick McCabe) + + HDFS-5450. better API for getting the cached blocks locations. Contributed + by Andrew Wang. + + HDFS-5485. add command-line support for modifyDirective (cmccabe) + + HDFS-5366. recaching improvements (cmccabe) + + HDFS-5520. loading cache path directives from edit log doesnt update + nextEntryId (cmccabe) + + HDFS-5512. CacheAdmin -listPools fails with NPE when user lacks permissions + to view all pools (awang via cmccabe) + + HDFS-5513. CacheAdmin commands fail when using . as the path. Contributed + by Andrew Wang. + + HDFS-5511. improve CacheManipulator interface to allow better unit testing + (cmccabe) + + HDFS-5451. Add byte and file statistics to PathBasedCacheEntry. Contributed + by Colin Patrick McCabe. + + HDFS-5473. Consistent naming of user-visible caching classes and methods + (cmccabe) + + HDFS-5543. Fix narrow race condition in TestPathBasedCacheRequests + (cmccabe) + + HDFS-5565. CacheAdmin help should match against non-dashed commands (wang + via cmccabe) + + HDFS-5556. Add some more NameNode cache statistics, cache pool stats + (cmccabe) + + HDFS-5562. TestCacheDirectives and TestFsDatasetCache should stub out + native mlock. Contributed by Colin Patrick McCabe and Akira Ajisaka. + + HDFS-5430. Support TTL on CacheDirectives. Contributed by Andrew Wang. + + HDFS-5555. CacheAdmin commands fail when first listed NameNode is in + Standby (jxiang via cmccabe) + + HDFS-5626. dfsadmin report shows incorrect values (cmccabe) + + HDFS-5630. Hook up cache directive and pool usage statistics. (wang) + + HDFS-5665. Remove the unnecessary writeLock while initializing CacheManager + in FsNameSystem Ctor. (Uma Maheswara Rao G via Andrew Wang) + + HDFS-5431. Support cachepool-based limit management in path-based caching. + (awang via cmccabe) + + HDFS-5679. TestCacheDirectives should handle the case where native code is + not available. (wang) + + HDFS-5636. Enforce a max TTL per cache pool (awang via cmccabe) + + HDFS-5701. Fix the CacheAdmin -addPool -maxTtl option name. Contributed by + Stephen Chu. + + HDFS-5708. The CacheManager throws a NPE in the DataNode logs when + processing cache reports that refer to a block not known to the BlockManager. + Contributed by Colin Patrick McCabe. + + HDFS-5659. dfsadmin -report doesn't output cache information properly. + Contributed by Andrew Wang. + + HDFS-5651. Remove dfs.namenode.caching.enabled and improve CRM locking. + Contributed by Colin Patrick McCabe. + + HDFS-5589. Namenode loops caching and uncaching when data should be + uncached. (awang via cmccabe) + + HDFS-5724. modifyCacheDirective logging audit log command wrongly as + addCacheDirective (Uma Maheswara Rao G via Colin Patrick McCabe) + + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -799,7 +1006,7 @@ Release 2.1.1-beta - 2013-09-23 HDFS-5091. Support for spnego keytab separate from the JournalNode keytab for secure HA. (jing9) - HDFS-5051. nn fails to download checkpointed image from snn in some + HDFS-5055. nn fails to download checkpointed image from snn in some setups. (Vinay and suresh via suresh) HDFS-4898. BlockPlacementPolicyWithNodeGroup.chooseRemoteRack() fails to Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml Wed Jan 22 21:43:00 2014 @@ -346,4 +346,20 @@ + + + + + + + + + + + + + + + + Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs Wed Jan 22 21:43:00 2014 @@ -59,6 +59,7 @@ function print_usage(){ echo " Use -help to see options" echo " portmap run a portmap service" echo " nfs3 run an NFS version 3 gateway" + echo " cacheadmin configure the HDFS cache" echo "" echo "Most commands print help when invoked w/o parameters." } @@ -155,6 +156,8 @@ elif [ "$COMMAND" = "portmap" ] ; then CLASS=org.apache.hadoop.portmap.Portmap elif [ "$COMMAND" = "nfs3" ] ; then CLASS=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 +elif [ "$COMMAND" = "cacheadmin" ] ; then + CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin else CLASS="$COMMAND" fi Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/CacheFlag.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/CacheFlag.java?rev=1560522&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/CacheFlag.java (added) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/CacheFlag.java Wed Jan 22 21:43:00 2014 @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Specifies semantics for CacheDirective operations. Multiple flags can + * be combined in an EnumSet. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public enum CacheFlag { + + /** + * Ignore cache pool resource limits when performing this operation. + */ + FORCE((short) 0x01); + private final short mode; + + private CacheFlag(short mode) { + this.mode = mode; + } + + short getMode() { + return mode; + } +} Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/CacheFlag.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java Wed Jan 22 21:43:00 2014 @@ -37,8 +37,7 @@ public class HdfsBlockLocation extends B public HdfsBlockLocation(BlockLocation loc, LocatedBlock block) throws IOException { // Initialize with data from passed in BlockLocation - super(loc.getNames(), loc.getHosts(), loc.getTopologyPaths(), - loc.getOffset(), loc.getLength(), loc.isCorrupt()); + super(loc); this.block = block; } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Wed Jan 22 21:43:00 2014 @@ -87,6 +87,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStorageLocation; +import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; @@ -100,6 +101,7 @@ import org.apache.hadoop.fs.MD5MD5CRC32C import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; @@ -109,6 +111,11 @@ import org.apache.hadoop.fs.permission.F import org.apache.hadoop.hdfs.client.ClientMmapManager; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveIterator; +import org.apache.hadoop.hdfs.protocol.CachePoolEntry; +import org.apache.hadoop.hdfs.protocol.CachePoolInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolIterator; import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks; import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; @@ -117,6 +124,7 @@ import org.apache.hadoop.hdfs.protocol.D import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; @@ -2312,7 +2320,73 @@ public class DFSClient implements java.i throw re.unwrapRemoteException(); } } + + public long addCacheDirective( + CacheDirectiveInfo info, EnumSet flags) throws IOException { + checkOpen(); + try { + return namenode.addCacheDirective(info, flags); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + + public void modifyCacheDirective( + CacheDirectiveInfo info, EnumSet flags) throws IOException { + checkOpen(); + try { + namenode.modifyCacheDirective(info, flags); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + + public void removeCacheDirective(long id) + throws IOException { + checkOpen(); + try { + namenode.removeCacheDirective(id); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + public RemoteIterator listCacheDirectives( + CacheDirectiveInfo filter) throws IOException { + return new CacheDirectiveIterator(namenode, filter); + } + + public void addCachePool(CachePoolInfo info) throws IOException { + checkOpen(); + try { + namenode.addCachePool(info); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + + public void modifyCachePool(CachePoolInfo info) throws IOException { + checkOpen(); + try { + namenode.modifyCachePool(info); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + + public void removeCachePool(String poolName) throws IOException { + checkOpen(); + try { + namenode.removeCachePool(poolName); + } catch (RemoteException re) { + throw re.unwrapRemoteException(); + } + } + + public RemoteIterator listCachePools() throws IOException { + return new CachePoolIterator(namenode); + } + /** * Save namespace image. * Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Wed Jan 22 21:43:00 2014 @@ -104,6 +104,13 @@ public class DFSConfigKeys extends Commo public static final boolean DFS_DATANODE_DROP_CACHE_BEHIND_READS_DEFAULT = false; public static final String DFS_DATANODE_USE_DN_HOSTNAME = "dfs.datanode.use.datanode.hostname"; public static final boolean DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT = false; + public static final String DFS_DATANODE_MAX_LOCKED_MEMORY_KEY = "dfs.datanode.max.locked.memory"; + public static final long DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0; + public static final String DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume"; + public static final int DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4; + public static final String DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT = + "dfs.namenode.path.based.cache.block.map.allocation.percent"; + public static final float DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT = 0.25f; public static final String DFS_NAMENODE_HTTP_PORT_KEY = "dfs.http.port"; public static final int DFS_NAMENODE_HTTP_PORT_DEFAULT = 50070; @@ -210,6 +217,16 @@ public class DFSConfigKeys extends Commo public static final String DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_KEY = "dfs.namenode.datanode.registration.ip-hostname-check"; public static final boolean DFS_NAMENODE_DATANODE_REGISTRATION_IP_HOSTNAME_CHECK_DEFAULT = true; + + public static final String DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES = + "dfs.namenode.list.cache.pools.num.responses"; + public static final int DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES_DEFAULT = 100; + public static final String DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES = + "dfs.namenode.list.cache.directives.num.responses"; + public static final int DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES_DEFAULT = 100; + public static final String DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS = + "dfs.namenode.path.based.cache.refresh.interval.ms"; + public static final long DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS_DEFAULT = 300000L; // Whether to enable datanode's stale state detection and usage for reads public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = "dfs.namenode.avoid.read.stale.datanode"; @@ -335,6 +352,8 @@ public class DFSConfigKeys extends Commo public static final boolean DFS_DATANODE_TRANSFERTO_ALLOWED_DEFAULT = true; public static final String DFS_HEARTBEAT_INTERVAL_KEY = "dfs.heartbeat.interval"; public static final long DFS_HEARTBEAT_INTERVAL_DEFAULT = 3; + public static final String DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS = "dfs.namenode.path.based.cache.retry.interval.ms"; + public static final long DFS_NAMENODE_PATH_BASED_CACHE_RETRY_INTERVAL_MS_DEFAULT = 60000L; public static final String DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY = "dfs.namenode.decommission.interval"; public static final int DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT = 30; public static final String DFS_NAMENODE_DECOMMISSION_NODES_PER_INTERVAL_KEY = "dfs.namenode.decommission.nodes.per.interval"; @@ -378,6 +397,8 @@ public class DFSConfigKeys extends Commo public static final long DFS_BLOCKREPORT_INTERVAL_MSEC_DEFAULT = 60 * 60 * 1000; public static final String DFS_BLOCKREPORT_INITIAL_DELAY_KEY = "dfs.blockreport.initialDelay"; public static final int DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT = 0; + public static final String DFS_CACHEREPORT_INTERVAL_MSEC_KEY = "dfs.cachereport.intervalMsec"; + public static final long DFS_CACHEREPORT_INTERVAL_MSEC_DEFAULT = 10 * 1000; public static final String DFS_BLOCK_INVALIDATE_LIMIT_KEY = "dfs.block.invalidate.limit"; public static final int DFS_BLOCK_INVALIDATE_LIMIT_DEFAULT = 1000; public static final String DFS_DEFAULT_MAX_CORRUPT_FILES_RETURNED_KEY = "dfs.corruptfilesreturned.max"; Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Wed Jan 22 21:43:00 2014 @@ -41,13 +41,15 @@ import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; import java.security.SecureRandom; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; +import java.util.Date; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.Set; @@ -449,7 +451,13 @@ public class DFSUtil { locations[hCnt].getNetworkLocation()); racks[hCnt] = node.toString(); } - blkLocations[idx] = new BlockLocation(xferAddrs, hosts, racks, + DatanodeInfo[] cachedLocations = blk.getCachedLocations(); + String[] cachedHosts = new String[cachedLocations.length]; + for (int i=0; i flags) throws IOException { + Preconditions.checkNotNull(info.getPath()); + Path path = new Path(getPathName(fixRelativePart(info.getPath()))). + makeQualified(getUri(), getWorkingDirectory()); + return dfs.addCacheDirective( + new CacheDirectiveInfo.Builder(info). + setPath(path). + build(), + flags); + } + + /** + * @see {@link #modifyCacheDirective(CacheDirectiveInfo, EnumSet)} + */ + public void modifyCacheDirective(CacheDirectiveInfo info) throws IOException { + modifyCacheDirective(info, EnumSet.noneOf(CacheFlag.class)); + } + + /** + * Modify a CacheDirective. + * + * @param info Information about the directive to modify. You must set the ID + * to indicate which CacheDirective you want to modify. + * @param flags {@link CacheFlag}s to use for this operation. + * @throws IOException if the directive could not be modified + */ + public void modifyCacheDirective( + CacheDirectiveInfo info, EnumSet flags) throws IOException { + if (info.getPath() != null) { + info = new CacheDirectiveInfo.Builder(info). + setPath(new Path(getPathName(fixRelativePart(info.getPath()))). + makeQualified(getUri(), getWorkingDirectory())).build(); + } + dfs.modifyCacheDirective(info, flags); + } + + /** + * Remove a CacheDirectiveInfo. + * + * @param id identifier of the CacheDirectiveInfo to remove + * @throws IOException if the directive could not be removed + */ + public void removeCacheDirective(long id) + throws IOException { + dfs.removeCacheDirective(id); + } + /** + * List cache directives. Incrementally fetches results from the server. + * + * @param filter Filter parameters to use when listing the directives, null to + * list all directives visible to us. + * @return A RemoteIterator which returns CacheDirectiveInfo objects. + */ + public RemoteIterator listCacheDirectives( + CacheDirectiveInfo filter) throws IOException { + if (filter == null) { + filter = new CacheDirectiveInfo.Builder().build(); + } + if (filter.getPath() != null) { + filter = new CacheDirectiveInfo.Builder(filter). + setPath(new Path(getPathName(fixRelativePart(filter.getPath())))). + build(); + } + final RemoteIterator iter = + dfs.listCacheDirectives(filter); + return new RemoteIterator() { + @Override + public boolean hasNext() throws IOException { + return iter.hasNext(); + } + + @Override + public CacheDirectiveEntry next() throws IOException { + // Although the paths we get back from the NameNode should always be + // absolute, we call makeQualified to add the scheme and authority of + // this DistributedFilesystem. + CacheDirectiveEntry desc = iter.next(); + CacheDirectiveInfo info = desc.getInfo(); + Path p = info.getPath().makeQualified(getUri(), getWorkingDirectory()); + return new CacheDirectiveEntry( + new CacheDirectiveInfo.Builder(info).setPath(p).build(), + desc.getStats()); + } + }; + } + + /** + * Add a cache pool. + * + * @param info + * The request to add a cache pool. + * @throws IOException + * If the request could not be completed. + */ + public void addCachePool(CachePoolInfo info) throws IOException { + CachePoolInfo.validate(info); + dfs.addCachePool(info); + } + + /** + * Modify an existing cache pool. + * + * @param info + * The request to modify a cache pool. + * @throws IOException + * If the request could not be completed. + */ + public void modifyCachePool(CachePoolInfo info) throws IOException { + CachePoolInfo.validate(info); + dfs.modifyCachePool(info); + } + + /** + * Remove a cache pool. + * + * @param poolName + * Name of the cache pool to remove. + * @throws IOException + * if the cache pool did not exist, or could not be removed. + */ + public void removeCachePool(String poolName) throws IOException { + CachePoolInfo.validateName(poolName); + dfs.removeCachePool(poolName); + } + + /** + * List all cache pools. + * + * @return A remote iterator from which you can get CachePoolEntry objects. + * Requests will be made as needed. + * @throws IOException + * If there was an error listing cache pools. + */ + public RemoteIterator listCachePools() throws IOException { + return dfs.listCachePools(); + } } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ClientMmap.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ClientMmap.java?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ClientMmap.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/ClientMmap.java Wed Jan 22 21:43:00 2014 @@ -22,6 +22,7 @@ import java.io.FileInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.io.nativeio.NativeIO; import java.io.IOException; import java.lang.ref.WeakReference; @@ -147,20 +148,9 @@ public class ClientMmap { /** * Unmap the memory region. - * - * There isn't any portable way to unmap a memory region in Java. - * So we use the sun.nio method here. - * Note that unmapping a memory region could cause crashes if code - * continues to reference the unmapped code. However, if we don't - * manually unmap the memory, we are dependent on the finalizer to - * do it, and we have no idea when the finalizer will run. */ void unmap() { assert(refCount.get() == 0); - if (map instanceof sun.nio.ch.DirectBuffer) { - final sun.misc.Cleaner cleaner = - ((sun.nio.ch.DirectBuffer) map).cleaner(); - cleaner.clean(); - } + NativeIO.POSIX.munmap(map); } } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java?rev=1560522&r1=1560521&r2=1560522&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java Wed Jan 22 21:43:00 2014 @@ -19,13 +19,20 @@ package org.apache.hadoop.hdfs.client; import java.io.IOException; import java.net.URI; +import java.util.EnumSet; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CachePoolEntry; +import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.tools.DFSAdmin; @@ -121,4 +128,101 @@ public class HdfsAdmin { public void disallowSnapshot(Path path) throws IOException { dfs.disallowSnapshot(path); } + + /** + * Add a new CacheDirectiveInfo. + * + * @param info Information about a directive to add. + * @param flags {@link CacheFlag}s to use for this operation. + * @return the ID of the directive that was created. + * @throws IOException if the directive could not be added + */ + public long addCacheDirective(CacheDirectiveInfo info, + EnumSet flags) throws IOException { + return dfs.addCacheDirective(info, flags); + } + + /** + * Modify a CacheDirective. + * + * @param info Information about the directive to modify. You must set the ID + * to indicate which CacheDirective you want to modify. + * @param flags {@link CacheFlag}s to use for this operation. + * @throws IOException if the directive could not be modified + */ + public void modifyCacheDirective(CacheDirectiveInfo info, + EnumSet flags) throws IOException { + dfs.modifyCacheDirective(info, flags); + } + + /** + * Remove a CacheDirective. + * + * @param id identifier of the CacheDirectiveInfo to remove + * @throws IOException if the directive could not be removed + */ + public void removeCacheDirective(long id) + throws IOException { + dfs.removeCacheDirective(id); + } + + /** + * List cache directives. Incrementally fetches results from the server. + * + * @param filter Filter parameters to use when listing the directives, null to + * list all directives visible to us. + * @return A RemoteIterator which returns CacheDirectiveInfo objects. + */ + public RemoteIterator listCacheDirectives( + CacheDirectiveInfo filter) throws IOException { + return dfs.listCacheDirectives(filter); + } + + /** + * Add a cache pool. + * + * @param info + * The request to add a cache pool. + * @throws IOException + * If the request could not be completed. + */ + public void addCachePool(CachePoolInfo info) throws IOException { + dfs.addCachePool(info); + } + + /** + * Modify an existing cache pool. + * + * @param info + * The request to modify a cache pool. + * @throws IOException + * If the request could not be completed. + */ + public void modifyCachePool(CachePoolInfo info) throws IOException { + dfs.modifyCachePool(info); + } + + /** + * Remove a cache pool. + * + * @param poolName + * Name of the cache pool to remove. + * @throws IOException + * if the cache pool did not exist, or could not be removed. + */ + public void removeCachePool(String poolName) throws IOException { + dfs.removeCachePool(poolName); + } + + /** + * List all cache pools. + * + * @return A remote iterator from which you can get CachePoolEntry objects. + * Requests will be made as needed. + * @throws IOException + * If there was an error listing cache pools. + */ + public RemoteIterator listCachePools() throws IOException { + return dfs.listCachePools(); + } } Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java?rev=1560522&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java (added) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java Wed Jan 22 21:43:00 2014 @@ -0,0 +1,268 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Date; + +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSUtil; +import org.apache.hadoop.hdfs.server.namenode.CachePool; +import org.apache.hadoop.util.IntrusiveCollection; +import org.apache.hadoop.util.IntrusiveCollection.Element; + +import com.google.common.base.Preconditions; + +/** + * Namenode class that tracks state related to a cached path. + * + * This is an implementation class, not part of the public API. + */ +@InterfaceAudience.Private +public final class CacheDirective implements IntrusiveCollection.Element { + private final long id; + private final String path; + private final short replication; + private CachePool pool; + private final long expiryTime; + + private long bytesNeeded; + private long bytesCached; + private long filesNeeded; + private long filesCached; + + private Element prev; + private Element next; + + public CacheDirective(CacheDirectiveInfo info) { + this( + info.getId(), + info.getPath().toUri().getPath(), + info.getReplication(), + info.getExpiration().getAbsoluteMillis()); + } + + public CacheDirective(long id, String path, + short replication, long expiryTime) { + Preconditions.checkArgument(id > 0); + this.id = id; + this.path = checkNotNull(path); + Preconditions.checkArgument(replication > 0); + this.replication = replication; + this.expiryTime = expiryTime; + } + + public long getId() { + return id; + } + + public String getPath() { + return path; + } + + public short getReplication() { + return replication; + } + + public CachePool getPool() { + return pool; + } + + /** + * @return When this directive expires, in milliseconds since Unix epoch + */ + public long getExpiryTime() { + return expiryTime; + } + + /** + * @return When this directive expires, as an ISO-8601 formatted string. + */ + public String getExpiryTimeString() { + return DFSUtil.dateToIso8601String(new Date(expiryTime)); + } + + /** + * Returns a {@link CacheDirectiveInfo} based on this CacheDirective. + *

+ * This always sets an absolute expiry time, never a relative TTL. + */ + public CacheDirectiveInfo toInfo() { + return new CacheDirectiveInfo.Builder(). + setId(id). + setPath(new Path(path)). + setReplication(replication). + setPool(pool.getPoolName()). + setExpiration(CacheDirectiveInfo.Expiration.newAbsolute(expiryTime)). + build(); + } + + public CacheDirectiveStats toStats() { + return new CacheDirectiveStats.Builder(). + setBytesNeeded(bytesNeeded). + setBytesCached(bytesCached). + setFilesNeeded(filesNeeded). + setFilesCached(filesCached). + setHasExpired(new Date().getTime() > expiryTime). + build(); + } + + public CacheDirectiveEntry toEntry() { + return new CacheDirectiveEntry(toInfo(), toStats()); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("{ id:").append(id). + append(", path:").append(path). + append(", replication:").append(replication). + append(", pool:").append(pool). + append(", expiryTime: ").append(getExpiryTimeString()). + append(", bytesNeeded:").append(bytesNeeded). + append(", bytesCached:").append(bytesCached). + append(", filesNeeded:").append(filesNeeded). + append(", filesCached:").append(filesCached). + append(" }"); + return builder.toString(); + } + + @Override + public boolean equals(Object o) { + if (o == null) { return false; } + if (o == this) { return true; } + if (o.getClass() != this.getClass()) { + return false; + } + CacheDirective other = (CacheDirective)o; + return id == other.id; + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(id).toHashCode(); + } + + // + // Stats related getters and setters + // + + /** + * Resets the byte and file statistics being tracked by this CacheDirective. + */ + public void resetStatistics() { + bytesNeeded = 0; + bytesCached = 0; + filesNeeded = 0; + filesCached = 0; + } + + public long getBytesNeeded() { + return bytesNeeded; + } + + public void addBytesNeeded(long bytes) { + this.bytesNeeded += bytes; + pool.addBytesNeeded(bytes); + } + + public long getBytesCached() { + return bytesCached; + } + + public void addBytesCached(long bytes) { + this.bytesCached += bytes; + pool.addBytesCached(bytes); + } + + public long getFilesNeeded() { + return filesNeeded; + } + + public void addFilesNeeded(long files) { + this.filesNeeded += files; + pool.addFilesNeeded(files); + } + + public long getFilesCached() { + return filesCached; + } + + public void addFilesCached(long files) { + this.filesCached += files; + pool.addFilesCached(files); + } + + // + // IntrusiveCollection.Element implementation + // + + @SuppressWarnings("unchecked") + @Override // IntrusiveCollection.Element + public void insertInternal(IntrusiveCollection list, + Element prev, Element next) { + assert this.pool == null; + this.pool = ((CachePool.DirectiveList)list).getCachePool(); + this.prev = prev; + this.next = next; + } + + @Override // IntrusiveCollection.Element + public void setPrev(IntrusiveCollection list, Element prev) { + assert list == pool.getDirectiveList(); + this.prev = prev; + } + + @Override // IntrusiveCollection.Element + public void setNext(IntrusiveCollection list, Element next) { + assert list == pool.getDirectiveList(); + this.next = next; + } + + @Override // IntrusiveCollection.Element + public void removeInternal(IntrusiveCollection list) { + assert list == pool.getDirectiveList(); + this.pool = null; + this.prev = null; + this.next = null; + } + + @Override // IntrusiveCollection.Element + public Element getPrev(IntrusiveCollection list) { + if (list != pool.getDirectiveList()) { + return null; + } + return this.prev; + } + + @Override // IntrusiveCollection.Element + public Element getNext(IntrusiveCollection list) { + if (list != pool.getDirectiveList()) { + return null; + } + return this.next; + } + + @Override // IntrusiveCollection.Element + public boolean isInList(IntrusiveCollection list) { + return pool == null ? false : list == pool.getDirectiveList(); + } +}; Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java ------------------------------------------------------------------------------ svn:eol-style = native Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveEntry.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveEntry.java?rev=1560522&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveEntry.java (added) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveEntry.java Wed Jan 22 21:43:00 2014 @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Describes a path-based cache directive entry. + */ +@InterfaceStability.Evolving +@InterfaceAudience.Public +public class CacheDirectiveEntry { + private final CacheDirectiveInfo info; + private final CacheDirectiveStats stats; + + public CacheDirectiveEntry(CacheDirectiveInfo info, + CacheDirectiveStats stats) { + this.info = info; + this.stats = stats; + } + + public CacheDirectiveInfo getInfo() { + return info; + } + + public CacheDirectiveStats getStats() { + return stats; + } +}; Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveEntry.java ------------------------------------------------------------------------------ svn:eol-style = native Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java?rev=1560522&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java (added) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java Wed Jan 22 21:43:00 2014 @@ -0,0 +1,358 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.util.Date; + +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSUtil; + +import com.google.common.base.Preconditions; + +/** + * Describes a path-based cache directive. + */ +@InterfaceStability.Evolving +@InterfaceAudience.Public +public class CacheDirectiveInfo { + /** + * A builder for creating new CacheDirectiveInfo instances. + */ + public static class Builder { + private Long id; + private Path path; + private Short replication; + private String pool; + private Expiration expiration; + + /** + * Builds a new CacheDirectiveInfo populated with the set properties. + * + * @return New CacheDirectiveInfo. + */ + public CacheDirectiveInfo build() { + return new CacheDirectiveInfo(id, path, replication, pool, expiration); + } + + /** + * Creates an empty builder. + */ + public Builder() { + } + + /** + * Creates a builder with all elements set to the same values as the + * given CacheDirectiveInfo. + */ + public Builder(CacheDirectiveInfo directive) { + this.id = directive.getId(); + this.path = directive.getPath(); + this.replication = directive.getReplication(); + this.pool = directive.getPool(); + this.expiration = directive.getExpiration(); + } + + /** + * Sets the id used in this request. + * + * @param id The id used in this request. + * @return This builder, for call chaining. + */ + public Builder setId(Long id) { + this.id = id; + return this; + } + + /** + * Sets the path used in this request. + * + * @param path The path used in this request. + * @return This builder, for call chaining. + */ + public Builder setPath(Path path) { + this.path = path; + return this; + } + + /** + * Sets the replication used in this request. + * + * @param replication The replication used in this request. + * @return This builder, for call chaining. + */ + public Builder setReplication(Short replication) { + this.replication = replication; + return this; + } + + /** + * Sets the pool used in this request. + * + * @param pool The pool used in this request. + * @return This builder, for call chaining. + */ + public Builder setPool(String pool) { + this.pool = pool; + return this; + } + + /** + * Sets when the CacheDirective should expire. A + * {@link CacheDirectiveInfo.Expiration} can specify either an absolute or + * relative expiration time. + * + * @param expiration when this CacheDirective should expire + * @return This builder, for call chaining + */ + public Builder setExpiration(Expiration expiration) { + this.expiration = expiration; + return this; + } + } + + /** + * Denotes a relative or absolute expiration time for a CacheDirective. Use + * factory methods {@link CacheDirectiveInfo.Expiration#newAbsolute(Date)} and + * {@link CacheDirectiveInfo.Expiration#newRelative(long)} to create an + * Expiration. + *

+ * In either case, the server-side clock is used to determine when a + * CacheDirective expires. + */ + public static class Expiration { + + /** + * The maximum value we accept for a relative expiry. + */ + public static final long MAX_RELATIVE_EXPIRY_MS = + Long.MAX_VALUE / 4; // This helps prevent weird overflow bugs + + /** + * An relative Expiration that never expires. + */ + public static final Expiration NEVER = newRelative(MAX_RELATIVE_EXPIRY_MS); + + /** + * Create a new relative Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. + * + * @param ms how long until the CacheDirective expires, in milliseconds + * @return A relative Expiration + */ + public static Expiration newRelative(long ms) { + return new Expiration(ms, true); + } + + /** + * Create a new absolute Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. + * + * @param date when the CacheDirective expires + * @return An absolute Expiration + */ + public static Expiration newAbsolute(Date date) { + return new Expiration(date.getTime(), false); + } + + /** + * Create a new absolute Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. + * + * @param ms when the CacheDirective expires, in milliseconds since the Unix + * epoch. + * @return An absolute Expiration + */ + public static Expiration newAbsolute(long ms) { + return new Expiration(ms, false); + } + + private final long ms; + private final boolean isRelative; + + private Expiration(long ms, boolean isRelative) { + if (isRelative) { + Preconditions.checkArgument(ms <= MAX_RELATIVE_EXPIRY_MS, + "Expiration time is too far in the future!"); + } + this.ms = ms; + this.isRelative = isRelative; + } + + /** + * @return true if Expiration was specified as a relative duration, false if + * specified as an absolute time. + */ + public boolean isRelative() { + return isRelative; + } + + /** + * @return The raw underlying millisecond value, either a relative duration + * or an absolute time as milliseconds since the Unix epoch. + */ + public long getMillis() { + return ms; + } + + /** + * @return Expiration time as a {@link Date} object. This converts a + * relative Expiration into an absolute Date based on the local + * clock. + */ + public Date getAbsoluteDate() { + return new Date(getAbsoluteMillis()); + } + + /** + * @return Expiration time in milliseconds from the Unix epoch. This + * converts a relative Expiration into an absolute time based on the + * local clock. + */ + public long getAbsoluteMillis() { + if (!isRelative) { + return ms; + } else { + return new Date().getTime() + ms; + } + } + + @Override + public String toString() { + if (isRelative) { + return DFSUtil.durationToString(ms); + } + return DFSUtil.dateToIso8601String(new Date(ms)); + } + } + + private final Long id; + private final Path path; + private final Short replication; + private final String pool; + private final Expiration expiration; + + CacheDirectiveInfo(Long id, Path path, Short replication, String pool, + Expiration expiration) { + this.id = id; + this.path = path; + this.replication = replication; + this.pool = pool; + this.expiration = expiration; + } + + /** + * @return The ID of this directive. + */ + public Long getId() { + return id; + } + + /** + * @return The path used in this request. + */ + public Path getPath() { + return path; + } + + /** + * @return The number of times the block should be cached. + */ + public Short getReplication() { + return replication; + } + + /** + * @return The pool used in this request. + */ + public String getPool() { + return pool; + } + + /** + * @return When this directive expires. + */ + public Expiration getExpiration() { + return expiration; + } + + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } + if (getClass() != o.getClass()) { + return false; + } + CacheDirectiveInfo other = (CacheDirectiveInfo)o; + return new EqualsBuilder().append(getId(), other.getId()). + append(getPath(), other.getPath()). + append(getReplication(), other.getReplication()). + append(getPool(), other.getPool()). + append(getExpiration(), other.getExpiration()). + isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(id). + append(path). + append(replication). + append(pool). + append(expiration). + hashCode(); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("{"); + String prefix = ""; + if (id != null) { + builder.append(prefix).append("id: ").append(id); + prefix = ", "; + } + if (path != null) { + builder.append(prefix).append("path: ").append(path); + prefix = ", "; + } + if (replication != null) { + builder.append(prefix).append("replication: ").append(replication); + prefix = ", "; + } + if (pool != null) { + builder.append(prefix).append("pool: ").append(pool); + prefix = ", "; + } + if (expiration != null) { + builder.append(prefix).append("expiration: ").append(expiration); + prefix = ", "; + } + builder.append("}"); + return builder.toString(); + } +}; Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java ------------------------------------------------------------------------------ svn:eol-style = native Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java?rev=1560522&view=auto ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java (added) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java Wed Jan 22 21:43:00 2014 @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs.protocol; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.BatchedRemoteIterator; + +/** + * CacheDirectiveIterator is a remote iterator that iterates cache directives. + * It supports retrying in case of namenode failover. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class CacheDirectiveIterator + extends BatchedRemoteIterator { + + private final CacheDirectiveInfo filter; + private final ClientProtocol namenode; + + public CacheDirectiveIterator(ClientProtocol namenode, + CacheDirectiveInfo filter) { + super(Long.valueOf(0)); + this.namenode = namenode; + this.filter = filter; + } + + @Override + public BatchedEntries makeRequest(Long prevKey) + throws IOException { + return namenode.listCacheDirectives(prevKey, filter); + } + + @Override + public Long elementToPrevKey(CacheDirectiveEntry entry) { + return entry.getInfo().getId(); + } +} Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java ------------------------------------------------------------------------------ svn:eol-style = native