Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 688E610F3D for ; Mon, 23 Dec 2013 22:19:48 +0000 (UTC) Received: (qmail 69102 invoked by uid 500); 23 Dec 2013 22:19:48 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 69062 invoked by uid 500); 23 Dec 2013 22:19:48 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 69043 invoked by uid 99); 23 Dec 2013 22:19:48 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Dec 2013 22:19:48 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Dec 2013 22:19:38 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 187772388980; Mon, 23 Dec 2013 22:19:15 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1553226 [1/2] - in /hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/ src/main/java/org/apache/hadoop/hdfs/ src/main/java/org/apache/hadoop/hdfs/protocol/ src/main/java/org/apache/hadoop/hdfs/protocolPB/ s... Date: Mon, 23 Dec 2013 22:19:14 -0000 To: hdfs-commits@hadoop.apache.org From: cnauroth@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131223221915.187772388980@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cnauroth Date: Mon Dec 23 22:19:12 2013 New Revision: 1553226 URL: http://svn.apache.org/r1553226 Log: Merge trunk to HDFS-4685. Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed) hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1552466-1553224 Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Dec 23 22:19:12 2013 @@ -241,6 +241,8 @@ Trunk (Unreleased) HDFS-5431. Support cachepool-based limit management in path-based caching (awang via cmccabe) + HDFS-5636. Enforce a max TTL per cache pool. (awang via cmccabe) + OPTIMIZATIONS HDFS-5349. DNA_CACHE and DNA_UNCACHE should be by blockId only. (cmccabe) @@ -763,6 +765,9 @@ Release 2.4.0 - UNRELEASED HDFS-5540. Fix intermittent failure in TestBlocksWithNotEnoughRacks. (Binglin Chang via junping_du) + HDFS-2933. Improve DataNode Web UI Index Page. (Vivek Ganesan via + Arpit Agarwal) + OPTIMIZATIONS HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn) @@ -770,6 +775,8 @@ Release 2.4.0 - UNRELEASED HDFS-5341. Reduce fsdataset lock duration during directory scanning. (Qus-Jiawei via kihwal) + HDFS-5681. renewLease should not hold fsn write lock. (daryn via Kihwal) + BUG FIXES HDFS-5034. Remove debug prints from GetFileLinkInfo (Andrew Wang via Colin @@ -820,6 +827,12 @@ Release 2.4.0 - UNRELEASED HDFS-5676. fix inconsistent synchronization of CachingStrategy (cmccabe) + HDFS-5691. Fix typo in ShortCircuitLocalRead document. + (Akira Ajisaka via suresh) + + HDFS-5690. DataNode fails to start in secure mode when dfs.http.policy equals to + HTTP_ONLY. (Haohui Mai via jing9) + Release 2.3.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -1132,9 +1145,6 @@ Release 2.1.1-beta - 2013-09-23 HDFS-5047. Supress logging of full stack trace of quota and lease exceptions. (Robert Parker via kihwal) - HDFS-2933. Improve DataNode Web UI Index Page. (Vivek Ganesan via - Arpit Agarwal) - HDFS-5111. Remove duplicated error message for snapshot commands when processing invalid arguments. (jing9) Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1552466-1553224 Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Mon Dec 23 22:19:12 2013 @@ -1546,7 +1546,11 @@ public class DFSUtil { * Converts a time duration in milliseconds into DDD:HH:MM:SS format. */ public static String durationToString(long durationMs) { - Preconditions.checkArgument(durationMs >= 0, "Invalid negative duration"); + boolean negative = false; + if (durationMs < 0) { + negative = true; + durationMs = -durationMs; + } // Chop off the milliseconds long durationSec = durationMs / 1000; final int secondsPerMinute = 60; @@ -1559,7 +1563,12 @@ public class DFSUtil { final long minutes = durationSec / secondsPerMinute; durationSec -= minutes * secondsPerMinute; final long seconds = durationSec; - return String.format("%03d:%02d:%02d:%02d", days, hours, minutes, seconds); + final long milliseconds = durationMs % 1000; + String format = "%03d:%02d:%02d:%02d.%03d"; + if (negative) { + format = "-" + format; + } + return String.format(format, days, hours, minutes, seconds, milliseconds); } /** @@ -1571,9 +1580,9 @@ public class DFSUtil { + ": too short"); } String ttlString = relTime.substring(0, relTime.length()-1); - int ttl; + long ttl; try { - ttl = Integer.parseInt(ttlString); + ttl = Long.parseLong(ttlString); } catch (NumberFormatException e) { throw new IOException("Unable to parse relative time value of " + relTime + ": " + ttlString + " is not a number"); Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirective.java Mon Dec 23 22:19:12 2013 @@ -52,6 +52,14 @@ public final class CacheDirective implem private Element prev; private Element next; + public CacheDirective(CacheDirectiveInfo info) { + this( + info.getId(), + info.getPath().toUri().getPath(), + info.getReplication(), + info.getExpiration().getAbsoluteMillis()); + } + public CacheDirective(long id, String path, short replication, long expiryTime) { Preconditions.checkArgument(id > 0); Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveInfo.java Mon Dec 23 22:19:12 2013 @@ -26,6 +26,8 @@ import org.apache.hadoop.classification. import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSUtil; +import com.google.common.base.Preconditions; + /** * Describes a path-based cache directive. */ @@ -138,11 +140,22 @@ public class CacheDirectiveInfo { */ public static class Expiration { - /** Denotes a CacheDirectiveInfo that never expires **/ - public static final int EXPIRY_NEVER = -1; + /** + * The maximum value we accept for a relative expiry. + */ + public static final long MAX_RELATIVE_EXPIRY_MS = + Long.MAX_VALUE / 4; // This helps prevent weird overflow bugs + + /** + * An relative Expiration that never expires. + */ + public static final Expiration NEVER = newRelative(MAX_RELATIVE_EXPIRY_MS); /** * Create a new relative Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. * * @param ms how long until the CacheDirective expires, in milliseconds * @return A relative Expiration @@ -153,6 +166,9 @@ public class CacheDirectiveInfo { /** * Create a new absolute Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. * * @param date when the CacheDirective expires * @return An absolute Expiration @@ -163,6 +179,9 @@ public class CacheDirectiveInfo { /** * Create a new absolute Expiration. + *

+ * Use {@link Expiration#NEVER} to indicate an Expiration that never + * expires. * * @param ms when the CacheDirective expires, in milliseconds since the Unix * epoch. @@ -176,6 +195,10 @@ public class CacheDirectiveInfo { private final boolean isRelative; private Expiration(long ms, boolean isRelative) { + if (isRelative) { + Preconditions.checkArgument(ms <= MAX_RELATIVE_EXPIRY_MS, + "Expiration time is too far in the future!"); + } this.ms = ms; this.isRelative = isRelative; } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolInfo.java Mon Dec 23 22:19:12 2013 @@ -30,6 +30,7 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.InvalidRequestException; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; /** * CachePoolInfo describes a cache pool. @@ -42,6 +43,20 @@ import org.apache.hadoop.fs.permission.F public class CachePoolInfo { public static final Log LOG = LogFactory.getLog(CachePoolInfo.class); + /** + * Indicates that the pool does not have a maximum relative expiry. + */ + public static final long RELATIVE_EXPIRY_NEVER = + Expiration.MAX_RELATIVE_EXPIRY_MS; + /** + * Default max relative expiry for cache pools. + */ + public static final long DEFAULT_MAX_RELATIVE_EXPIRY = + RELATIVE_EXPIRY_NEVER; + + public static final long LIMIT_UNLIMITED = Long.MAX_VALUE; + public static final long DEFAULT_LIMIT = LIMIT_UNLIMITED; + final String poolName; @Nullable @@ -56,14 +71,24 @@ public class CachePoolInfo { @Nullable Long limit; + @Nullable + Long maxRelativeExpiryMs; + public CachePoolInfo(String poolName) { this.poolName = poolName; } - + + /** + * @return Name of the pool. + */ public String getPoolName() { return poolName; } + /** + * @return The owner of the pool. Along with the group and mode, determines + * who has access to view and modify the pool. + */ public String getOwnerName() { return ownerName; } @@ -73,6 +98,10 @@ public class CachePoolInfo { return this; } + /** + * @return The group of the pool. Along with the owner and mode, determines + * who has access to view and modify the pool. + */ public String getGroupName() { return groupName; } @@ -81,7 +110,11 @@ public class CachePoolInfo { this.groupName = groupName; return this; } - + + /** + * @return Unix-style permissions of the pool. Along with the owner and group, + * determines who has access to view and modify the pool. + */ public FsPermission getMode() { return mode; } @@ -91,6 +124,10 @@ public class CachePoolInfo { return this; } + /** + * @return The maximum aggregate number of bytes that can be cached by + * directives in this pool. + */ public Long getLimit() { return limit; } @@ -100,6 +137,26 @@ public class CachePoolInfo { return this; } + /** + * @return The maximum relative expiration of directives of this pool in + * milliseconds + */ + public Long getMaxRelativeExpiryMs() { + return maxRelativeExpiryMs; + } + + /** + * Set the maximum relative expiration of directives of this pool in + * milliseconds. + * + * @param ms in milliseconds + * @return This builder, for call chaining. + */ + public CachePoolInfo setMaxRelativeExpiryMs(Long ms) { + this.maxRelativeExpiryMs = ms; + return this; + } + public String toString() { return new StringBuilder().append("{"). append("poolName:").append(poolName). @@ -108,6 +165,7 @@ public class CachePoolInfo { append(", mode:").append((mode == null) ? "null" : String.format("0%03o", mode.toShort())). append(", limit:").append(limit). + append(", maxRelativeExpiryMs:").append(maxRelativeExpiryMs). append("}").toString(); } @@ -125,6 +183,7 @@ public class CachePoolInfo { append(groupName, other.groupName). append(mode, other.mode). append(limit, other.limit). + append(maxRelativeExpiryMs, other.maxRelativeExpiryMs). isEquals(); } @@ -136,6 +195,7 @@ public class CachePoolInfo { append(groupName). append(mode). append(limit). + append(maxRelativeExpiryMs). hashCode(); } @@ -146,6 +206,15 @@ public class CachePoolInfo { if ((info.getLimit() != null) && (info.getLimit() < 0)) { throw new InvalidRequestException("Limit is negative."); } + if (info.getMaxRelativeExpiryMs() != null) { + long maxRelativeExpiryMs = info.getMaxRelativeExpiryMs(); + if (maxRelativeExpiryMs < 0l) { + throw new InvalidRequestException("Max relative expiry is negative."); + } + if (maxRelativeExpiryMs > Expiration.MAX_RELATIVE_EXPIRY_MS) { + throw new InvalidRequestException("Max relative expiry is too big."); + } + } validateName(info.poolName); } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java Mon Dec 23 22:19:12 2013 @@ -1839,6 +1839,9 @@ public class PBHelper { if (info.getLimit() != null) { builder.setLimit(info.getLimit()); } + if (info.getMaxRelativeExpiryMs() != null) { + builder.setMaxRelativeExpiry(info.getMaxRelativeExpiryMs()); + } return builder.build(); } @@ -1858,6 +1861,9 @@ public class PBHelper { if (proto.hasLimit()) { info.setLimit(proto.getLimit()); } + if (proto.hasMaxRelativeExpiry()) { + info.setMaxRelativeExpiryMs(proto.getMaxRelativeExpiry()); + } return info; } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java Mon Dec 23 22:19:12 2013 @@ -365,7 +365,7 @@ public class CacheReplicationMonitor ext if (directive.getExpiryTime() > 0 && directive.getExpiryTime() <= now) { if (LOG.isDebugEnabled()) { LOG.debug("Skipping directive id " + directive.getId() - + " because it has expired (" + directive.getExpiryTime() + ">=" + + " because it has expired (" + directive.getExpiryTime() + "<=" + now + ")"); } continue; Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/SecureDataNodeStarter.java Mon Dec 23 22:19:12 2013 @@ -87,6 +87,7 @@ public class SecureDataNodeStarter imple public static SecureResources getSecureResources(Configuration conf) throws Exception { HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); + boolean isSecure = UserGroupInformation.isSecurityEnabled(); // Obtain secure port for data streaming to datanode InetSocketAddress streamingAddr = DataNode.getStreamingAddr(conf); @@ -106,6 +107,11 @@ public class SecureDataNodeStarter imple + ss.getLocalPort()); } + if (ss.getLocalPort() > 1023 && isSecure) { + throw new RuntimeException( + "Cannot start secure datanode with unprivileged RPC ports"); + } + System.err.println("Opened streaming server at " + streamingAddr); // Bind a port for the web server. The code intends to bind HTTP server to @@ -126,9 +132,9 @@ public class SecureDataNodeStarter imple System.err.println("Successfully obtained privileged resources (streaming port = " + ss + " ) (http listener port = " + listener.getConnection() +")"); - if ((ss.getLocalPort() > 1023 || listener.getPort() > 1023) && - UserGroupInformation.isSecurityEnabled()) { - throw new RuntimeException("Cannot start secure datanode with unprivileged ports"); + if (listener.getPort() > 1023 && isSecure) { + throw new RuntimeException( + "Cannot start secure datanode with unprivileged HTTP ports"); } System.err.println("Opened info server at " + infoSocAddr); } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CacheManager.java Mon Dec 23 22:19:12 2013 @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.EnumSet; import java.util.Iterator; import java.util.LinkedList; @@ -55,6 +56,7 @@ import org.apache.hadoop.hdfs.protocol.B import org.apache.hadoop.hdfs.protocol.CacheDirective; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; @@ -322,27 +324,48 @@ public final class CacheManager { * {@link CacheDirectiveInfo.Expiration}. This converts a relative Expiration * into an absolute time based on the local clock. * - * @param directive from which to get the expiry time - * @param defaultValue to use if Expiration is not set - * @return Absolute expiry time in milliseconds since Unix epoch - * @throws InvalidRequestException if the Expiration is invalid - */ - private static long validateExpiryTime(CacheDirectiveInfo directive, - long defaultValue) throws InvalidRequestException { - long expiryTime; - CacheDirectiveInfo.Expiration expiration = directive.getExpiration(); - if (expiration != null) { - if (expiration.getMillis() < 0) { - throw new InvalidRequestException("Cannot set a negative expiration: " - + expiration.getMillis()); - } - // Converts a relative duration into an absolute time based on the local - // clock - expiryTime = expiration.getAbsoluteMillis(); + * @param info to validate. + * @param maxRelativeExpiryTime of the info's pool. + * @return the expiration time, or the pool's max absolute expiration if the + * info's expiration was not set. + * @throws InvalidRequestException if the info's Expiration is invalid. + */ + private static long validateExpiryTime(CacheDirectiveInfo info, + long maxRelativeExpiryTime) throws InvalidRequestException { + if (LOG.isTraceEnabled()) { + LOG.trace("Validating directive " + info + + " pool maxRelativeExpiryTime " + maxRelativeExpiryTime); + } + final long now = new Date().getTime(); + final long maxAbsoluteExpiryTime = now + maxRelativeExpiryTime; + if (info == null || info.getExpiration() == null) { + return maxAbsoluteExpiryTime; + } + Expiration expiry = info.getExpiration(); + if (expiry.getMillis() < 0l) { + throw new InvalidRequestException("Cannot set a negative expiration: " + + expiry.getMillis()); + } + long relExpiryTime, absExpiryTime; + if (expiry.isRelative()) { + relExpiryTime = expiry.getMillis(); + absExpiryTime = now + relExpiryTime; } else { - expiryTime = defaultValue; + absExpiryTime = expiry.getMillis(); + relExpiryTime = absExpiryTime - now; } - return expiryTime; + // Need to cap the expiry so we don't overflow a long when doing math + if (relExpiryTime > Expiration.MAX_RELATIVE_EXPIRY_MS) { + throw new InvalidRequestException("Expiration " + + expiry.toString() + " is too far in the future!"); + } + // Fail if the requested expiry is greater than the max + if (relExpiryTime > maxRelativeExpiryTime) { + throw new InvalidRequestException("Expiration " + expiry.toString() + + " exceeds the max relative expiration time of " + + maxRelativeExpiryTime + " ms."); + } + return absExpiryTime; } /** @@ -357,6 +380,9 @@ public final class CacheManager { private void checkLimit(CachePool pool, String path, short replication) throws InvalidRequestException { CacheDirectiveStats stats = computeNeeded(path, replication); + if (pool.getLimit() == CachePoolInfo.LIMIT_UNLIMITED) { + return; + } if (pool.getBytesNeeded() + (stats.getBytesNeeded() * replication) > pool .getLimit()) { throw new InvalidRequestException("Caching path " + path + " of size " @@ -461,17 +487,13 @@ public final class CacheManager { } /** - * To be called only from the edit log loading code + * Adds a directive, skipping most error checking. This should only be called + * internally in special scenarios like edit log replay. */ CacheDirectiveInfo addDirectiveFromEditLog(CacheDirectiveInfo directive) throws InvalidRequestException { long id = directive.getId(); - CacheDirective entry = - new CacheDirective( - directive.getId(), - directive.getPath().toUri().getPath(), - directive.getReplication(), - directive.getExpiration().getAbsoluteMillis()); + CacheDirective entry = new CacheDirective(directive); CachePool pool = cachePools.get(directive.getPool()); addInternal(entry, pool); if (nextDirectiveId <= id) { @@ -490,8 +512,7 @@ public final class CacheManager { checkWritePermission(pc, pool); String path = validatePath(info); short replication = validateReplication(info, (short)1); - long expiryTime = validateExpiryTime(info, - CacheDirectiveInfo.Expiration.EXPIRY_NEVER); + long expiryTime = validateExpiryTime(info, pool.getMaxRelativeExpiryMs()); // Do quota validation if required if (!flags.contains(CacheFlag.FORCE)) { // Can't kick and wait if caching is disabled @@ -513,6 +534,56 @@ public final class CacheManager { return directive.toInfo(); } + /** + * Factory method that makes a new CacheDirectiveInfo by applying fields in a + * CacheDirectiveInfo to an existing CacheDirective. + * + * @param info with some or all fields set. + * @param defaults directive providing default values for unset fields in + * info. + * + * @return new CacheDirectiveInfo of the info applied to the defaults. + */ + private static CacheDirectiveInfo createFromInfoAndDefaults( + CacheDirectiveInfo info, CacheDirective defaults) { + // Initialize the builder with the default values + CacheDirectiveInfo.Builder builder = + new CacheDirectiveInfo.Builder(defaults.toInfo()); + // Replace default with new value if present + if (info.getPath() != null) { + builder.setPath(info.getPath()); + } + if (info.getReplication() != null) { + builder.setReplication(info.getReplication()); + } + if (info.getPool() != null) { + builder.setPool(info.getPool()); + } + if (info.getExpiration() != null) { + builder.setExpiration(info.getExpiration()); + } + return builder.build(); + } + + /** + * Modifies a directive, skipping most error checking. This is for careful + * internal use only. modifyDirective can be non-deterministic since its error + * checking depends on current system time, which poses a problem for edit log + * replay. + */ + void modifyDirectiveFromEditLog(CacheDirectiveInfo info) + throws InvalidRequestException { + // Check for invalid IDs. + Long id = info.getId(); + if (id == null) { + throw new InvalidRequestException("Must supply an ID."); + } + CacheDirective prevEntry = getById(id); + CacheDirectiveInfo newInfo = createFromInfoAndDefaults(info, prevEntry); + removeInternal(prevEntry); + addInternal(new CacheDirective(newInfo), getCachePool(newInfo.getPool())); + } + public void modifyDirective(CacheDirectiveInfo info, FSPermissionChecker pc, EnumSet flags) throws IOException { assert namesystem.hasWriteLock(); @@ -527,33 +598,38 @@ public final class CacheManager { } CacheDirective prevEntry = getById(id); checkWritePermission(pc, prevEntry.getPool()); - String path = prevEntry.getPath(); - if (info.getPath() != null) { - path = validatePath(info); - } - - short replication = prevEntry.getReplication(); - replication = validateReplication(info, replication); - - long expiryTime = prevEntry.getExpiryTime(); - expiryTime = validateExpiryTime(info, expiryTime); - CachePool pool = prevEntry.getPool(); - if (info.getPool() != null) { - pool = getCachePool(validatePoolName(info)); - checkWritePermission(pc, pool); + // Fill in defaults + CacheDirectiveInfo infoWithDefaults = + createFromInfoAndDefaults(info, prevEntry); + CacheDirectiveInfo.Builder builder = + new CacheDirectiveInfo.Builder(infoWithDefaults); + + // Do validation + validatePath(infoWithDefaults); + validateReplication(infoWithDefaults, (short)-1); + // Need to test the pool being set here to avoid rejecting a modify for a + // directive that's already been forced into a pool + CachePool srcPool = prevEntry.getPool(); + CachePool destPool = getCachePool(validatePoolName(infoWithDefaults)); + if (!srcPool.getPoolName().equals(destPool.getPoolName())) { + checkWritePermission(pc, destPool); if (!flags.contains(CacheFlag.FORCE)) { - // Can't kick and wait if caching is disabled - if (monitor != null) { - monitor.waitForRescan(); - } - checkLimit(pool, path, replication); + checkLimit(destPool, infoWithDefaults.getPath().toUri().getPath(), + infoWithDefaults.getReplication()); } } + // Verify the expiration against the destination pool + validateExpiryTime(infoWithDefaults, destPool.getMaxRelativeExpiryMs()); + + // Indicate changes to the CRM + if (monitor != null) { + monitor.setNeedsRescan(); + } + + // Validation passed removeInternal(prevEntry); - CacheDirective newEntry = - new CacheDirective(id, path, replication, expiryTime); - addInternal(newEntry, pool); + addInternal(new CacheDirective(builder.build()), destPool); } catch (IOException e) { LOG.warn("modifyDirective of " + idString + " failed: ", e); throw e; @@ -562,7 +638,7 @@ public final class CacheManager { info+ "."); } - public void removeInternal(CacheDirective directive) + private void removeInternal(CacheDirective directive) throws InvalidRequestException { assert namesystem.hasWriteLock(); // Remove the corresponding entry in directivesByPath. @@ -734,6 +810,13 @@ public final class CacheManager { monitor.setNeedsRescan(); } } + if (info.getMaxRelativeExpiryMs() != null) { + final Long maxRelativeExpiry = info.getMaxRelativeExpiryMs(); + pool.setMaxRelativeExpiryMs(maxRelativeExpiry); + bld.append(prefix).append("set maxRelativeExpiry to " + + maxRelativeExpiry); + prefix = "; "; + } if (prefix.isEmpty()) { bld.append("no changes."); } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CachePool.java Mon Dec 23 22:19:12 2013 @@ -49,8 +49,6 @@ import com.google.common.base.Preconditi public final class CachePool { public static final Log LOG = LogFactory.getLog(CachePool.class); - public static final long DEFAULT_LIMIT = Long.MAX_VALUE; - @Nonnull private final String poolName; @@ -76,6 +74,12 @@ public final class CachePool { */ private long limit; + /** + * Maximum duration that a CacheDirective in this pool remains valid, + * in milliseconds. + */ + private long maxRelativeExpiryMs; + private long bytesNeeded; private long bytesCached; private long filesNeeded; @@ -122,9 +126,12 @@ public final class CachePool { FsPermission mode = (info.getMode() == null) ? FsPermission.getCachePoolDefault() : info.getMode(); long limit = info.getLimit() == null ? - DEFAULT_LIMIT : info.getLimit(); + CachePoolInfo.DEFAULT_LIMIT : info.getLimit(); + long maxRelativeExpiry = info.getMaxRelativeExpiryMs() == null ? + CachePoolInfo.DEFAULT_MAX_RELATIVE_EXPIRY : + info.getMaxRelativeExpiryMs(); return new CachePool(info.getPoolName(), - ownerName, groupName, mode, limit); + ownerName, groupName, mode, limit, maxRelativeExpiry); } /** @@ -134,11 +141,11 @@ public final class CachePool { static CachePool createFromInfo(CachePoolInfo info) { return new CachePool(info.getPoolName(), info.getOwnerName(), info.getGroupName(), - info.getMode(), info.getLimit()); + info.getMode(), info.getLimit(), info.getMaxRelativeExpiryMs()); } CachePool(String poolName, String ownerName, String groupName, - FsPermission mode, long limit) { + FsPermission mode, long limit, long maxRelativeExpiry) { Preconditions.checkNotNull(poolName); Preconditions.checkNotNull(ownerName); Preconditions.checkNotNull(groupName); @@ -148,6 +155,7 @@ public final class CachePool { this.groupName = groupName; this.mode = new FsPermission(mode); this.limit = limit; + this.maxRelativeExpiryMs = maxRelativeExpiry; } public String getPoolName() { @@ -190,6 +198,15 @@ public final class CachePool { return this; } + public long getMaxRelativeExpiryMs() { + return maxRelativeExpiryMs; + } + + public CachePool setMaxRelativeExpiryMs(long expiry) { + this.maxRelativeExpiryMs = expiry; + return this; + } + /** * Get either full or partial information about this CachePool. * @@ -207,7 +224,8 @@ public final class CachePool { return info.setOwnerName(ownerName). setGroupName(groupName). setMode(new FsPermission(mode)). - setLimit(limit); + setLimit(limit). + setMaxRelativeExpiryMs(maxRelativeExpiryMs); } /** @@ -300,6 +318,7 @@ public final class CachePool { append(", groupName:").append(groupName). append(", mode:").append(mode). append(", limit:").append(limit). + append(", maxRelativeExpiryMs:").append(maxRelativeExpiryMs). append(" }").toString(); } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Mon Dec 23 22:19:12 2013 @@ -652,8 +652,8 @@ public class FSEditLogLoader { case OP_MODIFY_CACHE_DIRECTIVE: { ModifyCacheDirectiveInfoOp modifyOp = (ModifyCacheDirectiveInfoOp) op; - fsNamesys.getCacheManager().modifyDirective( - modifyOp.directive, null, EnumSet.of(CacheFlag.FORCE)); + fsNamesys.getCacheManager().modifyDirectiveFromEditLog( + modifyOp.directive); if (toAddRetryCache) { fsNamesys.addCacheEntry(op.rpcClientId, op.rpcCallId); } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java Mon Dec 23 22:19:12 2013 @@ -604,18 +604,22 @@ public class FSImageSerialization { final String groupName = info.getGroupName(); final Long limit = info.getLimit(); final FsPermission mode = info.getMode(); + final Long maxRelativeExpiry = info.getMaxRelativeExpiryMs(); - boolean hasOwner, hasGroup, hasMode, hasLimit; + boolean hasOwner, hasGroup, hasMode, hasLimit, hasMaxRelativeExpiry; hasOwner = ownerName != null; hasGroup = groupName != null; hasMode = mode != null; hasLimit = limit != null; + hasMaxRelativeExpiry = maxRelativeExpiry != null; int flags = (hasOwner ? 0x1 : 0) | (hasGroup ? 0x2 : 0) | (hasMode ? 0x4 : 0) | - (hasLimit ? 0x8 : 0); + (hasLimit ? 0x8 : 0) | + (hasMaxRelativeExpiry ? 0x10 : 0); + writeInt(flags, out); if (hasOwner) { @@ -630,6 +634,9 @@ public class FSImageSerialization { if (hasLimit) { writeLong(limit, out); } + if (hasMaxRelativeExpiry) { + writeLong(maxRelativeExpiry, out); + } } public static CachePoolInfo readCachePoolInfo(DataInput in) @@ -649,7 +656,10 @@ public class FSImageSerialization { if ((flags & 0x8) != 0) { info.setLimit(readLong(in)); } - if ((flags & ~0xF) != 0) { + if ((flags & 0x10) != 0) { + info.setMaxRelativeExpiryMs(readLong(in)); + } + if ((flags & ~0x1F) != 0) { throw new IOException("Unknown flag in CachePoolInfo: " + flags); } return info; @@ -663,6 +673,7 @@ public class FSImageSerialization { final String groupName = info.getGroupName(); final Long limit = info.getLimit(); final FsPermission mode = info.getMode(); + final Long maxRelativeExpiry = info.getMaxRelativeExpiryMs(); if (ownerName != null) { XMLUtils.addSaxString(contentHandler, "OWNERNAME", ownerName); @@ -677,6 +688,10 @@ public class FSImageSerialization { XMLUtils.addSaxString(contentHandler, "LIMIT", Long.toString(limit)); } + if (maxRelativeExpiry != null) { + XMLUtils.addSaxString(contentHandler, "MAXRELATIVEEXPIRY", + Long.toString(maxRelativeExpiry)); + } } public static CachePoolInfo readCachePoolInfo(Stanza st) @@ -695,6 +710,10 @@ public class FSImageSerialization { if (st.hasChildren("LIMIT")) { info.setLimit(Long.parseLong(st.getValue("LIMIT"))); } + if (st.hasChildren("MAXRELATIVEEXPIRY")) { + info.setMaxRelativeExpiryMs( + Long.parseLong(st.getValue("MAXRELATIVEEXPIRY"))); + } return info; } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Dec 23 22:19:12 2013 @@ -4001,13 +4001,13 @@ public class FSNamesystem implements Nam */ void renewLease(String holder) throws IOException { checkOperation(OperationCategory.WRITE); - writeLock(); + readLock(); try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot renew lease for " + holder); leaseManager.renewLease(holder); } finally { - writeUnlock(); + readUnlock(); } } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java Mon Dec 23 22:19:12 2013 @@ -35,14 +35,12 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; +import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo.Expiration; import org.apache.hadoop.hdfs.protocol.CacheDirectiveStats; import org.apache.hadoop.hdfs.protocol.CachePoolEntry; import org.apache.hadoop.hdfs.protocol.CachePoolInfo; import org.apache.hadoop.hdfs.protocol.CachePoolStats; -import org.apache.hadoop.hdfs.server.namenode.CachePool; import org.apache.hadoop.hdfs.tools.TableListing.Justification; -import org.apache.hadoop.ipc.RemoteException; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; @@ -120,6 +118,23 @@ public class CacheAdmin extends Configur return listing; } + /** + * Parses a time-to-live value from a string + * @return The ttl in milliseconds + * @throws IOException if it could not be parsed + */ + private static Long parseTtlString(String maxTtlString) throws IOException { + Long maxTtl = null; + if (maxTtlString != null) { + if (maxTtlString.equalsIgnoreCase("never")) { + maxTtl = CachePoolInfo.RELATIVE_EXPIRY_NEVER; + } else { + maxTtl = DFSUtil.parseRelativeTime(maxTtlString); + } + } + return maxTtl; + } + interface Command { String getName(); String getShortUsage(); @@ -154,7 +169,7 @@ public class CacheAdmin extends Configur listing.addRow("", "The cache replication factor to use. " + "Defaults to 1."); listing.addRow("", "How long the directive is " + - "valid. Can be specified in minutes, hours, and days via e.g. " + + "valid. Can be specified in minutes, hours, and days, e.g. " + "30m, 4h, 2d. Valid units are [smhd]." + " If unspecified, the directive never expires."); return getShortUsage() + "\n" + @@ -309,7 +324,7 @@ public class CacheAdmin extends Configur "added. You must have write permission on the cache pool " + "in order to move a directive into it. (optional)"); listing.addRow("", "How long the directive is " + - "valid. Can be specified in minutes, hours, and days via e.g. " + + "valid. Can be specified in minutes, hours, and days, e.g. " + "30m, 4h, 2d. Valid units are [smhd]." + " If unspecified, the directive never expires."); return getShortUsage() + "\n" + @@ -419,22 +434,27 @@ public class CacheAdmin extends Configur System.err.println("Usage is " + getShortUsage()); return 1; } - DistributedFileSystem dfs = getDFS(conf); - RemoteIterator iter = - dfs.listCacheDirectives( - new CacheDirectiveInfo.Builder(). - setPath(new Path(path)).build()); int exitCode = 0; - while (iter.hasNext()) { - CacheDirectiveEntry entry = iter.next(); - try { - dfs.removeCacheDirective(entry.getInfo().getId()); - System.out.println("Removed cache directive " + - entry.getInfo().getId()); - } catch (IOException e) { - System.err.println(prettifyException(e)); - exitCode = 2; + try { + DistributedFileSystem dfs = getDFS(conf); + RemoteIterator iter = + dfs.listCacheDirectives( + new CacheDirectiveInfo.Builder(). + setPath(new Path(path)).build()); + while (iter.hasNext()) { + CacheDirectiveEntry entry = iter.next(); + try { + dfs.removeCacheDirective(entry.getInfo().getId()); + System.out.println("Removed cache directive " + + entry.getInfo().getId()); + } catch (IOException e) { + System.err.println(prettifyException(e)); + exitCode = 2; + } } + } catch (IOException e) { + System.err.println(prettifyException(e)); + exitCode = 2; } if (exitCode == 0) { System.out.println("Removed every cache directive with path " + @@ -500,41 +520,46 @@ public class CacheAdmin extends Configur addField("FILES_CACHED", Justification.RIGHT); } TableListing tableListing = tableBuilder.build(); - - DistributedFileSystem dfs = getDFS(conf); - RemoteIterator iter = - dfs.listCacheDirectives(builder.build()); - int numEntries = 0; - while (iter.hasNext()) { - CacheDirectiveEntry entry = iter.next(); - CacheDirectiveInfo directive = entry.getInfo(); - CacheDirectiveStats stats = entry.getStats(); - List row = new LinkedList(); - row.add("" + directive.getId()); - row.add(directive.getPool()); - row.add("" + directive.getReplication()); - String expiry; - if (directive.getExpiration().getMillis() == - CacheDirectiveInfo.Expiration.EXPIRY_NEVER) { - expiry = "never"; - } else { - expiry = directive.getExpiration().toString(); + try { + DistributedFileSystem dfs = getDFS(conf); + RemoteIterator iter = + dfs.listCacheDirectives(builder.build()); + int numEntries = 0; + while (iter.hasNext()) { + CacheDirectiveEntry entry = iter.next(); + CacheDirectiveInfo directive = entry.getInfo(); + CacheDirectiveStats stats = entry.getStats(); + List row = new LinkedList(); + row.add("" + directive.getId()); + row.add(directive.getPool()); + row.add("" + directive.getReplication()); + String expiry; + // This is effectively never, round for nice printing + if (directive.getExpiration().getMillis() > + Expiration.MAX_RELATIVE_EXPIRY_MS / 2) { + expiry = "never"; + } else { + expiry = directive.getExpiration().toString(); + } + row.add(expiry); + row.add(directive.getPath().toUri().getPath()); + if (printStats) { + row.add("" + stats.getBytesNeeded()); + row.add("" + stats.getBytesCached()); + row.add("" + stats.getFilesNeeded()); + row.add("" + stats.getFilesCached()); + } + tableListing.addRow(row.toArray(new String[0])); + numEntries++; } - row.add(expiry); - row.add(directive.getPath().toUri().getPath()); - if (printStats) { - row.add("" + stats.getBytesNeeded()); - row.add("" + stats.getBytesCached()); - row.add("" + stats.getFilesNeeded()); - row.add("" + stats.getFilesCached()); + System.out.print(String.format("Found %d entr%s\n", + numEntries, numEntries == 1 ? "y" : "ies")); + if (numEntries > 0) { + System.out.print(tableListing); } - tableListing.addRow(row.toArray(new String[0])); - numEntries++; - } - System.out.print(String.format("Found %d entr%s\n", - numEntries, numEntries == 1 ? "y" : "ies")); - if (numEntries > 0) { - System.out.print(tableListing); + } catch (IOException e) { + System.err.println(prettifyException(e)); + return 2; } return 0; } @@ -552,7 +577,8 @@ public class CacheAdmin extends Configur @Override public String getShortUsage() { return "[" + NAME + " [-owner ] " + - "[-group ] [-mode ] [-limit ]]\n"; + "[-group ] [-mode ] [-limit ] " + + "[-maxttl ]\n"; } @Override @@ -571,7 +597,11 @@ public class CacheAdmin extends Configur listing.addRow("", "The maximum number of bytes that can be " + "cached by directives in this pool, in aggregate. By default, " + "no limit is set."); - + listing.addRow("", "The maximum allowed time-to-live for " + + "directives being added to the pool. This can be specified in " + + "seconds, minutes, hours, and days, e.g. 120s, 30m, 4h, 2d. " + + "Valid units are [smhd]. By default, no maximum is set. " + + "This can also be manually specified by \"never\"."); return getShortUsage() + "\n" + "Add a new cache pool.\n\n" + listing.toString(); @@ -605,6 +635,18 @@ public class CacheAdmin extends Configur long limit = Long.parseLong(limitString); info.setLimit(limit); } + String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args); + try { + Long maxTtl = parseTtlString(maxTtlString); + if (maxTtl != null) { + info.setMaxRelativeExpiryMs(maxTtl); + } + } catch (IOException e) { + System.err.println( + "Error while parsing maxTtl value: " + e.getMessage()); + return 1; + } + if (!args.isEmpty()) { System.err.print("Can't understand arguments: " + Joiner.on(" ").join(args) + "\n"); @@ -615,7 +657,8 @@ public class CacheAdmin extends Configur try { dfs.addCachePool(info); } catch (IOException e) { - throw new RemoteException(e.getClass().getName(), e.getMessage()); + System.err.println(prettifyException(e)); + return 2; } System.out.println("Successfully added cache pool " + name + "."); return 0; @@ -632,7 +675,8 @@ public class CacheAdmin extends Configur @Override public String getShortUsage() { return "[" + getName() + " [-owner ] " + - "[-group ] [-mode ] [-limit ]]\n"; + "[-group ] [-mode ] [-limit ] " + + "[-maxTtl ]]\n"; } @Override @@ -645,6 +689,8 @@ public class CacheAdmin extends Configur listing.addRow("", "Unix-style permissions of the pool in octal."); listing.addRow("", "Maximum number of bytes that can be cached " + "by this pool."); + listing.addRow("", "The maximum allowed time-to-live for " + + "directives being added to the pool."); return getShortUsage() + "\n" + WordUtils.wrap("Modifies the metadata of an existing cache pool. " + @@ -663,6 +709,15 @@ public class CacheAdmin extends Configur String limitString = StringUtils.popOptionWithArgument("-limit", args); Long limit = (limitString == null) ? null : Long.parseLong(limitString); + String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args); + Long maxTtl = null; + try { + maxTtl = parseTtlString(maxTtlString); + } catch (IOException e) { + System.err.println( + "Error while parsing maxTtl value: " + e.getMessage()); + return 1; + } String name = StringUtils.popFirstNonOption(args); if (name == null) { System.err.println("You must specify a name when creating a " + @@ -693,6 +748,10 @@ public class CacheAdmin extends Configur info.setLimit(limit); changed = true; } + if (maxTtl != null) { + info.setMaxRelativeExpiryMs(maxTtl); + changed = true; + } if (!changed) { System.err.println("You must specify at least one attribute to " + "change in the cache pool."); @@ -702,7 +761,8 @@ public class CacheAdmin extends Configur try { dfs.modifyCachePool(info); } catch (IOException e) { - throw new RemoteException(e.getClass().getName(), e.getMessage()); + System.err.println(prettifyException(e)); + return 2; } System.out.print("Successfully modified cache pool " + name); String prefix = " to have "; @@ -722,6 +782,9 @@ public class CacheAdmin extends Configur System.out.print(prefix + "limit " + limit); prefix = " and "; } + if (maxTtl != null) { + System.out.print(prefix + "max time-to-live " + maxTtlString); + } System.out.print("\n"); return 0; } @@ -765,7 +828,8 @@ public class CacheAdmin extends Configur try { dfs.removeCachePool(name); } catch (IOException e) { - throw new RemoteException(e.getClass().getName(), e.getMessage()); + System.err.println(prettifyException(e)); + return 2; } System.out.println("Successfully removed cache pool " + name + "."); return 0; @@ -813,7 +877,8 @@ public class CacheAdmin extends Configur addField("OWNER", Justification.LEFT). addField("GROUP", Justification.LEFT). addField("MODE", Justification.LEFT). - addField("LIMIT", Justification.RIGHT); + addField("LIMIT", Justification.RIGHT). + addField("MAXTTL", Justification.RIGHT); if (printStats) { builder. addField("BYTES_NEEDED", Justification.RIGHT). @@ -837,12 +902,23 @@ public class CacheAdmin extends Configur row.add(info.getMode() != null ? info.getMode().toString() : null); Long limit = info.getLimit(); String limitString; - if (limit != null && limit.equals(CachePool.DEFAULT_LIMIT)) { + if (limit != null && limit.equals(CachePoolInfo.LIMIT_UNLIMITED)) { limitString = "unlimited"; } else { limitString = "" + limit; } row.add(limitString); + Long maxTtl = info.getMaxRelativeExpiryMs(); + String maxTtlString = null; + + if (maxTtl != null) { + if (maxTtl.longValue() == CachePoolInfo.RELATIVE_EXPIRY_NEVER) { + maxTtlString = "never"; + } else { + maxTtlString = DFSUtil.durationToString(maxTtl); + } + } + row.add(maxTtlString); if (printStats) { CachePoolStats stats = entry.getStats(); row.add(Long.toString(stats.getBytesNeeded())); @@ -859,7 +935,8 @@ public class CacheAdmin extends Configur } } } catch (IOException e) { - throw new RemoteException(e.getClass().getName(), e.getMessage()); + System.err.println(prettifyException(e)); + return 2; } System.out.print(String.format("Found %d result%s.\n", numResults, (numResults == 1 ? "" : "s"))); Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1552466-1553224 Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto Mon Dec 23 22:19:12 2013 @@ -435,6 +435,7 @@ message CachePoolInfoProto { optional string groupName = 3; optional int32 mode = 4; optional int64 limit = 5; + optional int64 maxRelativeExpiry = 6; } message CachePoolStatsProto { Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1552466-1553224 Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1552466-1553224 Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1552466-1553224 Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ShortCircuitLocalReads.apt.vm Mon Dec 23 22:19:12 2013 @@ -73,7 +73,7 @@ HDFS Short-Circuit Local Reads This configuration parameter turns on short-circuit local reads. - * dfs.client.read.shortcircuit.skip.checkusm + * dfs.client.read.shortcircuit.skip.checksum If this configuration parameter is set, short-circuit local reads will skip checksums. This is normally not recommended, but it may be useful for Propchange: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1552466-1553224 Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Mon Dec 23 22:19:12 2013 @@ -62,7 +62,6 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.junit.Assume; import org.junit.Before; @@ -730,16 +729,15 @@ public class TestDFSUtil { @Test(timeout=1000) public void testDurationToString() throws Exception { - assertEquals("000:00:00:00", DFSUtil.durationToString(0)); - try { - DFSUtil.durationToString(-199); - } catch (IllegalArgumentException e) { - GenericTestUtils.assertExceptionContains("Invalid negative duration", e); - } - assertEquals("001:01:01:01", + assertEquals("000:00:00:00.000", DFSUtil.durationToString(0)); + assertEquals("001:01:01:01.000", DFSUtil.durationToString(((24*60*60)+(60*60)+(60)+1)*1000)); - assertEquals("000:23:59:59", - DFSUtil.durationToString(((23*60*60)+(59*60)+(59))*1000)); + assertEquals("000:23:59:59.999", + DFSUtil.durationToString(((23*60*60)+(59*60)+(59))*1000+999)); + assertEquals("-001:01:01:01.000", + DFSUtil.durationToString(-((24*60*60)+(60*60)+(60)+1)*1000)); + assertEquals("-000:23:59:59.574", + DFSUtil.durationToString(-(((23*60*60)+(59*60)+(59))*1000+574))); } @Test(timeout=5000) @@ -763,7 +761,7 @@ public class TestDFSUtil { assertEquals(61*60*1000, DFSUtil.parseRelativeTime("61m")); assertEquals(0, DFSUtil.parseRelativeTime("0s")); assertEquals(25*60*60*1000, DFSUtil.parseRelativeTime("25h")); - assertEquals(4*24*60*60*1000, DFSUtil.parseRelativeTime("4d")); - assertEquals(999*24*60*60*1000, DFSUtil.parseRelativeTime("999d")); + assertEquals(4*24*60*60*1000l, DFSUtil.parseRelativeTime("4d")); + assertEquals(999*24*60*60*1000l, DFSUtil.parseRelativeTime("999d")); } } Modified: hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java?rev=1553226&r1=1553225&r2=1553226&view=diff ============================================================================== --- hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java (original) +++ hadoop/common/branches/HDFS-4685/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java Mon Dec 23 22:19:12 2013 @@ -23,6 +23,8 @@ import static org.apache.hadoop.hdfs.DFS import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CACHING_ENABLED_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_PATH_BASED_CACHE_REFRESH_INTERVAL_MS; +import static org.apache.hadoop.hdfs.protocol.CachePoolInfo.RELATIVE_EXPIRY_NEVER; +import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -137,6 +139,8 @@ public class TestCacheDirectives { NativeIO.POSIX.setCacheManipulator(new NoMlockCacheManipulator()); LogManager.getLogger(CacheReplicationMonitor.class.getName()).setLevel( Level.TRACE); + LogManager.getLogger(CacheManager.class.getName()).setLevel( + Level.TRACE); } @After @@ -1189,4 +1193,185 @@ public class TestCacheDirectives { new CacheDirectiveInfo.Builder().setPool(inadequate.getPoolName()) .setPath(path1).build(), EnumSet.of(CacheFlag.FORCE)); } + + @Test(timeout=30000) + public void testMaxRelativeExpiry() throws Exception { + // Test that negative and really big max expirations can't be set during add + try { + dfs.addCachePool(new CachePoolInfo("failpool").setMaxRelativeExpiryMs(-1l)); + fail("Added a pool with a negative max expiry."); + } catch (InvalidRequestException e) { + GenericTestUtils.assertExceptionContains("negative", e); + } + try { + dfs.addCachePool(new CachePoolInfo("failpool") + .setMaxRelativeExpiryMs(Long.MAX_VALUE - 1)); + fail("Added a pool with too big of a max expiry."); + } catch (InvalidRequestException e) { + GenericTestUtils.assertExceptionContains("too big", e); + } + // Test that setting a max relative expiry on a pool works + CachePoolInfo coolPool = new CachePoolInfo("coolPool"); + final long poolExpiration = 1000 * 60 * 10l; + dfs.addCachePool(coolPool.setMaxRelativeExpiryMs(poolExpiration)); + RemoteIterator poolIt = dfs.listCachePools(); + CachePoolInfo listPool = poolIt.next().getInfo(); + assertFalse("Should only be one pool", poolIt.hasNext()); + assertEquals("Expected max relative expiry to match set value", + poolExpiration, listPool.getMaxRelativeExpiryMs().longValue()); + // Test that negative and really big max expirations can't be modified + try { + dfs.addCachePool(coolPool.setMaxRelativeExpiryMs(-1l)); + fail("Added a pool with a negative max expiry."); + } catch (InvalidRequestException e) { + assertExceptionContains("negative", e); + } + try { + dfs.modifyCachePool(coolPool + .setMaxRelativeExpiryMs(CachePoolInfo.RELATIVE_EXPIRY_NEVER+1)); + fail("Added a pool with too big of a max expiry."); + } catch (InvalidRequestException e) { + assertExceptionContains("too big", e); + } + // Test that adding a directives without an expiration uses the pool's max + CacheDirectiveInfo defaultExpiry = new CacheDirectiveInfo.Builder() + .setPath(new Path("/blah")) + .setPool(coolPool.getPoolName()) + .build(); + dfs.addCacheDirective(defaultExpiry); + RemoteIterator dirIt = + dfs.listCacheDirectives(defaultExpiry); + CacheDirectiveInfo listInfo = dirIt.next().getInfo(); + assertFalse("Should only have one entry in listing", dirIt.hasNext()); + long listExpiration = listInfo.getExpiration().getAbsoluteMillis() + - new Date().getTime(); + assertTrue("Directive expiry should be approximately the pool's max expiry", + Math.abs(listExpiration - poolExpiration) < 10*1000); + // Test that the max is enforced on add for relative and absolute + CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder() + .setPath(new Path("/lolcat")) + .setPool(coolPool.getPoolName()); + try { + dfs.addCacheDirective(builder + .setExpiration(Expiration.newRelative(poolExpiration+1)) + .build()); + fail("Added a directive that exceeds pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + try { + dfs.addCacheDirective(builder + .setExpiration(Expiration.newAbsolute( + new Date().getTime() + poolExpiration + (10*1000))) + .build()); + fail("Added a directive that exceeds pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + // Test that max is enforced on modify for relative and absolute Expirations + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setExpiration(Expiration.newRelative(poolExpiration+1)) + .build()); + fail("Modified a directive to exceed pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setExpiration(Expiration.newAbsolute( + new Date().getTime() + poolExpiration + (10*1000))) + .build()); + fail("Modified a directive to exceed pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + // Test some giant limit values with add + try { + dfs.addCacheDirective(builder + .setExpiration(Expiration.newRelative( + Long.MAX_VALUE)) + .build()); + fail("Added a directive with a gigantic max value"); + } catch (IllegalArgumentException e) { + assertExceptionContains("is too far in the future", e); + } + try { + dfs.addCacheDirective(builder + .setExpiration(Expiration.newAbsolute( + Long.MAX_VALUE)) + .build()); + fail("Added a directive with a gigantic max value"); + } catch (InvalidRequestException e) { + assertExceptionContains("is too far in the future", e); + } + // Test some giant limit values with modify + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setExpiration(Expiration.NEVER) + .build()); + fail("Modified a directive to exceed pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setExpiration(Expiration.newAbsolute( + Long.MAX_VALUE)) + .build()); + fail("Modified a directive to exceed pool's max relative expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("is too far in the future", e); + } + // Test that the max is enforced on modify correctly when changing pools + CachePoolInfo destPool = new CachePoolInfo("destPool"); + dfs.addCachePool(destPool.setMaxRelativeExpiryMs(poolExpiration / 2)); + try { + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setPool(destPool.getPoolName()) + .build()); + fail("Modified a directive to a pool with a lower max expiration"); + } catch (InvalidRequestException e) { + assertExceptionContains("exceeds the max relative expiration", e); + } + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder(defaultExpiry) + .setId(listInfo.getId()) + .setPool(destPool.getPoolName()) + .setExpiration(Expiration.newRelative(poolExpiration / 2)) + .build()); + dirIt = dfs.listCacheDirectives(new CacheDirectiveInfo.Builder() + .setPool(destPool.getPoolName()) + .build()); + listInfo = dirIt.next().getInfo(); + listExpiration = listInfo.getExpiration().getAbsoluteMillis() + - new Date().getTime(); + assertTrue("Unexpected relative expiry " + listExpiration + + " expected approximately " + poolExpiration/2, + Math.abs(poolExpiration/2 - listExpiration) < 10*1000); + // Test that cache pool and directive expiry can be modified back to never + dfs.modifyCachePool(destPool + .setMaxRelativeExpiryMs(CachePoolInfo.RELATIVE_EXPIRY_NEVER)); + poolIt = dfs.listCachePools(); + listPool = poolIt.next().getInfo(); + while (!listPool.getPoolName().equals(destPool.getPoolName())) { + listPool = poolIt.next().getInfo(); + } + assertEquals("Expected max relative expiry to match set value", + CachePoolInfo.RELATIVE_EXPIRY_NEVER, + listPool.getMaxRelativeExpiryMs().longValue()); + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder() + .setId(listInfo.getId()) + .setExpiration(Expiration.newRelative(RELATIVE_EXPIRY_NEVER)) + .build()); + // Test modifying close to the limit + dfs.modifyCacheDirective(new CacheDirectiveInfo.Builder() + .setId(listInfo.getId()) + .setExpiration(Expiration.newRelative(RELATIVE_EXPIRY_NEVER - 1)) + .build()); + } }