hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject svn commit: r1539898 [3/3] - in /hadoop/common/branches/HDFS-2832/hadoop-hdfs-project: hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/ hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/ hadoop-hdfs-nfs/src/test/java/org/apache/had...
Date Fri, 08 Nov 2013 01:44:26 GMT
Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPathBasedCacheRequests.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPathBasedCacheRequests.java?rev=1539898&r1=1539897&r2=1539898&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPathBasedCacheRequests.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestPathBasedCacheRequests.java
Fri Nov  8 01:44:24 2013
@@ -42,6 +42,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.IdNotFoundException;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -49,17 +50,12 @@ import org.apache.hadoop.hdfs.DFSConfigK
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.protocol.AddPathBasedCacheDirectiveException.InvalidPathNameError;
-import org.apache.hadoop.hdfs.protocol.AddPathBasedCacheDirectiveException.InvalidPoolNameError;
-import org.apache.hadoop.hdfs.protocol.AddPathBasedCacheDirectiveException.PoolWritePermissionDeniedError;
 import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
-import org.apache.hadoop.hdfs.protocol.PathBasedCacheDescriptor;
 import org.apache.hadoop.hdfs.protocol.PathBasedCacheDirective;
-import org.apache.hadoop.hdfs.protocol.RemovePathBasedCacheDescriptorException.InvalidIdException;
-import org.apache.hadoop.hdfs.protocol.RemovePathBasedCacheDescriptorException.NoSuchIdException;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.CachedBlocksList.Type;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
 import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.GSet;
@@ -86,7 +82,7 @@ public class TestPathBasedCacheRequests 
     conf = new HdfsConfiguration();
     // set low limits here for testing purposes
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_POOLS_NUM_RESPONSES, 2);
-    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_DESCRIPTORS_NUM_RESPONSES, 2);
+    conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_CACHE_DIRECTIVES_NUM_RESPONSES, 2);
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
     cluster.waitActive();
     dfs = cluster.getFileSystem();
@@ -296,21 +292,21 @@ public class TestPathBasedCacheRequests 
   }
 
   private static void validateListAll(
-      RemoteIterator<PathBasedCacheDescriptor> iter,
-      PathBasedCacheDescriptor... descriptors) throws Exception {
-    for (PathBasedCacheDescriptor descriptor: descriptors) {
+      RemoteIterator<PathBasedCacheDirective> iter,
+      Long... ids) throws Exception {
+    for (Long id: ids) {
       assertTrue("Unexpectedly few elements", iter.hasNext());
-      assertEquals("Unexpected descriptor", descriptor, iter.next());
+      assertEquals("Unexpected directive ID", id, iter.next().getId());
     }
     assertFalse("Unexpectedly many list elements", iter.hasNext());
   }
 
-  private static PathBasedCacheDescriptor addAsUnprivileged(
+  private static long addAsUnprivileged(
       final PathBasedCacheDirective directive) throws Exception {
     return unprivilegedUser
-        .doAs(new PrivilegedExceptionAction<PathBasedCacheDescriptor>() {
+        .doAs(new PrivilegedExceptionAction<Long>() {
           @Override
-          public PathBasedCacheDescriptor run() throws IOException {
+          public Long run() throws IOException {
             DistributedFileSystem myDfs =
                 (DistributedFileSystem) FileSystem.get(conf);
             return myDfs.addPathBasedCacheDirective(directive);
@@ -342,12 +338,12 @@ public class TestPathBasedCacheRequests 
         setPool("pool1").
         build();
 
-    PathBasedCacheDescriptor alphaD = addAsUnprivileged(alpha);
-    PathBasedCacheDescriptor alphaD2 = addAsUnprivileged(alpha);
-    assertFalse("Expected to get unique descriptors when re-adding an "
+    long alphaId = addAsUnprivileged(alpha);
+    long alphaId2 = addAsUnprivileged(alpha);
+    assertFalse("Expected to get unique directives when re-adding an "
         + "existing PathBasedCacheDirective",
-        alphaD.getEntryId() == alphaD2.getEntryId());
-    PathBasedCacheDescriptor betaD = addAsUnprivileged(beta);
+        alphaId == alphaId2);
+    long betaId = addAsUnprivileged(beta);
 
     try {
       addAsUnprivileged(new PathBasedCacheDirective.Builder().
@@ -355,8 +351,8 @@ public class TestPathBasedCacheRequests 
           setPool("no_such_pool").
           build());
       fail("expected an error when adding to a non-existent pool.");
-    } catch (IOException ioe) {
-      assertTrue(ioe instanceof InvalidPoolNameError);
+    } catch (IdNotFoundException ioe) {
+      GenericTestUtils.assertExceptionContains("no such pool as", ioe);
     }
 
     try {
@@ -366,8 +362,9 @@ public class TestPathBasedCacheRequests 
           build());
       fail("expected an error when adding to a pool with " +
           "mode 0 (no permissions for anyone).");
-    } catch (IOException ioe) {
-      assertTrue(ioe instanceof PoolWritePermissionDeniedError);
+    } catch (AccessControlException e) {
+      GenericTestUtils.
+          assertExceptionContains("permission denied for pool", e);
     }
 
     try {
@@ -378,7 +375,7 @@ public class TestPathBasedCacheRequests 
       fail("expected an error when adding a malformed path " +
           "to the cache directives.");
     } catch (IllegalArgumentException e) {
-      // expected
+      GenericTestUtils.assertExceptionContains("is not a valid DFS filename", e);
     }
 
     try {
@@ -389,59 +386,74 @@ public class TestPathBasedCacheRequests 
           build());
       Assert.fail("expected an error when adding a PathBasedCache " +
           "directive with an empty pool name.");
-    } catch (IOException ioe) {
-      Assert.assertTrue(ioe instanceof InvalidPoolNameError);
+    } catch (IdNotFoundException e) {
+      GenericTestUtils.assertExceptionContains("pool name was empty", e);
     }
 
-    PathBasedCacheDescriptor deltaD = addAsUnprivileged(delta);
+    long deltaId = addAsUnprivileged(delta);
 
     // We expect the following to succeed, because DistributedFileSystem
     // qualifies the path.
-    PathBasedCacheDescriptor relativeD = addAsUnprivileged(
+    long relativeId = addAsUnprivileged(
         new PathBasedCacheDirective.Builder().
             setPath(new Path("relative")).
             setPool("pool1").
             build());
 
-    RemoteIterator<PathBasedCacheDescriptor> iter;
-    iter = dfs.listPathBasedCacheDescriptors(null, null);
-    validateListAll(iter, alphaD, alphaD2, betaD, deltaD, relativeD);
-    iter = dfs.listPathBasedCacheDescriptors("pool3", null);
+    RemoteIterator<PathBasedCacheDirective> iter;
+    iter = dfs.listPathBasedCacheDirectives(null);
+    validateListAll(iter, alphaId, alphaId2, betaId, deltaId, relativeId );
+    iter = dfs.listPathBasedCacheDirectives(
+        new PathBasedCacheDirective.Builder().setPool("pool3").build());
     Assert.assertFalse(iter.hasNext());
-    iter = dfs.listPathBasedCacheDescriptors("pool1", null);
-    validateListAll(iter, alphaD, alphaD2, deltaD, relativeD);
-    iter = dfs.listPathBasedCacheDescriptors("pool2", null);
-    validateListAll(iter, betaD);
-
-    dfs.removePathBasedCacheDescriptor(betaD);
-    iter = dfs.listPathBasedCacheDescriptors("pool2", null);
+    iter = dfs.listPathBasedCacheDirectives(
+        new PathBasedCacheDirective.Builder().setPool("pool1").build());
+    validateListAll(iter, alphaId, alphaId2, deltaId, relativeId );
+    iter = dfs.listPathBasedCacheDirectives(
+        new PathBasedCacheDirective.Builder().setPool("pool2").build());
+    validateListAll(iter, betaId);
+
+    dfs.removePathBasedCacheDirective(betaId);
+    iter = dfs.listPathBasedCacheDirectives(
+        new PathBasedCacheDirective.Builder().setPool("pool2").build());
     Assert.assertFalse(iter.hasNext());
 
     try {
-      dfs.removePathBasedCacheDescriptor(betaD);
+      dfs.removePathBasedCacheDirective(betaId);
       Assert.fail("expected an error when removing a non-existent ID");
-    } catch (IOException ioe) {
-      Assert.assertTrue(ioe instanceof NoSuchIdException);
+    } catch (IdNotFoundException e) {
+      GenericTestUtils.assertExceptionContains("id not found", e);
     }
 
     try {
-      proto.removePathBasedCacheDescriptor(-42l);
+      proto.removePathBasedCacheDirective(-42l);
       Assert.fail("expected an error when removing a negative ID");
-    } catch (IOException ioe) {
-      Assert.assertTrue(ioe instanceof InvalidIdException);
+    } catch (IdNotFoundException e) {
+      GenericTestUtils.assertExceptionContains(
+          "invalid non-positive directive ID", e);
     }
     try {
-      proto.removePathBasedCacheDescriptor(43l);
+      proto.removePathBasedCacheDirective(43l);
       Assert.fail("expected an error when removing a non-existent ID");
-    } catch (IOException ioe) {
-      Assert.assertTrue(ioe instanceof NoSuchIdException);
+    } catch (IdNotFoundException e) {
+      GenericTestUtils.assertExceptionContains("id not found", e);
     }
 
-    dfs.removePathBasedCacheDescriptor(alphaD);
-    dfs.removePathBasedCacheDescriptor(alphaD2);
-    dfs.removePathBasedCacheDescriptor(deltaD);
-    dfs.removePathBasedCacheDescriptor(relativeD);
-    iter = dfs.listPathBasedCacheDescriptors(null, null);
+    dfs.removePathBasedCacheDirective(alphaId);
+    dfs.removePathBasedCacheDirective(alphaId2);
+    dfs.removePathBasedCacheDirective(deltaId);
+
+    dfs.modifyPathBasedCacheDirective(new PathBasedCacheDirective.Builder().
+        setId(relativeId).
+        setReplication((short)555).
+        build());
+    iter = dfs.listPathBasedCacheDirectives(null);
+    assertTrue(iter.hasNext());
+    PathBasedCacheDirective modified = iter.next();
+    assertEquals(relativeId, modified.getId().longValue());
+    assertEquals((short)555, modified.getReplication().shortValue());
+    dfs.removePathBasedCacheDirective(relativeId);
+    iter = dfs.listPathBasedCacheDirectives(null);
     assertFalse(iter.hasNext());
   }
 
@@ -481,16 +493,16 @@ public class TestPathBasedCacheRequests 
           new PathBasedCacheDirective.Builder().
             setPath(new Path(entryPrefix + i)).setPool(pool).build());
     }
-    RemoteIterator<PathBasedCacheDescriptor> dit
-        = dfs.listPathBasedCacheDescriptors(null, null);
+    RemoteIterator<PathBasedCacheDirective> dit
+        = dfs.listPathBasedCacheDirectives(null);
     for (int i=0; i<numEntries; i++) {
       assertTrue("Unexpected # of cache entries: " + i, dit.hasNext());
-      PathBasedCacheDescriptor cd = dit.next();
-      assertEquals(i+1, cd.getEntryId());
+      PathBasedCacheDirective cd = dit.next();
+      assertEquals(i+1, cd.getId().longValue());
       assertEquals(entryPrefix + i, cd.getPath().toUri().getPath());
       assertEquals(pool, cd.getPool());
     }
-    assertFalse("Unexpected # of cache descriptors found", dit.hasNext());
+    assertFalse("Unexpected # of cache directives found", dit.hasNext());
   
     // Restart namenode
     cluster.restartNameNode();
@@ -506,15 +518,15 @@ public class TestPathBasedCacheRequests 
     assertEquals(weight, (int)info.getWeight());
     assertFalse("Unexpected # of cache pools found", pit.hasNext());
   
-    dit = dfs.listPathBasedCacheDescriptors(null, null);
+    dit = dfs.listPathBasedCacheDirectives(null);
     for (int i=0; i<numEntries; i++) {
       assertTrue("Unexpected # of cache entries: " + i, dit.hasNext());
-      PathBasedCacheDescriptor cd = dit.next();
-      assertEquals(i+1, cd.getEntryId());
+      PathBasedCacheDirective cd = dit.next();
+      assertEquals(i+1, cd.getId().longValue());
       assertEquals(entryPrefix + i, cd.getPath().toUri().getPath());
       assertEquals(pool, cd.getPool());
     }
-    assertFalse("Unexpected # of cache descriptors found", dit.hasNext());
+    assertFalse("Unexpected # of cache directives found", dit.hasNext());
   }
 
   private static void waitForCachedBlocks(NameNode nn,
@@ -625,21 +637,16 @@ public class TestPathBasedCacheRequests 
               setPath(new Path(paths.get(i))).
               setPool(pool).
               build();
-        PathBasedCacheDescriptor descriptor =
-            nnRpc.addPathBasedCacheDirective(directive);
-        assertEquals("Descriptor does not match requested path",
-            new Path(paths.get(i)), descriptor.getPath());
-        assertEquals("Descriptor does not match requested pool", pool,
-            descriptor.getPool());
+        nnRpc.addPathBasedCacheDirective(directive);
         expected += numBlocksPerFile;
         waitForCachedBlocks(namenode, expected, expected);
       }
       // Uncache and check each path in sequence
-      RemoteIterator<PathBasedCacheDescriptor> entries =
-          nnRpc.listPathBasedCacheDescriptors(0, null, null);
+      RemoteIterator<PathBasedCacheDirective> entries =
+          nnRpc.listPathBasedCacheDirectives(0, null);
       for (int i=0; i<numFiles; i++) {
-        PathBasedCacheDescriptor descriptor = entries.next();
-        nnRpc.removePathBasedCacheDescriptor(descriptor.getEntryId());
+        PathBasedCacheDirective directive = entries.next();
+        nnRpc.removePathBasedCacheDirective(directive.getId());
         expected -= numBlocksPerFile;
         waitForCachedBlocks(namenode, expected, expected);
       }
@@ -723,17 +730,15 @@ public class TestPathBasedCacheRequests 
       }
       waitForCachedBlocks(namenode, 0, 0);
       // cache entire directory
-      PathBasedCacheDescriptor descriptor = dfs.addPathBasedCacheDirective(
+      long id = dfs.addPathBasedCacheDirective(
             new PathBasedCacheDirective.Builder().
               setPath(new Path("/foo")).
               setReplication((short)2).
               setPool(pool).
               build());
-      assertEquals("Descriptor does not match requested pool", pool,
-          descriptor.getPool());
       waitForCachedBlocks(namenode, 4, 8);
       // remove and watch numCached go to 0
-      dfs.removePathBasedCacheDescriptor(descriptor);
+      dfs.removePathBasedCacheDirective(id);
       waitForCachedBlocks(namenode, 0, 0);
     } finally {
       cluster.shutdown();

Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java?rev=1539898&r1=1539897&r2=1539898&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
Fri Nov  8 01:44:24 2013
@@ -61,7 +61,6 @@ import org.apache.hadoop.hdfs.protocol.E
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.protocol.PathBasedCacheDescriptor;
 import org.apache.hadoop.hdfs.protocol.PathBasedCacheDirective;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -151,7 +150,7 @@ public class TestRetryCacheWithHA {
     FSNamesystem fsn0 = cluster.getNamesystem(0);
     LightWeightCache<CacheEntry, CacheEntry> cacheSet = 
         (LightWeightCache<CacheEntry, CacheEntry>) fsn0.getRetryCache().getCacheSet();
-    assertEquals(19, cacheSet.size());
+    assertEquals(20, cacheSet.size());
     
     Map<CacheEntry, CacheEntry> oldEntries = 
         new HashMap<CacheEntry, CacheEntry>();
@@ -172,7 +171,7 @@ public class TestRetryCacheWithHA {
     FSNamesystem fsn1 = cluster.getNamesystem(1);
     cacheSet = (LightWeightCache<CacheEntry, CacheEntry>) fsn1
         .getRetryCache().getCacheSet();
-    assertEquals(19, cacheSet.size());
+    assertEquals(20, cacheSet.size());
     iter = cacheSet.iterator();
     while (iter.hasNext()) {
       CacheEntry entry = iter.next();
@@ -740,35 +739,34 @@ public class TestRetryCacheWithHA {
   
   /** addPathBasedCacheDirective */
   class AddPathBasedCacheDirectiveOp extends AtMostOnceOp {
-    private String pool;
-    private String path;
-    private PathBasedCacheDescriptor descriptor;
+    private PathBasedCacheDirective directive;
+    private Long result;
 
-    AddPathBasedCacheDirectiveOp(DFSClient client, String pool, String path) {
+    AddPathBasedCacheDirectiveOp(DFSClient client,
+        PathBasedCacheDirective directive) {
       super("addPathBasedCacheDirective", client);
-      this.pool = pool;
-      this.path = path;
+      this.directive = directive;
     }
 
     @Override
     void prepare() throws Exception {
-      dfs.addCachePool(new CachePoolInfo(pool));
+      dfs.addCachePool(new CachePoolInfo(directive.getPool()));
     }
 
     @Override
     void invoke() throws Exception {
-      descriptor = client.addPathBasedCacheDirective(
-          new PathBasedCacheDirective.Builder().
-              setPath(new Path(path)).
-              setPool(pool).
-              build());
+      result = client.addPathBasedCacheDirective(directive);
     }
 
     @Override
     boolean checkNamenodeBeforeReturn() throws Exception {
       for (int i = 0; i < CHECKTIMES; i++) {
-        RemoteIterator<PathBasedCacheDescriptor> iter =
-            dfs.listPathBasedCacheDescriptors(pool, new Path(path));
+        RemoteIterator<PathBasedCacheDirective> iter =
+            dfs.listPathBasedCacheDirectives(
+                new PathBasedCacheDirective.Builder().
+                    setPool(directive.getPool()).
+                    setPath(directive.getPath()).
+                    build());
         if (iter.hasNext()) {
           return true;
         }
@@ -779,43 +777,99 @@ public class TestRetryCacheWithHA {
 
     @Override
     Object getResult() {
-      return descriptor;
+      return result;
     }
   }
 
-  /** removePathBasedCacheDescriptor */
-  class RemovePathBasedCacheDescriptorOp extends AtMostOnceOp {
-    private String pool;
-    private String path;
-    private PathBasedCacheDescriptor descriptor;
+  /** modifyPathBasedCacheDirective */
+  class ModifyPathBasedCacheDirectiveOp extends AtMostOnceOp {
+    private final PathBasedCacheDirective directive;
+    private final short newReplication;
+    private long id;
 
-    RemovePathBasedCacheDescriptorOp(DFSClient client, String pool,
-        String path) {
-      super("removePathBasedCacheDescriptor", client);
-      this.pool = pool;
-      this.path = path;
+    ModifyPathBasedCacheDirectiveOp(DFSClient client,
+        PathBasedCacheDirective directive, short newReplication) {
+      super("modifyPathBasedCacheDirective", client);
+      this.directive = directive;
+      this.newReplication = newReplication;
     }
 
     @Override
     void prepare() throws Exception {
-      dfs.addCachePool(new CachePoolInfo(pool));
-      descriptor = dfs.addPathBasedCacheDirective(
+      dfs.addCachePool(new CachePoolInfo(directive.getPool()));
+      id = client.addPathBasedCacheDirective(directive);
+    }
+
+    @Override
+    void invoke() throws Exception {
+      client.modifyPathBasedCacheDirective(
           new PathBasedCacheDirective.Builder().
-              setPath(new Path(path)).
-              setPool(pool).
+              setId(id).
+              setReplication(newReplication).
               build());
     }
 
     @Override
+    boolean checkNamenodeBeforeReturn() throws Exception {
+      for (int i = 0; i < CHECKTIMES; i++) {
+        RemoteIterator<PathBasedCacheDirective> iter =
+            dfs.listPathBasedCacheDirectives(
+                new PathBasedCacheDirective.Builder().
+                    setPool(directive.getPool()).
+                    setPath(directive.getPath()).
+                    build());
+        while (iter.hasNext()) {
+          PathBasedCacheDirective result = iter.next();
+          if ((result.getId() == id) &&
+              (result.getReplication().shortValue() == newReplication)) {
+            return true;
+          }
+        }
+        Thread.sleep(1000);
+      }
+      return false;
+    }
+
+    @Override
+    Object getResult() {
+      return null;
+    }
+  }
+
+  /** removePathBasedCacheDirective */
+  class RemovePathBasedCacheDirectiveOp extends AtMostOnceOp {
+    private PathBasedCacheDirective directive;
+    private long id;
+
+    RemovePathBasedCacheDirectiveOp(DFSClient client, String pool,
+        String path) {
+      super("removePathBasedCacheDirective", client);
+      this.directive = new PathBasedCacheDirective.Builder().
+          setPool(pool).
+          setPath(new Path(path)).
+          build();
+    }
+
+    @Override
+    void prepare() throws Exception {
+      dfs.addCachePool(new CachePoolInfo(directive.getPool()));
+      id = dfs.addPathBasedCacheDirective(directive);
+    }
+
+    @Override
     void invoke() throws Exception {
-      client.removePathBasedCacheDescriptor(descriptor.getEntryId());
+      client.removePathBasedCacheDirective(id);
     }
 
     @Override
     boolean checkNamenodeBeforeReturn() throws Exception {
       for (int i = 0; i < CHECKTIMES; i++) {
-        RemoteIterator<PathBasedCacheDescriptor> iter =
-            dfs.listPathBasedCacheDescriptors(pool, new Path(path));
+        RemoteIterator<PathBasedCacheDirective> iter =
+            dfs.listPathBasedCacheDirectives(
+                new PathBasedCacheDirective.Builder().
+                  setPool(directive.getPool()).
+                  setPath(directive.getPath()).
+                  build());
         if (!iter.hasNext()) {
           return true;
         }
@@ -1020,14 +1074,30 @@ public class TestRetryCacheWithHA {
   @Test (timeout=60000)
   public void testAddPathBasedCacheDirective() throws Exception {
     DFSClient client = genClientWithDummyHandler();
-    AtMostOnceOp op = new AddPathBasedCacheDirectiveOp(client, "pool", "/path");
+    AtMostOnceOp op = new AddPathBasedCacheDirectiveOp(client, 
+        new PathBasedCacheDirective.Builder().
+            setPool("pool").
+            setPath(new Path("/path")).
+            build());
+    testClientRetryWithFailover(op);
+  }
+
+  @Test (timeout=60000)
+  public void testModifyPathBasedCacheDirective() throws Exception {
+    DFSClient client = genClientWithDummyHandler();
+    AtMostOnceOp op = new ModifyPathBasedCacheDirectiveOp(client, 
+        new PathBasedCacheDirective.Builder().
+            setPool("pool").
+            setPath(new Path("/path")).
+            setReplication((short)1).build(),
+        (short)555);
     testClientRetryWithFailover(op);
   }
 
   @Test (timeout=60000)
   public void testRemovePathBasedCacheDescriptor() throws Exception {
     DFSClient client = genClientWithDummyHandler();
-    AtMostOnceOp op = new RemovePathBasedCacheDescriptorOp(client, "pool",
+    AtMostOnceOp op = new RemovePathBasedCacheDirectiveOp(client, "pool",
         "/path");
     testClientRetryWithFailover(op);
   }

Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java?rev=1539898&r1=1539897&r2=1539898&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
Fri Nov  8 01:44:24 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.DFSConfigK
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -274,4 +275,76 @@ public class TestSnapshotBlocksMap {
         "s2/bar");
     DFSTestUtil.readFile(hdfs, new Path(bar2SnapshotPath));
   }
+
+  /**
+   * Make sure we delete 0-sized block when deleting an INodeFileUCWithSnapshot
+   */
+  @Test
+  public void testDeletionWithZeroSizeBlock() throws Exception {
+    final Path foo = new Path("/foo");
+    final Path bar = new Path(foo, "bar");
+    DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPLICATION, 0L);
+
+    SnapshotTestHelper.createSnapshot(hdfs, foo, "s0");
+    hdfs.append(bar);
+
+    INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
+    BlockInfo[] blks = barNode.getBlocks();
+    assertEquals(1, blks.length);
+    assertEquals(BLOCKSIZE, blks[0].getNumBytes());
+    ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
+    cluster.getNameNodeRpc()
+        .addBlock(bar.toString(), hdfs.getClient().getClientName(), previous,
+            null, barNode.getId(), null);
+
+    SnapshotTestHelper.createSnapshot(hdfs, foo, "s1");
+
+    barNode = fsdir.getINode4Write(bar.toString()).asFile();
+    blks = barNode.getBlocks();
+    assertEquals(2, blks.length);
+    assertEquals(BLOCKSIZE, blks[0].getNumBytes());
+    assertEquals(0, blks[1].getNumBytes());
+
+    hdfs.delete(bar, true);
+    final Path sbar = SnapshotTestHelper.getSnapshotPath(foo, "s1",
+        bar.getName());
+    barNode = fsdir.getINode(sbar.toString()).asFile();
+    blks = barNode.getBlocks();
+    assertEquals(1, blks.length);
+    assertEquals(BLOCKSIZE, blks[0].getNumBytes());
+  }
+
+  /** Make sure we delete 0-sized block when deleting an INodeFileUC */
+  @Test
+  public void testDeletionWithZeroSizeBlock2() throws Exception {
+    final Path foo = new Path("/foo");
+    final Path subDir = new Path(foo, "sub");
+    final Path bar = new Path(subDir, "bar");
+    DFSTestUtil.createFile(hdfs, bar, BLOCKSIZE, REPLICATION, 0L);
+
+    hdfs.append(bar);
+
+    INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
+    BlockInfo[] blks = barNode.getBlocks();
+    assertEquals(1, blks.length);
+    ExtendedBlock previous = new ExtendedBlock(fsn.getBlockPoolId(), blks[0]);
+    cluster.getNameNodeRpc()
+        .addBlock(bar.toString(), hdfs.getClient().getClientName(), previous,
+            null, barNode.getId(), null);
+
+    SnapshotTestHelper.createSnapshot(hdfs, foo, "s1");
+
+    barNode = fsdir.getINode4Write(bar.toString()).asFile();
+    blks = barNode.getBlocks();
+    assertEquals(2, blks.length);
+    assertEquals(BLOCKSIZE, blks[0].getNumBytes());
+    assertEquals(0, blks[1].getNumBytes());
+
+    hdfs.delete(subDir, true);
+    final Path sbar = SnapshotTestHelper.getSnapshotPath(foo, "s1", "sub/bar");
+    barNode = fsdir.getINode(sbar.toString()).asFile();
+    blks = barNode.getBlocks();
+    assertEquals(1, blks.length);
+    assertEquals(BLOCKSIZE, blks[0].getNumBytes());
+  }
 }

Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored?rev=1539898&r1=1539897&r2=1539898&view=diff
==============================================================================
Binary files - no diff available.

Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml?rev=1539898&r1=1539897&r2=1539898&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
Fri Nov  8 01:44:24 2013
@@ -843,6 +843,7 @@
     <OPCODE>OP_ADD_PATH_BASED_CACHE_DIRECTIVE</OPCODE>
     <DATA>
       <TXID>63</TXID>
+      <ID>1</ID>
       <PATH>/bar</PATH>
       <REPLICATION>1</REPLICATION>
       <POOL>poolparty</POOL>
@@ -851,10 +852,20 @@
     </DATA>
   </RECORD>
   <RECORD>
-    <OPCODE>OP_REMOVE_PATH_BASED_CACHE_DESCRIPTOR</OPCODE>
+    <OPCODE>OP_MODIFY_PATH_BASED_CACHE_DIRECTIVE</OPCODE>
     <DATA>
       <TXID>64</TXID>
       <ID>1</ID>
+      <REPLICATION>2</REPLICATION>
+      <RPC_CLIENTID></RPC_CLIENTID>
+      <RPC_CALLID>-2</RPC_CALLID>
+    </DATA>
+  </RECORD>
+  <RECORD>
+    <OPCODE>OP_REMOVE_PATH_BASED_CACHE_DIRECTIVE</OPCODE>
+    <DATA>
+      <TXID>65</TXID>
+      <ID>1</ID>
       <RPC_CLIENTID>27ac79f0-d378-4933-824b-c2a188968d97</RPC_CLIENTID>
       <RPC_CALLID>78</RPC_CALLID>
     </DATA>
@@ -862,7 +873,7 @@
   <RECORD>
     <OPCODE>OP_REMOVE_CACHE_POOL</OPCODE>
     <DATA>
-      <TXID>65</TXID>
+      <TXID>66</TXID>
       <POOLNAME>poolparty</POOLNAME>
       <RPC_CLIENTID>27ac79f0-d378-4933-824b-c2a188968d97</RPC_CLIENTID>
       <RPC_CALLID>79</RPC_CALLID>
@@ -871,7 +882,7 @@
   <RECORD>
     <OPCODE>OP_ADD</OPCODE>
     <DATA>
-      <TXID>66</TXID>
+      <TXID>67</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>16393</INODEID>
       <PATH>/hard-lease-recovery-test</PATH>
@@ -893,21 +904,21 @@
   <RECORD>
     <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
     <DATA>
-      <TXID>67</TXID>
+      <TXID>68</TXID>
       <BLOCK_ID>1073741834</BLOCK_ID>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>68</TXID>
+      <TXID>69</TXID>
       <GENSTAMPV2>1010</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_UPDATE_BLOCKS</OPCODE>
     <DATA>
-      <TXID>69</TXID>
+      <TXID>70</TXID>
       <PATH>/hard-lease-recovery-test</PATH>
       <BLOCK>
         <BLOCK_ID>1073741834</BLOCK_ID>
@@ -921,7 +932,7 @@
   <RECORD>
     <OPCODE>OP_UPDATE_BLOCKS</OPCODE>
     <DATA>
-      <TXID>70</TXID>
+      <TXID>71</TXID>
       <PATH>/hard-lease-recovery-test</PATH>
       <BLOCK>
         <BLOCK_ID>1073741834</BLOCK_ID>
@@ -935,14 +946,14 @@
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>71</TXID>
+      <TXID>72</TXID>
       <GENSTAMPV2>1011</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REASSIGN_LEASE</OPCODE>
     <DATA>
-      <TXID>72</TXID>
+      <TXID>73</TXID>
       <LEASEHOLDER>DFSClient_NONMAPREDUCE_-134124999_1</LEASEHOLDER>
       <PATH>/hard-lease-recovery-test</PATH>
       <NEWHOLDER>HDFS_NameNode</NEWHOLDER>
@@ -951,14 +962,14 @@
   <RECORD>
     <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
     <DATA>
-      <TXID>73</TXID>
+      <TXID>74</TXID>
       <GENSTAMPV2>1012</GENSTAMPV2>
     </DATA>
   </RECORD>
   <RECORD>
     <OPCODE>OP_REASSIGN_LEASE</OPCODE>
     <DATA>
-      <TXID>74</TXID>
+      <TXID>75</TXID>
       <LEASEHOLDER>HDFS_NameNode</LEASEHOLDER>
       <PATH>/hard-lease-recovery-test</PATH>
       <NEWHOLDER>HDFS_NameNode</NEWHOLDER>
@@ -967,7 +978,7 @@
   <RECORD>
     <OPCODE>OP_CLOSE</OPCODE>
     <DATA>
-      <TXID>75</TXID>
+      <TXID>76</TXID>
       <LENGTH>0</LENGTH>
       <INODEID>0</INODEID>
       <PATH>/hard-lease-recovery-test</PATH>
@@ -992,7 +1003,7 @@
   <RECORD>
     <OPCODE>OP_END_LOG_SEGMENT</OPCODE>
     <DATA>
-      <TXID>76</TXID>
+      <TXID>77</TXID>
     </DATA>
   </RECORD>
 </EDITS>

Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml?rev=1539898&r1=1539897&r2=1539898&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
(original)
+++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
Fri Nov  8 01:44:24 2013
@@ -358,5 +358,20 @@
       </comparators>
     </test>
 
+    <test>
+      <description>Testing the help usage</description>
+      <test-commands>
+        <cache-admin-command>-help -addPool</cache-admin-command>
+      </test-commands>
+      <cleanup-commands>
+      </cleanup-commands>
+      <comparators>
+        <comparator>
+          <type>SubstringComparator</type>
+          <expected-output>Add a new cache pool.</expected-output>
+        </comparator>
+      </comparators>
+    </test>
+
   </tests>
 </configuration>



Mime
View raw message