Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7CD691091B for ; Fri, 22 Nov 2013 20:51:33 +0000 (UTC) Received: (qmail 95647 invoked by uid 500); 22 Nov 2013 20:51:33 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 95598 invoked by uid 500); 22 Nov 2013 20:51:33 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 95590 invoked by uid 99); 22 Nov 2013 20:51:33 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 Nov 2013 20:51:33 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,WEIRD_QUOTING X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 Nov 2013 20:51:29 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7091B2388A74; Fri, 22 Nov 2013 20:51:09 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1544672 [2/2] - in /hadoop/common/branches/HDFS-2832/hadoop-hdfs-project: hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/ hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/ hadoop-hdfs/ hadoop-hdfs/src/main/java/ hado... Date: Fri, 22 Nov 2013 20:51:08 -0000 To: hdfs-commits@hadoop.apache.org From: arp@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131122205109.7091B2388A74@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js Fri Nov 22 20:51:06 2013 @@ -1,8 +1,9 @@ -(function(k){function n(b){b=b.f();return"object"===typeof b&&!0===b.h}function p(b,c){return"function"===typeof c?c.toString():c}function l(b,c,d,a,e){a=a||{};var m=d.a,g,f,l=a.d||"";if("undefined"!==typeof a.key)g=k.b.c(a.key,b,c);else if(n(c))g=c.f().i,c.f().g&&(e=function(){return!1});else return h.log("No key specified for filter in:"+l+" helper "),b;f=k.b.c(a.value,b,c);if(e(q(f,a.type,c),q(g,a.type,c))){n(c)&&(c.f().g=!0);if(m)return b.e(m,c);h.log("Missing body block in the "+l+" helper ")}else if(d["else"])return b.e(d["else"], -c);return b}function q(b,c,d){if(b)switch(c||typeof b){case "number":return+b;case "string":return String(b);case "boolean":return Boolean("false"===b?!1:b);case "date":return new Date(b);case "context":return d.get(b)}return b}var h="undefined"!==typeof console?console:{log:function(){}};k.b={tap:function(b,c,d){var a=b;"function"===typeof b&&(!0===b.l?a=b():(a="",c.c(function(b){a+=b;return""}).e(b,d).p(),""===a&&(a=!1)));return a},sep:function(b,c,d){return c.stack.index===c.stack.m-1?b:d.a?d.a(b, -c):b},idx:function(b,c,d){return d.a?d.a(b,c.push(c.stack.index)):b},contextDump:function(b,c,d,a){a=a||{};d=a.o||"output";a=a.key||"current";d=k.b.c(d,b,c);a=k.b.c(a,b,c);c="full"===a?JSON.stringify(c.stack,p,2):JSON.stringify(c.stack.head,p,2);return"console"===d?(h.log(c),b):b.write(c)},"if":function(b,c,d,a){var e=d.a,m=d["else"];if(a&&a.j){a=a.j;a=k.b.c(a,b,c);if(eval(a)){if(e)return b.e(d.a,c);h.log("Missing body block in the if helper!");return b}if(m)return b.e(d["else"],c)}else h.log("No condition given in the if helper!"); -return b},math:function(b,c,d,a){if(a&&"undefined"!==typeof a.key&&a.method){var e=a.key,m=a.method,g=a.n;a=a.round;var f=null,e=k.b.c(e,b,c),g=k.b.c(g,b,c);switch(m){case "mod":0!==g&&-0!==g||h.log("operand for divide operation is 0/-0: expect Nan!");f=parseFloat(e)%parseFloat(g);break;case "add":f=parseFloat(e)+parseFloat(g);break;case "subtract":f=parseFloat(e)-parseFloat(g);break;case "multiply":f=parseFloat(e)*parseFloat(g);break;case "divide":0!==g&&-0!==g||h.log("operand for divide operation is 0/-0: expect Nan/Infinity!"); -f=parseFloat(e)/parseFloat(g);break;case "ceil":f=Math.ceil(parseFloat(e));break;case "floor":f=Math.floor(parseFloat(e));break;case "round":f=Math.round(parseFloat(e));break;case "abs":f=Math.abs(parseFloat(e));break;default:h.log("method passed is not supported")}if(null!==f)return a&&(f=Math.round(f)),d&&d.a?b.e(d.a,c.push({h:!0,g:!1,i:f})):b.write(f)}else h.log("Key is a required parameter for math helper along with method/operand!");return b},select:function(b,c,d,a){var e=d.a;if(a&&"undefined"!== -typeof a.key){a=k.b.c(a.key,b,c);if(e)return b.e(d.a,c.push({h:!0,g:!1,i:a}));h.log("Missing body block in the select helper ")}else h.log("No key given in the select helper!");return b},eq:function(b,c,d,a){a&&(a.d="eq");return l(b,c,d,a,function(a,b){return b===a})},ne:function(b,c,d,a){return a?(a.d="ne",l(b,c,d,a,function(a,b){return b!==a})):b},lt:function(b,c,d,a){if(a)return a.d="lt",l(b,c,d,a,function(a,b){return ba})):b},gte:function(b,c,d,a){return a?(a.d="gte",l(b,c,d,a,function(a,b){return b>=a})):b},"default":function(b,c,d,a){a&&(a.d="default");return l(b,c,d,a,function(){return!0})},size:function(b,c,d,a){c=0;var e;a=a||{};if((a=a.key)&&!0!==a)if(k.isArray(a))c=a.length;else if(!isNaN(parseFloat(a))&&isFinite(a))c=a;else if("object"===typeof a)for(e in c=0,a)Object.hasOwnProperty.call(a,e)&&c++;else c=(a+"").length;else c= -0;return b.write(c)}}})("undefined"!==typeof exports?module.k=require("dustjs-linkedin"):dust); +(function(k){function n(b){b=b.current();return"object"===typeof b&&!0===b.isSelect}function p(b,c){return"function"===typeof c?c.toString():c}function l(b,c,d,a,e){a=a||{};var m=d.block,g,f,l=a.filterOpType||"";if("undefined"!==typeof a.key)g=k.helpers.tap(a.key,b,c);else if(n(c))g=c.current().selectKey,c.current().isResolved&&(e=function(){return!1});else return h.log("No key specified for filter in:"+l+" helper "),b;f=k.helpers.tap(a.value,b,c);if(e(q(f,a.type,c),q(g,a.type,c))){n(c)&&(c.current().isResolved= +!0);if(m)return b.render(m,c);h.log("Missing body block in the "+l+" helper ")}else if(d["else"])return b.render(d["else"],c);return b}function q(b,c,d){if(b)switch(c||typeof b){case "number":return+b;case "string":return String(b);case "boolean":return Boolean("false"===b?!1:b);case "date":return new Date(b);case "context":return d.get(b)}return b}var h="undefined"!==typeof console?console:{log:function(){}};k.helpers={tap:function(b,c,d){var a=b;"function"===typeof b&&(!0===b.isFunction?a=b():(a= +"",c.tap(function(b){a+=b;return""}).render(b,d).untap(),""===a&&(a=!1)));return a},sep:function(b,c,d){return c.stack.index===c.stack.of-1?b:d.block?d.block(b,c):b},idx:function(b,c,d){return d.block?d.block(b,c.push(c.stack.index)):b},contextDump:function(b,c,d,a){a=a||{};d=a.to||"output";a=a.key||"current";d=k.helpers.tap(d,b,c);a=k.helpers.tap(a,b,c);c="full"===a?JSON.stringify(c.stack,p,2):JSON.stringify(c.stack.head,p,2);return"console"===d?(h.log(c),b):b.write(c)},"if":function(b,c,d,a){var e= +d.block,m=d["else"];if(a&&a.cond){a=a.cond;a=k.helpers.tap(a,b,c);if(eval(a)){if(e)return b.render(d.block,c);h.log("Missing body block in the if helper!");return b}if(m)return b.render(d["else"],c)}else h.log("No condition given in the if helper!");return b},math:function(b,c,d,a){if(a&&"undefined"!==typeof a.key&&a.method){var e=a.key,m=a.method,g=a.operand;a=a.round;var f=null,e=k.helpers.tap(e,b,c),g=k.helpers.tap(g,b,c);switch(m){case "mod":0!==g&&-0!==g||h.log("operand for divide operation is 0/-0: expect Nan!"); +f=parseFloat(e)%parseFloat(g);break;case "add":f=parseFloat(e)+parseFloat(g);break;case "subtract":f=parseFloat(e)-parseFloat(g);break;case "multiply":f=parseFloat(e)*parseFloat(g);break;case "divide":0!==g&&-0!==g||h.log("operand for divide operation is 0/-0: expect Nan/Infinity!");f=parseFloat(e)/parseFloat(g);break;case "ceil":f=Math.ceil(parseFloat(e));break;case "floor":f=Math.floor(parseFloat(e));break;case "round":f=Math.round(parseFloat(e));break;case "abs":f=Math.abs(parseFloat(e));break; +default:h.log("method passed is not supported")}if(null!==f)return a&&(f=Math.round(f)),d&&d.block?b.render(d.block,c.push({isSelect:!0,isResolved:!1,selectKey:f})):b.write(f)}else h.log("Key is a required parameter for math helper along with method/operand!");return b},select:function(b,c,d,a){var e=d.block;if(a&&"undefined"!==typeof a.key){a=k.helpers.tap(a.key,b,c);if(e)return b.render(d.block,c.push({isSelect:!0,isResolved:!1,selectKey:a}));h.log("Missing body block in the select helper ")}else h.log("No key given in the select helper!"); +return b},eq:function(b,c,d,a){a&&(a.filterOpType="eq");return l(b,c,d,a,function(a,b){return b===a})},ne:function(b,c,d,a){return a?(a.filterOpType="ne",l(b,c,d,a,function(a,b){return b!==a})):b},lt:function(b,c,d,a){if(a)return a.filterOpType="lt",l(b,c,d,a,function(a,b){return ba})):b},gte:function(b,c,d,a){return a? +(a.filterOpType="gte",l(b,c,d,a,function(a,b){return b>=a})):b},"default":function(b,c,d,a){a&&(a.filterOpType="default");return l(b,c,d,a,function(a,b){return!0})},size:function(b,c,d,a){c=0;var e;a=a||{};if((a=a.key)&&!0!==a)if(k.isArray(a))c=a.length;else if(!isNaN(parseFloat(a))&&isFinite(a))c=a;else if("object"===typeof a)for(e in c=0,a)Object.hasOwnProperty.call(a,e)&&c++;else c=(a+"").length;else c=0;return b.write(c)}}})("undefined"!==typeof exports?module.exports=require("dustjs-linkedin"): +dust); Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSOutputSummer.java Fri Nov 22 20:51:06 2013 @@ -135,4 +135,25 @@ public class TestFSOutputSummer { cluster.shutdown(); } } + + @Test + public void TestDFSCheckSumType() throws Exception{ + Configuration conf = new HdfsConfiguration(); + conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE); + conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, BYTES_PER_CHECKSUM); + conf.set(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, "NULL"); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(NUM_OF_DATANODES) + .build(); + fileSys = cluster.getFileSystem(); + try { + Path file = new Path("try.dat"); + Random rand = new Random(seed); + rand.nextBytes(expected); + writeFile1(file); + } finally { + fileSys.close(); + cluster.shutdown(); + } + } } Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Fri Nov 22 20:51:06 2013 @@ -1157,10 +1157,11 @@ public class TestReplicationPolicy { assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0); final BlockInfo info = new BlockInfo(block1, 1); - final MutableBlockCollection mbc = mock(MutableBlockCollection.class); + final BlockCollection mbc = mock(BlockCollection.class); when(mbc.getLastBlock()).thenReturn(info); when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1); when(mbc.getBlockReplication()).thenReturn((short)1); + when(mbc.isUnderConstruction()).thenReturn(true); ContentSummary cs = mock(ContentSummary.class); when(cs.getLength()).thenReturn((long)1); when(mbc.computeContentSummary()).thenReturn(cs); Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/CreateEditsLog.java Fri Nov 22 20:51:06 2013 @@ -82,9 +82,10 @@ public class CreateEditsLog { blocks[iB].setBlockId(currentBlockId++); } - INodeFileUnderConstruction inode = new INodeFileUnderConstruction( - inodeId.nextValue(), null, replication, 0, blockSize, blocks, p, "", - "", null); + final INodeFile inode = new INodeFile(inodeId.nextValue(), null, + p, 0L, 0L, blocks, replication, blockSize); + inode.toUnderConstruction("", "", null); + // Append path to filename with information about blockIDs String path = "_" + iF + "_B" + blocks[0].getBlockId() + "_to_B" + blocks[blocksPerFile-1].getBlockId() + "_"; @@ -96,9 +97,10 @@ public class CreateEditsLog { dirInode = new INodeDirectory(inodeId.nextValue(), null, p, 0L); editLog.logMkDir(currentDir, dirInode); } - editLog.logOpenFile(filePath, - new INodeFileUnderConstruction(inodeId.nextValue(), p, replication, - 0, blockSize, "", "", null), false); + INodeFile fileUc = new INodeFile(inodeId.nextValue(), null, + p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize); + fileUc.toUnderConstruction("", "", null); + editLog.logOpenFile(filePath, fileUc, false); editLog.logCloseFile(filePath, inode); if (currentBlockId - bidAtSync >= 2000) { // sync every 2K blocks Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java Fri Nov 22 20:51:06 2013 @@ -614,6 +614,47 @@ public class TestCacheDirectives { }, 500, 60000); } + private static void waitForCachedStats(final DistributedFileSystem dfs, + final long targetFilesAffected, final long targetBytesNeeded, + final long targetBytesCached, + final CacheDirectiveInfo filter, final String infoString) + throws Exception { + LOG.info("Polling listDirectives{" + + ((filter == null) ? "ALL" : filter.toString()) + + " for " + targetFilesAffected + " targetFilesAffected, " + + targetBytesNeeded + " targetBytesNeeded, " + + targetBytesCached + " targetBytesCached"); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + RemoteIterator iter = null; + CacheDirectiveEntry entry = null; + try { + iter = dfs.listCacheDirectives(filter); + entry = iter.next(); + } catch (IOException e) { + fail("got IOException while calling " + + "listCacheDirectives: " + e.getMessage()); + } + Assert.assertNotNull(entry); + CacheDirectiveStats stats = entry.getStats(); + if ((targetFilesAffected == stats.getFilesAffected()) && + (targetBytesNeeded == stats.getBytesNeeded()) && + (targetBytesCached == stats.getBytesCached())) { + return true; + } else { + LOG.info(infoString + ": filesAffected: " + + stats.getFilesAffected() + "/" + targetFilesAffected + + ", bytesNeeded: " + + stats.getBytesNeeded() + "/" + targetBytesNeeded + + ", bytesCached: " + + stats.getBytesCached() + "/" + targetBytesCached); + return false; + } + } + }, 500, 60000); + } + private static void checkNumCachedReplicas(final DistributedFileSystem dfs, final List paths, final int expectedBlocks, final int expectedReplicas) @@ -804,21 +845,12 @@ public class TestCacheDirectives { waitForCachedBlocks(namenode, 4, 8, "testWaitForCachedReplicasInDirectory:1"); // Verify that listDirectives gives the stats we want. - RemoteIterator iter = - dfs.listCacheDirectives(new CacheDirectiveInfo.Builder(). - setPath(new Path("/foo")). - build()); - CacheDirectiveEntry entry = iter.next(); - CacheDirectiveStats stats = entry.getStats(); - Assert.assertEquals(Long.valueOf(2), - stats.getFilesAffected()); - Assert.assertEquals(Long.valueOf( - 2 * numBlocksPerFile * BLOCK_SIZE * 2), - stats.getBytesNeeded()); - Assert.assertEquals(Long.valueOf( - 2 * numBlocksPerFile * BLOCK_SIZE * 2), - stats.getBytesCached()); - + waitForCachedStats(dfs, 2, + 8 * BLOCK_SIZE, 8 * BLOCK_SIZE, + new CacheDirectiveInfo.Builder(). + setPath(new Path("/foo")). + build(), + "testWaitForCachedReplicasInDirectory:2"); long id2 = dfs.addCacheDirective( new CacheDirectiveInfo.Builder(). setPath(new Path("/foo/bar")). @@ -827,44 +859,28 @@ public class TestCacheDirectives { build()); // wait for an additional 2 cached replicas to come up waitForCachedBlocks(namenode, 4, 10, - "testWaitForCachedReplicasInDirectory:2"); + "testWaitForCachedReplicasInDirectory:3"); // the directory directive's stats are unchanged - iter = dfs.listCacheDirectives( + waitForCachedStats(dfs, 2, + 8 * BLOCK_SIZE, 8 * BLOCK_SIZE, new CacheDirectiveInfo.Builder(). - setPath(new Path("/foo")). - build()); - entry = iter.next(); - stats = entry.getStats(); - Assert.assertEquals(Long.valueOf(2), - stats.getFilesAffected()); - Assert.assertEquals(Long.valueOf( - 2 * numBlocksPerFile * BLOCK_SIZE * 2), - stats.getBytesNeeded()); - Assert.assertEquals(Long.valueOf( - 2 * numBlocksPerFile * BLOCK_SIZE * 2), - stats.getBytesCached()); + setPath(new Path("/foo")). + build(), + "testWaitForCachedReplicasInDirectory:4"); // verify /foo/bar's stats - iter = dfs.listCacheDirectives( + waitForCachedStats(dfs, 1, + 4 * numBlocksPerFile * BLOCK_SIZE, + // only 3 because the file only has 3 replicas, not 4 as requested. + 3 * numBlocksPerFile * BLOCK_SIZE, new CacheDirectiveInfo.Builder(). - setPath(new Path("/foo/bar")). - build()); - entry = iter.next(); - stats = entry.getStats(); - Assert.assertEquals(Long.valueOf(1), - stats.getFilesAffected()); - Assert.assertEquals(Long.valueOf( - 4 * numBlocksPerFile * BLOCK_SIZE), - stats.getBytesNeeded()); - // only 3 because the file only has 3 replicas, not 4 as requested. - Assert.assertEquals(Long.valueOf( - 3 * numBlocksPerFile * BLOCK_SIZE), - stats.getBytesCached()); - + setPath(new Path("/foo/bar")). + build(), + "testWaitForCachedReplicasInDirectory:5"); // remove and watch numCached go to 0 dfs.removeCacheDirective(id); dfs.removeCacheDirective(id2); waitForCachedBlocks(namenode, 0, 0, - "testWaitForCachedReplicasInDirectory:3"); + "testWaitForCachedReplicasInDirectory:6"); } finally { cluster.shutdown(); } Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java Fri Nov 22 20:51:06 2013 @@ -43,8 +43,7 @@ public class TestCommitBlockSynchronizat private static final long length = 200; private static final long genStamp = 300; - private FSNamesystem makeNameSystemSpy(Block block, - INodeFileUnderConstruction file) + private FSNamesystem makeNameSystemSpy(Block block, INodeFile file) throws IOException { Configuration conf = new Configuration(); FSImage image = new FSImage(conf); @@ -58,21 +57,26 @@ public class TestCommitBlockSynchronizat blockInfo.setGenerationStamp(genStamp); blockInfo.initializeBlockRecovery(genStamp); doReturn(true).when(file).removeLastBlock(any(Block.class)); + doReturn(true).when(file).isUnderConstruction(); doReturn(blockInfo).when(namesystemSpy).getStoredBlock(any(Block.class)); doReturn("").when(namesystemSpy).closeFileCommitBlocks( - any(INodeFileUnderConstruction.class), - any(BlockInfo.class)); + any(INodeFile.class), any(BlockInfo.class)); doReturn("").when(namesystemSpy).persistBlocks( - any(INodeFileUnderConstruction.class), anyBoolean()); + any(INodeFile.class), anyBoolean()); doReturn(mock(FSEditLog.class)).when(namesystemSpy).getEditLog(); return namesystemSpy; } + private INodeFile mockFileUnderConstruction() { + INodeFile file = mock(INodeFile.class); + return file; + } + @Test public void testCommitBlockSynchronization() throws IOException { - INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class); + INodeFile file = mockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = makeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[0]; @@ -100,7 +104,7 @@ public class TestCommitBlockSynchronizat @Test public void testCommitBlockSynchronization2() throws IOException { - INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class); + INodeFile file = mockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = makeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[0]; @@ -124,7 +128,7 @@ public class TestCommitBlockSynchronizat @Test public void testCommitBlockSynchronizationWithDelete() throws IOException { - INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class); + INodeFile file = mockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = makeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[0]; @@ -144,7 +148,7 @@ public class TestCommitBlockSynchronizat @Test public void testCommitBlockSynchronizationWithClose() throws IOException { - INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class); + INodeFile file = mockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = makeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[0]; @@ -171,7 +175,7 @@ public class TestCommitBlockSynchronizat @Test public void testCommitBlockSynchronizationWithCloseAndNonExistantTarget() throws IOException { - INodeFileUnderConstruction file = mock(INodeFileUnderConstruction.class); + INodeFile file = mockFileUnderConstruction(); Block block = new Block(blockId, length, genStamp); FSNamesystem namesystemSpy = makeNameSystemSpy(block, file); DatanodeID[] newTargets = new DatanodeID[]{ Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java Fri Nov 22 20:51:06 2013 @@ -63,6 +63,7 @@ import org.apache.hadoop.hdfs.HdfsConfig import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; @@ -152,9 +153,10 @@ public class TestEditLog { FSEditLog editLog = namesystem.getEditLog(); for (int i = 0; i < numTransactions; i++) { - INodeFileUnderConstruction inode = new INodeFileUnderConstruction( - namesystem.allocateNewInodeId(), p, replication, blockSize, 0, "", - "", null); + INodeFile inode = new INodeFile(namesystem.allocateNewInodeId(), null, + p, 0L, 0L, BlockInfo.EMPTY_ARRAY, replication, blockSize); + inode.toUnderConstruction("", "", null); + editLog.logOpenFile("/filename" + (startIndex + i), inode, false); editLog.logCloseFile("/filename" + (startIndex + i), inode); editLog.logSync(); Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java Fri Nov 22 20:51:06 2013 @@ -29,6 +29,8 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import junit.framework.Assert; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -59,6 +61,7 @@ import org.apache.hadoop.hdfs.protocol.L import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.util.Time; import org.junit.Test; import org.mockito.Mockito; @@ -282,14 +285,6 @@ public class TestINodeFile { assertTrue(fnfe.getMessage().contains("File does not exist")); } - //cast to INodeFileUnderConstruction, should fail - try { - INodeFileUnderConstruction.valueOf(from, path); - fail(); - } catch(FileNotFoundException fnfe) { - assertTrue(fnfe.getMessage().contains("File does not exist")); - } - //cast to INodeDirectory, should fail try { INodeDirectory.valueOf(from, path); @@ -306,14 +301,6 @@ public class TestINodeFile { final INodeFile f = INodeFile.valueOf(from, path); assertTrue(f == from); - //cast to INodeFileUnderConstruction, should fail - try { - INodeFileUnderConstruction.valueOf(from, path); - fail(); - } catch(IOException ioe) { - assertTrue(ioe.getMessage().contains("File is not under construction")); - } - //cast to INodeDirectory, should fail try { INodeDirectory.valueOf(from, path); @@ -324,19 +311,14 @@ public class TestINodeFile { } {//cast from INodeFileUnderConstruction - final INode from = new INodeFileUnderConstruction( - INodeId.GRANDFATHER_INODE_ID, perm, replication, 0L, 0L, "client", - "machine", null); + final INode from = new INodeFile( + INodeId.GRANDFATHER_INODE_ID, null, perm, 0L, 0L, null, replication, 1024L); + from.asFile().toUnderConstruction("client", "machine", null); //cast to INodeFile, should success final INodeFile f = INodeFile.valueOf(from, path); assertTrue(f == from); - //cast to INodeFileUnderConstruction, should success - final INodeFileUnderConstruction u = INodeFileUnderConstruction.valueOf( - from, path); - assertTrue(u == from); - //cast to INodeDirectory, should fail try { INodeDirectory.valueOf(from, path); @@ -358,14 +340,6 @@ public class TestINodeFile { assertTrue(fnfe.getMessage().contains("Path is not a file")); } - //cast to INodeFileUnderConstruction, should fail - try { - INodeFileUnderConstruction.valueOf(from, path); - fail(); - } catch(FileNotFoundException fnfe) { - assertTrue(fnfe.getMessage().contains("Path is not a file")); - } - //cast to INodeDirectory, should success final INodeDirectory d = INodeDirectory.valueOf(from, path); assertTrue(d == from); @@ -1015,4 +989,24 @@ public class TestINodeFile { } } } + + @Test + public void testFileUnderConstruction() { + replication = 3; + final INodeFile file = new INodeFile(INodeId.GRANDFATHER_INODE_ID, null, + perm, 0L, 0L, null, replication, 1024L); + assertFalse(file.isUnderConstruction()); + + final String clientName = "client"; + final String clientMachine = "machine"; + file.toUnderConstruction(clientName, clientMachine, null); + assertTrue(file.isUnderConstruction()); + FileUnderConstructionFeature uc = file.getFileUnderConstructionFeature(); + assertEquals(clientName, uc.getClientName()); + assertEquals(clientMachine, uc.getClientMachine()); + Assert.assertNull(uc.getClientNode()); + + file.toCompleteFile(Time.now()); + assertFalse(file.isUnderConstruction()); + } } Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java Fri Nov 22 20:51:06 2013 @@ -66,7 +66,6 @@ import org.apache.hadoop.hdfs.protocol.C import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.INodeFile; -import org.apache.hadoop.hdfs.server.namenode.INodeFileUnderConstruction; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.IOUtils; @@ -720,8 +719,8 @@ public class TestRetryCacheWithHA { @Override boolean checkNamenodeBeforeReturn() throws Exception { - INodeFileUnderConstruction fileNode = (INodeFileUnderConstruction) cluster - .getNamesystem(0).getFSDirectory().getINode4Write(file).asFile(); + INodeFile fileNode = cluster.getNamesystem(0).getFSDirectory() + .getINode4Write(file).asFile(); BlockInfoUnderConstruction blkUC = (BlockInfoUnderConstruction) (fileNode.getBlocks())[1]; int datanodeNum = blkUC.getExpectedStorageLocations().length; Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java Fri Nov 22 20:51:06 2013 @@ -1227,8 +1227,9 @@ public class TestRenameWithSnapshots { out.write(content); fooRef = fsdir.getINode4Write(foo2.toString()); assertTrue(fooRef instanceof INodeReference.DstReference); - INode fooNode = fooRef.asFile(); - assertTrue(fooNode instanceof INodeFileUnderConstructionWithSnapshot); + INodeFile fooNode = fooRef.asFile(); + assertTrue(fooNode instanceof INodeFileWithSnapshot); + assertTrue(fooNode.isUnderConstruction()); } finally { if (out != null) { out.close(); @@ -1237,8 +1238,9 @@ public class TestRenameWithSnapshots { fooRef = fsdir.getINode4Write(foo2.toString()); assertTrue(fooRef instanceof INodeReference.DstReference); - INode fooNode = fooRef.asFile(); + INodeFile fooNode = fooRef.asFile(); assertTrue(fooNode instanceof INodeFileWithSnapshot); + assertFalse(fooNode.isUnderConstruction()); restartClusterAndCheckImage(true); } Modified: hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java?rev=1544672&r1=1544671&r2=1544672&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java (original) +++ hadoop/common/branches/HDFS-2832/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java Fri Nov 22 20:51:06 2013 @@ -314,7 +314,9 @@ public class TestSnapshotBlocksMap { assertEquals(BLOCKSIZE, blks[0].getNumBytes()); } - /** Make sure we delete 0-sized block when deleting an INodeFileUC */ + /** + * Make sure we delete 0-sized block when deleting an under-construction file + */ @Test public void testDeletionWithZeroSizeBlock2() throws Exception { final Path foo = new Path("/foo");