Return-Path: Delivered-To: apmail-lucene-hadoop-commits-archive@locus.apache.org Received: (qmail 28791 invoked from network); 20 Jul 2007 21:36:39 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 20 Jul 2007 21:36:39 -0000 Received: (qmail 1868 invoked by uid 500); 20 Jul 2007 21:36:40 -0000 Delivered-To: apmail-lucene-hadoop-commits-archive@lucene.apache.org Received: (qmail 1844 invoked by uid 500); 20 Jul 2007 21:36:40 -0000 Mailing-List: contact hadoop-commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hadoop-dev@lucene.apache.org Delivered-To: mailing list hadoop-commits@lucene.apache.org Received: (qmail 1835 invoked by uid 99); 20 Jul 2007 21:36:40 -0000 Received: from herse.apache.org (HELO herse.apache.org) (140.211.11.133) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Jul 2007 14:36:40 -0700 X-ASF-Spam-Status: No, hits=-99.5 required=10.0 tests=ALL_TRUSTED,NO_REAL_NAME X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO eris.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 20 Jul 2007 14:36:38 -0700 Received: by eris.apache.org (Postfix, from userid 65534) id B52611A981A; Fri, 20 Jul 2007 14:36:17 -0700 (PDT) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r558159 - in /lucene/hadoop/branches/branch-0.13: ./ src/java/org/apache/hadoop/dfs/ src/test/org/apache/hadoop/dfs/ Date: Fri, 20 Jul 2007 21:36:17 -0000 To: hadoop-commits@lucene.apache.org From: cutting@apache.org X-Mailer: svnmailer-1.1.0 Message-Id: <20070720213617.B52611A981A@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cutting Date: Fri Jul 20 14:36:15 2007 New Revision: 558159 URL: http://svn.apache.org/viewvc?view=rev&rev=558159 Log: HADOOP-1603. Fix a bug in namenode initialization where default replication is sometimes reset to one on restart. Contributed by Raghu. Modified: lucene/hadoop/branches/branch-0.13/CHANGES.txt lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/FSNamesystem.java lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/NameNode.java lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/SecondaryNameNode.java lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/dfs/TestCheckpoint.java Modified: lucene/hadoop/branches/branch-0.13/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/CHANGES.txt?view=diff&rev=558159&r1=558158&r2=558159 ============================================================================== --- lucene/hadoop/branches/branch-0.13/CHANGES.txt (original) +++ lucene/hadoop/branches/branch-0.13/CHANGES.txt Fri Jul 20 14:36:15 2007 @@ -6,6 +6,10 @@ 1. HADOOP-1623. Fix an infinite loop when copying directories into themselves. (Dhruba Borthakur via cutting) + 2. HADOOP-1603. Fix a bug in namenode initialization where + default replication is sometimes reset to one on restart. + (Raghu Angadi via cutting) + Release 0.13.0 - 2007-06-08 Modified: lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/FSNamesystem.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=558159&r1=558158&r2=558159 ============================================================================== --- lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original) +++ lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/FSNamesystem.java Fri Jul 20 14:36:15 2007 @@ -213,38 +213,7 @@ int port, NameNode nn, Configuration conf) throws IOException { fsNamesystemObject = this; - this.replicator = new ReplicationTargetChooser( - conf.getBoolean("dfs.replication.considerLoad", true), - this, - clusterMap, - LOG); - this.defaultReplication = conf.getInt("dfs.replication", 3); - this.maxReplication = conf.getInt("dfs.replication.max", 512); - this.minReplication = conf.getInt("dfs.replication.min", 1); - if (minReplication <= 0) - throw new IOException( - "Unexpected configuration parameters: dfs.replication.min = " - + minReplication - + " must be greater than 0"); - if (maxReplication >= (int)Short.MAX_VALUE) - throw new IOException( - "Unexpected configuration parameters: dfs.replication.max = " - + maxReplication + " must be less than " + (Short.MAX_VALUE)); - if (maxReplication < minReplication) - throw new IOException( - "Unexpected configuration parameters: dfs.replication.min = " - + minReplication - + " must be less than dfs.replication.max = " - + maxReplication); - this.maxReplicationStreams = conf.getInt("dfs.max-repl-streams", 2); - long heartbeatInterval = conf.getLong("dfs.heartbeat.interval", 3) * 1000; - this.heartbeatRecheckInterval = 5 * 60 * 1000; // 5 minutes - this.heartbeatExpireInterval = 2 * heartbeatRecheckInterval + - 10 * heartbeatInterval; - this.replicationRecheckInterval = 3 * 1000; // 3 second - this.decommissionRecheckInterval = conf.getInt( - "dfs.namenode.decommission.interval", - 5 * 60 * 1000); + setConfigurationParameters(conf); this.localMachine = hostname; this.port = port; @@ -300,9 +269,50 @@ * dirs is a list of directories where the filesystem directory state * is stored */ - FSNamesystem(FSImage fsImage) throws IOException { + FSNamesystem(FSImage fsImage, Configuration conf) throws IOException { fsNamesystemObject = this; + setConfigurationParameters(conf); this.dir = new FSDirectory(fsImage, this); + } + + /** + * Initializes some of the members from configuration + */ + private void setConfigurationParameters(Configuration conf) + throws IOException { + + this.replicator = new ReplicationTargetChooser( + conf.getBoolean("dfs.replication.considerLoad", true), + this, + clusterMap, + LOG); + this.defaultReplication = conf.getInt("dfs.replication", 3); + this.maxReplication = conf.getInt("dfs.replication.max", 512); + this.minReplication = conf.getInt("dfs.replication.min", 1); + if (minReplication <= 0) + throw new IOException( + "Unexpected configuration parameters: dfs.replication.min = " + + minReplication + + " must be greater than 0"); + if (maxReplication >= (int)Short.MAX_VALUE) + throw new IOException( + "Unexpected configuration parameters: dfs.replication.max = " + + maxReplication + " must be less than " + (Short.MAX_VALUE)); + if (maxReplication < minReplication) + throw new IOException( + "Unexpected configuration parameters: dfs.replication.min = " + + minReplication + + " must be less than dfs.replication.max = " + + maxReplication); + this.maxReplicationStreams = conf.getInt("dfs.max-repl-streams", 2); + long heartbeatInterval = conf.getLong("dfs.heartbeat.interval", 3) * 1000; + this.heartbeatRecheckInterval = 5 * 60 * 1000; // 5 minutes + this.heartbeatExpireInterval = 2 * heartbeatRecheckInterval + + 10 * heartbeatInterval; + this.replicationRecheckInterval = 3 * 1000; // 3 second + this.decommissionRecheckInterval = conf.getInt( + "dfs.namenode.decommission.interval", + 5 * 60 * 1000); } /** Return the FSNamesystem object Modified: lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/NameNode.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/NameNode.java?view=diff&rev=558159&r1=558158&r2=558159 ============================================================================== --- lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/NameNode.java (original) +++ lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/NameNode.java Fri Jul 20 14:36:15 2007 @@ -754,7 +754,7 @@ } } - FSNamesystem nsys = new FSNamesystem(new FSImage(dirsToFormat)); + FSNamesystem nsys = new FSNamesystem(new FSImage(dirsToFormat), conf); nsys.dir.fsImage.format(); return false; } Modified: lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/SecondaryNameNode.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/SecondaryNameNode.java?view=diff&rev=558159&r1=558158&r2=558159 ============================================================================== --- lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/SecondaryNameNode.java (original) +++ lucene/hadoop/branches/branch-0.13/src/java/org/apache/hadoop/dfs/SecondaryNameNode.java Fri Jul 20 14:36:15 2007 @@ -289,8 +289,8 @@ * DEST_FS_IMAGE */ private void doMerge() throws IOException { - FSNamesystem namesystem = new FSNamesystem( - new FSImage(checkpointDir)); + FSNamesystem namesystem = new FSNamesystem(new FSImage(checkpointDir), + conf); FSImage fsImage = namesystem.dir.fsImage; fsImage.loadFSImage(srcImage); fsImage.getEditLog().loadFSEdits(editFile); Modified: lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/dfs/TestCheckpoint.java URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/dfs/TestCheckpoint.java?view=diff&rev=558159&r1=558158&r2=558159 ============================================================================== --- lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/dfs/TestCheckpoint.java (original) +++ lucene/hadoop/branches/branch-0.13/src/test/org/apache/hadoop/dfs/TestCheckpoint.java Fri Jul 20 14:36:15 2007 @@ -34,7 +34,8 @@ static final long seed = 0xDEADBEEFL; static final int blockSize = 4096; static final int fileSize = 8192; - static final int numDatanodes = 1; + static final int numDatanodes = 3; + short replication = 3; private void writeFile(FileSystem fileSys, Path name, int repl) throws IOException { @@ -52,11 +53,9 @@ private void checkFile(FileSystem fileSys, Path name, int repl) throws IOException { assertTrue(fileSys.exists(name)); - String[][] locations = fileSys.getFileCacheHints(name, 0, fileSize); - for (int idx = 0; idx < locations.length; idx++) { - assertEquals("Number of replicas for block" + idx, - Math.min(numDatanodes, repl), locations[idx].length); - } + int replication = fileSys.getReplication(name); + assertEquals("replication for " + name, repl, replication); + //We should probably test for more of the file properties. } private void cleanupFile(FileSystem fileSys, Path name) @@ -150,8 +149,8 @@ // // Create a new file // - writeFile(fileSys, file1, 1); - checkFile(fileSys, file1, 1); + writeFile(fileSys, file1, replication); + checkFile(fileSys, file1, replication); } finally { fileSys.close(); cluster.shutdown(); @@ -167,7 +166,7 @@ cluster.waitActive(); fileSys = cluster.getFileSystem(); try { - checkFile(fileSys, file1, 1); + checkFile(fileSys, file1, replication); cleanupFile(fileSys, file1); SecondaryNameNode secondary = new SecondaryNameNode(conf); secondary.doCheckpoint(); @@ -209,8 +208,8 @@ // // Create a new file // - writeFile(fileSys, file1, 1); - checkFile(fileSys, file1, 1); + writeFile(fileSys, file1, replication); + checkFile(fileSys, file1, replication); } finally { fileSys.close(); cluster.shutdown(); @@ -226,7 +225,7 @@ cluster.waitActive(); fileSys = cluster.getFileSystem(); try { - checkFile(fileSys, file1, 1); + checkFile(fileSys, file1, replication); cleanupFile(fileSys, file1); SecondaryNameNode secondary = new SecondaryNameNode(conf); secondary.doCheckpoint(); @@ -246,6 +245,7 @@ Collection namedirs = null; Configuration conf = new Configuration(); + replication = (short)conf.getInt("dfs.replication", 3); MiniDFSCluster cluster = new MiniDFSCluster(conf, numDatanodes, true, null); cluster.waitActive(); FileSystem fileSys = cluster.getFileSystem(); @@ -261,8 +261,8 @@ // // Create file1 // - writeFile(fileSys, file1, 1); - checkFile(fileSys, file1, 1); + writeFile(fileSys, file1, replication); + checkFile(fileSys, file1, replication); // // Take a checkpoint @@ -283,12 +283,12 @@ fileSys = cluster.getFileSystem(); try { // check that file1 still exists - checkFile(fileSys, file1, 1); + checkFile(fileSys, file1, replication); cleanupFile(fileSys, file1); // create new file file2 - writeFile(fileSys, file2, 1); - checkFile(fileSys, file2, 1); + writeFile(fileSys, file2, replication); + checkFile(fileSys, file2, replication); // // Take a checkpoint @@ -313,7 +313,7 @@ try { // verify that file2 exists - checkFile(fileSys, file2, 1); + checkFile(fileSys, file2, replication); } finally { fileSys.close(); cluster.shutdown();