Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9CB119029 for ; Fri, 25 May 2012 03:58:21 +0000 (UTC) Received: (qmail 13600 invoked by uid 500); 25 May 2012 03:58:21 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 13567 invoked by uid 500); 25 May 2012 03:58:21 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 13559 invoked by uid 99); 25 May 2012 03:58:21 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 25 May 2012 03:58:21 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 25 May 2012 03:58:18 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 6499023888CD; Fri, 25 May 2012 03:57:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1342494 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project: ./ hadoop-hdfs/ hadoop-hdfs/src/contrib/bkjournal/ hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/ hadoop-hdfs/src/contrib/bkjournal/sr... Date: Fri, 25 May 2012 03:57:57 -0000 To: hdfs-commits@hadoop.apache.org From: eli@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120525035758.6499023888CD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: eli Date: Fri May 25 03:57:56 2012 New Revision: 1342494 URL: http://svn.apache.org/viewvc?rev=1342494&view=rev Log: Revert HDFS-3058 Removed: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/BKJMUtil.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperAsHASharedDir.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperHACheckpoints.java Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed) hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/ ------------------------------------------------------------------------------ Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project:r1342483 Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ ------------------------------------------------------------------------------ Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1342483 Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri May 25 03:57:56 2012 @@ -113,9 +113,6 @@ Release 2.0.1-alpha - UNRELEASED HDFS-3460. HttpFS proxyuser validation with Kerberos ON uses full principal name. (tucu) - HDFS-3058. HA: Bring BookKeeperJournalManager up to date with HA changes. - (Ivan Kelly via umamahesh) - Release 2.0.0-alpha - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml Fri May 25 03:57:56 2012 @@ -55,12 +55,6 @@ test - org.apache.hadoop - hadoop-common - test-jar - test - - org.apache.bookkeeper bookkeeper-server compile @@ -70,11 +64,6 @@ junit test - - org.mockito - mockito-all - test - Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java Fri May 25 03:57:56 2012 @@ -41,7 +41,6 @@ class BookKeeperEditLogInputStream exten private final long firstTxId; private final long lastTxId; private final int logVersion; - private final boolean inProgress; private final LedgerHandle lh; private final FSEditLogOp.Reader reader; @@ -70,7 +69,6 @@ class BookKeeperEditLogInputStream exten this.firstTxId = metadata.getFirstTxId(); this.lastTxId = metadata.getLastTxId(); this.logVersion = metadata.getVersion(); - this.inProgress = metadata.isInProgress(); BufferedInputStream bin = new BufferedInputStream( new LedgerInputStream(lh, firstBookKeeperEntry)); @@ -125,28 +123,10 @@ class BookKeeperEditLogInputStream exten lh.toString(), firstTxId, lastTxId); } + // TODO(HA): Test this. @Override public boolean isInProgress() { - return inProgress; - } - - /** - * Skip forward to specified transaction id. - * Currently we do this by just iterating forward. - * If this proves to be too expensive, this can be reimplemented - * with a binary search over bk entries - */ - public void skipTo(long txId) throws IOException { - long numToSkip = getFirstTxId() - txId; - - FSEditLogOp op = null; - for (long i = 0; i < numToSkip; i++) { - op = readOp(); - } - if (op != null && op.getTransactionId() != txId-1) { - throw new IOException("Corrupt stream, expected txid " - + (txId-1) + ", got " + op.getTransactionId()); - } + return true; } /** Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java Fri May 25 03:57:56 2012 @@ -233,14 +233,11 @@ public class BookKeeperJournalManager im */ l.write(zkc, znodePath); - maxTxId.store(txId); return new BookKeeperEditLogOutputStream(conf, currentLedger, wl); } catch (Exception e) { if (currentLedger != null) { try { - long id = currentLedger.getId(); currentLedger.close(); - bkc.deleteLedger(id); } catch (Exception e2) { //log & ignore, an IOException will be thrown soon LOG.error("Error closing ledger", e2); @@ -387,8 +384,8 @@ public class BookKeeperJournalManager im break; } } - count += (lastTxId - l.getFirstTxId()) + 1; - expectedStart = lastTxId + 1; + count += (l.getLastTxId() - l.getFirstTxId()) + 1; + expectedStart = l.getLastTxId() + 1; } } return count; @@ -407,7 +404,7 @@ public class BookKeeperJournalManager im String znode = ledgerPath + "/" + child; EditLogLedgerMetadata l = EditLogLedgerMetadata.read(zkc, znode); - long endTxId = recoverLastTxId(l, true); + long endTxId = recoverLastTxId(l); if (endTxId == HdfsConstants.INVALID_TXID) { LOG.error("Unrecoverable corruption has occurred in segment " + l.toString() + " at path " + znode @@ -477,19 +474,11 @@ public class BookKeeperJournalManager im * Find the id of the last edit log transaction writen to a edit log * ledger. */ - private long recoverLastTxId(EditLogLedgerMetadata l, boolean fence) - throws IOException { + private long recoverLastTxId(EditLogLedgerMetadata l) throws IOException { try { - LedgerHandle lh = null; - if (fence) { - lh = bkc.openLedger(l.getLedgerId(), - BookKeeper.DigestType.MAC, - digestpw.getBytes()); - } else { - lh = bkc.openLedgerNoRecovery(l.getLedgerId(), - BookKeeper.DigestType.MAC, - digestpw.getBytes()); - } + LedgerHandle lh = bkc.openLedger(l.getLedgerId(), + BookKeeper.DigestType.MAC, + digestpw.getBytes()); long lastAddConfirmed = lh.getLastAddConfirmed(); BookKeeperEditLogInputStream in = new BookKeeperEditLogInputStream(lh, l, lastAddConfirmed); Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java Fri May 25 03:57:56 2012 @@ -18,23 +18,55 @@ package org.apache.hadoop.contrib.bkjournal; import static org.junit.Assert.*; + +import java.net.URI; +import java.util.Collections; +import java.util.Arrays; +import java.util.List; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.apache.bookkeeper.proto.BookieServer; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.util.LocalBookKeeper; + +import java.io.RandomAccessFile; +import java.io.File; +import java.io.FilenameFilter; +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.IOException; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.security.SecurityUtil; import org.junit.Test; import org.junit.Before; import org.junit.After; import org.junit.BeforeClass; import org.junit.AfterClass; -import java.io.IOException; -import org.apache.hadoop.conf.Configuration; - +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import static org.apache.hadoop.hdfs.server.namenode.TestEditLog.setupEdits; import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream; import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; +import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; import org.apache.hadoop.hdfs.server.namenode.FSEditLogTestUtil; import org.apache.hadoop.hdfs.server.namenode.JournalManager; -import org.apache.bookkeeper.proto.BookieServer; import org.apache.zookeeper.ZooKeeper; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.KeeperException; + +import com.google.common.collect.ImmutableList; + +import java.util.zip.CheckedInputStream; +import java.util.zip.Checksum; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -43,26 +75,125 @@ public class TestBookKeeperJournalManage static final Log LOG = LogFactory.getLog(TestBookKeeperJournalManager.class); private static final long DEFAULT_SEGMENT_SIZE = 1000; + private static final String zkEnsemble = "localhost:2181"; + final static private int numBookies = 5; + private static Thread bkthread; protected static Configuration conf = new Configuration(); private ZooKeeper zkc; - private static BKJMUtil bkutil; - static int numBookies = 3; - @BeforeClass - public static void setupBookkeeper() throws Exception { - bkutil = new BKJMUtil(numBookies); - bkutil.start(); + + static int nextPort = 6000; // next port for additionally created bookies + + private static ZooKeeper connectZooKeeper(String ensemble) + throws IOException, KeeperException, InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + + ZooKeeper zkc = new ZooKeeper(zkEnsemble, 3600, new Watcher() { + public void process(WatchedEvent event) { + if (event.getState() == Watcher.Event.KeeperState.SyncConnected) { + latch.countDown(); + } + } + }); + if (!latch.await(3, TimeUnit.SECONDS)) { + throw new IOException("Zookeeper took too long to connect"); + } + return zkc; + } + + private static BookieServer newBookie() throws Exception { + int port = nextPort++; + ServerConfiguration bookieConf = new ServerConfiguration(); + bookieConf.setBookiePort(port); + File tmpdir = File.createTempFile("bookie" + Integer.toString(port) + "_", + "test"); + tmpdir.delete(); + tmpdir.mkdir(); + + bookieConf.setZkServers(zkEnsemble); + bookieConf.setJournalDirName(tmpdir.getPath()); + bookieConf.setLedgerDirNames(new String[] { tmpdir.getPath() }); + + BookieServer b = new BookieServer(bookieConf); + b.start(); + for (int i = 0; i < 10 && !b.isRunning(); i++) { + Thread.sleep(10000); + } + if (!b.isRunning()) { + throw new IOException("Bookie would not start"); + } + return b; } - @AfterClass - public static void teardownBookkeeper() throws Exception { - bkutil.teardown(); + /** + * Check that a number of bookies are available + * @param count number of bookies required + * @param timeout number of seconds to wait for bookies to start + * @throws IOException if bookies are not started by the time the timeout hits + */ + private static int checkBookiesUp(int count, int timeout) throws Exception { + ZooKeeper zkc = connectZooKeeper(zkEnsemble); + try { + boolean up = false; + int mostRecentSize = 0; + for (int i = 0; i < timeout; i++) { + try { + List children = zkc.getChildren("/ledgers/available", + false); + mostRecentSize = children.size(); + if (LOG.isDebugEnabled()) { + LOG.debug("Found " + mostRecentSize + " bookies up, " + + "waiting for " + count); + if (LOG.isTraceEnabled()) { + for (String child : children) { + LOG.trace(" server: " + child); + } + } + } + if (mostRecentSize == count) { + up = true; + break; + } + } catch (KeeperException e) { + // ignore + } + Thread.sleep(1000); + } + return mostRecentSize; + } finally { + zkc.close(); + } } + @BeforeClass + public static void setupBookkeeper() throws Exception { + bkthread = new Thread() { + public void run() { + try { + String[] args = new String[1]; + args[0] = String.valueOf(numBookies); + LOG.info("Starting bk"); + LocalBookKeeper.main(args); + } catch (InterruptedException e) { + // go away quietly + } catch (Exception e) { + LOG.error("Error starting local bk", e); + } + } + }; + bkthread.start(); + + if (!LocalBookKeeper.waitForServerUp(zkEnsemble, 10000)) { + throw new Exception("Error starting zookeeper/bookkeeper"); + } + assertEquals("Not all bookies started", + numBookies, checkBookiesUp(numBookies, 10)); + } + @Before public void setup() throws Exception { - zkc = BKJMUtil.connectZooKeeper(); + zkc = connectZooKeeper(zkEnsemble); } @After @@ -70,10 +201,18 @@ public class TestBookKeeperJournalManage zkc.close(); } + @AfterClass + public static void teardownBookkeeper() throws Exception { + if (bkthread != null) { + bkthread.interrupt(); + bkthread.join(); + } + } + @Test public void testSimpleWrite() throws Exception { BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simplewrite")); + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-simplewrite")); long txid = 1; EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { @@ -92,8 +231,8 @@ public class TestBookKeeperJournalManage @Test public void testNumberOfTransactions() throws Exception { - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-txncount")); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-txncount")); long txid = 1; EditLogOutputStream out = bkjm.startLogSegment(1); for (long i = 1 ; i <= 100; i++) { @@ -110,8 +249,8 @@ public class TestBookKeeperJournalManage @Test public void testNumberOfTransactionsWithGaps() throws Exception { - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-gaps")); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-gaps")); long txid = 1; for (long i = 0; i < 3; i++) { long start = txid; @@ -123,11 +262,9 @@ public class TestBookKeeperJournalManage } out.close(); bkjm.finalizeLogSegment(start, txid-1); - assertNotNull( - zkc.exists(bkjm.finalizedLedgerZNode(start, txid-1), false)); + assertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(start, txid-1), false)); } - zkc.delete(bkjm.finalizedLedgerZNode(DEFAULT_SEGMENT_SIZE+1, - DEFAULT_SEGMENT_SIZE*2), -1); + zkc.delete(bkjm.finalizedLedgerZNode(DEFAULT_SEGMENT_SIZE+1, DEFAULT_SEGMENT_SIZE*2), -1); long numTrans = bkjm.getNumberOfTransactions(1, true); assertEquals(DEFAULT_SEGMENT_SIZE, numTrans); @@ -145,8 +282,8 @@ public class TestBookKeeperJournalManage @Test public void testNumberOfTransactionsWithInprogressAtEnd() throws Exception { - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-inprogressAtEnd")); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-inprogressAtEnd")); long txid = 1; for (long i = 0; i < 3; i++) { long start = txid; @@ -159,8 +296,7 @@ public class TestBookKeeperJournalManage out.close(); bkjm.finalizeLogSegment(start, (txid-1)); - assertNotNull( - zkc.exists(bkjm.finalizedLedgerZNode(start, (txid-1)), false)); + assertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(start, (txid-1)), false)); } long start = txid; EditLogOutputStream out = bkjm.startLogSegment(start); @@ -184,8 +320,8 @@ public class TestBookKeeperJournalManage */ @Test public void testWriteRestartFrom1() throws Exception { - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1")); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-restartFrom1")); long txid = 1; long start = txid; EditLogOutputStream out = bkjm.startLogSegment(txid); @@ -239,10 +375,10 @@ public class TestBookKeeperJournalManage @Test public void testTwoWriters() throws Exception { long start = 1; - BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-dualWriter")); - BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-dualWriter")); + BookKeeperJournalManager bkjm1 = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-dualWriter")); + BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-dualWriter")); EditLogOutputStream out1 = bkjm1.startLogSegment(start); try { @@ -255,8 +391,8 @@ public class TestBookKeeperJournalManage @Test public void testSimpleRead() throws Exception { - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simpleread")); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-simpleread")); long txid = 1; final long numTransactions = 10000; EditLogOutputStream out = bkjm.startLogSegment(1); @@ -280,8 +416,8 @@ public class TestBookKeeperJournalManage @Test public void testSimpleRecovery() throws Exception { - BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery")); + BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, + URI.create("bookkeeper://" + zkEnsemble + "/hdfsjournal-simplerecovery")); EditLogOutputStream out = bkjm.startLogSegment(1); long txid = 1; for (long i = 1 ; i <= 100; i++) { @@ -312,13 +448,13 @@ public class TestBookKeeperJournalManage */ @Test public void testAllBookieFailure() throws Exception { - BookieServer bookieToFail = bkutil.newBookie(); + BookieServer bookieToFail = newBookie(); BookieServer replacementBookie = null; try { int ensembleSize = numBookies + 1; assertEquals("New bookie didn't start", - ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10)); + ensembleSize, checkBookiesUp(ensembleSize, 10)); // ensure that the journal manager has to use all bookies, // so that a failure will fail the journal manager @@ -329,7 +465,8 @@ public class TestBookKeeperJournalManage ensembleSize); long txid = 1; BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure")); + URI.create("bookkeeper://" + zkEnsemble + + "/hdfsjournal-allbookiefailure")); EditLogOutputStream out = bkjm.startLogSegment(txid); for (long i = 1 ; i <= 3; i++) { @@ -341,7 +478,7 @@ public class TestBookKeeperJournalManage out.flush(); bookieToFail.shutdown(); assertEquals("New bookie didn't die", - numBookies, bkutil.checkBookiesUp(numBookies, 10)); + numBookies, checkBookiesUp(numBookies, 10)); try { for (long i = 1 ; i <= 3; i++) { @@ -357,10 +494,10 @@ public class TestBookKeeperJournalManage assertTrue("Invalid exception message", ioe.getMessage().contains("Failed to write to bookkeeper")); } - replacementBookie = bkutil.newBookie(); + replacementBookie = newBookie(); assertEquals("New bookie didn't start", - numBookies+1, bkutil.checkBookiesUp(numBookies+1, 10)); + numBookies+1, checkBookiesUp(numBookies+1, 10)); out = bkjm.startLogSegment(txid); for (long i = 1 ; i <= 3; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); @@ -380,7 +517,7 @@ public class TestBookKeeperJournalManage } bookieToFail.shutdown(); - if (bkutil.checkBookiesUp(numBookies, 30) != numBookies) { + if (checkBookiesUp(numBookies, 30) != numBookies) { LOG.warn("Not all bookies from this test shut down, expect errors"); } } @@ -393,13 +530,13 @@ public class TestBookKeeperJournalManage */ @Test public void testOneBookieFailure() throws Exception { - BookieServer bookieToFail = bkutil.newBookie(); + BookieServer bookieToFail = newBookie(); BookieServer replacementBookie = null; try { int ensembleSize = numBookies + 1; assertEquals("New bookie didn't start", - ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10)); + ensembleSize, checkBookiesUp(ensembleSize, 10)); // ensure that the journal manager has to use all bookies, // so that a failure will fail the journal manager @@ -410,7 +547,8 @@ public class TestBookKeeperJournalManage ensembleSize); long txid = 1; BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, - BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure")); + URI.create("bookkeeper://" + zkEnsemble + + "/hdfsjournal-onebookiefailure")); EditLogOutputStream out = bkjm.startLogSegment(txid); for (long i = 1 ; i <= 3; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); @@ -420,12 +558,12 @@ public class TestBookKeeperJournalManage out.setReadyToFlush(); out.flush(); - replacementBookie = bkutil.newBookie(); + replacementBookie = newBookie(); assertEquals("replacement bookie didn't start", - ensembleSize+1, bkutil.checkBookiesUp(ensembleSize+1, 10)); + ensembleSize+1, checkBookiesUp(ensembleSize+1, 10)); bookieToFail.shutdown(); assertEquals("New bookie didn't die", - ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10)); + ensembleSize, checkBookiesUp(ensembleSize, 10)); for (long i = 1 ; i <= 3; i++) { FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance(); @@ -443,10 +581,10 @@ public class TestBookKeeperJournalManage } bookieToFail.shutdown(); - if (bkutil.checkBookiesUp(numBookies, 30) != numBookies) { + if (checkBookiesUp(numBookies, 30) != numBookies) { LOG.warn("Not all bookies from this test shut down, expect errors"); } } } -} +} \ No newline at end of file Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java Fri May 25 03:57:56 2012 @@ -36,9 +36,4 @@ public class FSEditLogTestUtil { FSEditLogLoader.EditLogValidation validation = FSEditLogLoader.validateEditLog(in); return (validation.getEndTxId() - in.getFirstTxId()) + 1; } - - public static void setRuntimeForEditLog(NameNode nn, Runtime rt) { - nn.setRuntimeForTesting(rt); - nn.getFSImage().getEditLog().setRuntimeForTesting(rt); - } } Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ ------------------------------------------------------------------------------ Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1342483 Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Fri May 25 03:57:56 2012 @@ -210,10 +210,6 @@ public class FSEditLog { DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_MINIMUM_DEFAULT); journalSet = new JournalSet(minimumRedundantJournals); - // set runtime so we can test starting with a faulty or unavailable - // shared directory - this.journalSet.setRuntimeForTesting(runtime); - for (URI u : dirs) { boolean required = FSNamesystem.getRequiredNamespaceEditsDirs(conf) .contains(u); @@ -825,7 +821,7 @@ public class FSEditLog { * Used only by unit tests. */ @VisibleForTesting - synchronized public void setRuntimeForTesting(Runtime runtime) { + synchronized void setRuntimeForTesting(Runtime runtime) { this.runtime = runtime; this.journalSet.setRuntimeForTesting(runtime); } Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ ------------------------------------------------------------------------------ Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1342483 Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ ------------------------------------------------------------------------------ Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1342483 Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ ------------------------------------------------------------------------------ Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1342483 Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ ------------------------------------------------------------------------------ Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1342483 Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ ------------------------------------------------------------------------------ Reverse-merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1342483 Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Fri May 25 03:57:56 2012 @@ -131,7 +131,6 @@ public class MiniDFSCluster { private int numDataNodes = 1; private boolean format = true; private boolean manageNameDfsDirs = true; - private boolean manageNameDfsSharedDirs = true; private boolean manageDataDfsDirs = true; private StartupOption option = null; private String[] racks = null; @@ -189,14 +188,6 @@ public class MiniDFSCluster { /** * Default: true */ - public Builder manageNameDfsSharedDirs(boolean val) { - this.manageNameDfsSharedDirs = val; - return this; - } - - /** - * Default: true - */ public Builder manageDataDfsDirs(boolean val) { this.manageDataDfsDirs = val; return this; @@ -295,7 +286,6 @@ public class MiniDFSCluster { builder.numDataNodes, builder.format, builder.manageNameDfsDirs, - builder.manageNameDfsSharedDirs, builder.manageDataDfsDirs, builder.option, builder.racks, @@ -535,7 +525,7 @@ public class MiniDFSCluster { long[] simulatedCapacities) throws IOException { this.nameNodes = new NameNodeInfo[1]; // Single namenode in the cluster initMiniDFSCluster(conf, numDataNodes, format, - manageNameDfsDirs, true, manageDataDfsDirs, operation, racks, hosts, + manageNameDfsDirs, manageDataDfsDirs, operation, racks, hosts, simulatedCapacities, null, true, false, MiniDFSNNTopology.simpleSingleNN(nameNodePort, 0)); } @@ -543,8 +533,7 @@ public class MiniDFSCluster { private void initMiniDFSCluster( Configuration conf, int numDataNodes, boolean format, boolean manageNameDfsDirs, - boolean manageNameDfsSharedDirs, boolean manageDataDfsDirs, - StartupOption operation, String[] racks, + boolean manageDataDfsDirs, StartupOption operation, String[] racks, String[] hosts, long[] simulatedCapacities, String clusterId, boolean waitSafeMode, boolean setupHostsFile, MiniDFSNNTopology nnTopology) @@ -583,8 +572,7 @@ public class MiniDFSCluster { federation = nnTopology.isFederated(); createNameNodesAndSetConf( - nnTopology, manageNameDfsDirs, manageNameDfsSharedDirs, - format, operation, clusterId, conf); + nnTopology, manageNameDfsDirs, format, operation, clusterId, conf); if (format) { if (data_dir.exists() && !FileUtil.fullyDelete(data_dir)) { @@ -605,8 +593,8 @@ public class MiniDFSCluster { } private void createNameNodesAndSetConf(MiniDFSNNTopology nnTopology, - boolean manageNameDfsDirs, boolean manageNameDfsSharedDirs, - boolean format, StartupOption operation, String clusterId, + boolean manageNameDfsDirs, boolean format, StartupOption operation, + String clusterId, Configuration conf) throws IOException { Preconditions.checkArgument(nnTopology.countNameNodes() > 0, "empty NN topology: no namenodes specified!"); @@ -651,7 +639,7 @@ public class MiniDFSCluster { if (nnIds.size() > 1) { conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, nameservice.getId()), Joiner.on(",").join(nnIds)); - if (manageNameDfsSharedDirs) { + if (manageNameDfsDirs) { URI sharedEditsUri = getSharedEditsDir(nnCounter, nnCounter+nnIds.size()-1); conf.set(DFS_NAMENODE_SHARED_EDITS_DIR_KEY, sharedEditsUri.toString()); } Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java?rev=1342494&r1=1342493&r2=1342494&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyCheckpoints.java Fri May 25 03:57:56 2012 @@ -54,9 +54,9 @@ import com.google.common.collect.Lists; public class TestStandbyCheckpoints { private static final int NUM_DIRS_IN_LOG = 200000; - protected MiniDFSCluster cluster; - protected NameNode nn0, nn1; - protected FileSystem fs; + private MiniDFSCluster cluster; + private NameNode nn0, nn1; + private FileSystem fs; @SuppressWarnings("rawtypes") @Before