zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tha...@apache.org
Subject svn commit: r1497929 [2/2] - in /zookeeper/trunk: ./ src/java/main/org/apache/zookeeper/server/ src/java/main/org/apache/zookeeper/server/persistence/ src/java/main/org/apache/zookeeper/server/quorum/ src/java/test/org/apache/zookeeper/server/quorum/ s...
Date Fri, 28 Jun 2013 21:22:36 GMT
Added: zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java
URL: http://svn.apache.org/viewvc/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java?rev=1497929&view=auto
==============================================================================
--- zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java (added)
+++ zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java Fri
Jun 28 21:22:35 2013
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.apache.jute.Record;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.PortAssignment;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZKTestCase;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooDefs.OpCode;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.SyncRequestProcessor;
+import org.apache.zookeeper.server.ZKDatabase;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.quorum.Leader.Proposal;
+import org.apache.zookeeper.server.util.SerializeUtils;
+import org.apache.zookeeper.txn.TxnHeader;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test loading committed proposal from txnlog. Learner uses these proposals to
+ * catch-up with leader
+ */
+public class GetProposalFromTxnTest extends ZKTestCase implements Watcher {
+    private static final Logger LOG = Logger
+            .getLogger(GetProposalFromTxnTest.class);
+    private static String HOSTPORT = "127.0.0.1:" + PortAssignment.unique();
+    private static final int CONNECTION_TIMEOUT = 3000;
+
+    private static final int MSG_COUNT = 2000;
+
+    /**
+     * Test loading proposal from txnlog
+     *
+     * @throws Exception
+     *             an exception might be thrown here
+     */
+    @Test
+    public void testGetProposalFromTxn() throws Exception {
+        File tmpDir = ClientBase.createTmpDir();
+        ClientBase.setupTestEnv();
+        ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
+        SyncRequestProcessor.setSnapCount(100);
+        final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
+        ServerCnxnFactory f = ServerCnxnFactory.createFactory(PORT, -1);
+        f.startup(zks);
+        Assert.assertTrue("waiting for server being up ",
+                ClientBase.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT));
+        ZooKeeper zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this);
+
+        // Generate transaction so we will have some txnlog
+        Long[] zxids = new Long[MSG_COUNT];
+        try {
+            String data = "data";
+            byte[] bytes = data.getBytes();
+            for (int i = 0; i < MSG_COUNT; i++) {
+                Stat stat = new Stat();
+                zk.create("/invalidsnap-" + i, bytes, Ids.OPEN_ACL_UNSAFE,
+                        CreateMode.PERSISTENT);
+                zk.getData("/invalidsnap-" + i, null, stat);
+                zxids[i] = stat.getCzxid();
+            }
+
+        } finally {
+            zk.close();
+        }
+
+        // shutdown and start zookeeper again
+        f.shutdown();
+        Assert.assertTrue("waiting for server to shutdown",
+                ClientBase.waitForServerDown(HOSTPORT, CONNECTION_TIMEOUT));
+        zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
+        zks.startdata();
+
+        ZKDatabase db = zks.getZKDatabase();
+
+        // Set sizeLimit to be very high number, so we can pull all transactions
+        // from txnlog
+        Iterator<Proposal> itr = db.getProposalsFromTxnLog(zxids[0], 10000000);
+
+        int createCount = 0;
+        ArrayList<Long> retrievedZxids = new ArrayList<Long>(MSG_COUNT);
+
+        // Get zxid of create requests
+        while (itr.hasNext()) {
+            Proposal proposal = itr.next();
+            TxnHeader hdr = new TxnHeader();
+            Record rec = SerializeUtils.deserializeTxn(
+                    proposal.packet.getData(), hdr);
+            if (hdr.getType() == OpCode.create) {
+                retrievedZxids.add(hdr.getZxid());
+                createCount++;
+            }
+        }
+
+        // All zxid should match what we created
+        Assert.assertTrue("Zxids missmatches",
+                Arrays.equals(zxids, retrievedZxids.toArray(new Long[0])));
+
+        // There should be 2000 create requests
+        Assert.assertTrue("create proposal count == " + MSG_COUNT,
+                (createCount == MSG_COUNT));
+
+        // We are requesting half the number of transaction from the snapshot
+        // this should exceed threshold (ZKDatabase.snapshotSizeFactor)
+        db.setSnapshotSizeFactor(0.33);
+        long sizeLimit = db.calculateTxnLogSizeLimit();
+
+        itr = db.getProposalsFromTxnLog(zxids[MSG_COUNT / 2], sizeLimit);
+        Assert.assertFalse("Expect empty proposal", (itr.hasNext()));
+    }
+
+    public void process(WatchedEvent event) {
+        // do nothing
+    }
+
+}

Propchange: zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: zookeeper/trunk/src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java
URL: http://svn.apache.org/viewvc/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java?rev=1497929&r1=1497928&r2=1497929&view=diff
==============================================================================
--- zookeeper/trunk/src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java (original)
+++ zookeeper/trunk/src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java Fri Jun 28
21:22:35 2013
@@ -46,6 +46,8 @@ import org.apache.zookeeper.server.ZooKe
 import org.apache.zookeeper.server.persistence.FileHeader;
 import org.apache.zookeeper.server.persistence.FileTxnLog;
 import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
+import org.apache.zookeeper.server.persistence.Util;
+import org.apache.zookeeper.server.persistence.FileTxnLog.FileTxnIterator;
 import org.apache.zookeeper.server.persistence.TxnLog.TxnIterator;
 import org.apache.zookeeper.txn.CreateTxn;
 import org.apache.zookeeper.txn.DeleteTxn;
@@ -104,6 +106,14 @@ public class LoadFromLogTest extends ZKT
         FileTxnLog txnLog = new FileTxnLog(logDir);
 
         TxnIterator itr = txnLog.read(0);
+        
+        // Check that storage space return some value
+        FileTxnIterator fileItr = (FileTxnIterator) itr;
+        long storageSize = fileItr.getStorageSize();
+        LOG.info("Txnlog size: " + storageSize + " bytes");
+        Assert.assertTrue("Storage size is greater than zero ",
+                (storageSize > 0));
+        
         long expectedZxid = 0;
         long lastZxid = 0;
         TxnHeader hdr;
@@ -118,8 +128,79 @@ public class LoadFromLogTest extends ZKT
         Assert.assertTrue("processed all transactions. " + expectedZxid + " == " + TOTAL_TRANSACTIONS,
(expectedZxid == TOTAL_TRANSACTIONS));
     }
 
+    /**
+     * test that we fail to load txnlog of a request zxid that is older
+     * than what exist on disk
+     * @throws Exception an exception might be thrown here
+     */
+    @Test
+    public void testLoadFailure() throws Exception {
+        // setup a single server cluster
+        File tmpDir = ClientBase.createTmpDir();
+        ClientBase.setupTestEnv();
+        ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
+        // So we have at least 4 logs
+        SyncRequestProcessor.setSnapCount(50);
+        final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
+        ServerCnxnFactory f = ServerCnxnFactory.createFactory(PORT, -1);
+        f.startup(zks);
+        Assert.assertTrue("waiting for server being up ",
+                ClientBase.waitForServerUp(HOSTPORT,CONNECTION_TIMEOUT));
+        ZooKeeper zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this);
 
+        // generate some transactions that will get logged
+        try {
+            for (int i = 0; i< NUM_MESSAGES; i++) {
+                zk.create("/data-", new byte[0], Ids.OPEN_ACL_UNSAFE,
+                        CreateMode.PERSISTENT_SEQUENTIAL);
+            }
+        } finally {
+            zk.close();
+        }
+        f.shutdown();
+        Assert.assertTrue("waiting for server to shutdown",
+                ClientBase.waitForServerDown(HOSTPORT, CONNECTION_TIMEOUT));
 
+        File logDir = new File(tmpDir, FileTxnSnapLog.version + FileTxnSnapLog.VERSION);
+        File[] logFiles = FileTxnLog.getLogFiles(logDir.listFiles(), 0);
+        // Verify that we have at least 4 txnlog
+        Assert.assertTrue(logFiles.length > 4);
+        // Delete the first log file, so we will fail to read it back from disk
+        Assert.assertTrue("delete the first log file", logFiles[0].delete());
+
+        // Find zxid for the second log
+        long secondStartZxid = Util.getZxidFromName(logFiles[1].getName(), "log");
+
+        FileTxnLog txnLog = new FileTxnLog(logDir);
+        TxnIterator itr = txnLog.read(1, false);
+
+        // Oldest log is already remove, so this should point to the start of
+        // of zxid on the second log
+        Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+
+        itr = txnLog.read(secondStartZxid, false);
+        Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+        Assert.assertTrue(itr.next());
+
+        // Trying to get a second txn on second txnlog give us the
+        // the start of second log, since the first one is removed
+        long nextZxid = itr.getHeader().getZxid();
+
+        itr = txnLog.read(nextZxid, false);
+        Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+
+        // Trying to get a first txn on the third give us the
+        // the start of second log, since the first one is removed
+        long thirdStartZxid = Util.getZxidFromName(logFiles[2].getName(), "log");
+        itr = txnLog.read(thirdStartZxid, false);
+        Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+        Assert.assertTrue(itr.next());
+
+        nextZxid = itr.getHeader().getZxid();
+        itr = txnLog.read(nextZxid, false);
+        Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+
+    }
 
     public void process(WatchedEvent event) {
     	switch (event.getType()) {



Mime
View raw message