Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 203B1200C5E for ; Sat, 8 Apr 2017 02:05:45 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 1EAC4160BB1; Sat, 8 Apr 2017 00:05:45 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id B3118160B97 for ; Sat, 8 Apr 2017 02:05:42 +0200 (CEST) Received: (qmail 36314 invoked by uid 500); 8 Apr 2017 00:05:41 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 36303 invoked by uid 99); 8 Apr 2017 00:05:41 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 08 Apr 2017 00:05:41 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BB5B7E00C5; Sat, 8 Apr 2017 00:05:41 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Sat, 08 Apr 2017 00:05:41 -0000 Message-Id: <7763d151f78847248225c291e76eedca@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/18] hbase-site git commit: Published site at 59e8b8e2ba4d403d042fe4cc02f8f9f80aad67af. archived-at: Sat, 08 Apr 2017 00:05:45 -0000 Repository: hbase-site Updated Branches: refs/heads/asf-site 6cfe10497 -> 799947ba1 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/799947ba/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.html ---------------------------------------------------------------------- diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.html index 82a82ad..cfccce7 100644 --- a/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.html +++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.html @@ -34,912 +34,911 @@ 026import java.util.Arrays; 027import java.util.Comparator; 028import java.util.HashSet; -029import java.util.List; -030import java.util.Set; -031 -032import org.apache.commons.logging.Log; -033import org.apache.commons.logging.LogFactory; -034import org.apache.hadoop.conf.Configuration; -035import org.apache.hadoop.fs.FileSystem; -036import org.apache.hadoop.fs.FileStatus; -037import org.apache.hadoop.fs.Path; -038import org.apache.hadoop.hbase.HBaseCommonTestingUtility; -039import org.apache.hadoop.hbase.procedure2.Procedure; -040import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; -041import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter; -042import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.TestProcedure; -043import org.apache.hadoop.hbase.procedure2.SequentialProcedure; -044import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; -045import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator; -046import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker; -047import org.apache.hadoop.hbase.testclassification.SmallTests; -048import org.apache.hadoop.hbase.testclassification.MasterTests; -049import org.apache.hadoop.hbase.util.Bytes; -050import org.apache.hadoop.io.IOUtils; -051 -052import org.junit.After; -053import org.junit.Before; -054import org.junit.Test; -055import org.junit.experimental.categories.Category; -056 -057import static org.junit.Assert.assertEquals; -058import static org.junit.Assert.assertFalse; -059import static org.junit.Assert.assertTrue; -060import static org.junit.Assert.fail; -061 -062@Category({MasterTests.class, SmallTests.class}) -063public class TestWALProcedureStore { -064 private static final Log LOG = LogFactory.getLog(TestWALProcedureStore.class); -065 -066 private static final int PROCEDURE_STORE_SLOTS = 1; -067 private static final Procedure NULL_PROC = null; -068 -069 private WALProcedureStore procStore; -070 -071 private HBaseCommonTestingUtility htu; -072 private FileSystem fs; -073 private Path testDir; -074 private Path logDir; -075 -076 private void setupConfig(final Configuration conf) { -077 conf.setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, true); -078 } -079 -080 @Before -081 public void setUp() throws IOException { -082 htu = new HBaseCommonTestingUtility(); -083 testDir = htu.getDataTestDir(); -084 fs = testDir.getFileSystem(htu.getConfiguration()); -085 assertTrue(testDir.depth() > 1); -086 -087 setupConfig(htu.getConfiguration()); -088 logDir = new Path(testDir, "proc-logs"); -089 procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir); -090 procStore.start(PROCEDURE_STORE_SLOTS); -091 procStore.recoverLease(); -092 procStore.load(new LoadCounter()); -093 } -094 -095 @After -096 public void tearDown() throws IOException { -097 procStore.stop(false); -098 fs.delete(logDir, true); -099 } -100 -101 private void storeRestart(ProcedureStore.ProcedureLoader loader) throws Exception { -102 ProcedureTestingUtility.storeRestart(procStore, loader); -103 } -104 -105 @Test -106 public void testEmptyRoll() throws Exception { -107 for (int i = 0; i < 10; ++i) { -108 procStore.periodicRollForTesting(); -109 } -110 assertEquals(1, procStore.getActiveLogs().size()); -111 FileStatus[] status = fs.listStatus(logDir); -112 assertEquals(1, status.length); -113 } -114 -115 @Test -116 public void testRestartWithoutData() throws Exception { -117 for (int i = 0; i < 10; ++i) { -118 final LoadCounter loader = new LoadCounter(); -119 storeRestart(loader); -120 } -121 LOG.info("ACTIVE WALs " + procStore.getActiveLogs()); -122 assertEquals(1, procStore.getActiveLogs().size()); -123 FileStatus[] status = fs.listStatus(logDir); -124 assertEquals(1, status.length); -125 } -126 -127 /** -128 * Tests that tracker for all old logs are loaded back after procedure store is restarted. -129 */ -130 @Test -131 public void trackersLoadedForAllOldLogs() throws Exception { -132 for (int i = 0; i <= 20; ++i) { -133 procStore.insert(new TestProcedure(i), null); -134 if (i > 0 && (i % 5) == 0) { -135 LoadCounter loader = new LoadCounter(); -136 storeRestart(loader); -137 } -138 } -139 assertEquals(5, procStore.getActiveLogs().size()); -140 for (int i = 0; i < procStore.getActiveLogs().size() - 1; ++i) { -141 ProcedureStoreTracker tracker = procStore.getActiveLogs().get(i).getTracker(); -142 assertTrue(tracker != null && !tracker.isEmpty()); -143 } -144 } -145 -146 @Test -147 public void testWalCleanerSequentialClean() throws Exception { -148 final Procedure[] procs = new Procedure[5]; -149 ArrayList<ProcedureWALFile> logs = null; -150 -151 // Insert procedures and roll wal after every insert. -152 for (int i = 0; i < procs.length; i++) { -153 procs[i] = new TestSequentialProcedure(); -154 procStore.insert(procs[i], null); -155 procStore.rollWriterForTesting(); -156 logs = procStore.getActiveLogs(); -157 assertEquals(logs.size(), i + 2); // Extra 1 for current ongoing wal. -158 } -159 -160 // Delete procedures in sequential order make sure that only the corresponding wal is deleted -161 // from logs list. -162 final int[] deleteOrder = new int[] { 0, 1, 2, 3, 4 }; -163 for (int i = 0; i < deleteOrder.length; i++) { -164 procStore.delete(procs[deleteOrder[i]].getProcId()); -165 procStore.removeInactiveLogsForTesting(); -166 assertFalse(logs.get(deleteOrder[i]).toString(), -167 procStore.getActiveLogs().contains(logs.get(deleteOrder[i]))); -168 assertEquals(procStore.getActiveLogs().size(), procs.length - i); -169 } -170 } +029import java.util.Set; +030 +031import org.apache.commons.logging.Log; +032import org.apache.commons.logging.LogFactory; +033import org.apache.hadoop.conf.Configuration; +034import org.apache.hadoop.fs.FileSystem; +035import org.apache.hadoop.fs.FileStatus; +036import org.apache.hadoop.fs.Path; +037import org.apache.hadoop.hbase.HBaseCommonTestingUtility; +038import org.apache.hadoop.hbase.procedure2.Procedure; +039import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +040import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter; +041import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.TestProcedure; +042import org.apache.hadoop.hbase.procedure2.SequentialProcedure; +043import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; +044import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator; +045import org.apache.hadoop.hbase.procedure2.store.ProcedureStoreTracker; +046import org.apache.hadoop.hbase.testclassification.SmallTests; +047import org.apache.hadoop.hbase.testclassification.MasterTests; +048import org.apache.hadoop.hbase.util.Bytes; +049import org.apache.hadoop.io.IOUtils; +050 +051import org.junit.After; +052import org.junit.Before; +053import org.junit.Test; +054import org.junit.experimental.categories.Category; +055 +056import static org.junit.Assert.assertEquals; +057import static org.junit.Assert.assertFalse; +058import static org.junit.Assert.assertTrue; +059import static org.junit.Assert.fail; +060 +061@Category({MasterTests.class, SmallTests.class}) +062public class TestWALProcedureStore { +063 private static final Log LOG = LogFactory.getLog(TestWALProcedureStore.class); +064 +065 private static final int PROCEDURE_STORE_SLOTS = 1; +066 private static final Procedure NULL_PROC = null; +067 +068 private WALProcedureStore procStore; +069 +070 private HBaseCommonTestingUtility htu; +071 private FileSystem fs; +072 private Path testDir; +073 private Path logDir; +074 +075 private void setupConfig(final Configuration conf) { +076 conf.setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, true); +077 } +078 +079 @Before +080 public void setUp() throws IOException { +081 htu = new HBaseCommonTestingUtility(); +082 testDir = htu.getDataTestDir(); +083 fs = testDir.getFileSystem(htu.getConfiguration()); +084 assertTrue(testDir.depth() > 1); +085 +086 setupConfig(htu.getConfiguration()); +087 logDir = new Path(testDir, "proc-logs"); +088 procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), fs, logDir); +089 procStore.start(PROCEDURE_STORE_SLOTS); +090 procStore.recoverLease(); +091 procStore.load(new LoadCounter()); +092 } +093 +094 @After +095 public void tearDown() throws IOException { +096 procStore.stop(false); +097 fs.delete(logDir, true); +098 } +099 +100 private void storeRestart(ProcedureStore.ProcedureLoader loader) throws Exception { +101 ProcedureTestingUtility.storeRestart(procStore, loader); +102 } +103 +104 @Test +105 public void testEmptyRoll() throws Exception { +106 for (int i = 0; i < 10; ++i) { +107 procStore.periodicRollForTesting(); +108 } +109 assertEquals(1, procStore.getActiveLogs().size()); +110 FileStatus[] status = fs.listStatus(logDir); +111 assertEquals(1, status.length); +112 } +113 +114 @Test +115 public void testRestartWithoutData() throws Exception { +116 for (int i = 0; i < 10; ++i) { +117 final LoadCounter loader = new LoadCounter(); +118 storeRestart(loader); +119 } +120 LOG.info("ACTIVE WALs " + procStore.getActiveLogs()); +121 assertEquals(1, procStore.getActiveLogs().size()); +122 FileStatus[] status = fs.listStatus(logDir); +123 assertEquals(1, status.length); +124 } +125 +126 /** +127 * Tests that tracker for all old logs are loaded back after procedure store is restarted. +128 */ +129 @Test +130 public void trackersLoadedForAllOldLogs() throws Exception { +131 for (int i = 0; i <= 20; ++i) { +132 procStore.insert(new TestProcedure(i), null); +133 if (i > 0 && (i % 5) == 0) { +134 LoadCounter loader = new LoadCounter(); +135 storeRestart(loader); +136 } +137 } +138 assertEquals(5, procStore.getActiveLogs().size()); +139 for (int i = 0; i < procStore.getActiveLogs().size() - 1; ++i) { +140 ProcedureStoreTracker tracker = procStore.getActiveLogs().get(i).getTracker(); +141 assertTrue(tracker != null && !tracker.isEmpty()); +142 } +143 } +144 +145 @Test +146 public void testWalCleanerSequentialClean() throws Exception { +147 final Procedure[] procs = new Procedure[5]; +148 ArrayList<ProcedureWALFile> logs = null; +149 +150 // Insert procedures and roll wal after every insert. +151 for (int i = 0; i < procs.length; i++) { +152 procs[i] = new TestSequentialProcedure(); +153 procStore.insert(procs[i], null); +154 procStore.rollWriterForTesting(); +155 logs = procStore.getActiveLogs(); +156 assertEquals(logs.size(), i + 2); // Extra 1 for current ongoing wal. +157 } +158 +159 // Delete procedures in sequential order make sure that only the corresponding wal is deleted +160 // from logs list. +161 final int[] deleteOrder = new int[] { 0, 1, 2, 3, 4 }; +162 for (int i = 0; i < deleteOrder.length; i++) { +163 procStore.delete(procs[deleteOrder[i]].getProcId()); +164 procStore.removeInactiveLogsForTesting(); +165 assertFalse(logs.get(deleteOrder[i]).toString(), +166 procStore.getActiveLogs().contains(logs.get(deleteOrder[i]))); +167 assertEquals(procStore.getActiveLogs().size(), procs.length - i); +168 } +169 } +170 171 -172 -173 // Test that wal cleaner doesn't create holes in wal files list i.e. it only deletes files if -174 // they are in the starting of the list. -175 @Test -176 public void testWalCleanerNoHoles() throws Exception { -177 final Procedure[] procs = new Procedure[5]; -178 ArrayList<ProcedureWALFile> logs = null; -179 // Insert procedures and roll wal after every insert. -180 for (int i = 0; i < procs.length; i++) { -181 procs[i] = new TestSequentialProcedure(); -182 procStore.insert(procs[i], null); -183 procStore.rollWriterForTesting(); -184 logs = procStore.getActiveLogs(); -185 assertEquals(i + 2, logs.size()); // Extra 1 for current ongoing wal. -186 } -187 -188 for (int i = 1; i < procs.length; i++) { -189 procStore.delete(procs[i].getProcId()); -190 } -191 assertEquals(procs.length + 1, procStore.getActiveLogs().size()); -192 procStore.delete(procs[0].getProcId()); -193 assertEquals(1, procStore.getActiveLogs().size()); -194 } -195 -196 @Test -197 public void testWalCleanerUpdates() throws Exception { -198 TestSequentialProcedure p1 = new TestSequentialProcedure(); -199 TestSequentialProcedure p2 = new TestSequentialProcedure(); -200 procStore.insert(p1, null); -201 procStore.insert(p2, null); -202 procStore.rollWriterForTesting(); -203 ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); -204 procStore.update(p1); -205 procStore.rollWriterForTesting(); -206 procStore.update(p2); -207 procStore.rollWriterForTesting(); -208 procStore.removeInactiveLogsForTesting(); -209 assertFalse(procStore.getActiveLogs().contains(firstLog)); -210 } -211 -212 @Test -213 public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { -214 TestSequentialProcedure p1 = new TestSequentialProcedure(); -215 TestSequentialProcedure p2 = new TestSequentialProcedure(); -216 procStore.insert(p1, null); -217 procStore.insert(p2, null); -218 procStore.rollWriterForTesting(); // generates first log with p1 + p2 -219 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); -220 procStore.update(p2); -221 procStore.rollWriterForTesting(); // generates second log with p2 -222 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); -223 procStore.update(p2); -224 procStore.rollWriterForTesting(); // generates third log with p2 -225 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. -226 assertEquals(4, procStore.getActiveLogs().size()); -227 procStore.update(p1); -228 procStore.rollWriterForTesting(); // generates fourth log with p1 -229 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. -230 assertEquals(3, procStore.getActiveLogs().size()); -231 assertFalse(procStore.getActiveLogs().contains(log1)); -232 assertFalse(procStore.getActiveLogs().contains(log2)); -233 } -234 -235 @Test -236 public void testWalCleanerWithEmptyRolls() throws Exception { -237 final Procedure[] procs = new Procedure[3]; -238 for (int i = 0; i < procs.length; ++i) { -239 procs[i] = new TestSequentialProcedure(); -240 procStore.insert(procs[i], null); -241 } -242 assertEquals(1, procStore.getActiveLogs().size()); -243 procStore.rollWriterForTesting(); -244 assertEquals(2, procStore.getActiveLogs().size()); -245 procStore.rollWriterForTesting(); -246 assertEquals(3, procStore.getActiveLogs().size()); -247 -248 for (int i = 0; i < procs.length; ++i) { -249 procStore.update(procs[i]); +172 // Test that wal cleaner doesn't create holes in wal files list i.e. it only deletes files if +173 // they are in the starting of the list. +174 @Test +175 public void testWalCleanerNoHoles() throws Exception { +176 final Procedure[] procs = new Procedure[5]; +177 ArrayList<ProcedureWALFile> logs = null; +178 // Insert procedures and roll wal after every insert. +179 for (int i = 0; i < procs.length; i++) { +180 procs[i] = new TestSequentialProcedure(); +181 procStore.insert(procs[i], null); +182 procStore.rollWriterForTesting(); +183 logs = procStore.getActiveLogs(); +184 assertEquals(i + 2, logs.size()); // Extra 1 for current ongoing wal. +185 } +186 +187 for (int i = 1; i < procs.length; i++) { +188 procStore.delete(procs[i].getProcId()); +189 } +190 assertEquals(procs.length + 1, procStore.getActiveLogs().size()); +191 procStore.delete(procs[0].getProcId()); +192 assertEquals(1, procStore.getActiveLogs().size()); +193 } +194 +195 @Test +196 public void testWalCleanerUpdates() throws Exception { +197 TestSequentialProcedure p1 = new TestSequentialProcedure(); +198 TestSequentialProcedure p2 = new TestSequentialProcedure(); +199 procStore.insert(p1, null); +200 procStore.insert(p2, null); +201 procStore.rollWriterForTesting(); +202 ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); +203 procStore.update(p1); +204 procStore.rollWriterForTesting(); +205 procStore.update(p2); +206 procStore.rollWriterForTesting(); +207 procStore.removeInactiveLogsForTesting(); +208 assertFalse(procStore.getActiveLogs().contains(firstLog)); +209 } +210 +211 @Test +212 public void testWalCleanerUpdatesDontLeaveHoles() throws Exception { +213 TestSequentialProcedure p1 = new TestSequentialProcedure(); +214 TestSequentialProcedure p2 = new TestSequentialProcedure(); +215 procStore.insert(p1, null); +216 procStore.insert(p2, null); +217 procStore.rollWriterForTesting(); // generates first log with p1 + p2 +218 ProcedureWALFile log1 = procStore.getActiveLogs().get(0); +219 procStore.update(p2); +220 procStore.rollWriterForTesting(); // generates second log with p2 +221 ProcedureWALFile log2 = procStore.getActiveLogs().get(1); +222 procStore.update(p2); +223 procStore.rollWriterForTesting(); // generates third log with p2 +224 procStore.removeInactiveLogsForTesting(); // Shouldn't remove 2nd log. +225 assertEquals(4, procStore.getActiveLogs().size()); +226 procStore.update(p1); +227 procStore.rollWriterForTesting(); // generates fourth log with p1 +228 procStore.removeInactiveLogsForTesting(); // Should remove first two logs. +229 assertEquals(3, procStore.getActiveLogs().size()); +230 assertFalse(procStore.getActiveLogs().contains(log1)); +231 assertFalse(procStore.getActiveLogs().contains(log2)); +232 } +233 +234 @Test +235 public void testWalCleanerWithEmptyRolls() throws Exception { +236 final Procedure[] procs = new Procedure[3]; +237 for (int i = 0; i < procs.length; ++i) { +238 procs[i] = new TestSequentialProcedure(); +239 procStore.insert(procs[i], null); +240 } +241 assertEquals(1, procStore.getActiveLogs().size()); +242 procStore.rollWriterForTesting(); +243 assertEquals(2, procStore.getActiveLogs().size()); +244 procStore.rollWriterForTesting(); +245 assertEquals(3, procStore.getActiveLogs().size()); +246 +247 for (int i = 0; i < procs.length; ++i) { +248 procStore.update(procs[i]); +249 procStore.rollWriterForTesting(); 250 procStore.rollWriterForTesting(); -251 procStore.rollWriterForTesting(); -252 if (i < (procs.length - 1)) { -253 assertEquals(3 + ((i + 1) * 2), procStore.getActiveLogs().size()); -254 } -255 } -256 assertEquals(7, procStore.getActiveLogs().size()); -257 -258 for (int i = 0; i < procs.length; ++i) { -259 procStore.delete(procs[i].getProcId()); -260 assertEquals(7 - ((i + 1) * 2), procStore.getActiveLogs().size()); -261 } -262 assertEquals(1, procStore.getActiveLogs().size()); -263 } -264 -265 @Test -266 public void testEmptyLogLoad() throws Exception { -267 LoadCounter loader = new LoadCounter(); -268 storeRestart(loader); -269 assertEquals(0, loader.getMaxProcId()); -270 assertEquals(0, loader.getLoadedCount()); -271 assertEquals(0, loader.getCorruptedCount()); -272 } -273 -274 @Test -275 public void testLoad() throws Exception { -276 Set<Long> procIds = new HashSet<>(); -277 -278 // Insert something in the log -279 Procedure proc1 = new TestSequentialProcedure(); -280 procIds.add(proc1.getProcId()); -281 procStore.insert(proc1, null); -282 -283 Procedure proc2 = new TestSequentialProcedure(); -284 Procedure[] child2 = new Procedure[2]; -285 child2[0] = new TestSequentialProcedure(); -286 child2[1] = new TestSequentialProcedure(); -287 -288 procIds.add(proc2.getProcId()); -289 procIds.add(child2[0].getProcId()); -290 procIds.add(child2[1].getProcId()); -291 procStore.insert(proc2, child2); -292 -293 // Verify that everything is there -294 verifyProcIdsOnRestart(procIds); -295 -296 // Update and delete something -297 procStore.update(proc1); -298 procStore.update(child2[1]); -299 procStore.delete(child2[1].getProcId()); -300 procIds.remove(child2[1].getProcId()); -301 -302 // Verify that everything is there -303 verifyProcIdsOnRestart(procIds); -304 -305 // Remove 4 byte from the trailers -306 procStore.stop(false); -307 FileStatus[] logs = fs.listStatus(logDir); -308 assertEquals(3, logs.length); -309 for (int i = 0; i < logs.length; ++i) { -310 corruptLog(logs[i], 4); -311 } -312 verifyProcIdsOnRestart(procIds); -313 } -314 -315 @Test -316 public void testNoTrailerDoubleRestart() throws Exception { -317 // log-0001: proc 0, 1 and 2 are inserted -318 Procedure proc0 = new TestSequentialProcedure(); -319 procStore.insert(proc0, null); -320 Procedure proc1 = new TestSequentialProcedure(); -321 procStore.insert(proc1, null); -322 Procedure proc2 = new TestSequentialProcedure(); -323 procStore.insert(proc2, null); -324 procStore.rollWriterForTesting(); -325 -326 // log-0002: proc 1 deleted -327 procStore.delete(proc1.getProcId()); -328 procStore.rollWriterForTesting(); -329 -330 // log-0003: proc 2 is update -331 procStore.update(proc2); -332 procStore.rollWriterForTesting(); -333 -334 // log-0004: proc 2 deleted -335 procStore.delete(proc2.getProcId()); -336 -337 // stop the store and remove the trailer -338 procStore.stop(false); -339 FileStatus[] logs = fs.listStatus(logDir); -340 assertEquals(4, logs.length); -341 for (int i = 0; i < logs.length; ++i) { -342 corruptLog(logs[i], 4); -343 } -344 -345 // Test Load 1 -346 // Restart the store (avoid cleaning up the files, to check the rebuilded trackers) -347 htu.getConfiguration().setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, false); -348 LoadCounter loader = new LoadCounter(); -349 storeRestart(loader); -350 assertEquals(1, loader.getLoadedCount()); -351 assertEquals(0, loader.getCorruptedCount()); -352 -353 // Test Load 2 -354 assertEquals(5, fs.listStatus(logDir).length); -355 loader = new LoadCounter(); -356 storeRestart(loader); -357 assertEquals(1, loader.getLoadedCount()); -358 assertEquals(0, loader.getCorruptedCount()); -359 -360 // remove proc-0 -361 procStore.delete(proc0.getProcId()); -362 procStore.periodicRollForTesting(); -363 assertEquals(1, fs.listStatus(logDir).length); -364 storeRestart(loader); -365 } -366 -367 @Test -368 public void testProcIdHoles() throws Exception { -369 // Insert -370 for (int i = 0; i < 100; i += 2) { -371 procStore.insert(new TestProcedure(i), null); -372 if (i > 0 && (i % 10) == 0) { -373 LoadCounter loader = new LoadCounter(); -374 storeRestart(loader); -375 assertEquals(0, loader.getCorruptedCount()); -376 assertEquals((i / 2) + 1, loader.getLoadedCount()); -377 } -378 } -379 assertEquals(10, procStore.getActiveLogs().size()); -380 -381 // Delete -382 for (int i = 0; i < 100; i += 2) { -383 procStore.delete(i); -384 } -385 assertEquals(1, procStore.getActiveLogs().size()); -386 -387 LoadCounter loader = new LoadCounter(); -388 storeRestart(loader); -389 assertEquals(0, loader.getLoadedCount()); -390 assertEquals(0, loader.getCorruptedCount()); -391 } -392 -393 @Test -394 public void testCorruptedTrailer() throws Exception { -395 // Insert something -396 for (int i = 0; i < 100; ++i) { -397 procStore.insert(new TestSequentialProcedure(), null); -398 } -399 -400 // Stop the store -401 procStore.stop(false); -402 -403 // Remove 4 byte from the trailer -404 FileStatus[] logs = fs.listStatus(logDir); -405 assertEquals(1, logs.length); -406 corruptLog(logs[0], 4); -407 -408 LoadCounter loader = new LoadCounter(); -409 storeRestart(loader); -410 assertEquals(100, loader.getLoadedCount()); -411 assertEquals(0, loader.getCorruptedCount()); -412 } -413 -414 private static void assertUpdated(final ProcedureStoreTracker tracker, -415 final Procedure[] procs, final int[] updatedProcs, final int[] nonUpdatedProcs) { -416 for (int index : updatedProcs) { -417 long procId = procs[index].getProcId(); -418 assertTrue("Procedure id : " + procId, tracker.isUpdated(procId)); -419 } -420 for (int index : nonUpdatedProcs) { -421 long procId = procs[index].getProcId(); -422 assertFalse("Procedure id : " + procId, tracker.isUpdated(procId)); -423 } -424 } -425 -426 private static void assertDeleted(final ProcedureStoreTracker tracker, -427 final Procedure[] procs, final int[] deletedProcs, final int[] nonDeletedProcs) { -428 for (int index : deletedProcs) { -429 long procId = procs[index].getProcId(); -430 assertEquals("Procedure id : " + procId, -431 ProcedureStoreTracker.DeleteState.YES, tracker.isDeleted(procId)); -432 } -433 for (int index : nonDeletedProcs) { -434 long procId = procs[index].getProcId(); -435 assertEquals("Procedure id : " + procId, -436 ProcedureStoreTracker.DeleteState.NO, tracker.isDeleted(procId)); -437 } -438 } -439 -440 @Test -441 public void testCorruptedTrailersRebuild() throws Exception { -442 final Procedure[] procs = new Procedure[6]; -443 for (int i = 0; i < procs.length; ++i) { -444 procs[i] = new TestSequentialProcedure(); -445 } -446 // Log State (I=insert, U=updated, D=delete) -447 // | log 1 | log 2 | log 3 | -448 // 0 | I, D | | | -449 // 1 | I | | | -450 // 2 | I | D | | -451 // 3 | I | U | | -452 // 4 | | I | D | -453 // 5 | | | I | -454 procStore.insert(procs[0], null); -455 procStore.insert(procs[1], null); -456 procStore.insert(procs[2], null); -457 procStore.insert(procs[3], null); -458 procStore.delete(procs[0].getProcId()); -459 procStore.rollWriterForTesting(); -460 procStore.delete(procs[2].getProcId()); -461 procStore.update(procs[3]); -462 procStore.insert(procs[4], null); -463 procStore.rollWriterForTesting(); -464 procStore.delete(procs[4].getProcId()); -465 procStore.insert(procs[5], null); -466 -467 // Stop the store -468 procStore.stop(false); -469 -470 // Remove 4 byte from the trailers -471 final FileStatus[] logs = fs.listStatus(logDir); -472 assertEquals(3, logs.length); -473 for (int i = 0; i < logs.length; ++i) { -474 corruptLog(logs[i], 4); -475 } -476 -477 // Restart the store (avoid cleaning up the files, to check the rebuilded trackers) -478 htu.getConfiguration().setBoolean(WALProcedureStore.EXEC_WAL_CLEANUP_ON_LOAD_CONF_KEY, false); -479 final LoadCounter loader = new LoadCounter(); -480 storeRestart(loader); -481 assertEquals(3, loader.getLoadedCount()); // procs 1, 3 and 5 -482 assertEquals(0, loader.getCorruptedCount()); -483 -484 // Check the Trackers -485 final ArrayList<ProcedureWALFile> walFiles = procStore.getActiveLogs(); -486 LOG.info("WALs " + walFiles); -487 assertEquals(4, walFiles.size()); -488 LOG.info("Checking wal " + walFiles.get(0)); -489 assertUpdated(walFiles.get(0).getTracker(), procs, new int[]{0, 1, 2, 3}, new int[] {4, 5}); -490 LOG.info("Checking wal " + walFiles.get(1)); -491 assertUpdated(walFiles.get(1).getTracker(), procs, new int[]{2, 3, 4}, new int[] {0, 1, 5}); -492 LOG.info("Checking wal " + walFiles.get(2)); -493 assertUpdated(walFiles.get(2).getTracker(), procs, new int[]{4, 5}, new int[] {0, 1, 2, 3}); -494 LOG.info("Checking global tracker "); -495 assertDeleted(procStore.getStoreTracker(), procs, new int[]{0, 2, 4}, new int[] {1, 3, 5}); -496 } -497 -498 @Test -499 public void testCorruptedEntries() throws Exception { -500 // Insert something -501 for (int i = 0; i < 100; ++i) { -502 procStore.insert(new TestSequentialProcedure(), null); -503 } -504 -505 // Stop the store -506 procStore.stop(false); -507 -508 // Remove some byte from the log -509 // (enough to cut the trailer and corrupt some entries) -510 FileStatus[] logs = fs.listStatus(logDir); -511 assertEquals(1, logs.length); -512 corruptLog(logs[0], 1823); -513 -514 LoadCounter loader = new LoadCounter(); -515 storeRestart(loader); -516 assertTrue(procStore.getCorruptedLogs() != null); -517 assertEquals(1, procStore.getCorruptedLogs().size()); -518 assertEquals(85, loader.getLoadedCount()); -519 assertEquals(0, loader.getCorruptedCount()); -520 } -521 -522 @Test -523 public void testCorruptedProcedures() throws Exception { -524 // Insert root-procedures -525 TestProcedure[] rootProcs = new TestProcedure[10]; -526 for (int i = 1; i <= rootProcs.length; i++) { -527 rootProcs[i-1] = new TestProcedure(i, 0); -528 procStore.insert(rootProcs[i-1], null); -529 rootProcs[i-1].addStackId(0); -530 procStore.update(rootProcs[i-1]); -531 } -532 // insert root-child txn -533 procStore.rollWriterForTesting(); -534 for (int i = 1; i <= rootProcs.length; i++) { -535 TestProcedure b = new TestProcedure(rootProcs.length + i, i); -536 rootProcs[i-1].addStackId(1); -537 procStore.insert(rootProcs[i-1], new Procedure[] { b }); -538 } -539 // insert child updates -540 procStore.rollWriterForTesting(); -541 for (int i = 1; i <= rootProcs.length; i++) { -542 procStore.update(new TestProcedure(rootProcs.length + i, i)); -543 } -544 -545 // Stop the store -546 procStore.stop(false); -547 -548 // the first log was removed, -549 // we have insert-txn and updates in the others so everything is fine -550 FileStatus[] logs = fs.listStatus(logDir); -551 assertEquals(Arrays.toString(logs), 2, logs.length); -552 Arrays.sort(logs, new Comparator<FileStatus>() { -553 @Override -554 public int compare(FileStatus o1, FileStatus o2) { -555 return o1.getPath().getName().compareTo(o2.getPath().getName()); -556 } -557 }); -558 -559 LoadCounter loader = new LoadCounter(); -560 storeRestart(loader); -561 assertEquals(rootProcs.length * 2, loader.getLoadedCount()); -562 assertEquals(0, loader.getCorruptedCount()); -563 -564 // Remove the second log, we have lost all the root/parent references -565 fs.delete(logs[0].getPath(), false); -566 loader.reset(); -567 storeRestart(loader); -568 assertEquals(0, loader.getLoadedCount()); -569 assertEquals(rootProcs.length, loader.getCorruptedCount()); -570 for (Procedure proc: loader.getCorrupted()) { -571 assertTrue(proc.toString(), proc.getParentProcId() <= rootProcs.length); -572 assertTrue(proc.toString(), -573 proc.getProcId() > rootProcs.length && -574 proc.getProcId() <= (rootProcs.length * 2)); -575 } -576 } -577 -578 @Test(timeout=60000) -579 public void testWalReplayOrder_AB_A() throws Exception { -580 /* -581 * | A B | -> | A | -582 */ -583 TestProcedure a = new TestProcedure(1, 0); -584 TestProcedure b = new TestProcedure(2, 1); -585 -586 procStore.insert(a, null); -587 a.addStackId(0); -588 procStore.update(a); -589 -590 procStore.insert(a, new Procedure[] { b }); -591 b.addStackId(1); -592 procStore.update(b); -593 -594 procStore.rollWriterForTesting(); -595 -596 a.addStackId(2); -597 procStore.update(a); -598 -599 storeRestart(new ProcedureStore.ProcedureLoader() { -600 @Override -601 public void setMaxProcId(long maxProcId) { -602 assertEquals(2, maxProcId); -603 } -604 -605 @Override -606 public void load(ProcedureIterator procIter) throws IOException { -607 assertTrue(procIter.hasNext()); -608 assertEquals(1, procIter.nextAsProcedureInfo().getProcId()); -609 assertTrue(procIter.hasNext()); -610 assertEquals(2, procIter.nextAsProcedureInfo().getProcId()); -611 assertFalse(procIter.hasNext()); -612 } -613 -614 @Override -615 public void handleCorrupted(ProcedureIterator procIter) throws IOException { -616 assertFalse(procIter.hasNext()); -617 } -618 }); -619 } -620 -621 @Test(timeout=60000) -622 public void testWalReplayOrder_ABC_BAD() throws Exception { -623 /* -624 * | A B C | -> | B A D | -625 */ -626 TestProcedure a = new TestProcedure(1, 0); -627 TestProcedure b = new TestProcedure(2, 1); -628 TestProcedure c = new TestProcedure(3, 2); -629 TestProcedure d = new TestProcedure(4, 0); -630 -631 procStore.insert(a, null); -632 a.addStackId(0); -633 procStore.update(a); -634 -635 procStore.insert(a, new Procedure[] { b }); -636 b.addStackId(1); -637 procStore.update(b); -638 -639 procStore.insert(b, new Procedure[] { c }); -640 b.addStackId(2); -641 procStore.update(b); -642 -643 procStore.rollWriterForTesting(); -644 -645 b.addStackId(3); -646 procStore.update(b); -647 -648 a.addStackId(4); -649 procStore.update(a); -650 -651 procStore.insert(d, null); -652 d.addStackId(0); -653 procStore.update(d); -654 -655 storeRestart(new ProcedureStore.ProcedureLoader() { -656 @Override -657 public void setMaxProcId(long maxProcId) { -658 assertEquals(4, maxProcId); -659 } -660 -661 @Override -662 public void load(ProcedureIterator procIter) throws IOException { -663 assertTrue(procIter.hasNext()); -664 assertEquals(4, procIter.nextAsProcedureInfo().getProcId()); -665 // TODO: This will be multiple call once we do fast-start -666 //assertFalse(procIter.hasNext()); -667 -668 assertTrue(procIter.hasNext()); -669 assertEquals(1, procIter.nextAsProcedureInfo().getProcId()); -670 assertTrue(procIter.hasNext()); -671 assertEquals(2, procIter.nextAsProcedureInfo().getProcId()); -672 assertTrue(procIter.hasNext()); -673 assertEquals(3, procIter.nextAsProcedureInfo().getProcId()); -674 assertFalse(procIter.hasNext()); -675 } -676 -677 @Override -678 public void handleCorrupted(ProcedureIterator procIter) throws IOException { -679 assertFalse(procIter.hasNext()); -680 } -681 }); -682 } -683 -684 @Test -685 public void testRollAndRemove() throws IOException { -686 // Insert something in the log -687 Procedure proc1 = new TestSequentialProcedure(); -688 procStore.insert(proc1, null); -689 -690 Procedure proc2 = new TestSequentialProcedure(); -691 procStore.insert(proc2, null); -692 -693 // roll the log, now we have 2 -694 procStore.rollWriterForTesting(); -695 assertEquals(2, procStore.getActiveLogs().size()); -696 -697 // everything will be up to date in the second log -698 // so we can remove the first one -699 procStore.update(proc1); -700 procStore.update(proc2); -701 assertEquals(1, procStore.getActiveLogs().size()); -702 -703 // roll the log, now we have 2 -704 procStore.rollWriterForTesting(); -705 assertEquals(2, procStore.getActiveLogs().size()); -706 -707 // remove everything active -708 // so we can remove all the logs -709 procStore.delete(proc1.getProcId()); -710 procStore.delete(proc2.getProcId()); -711 assertEquals(1, procStore.getActiveLogs().size()); -712 } -713 -714 @Test -715 public void testFileNotFoundDuringLeaseRecovery() throws IOException { -716 final TestProcedure[] procs = new TestProcedure[3]; -717 for (int i = 0; i < procs.length; ++i) { -718 procs[i] = new TestProcedure(i + 1, 0); -719 procStore.insert(procs[i], null); -720 } -721 procStore.rollWriterForTesting(); -722 for (int i = 0; i < procs.length; ++i) { -723 procStore.update(procs[i]); -724 procStore.rollWriterForTesting(); -