hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject [1/5] hbase git commit: HBASE-14465 Backport 'Allow rowlock to be reader/write' to branch-1
Date Wed, 30 Sep 2015 18:48:57 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 2ff8580d7 -> 4812d9a17


http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALReaderManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALReaderManager.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALReaderManager.java
index b4eacc5..40db3eb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALReaderManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationWALReaderManager.java
@@ -18,6 +18,18 @@
 
 package org.apache.hadoop.hbase.replication.regionserver;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -26,14 +38,15 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.wal.WAL;
 import org.apache.hadoop.hbase.wal.WALFactory;
 import org.apache.hadoop.hbase.wal.WALKey;
-import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -46,22 +59,12 @@ import org.junit.rules.TestName;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
-
-import static org.junit.Assert.*;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
-
-@Category(LargeTests.class)
+@Category({LargeTests.class})
 @RunWith(Parameterized.class)
 public class TestReplicationWALReaderManager {
 
   private static HBaseTestingUtility TEST_UTIL;
   private static Configuration conf;
-  private static Path hbaseDir;
   private static FileSystem fs;
   private static MiniDFSCluster cluster;
   private static final TableName tableName = TableName.valueOf("tablename");
@@ -76,8 +79,8 @@ public class TestReplicationWALReaderManager {
   private PathWatcher pathWatcher;
   private int nbRows;
   private int walEditKVs;
-  private final AtomicLong sequenceId = new AtomicLong(1);
   @Rule public TestName tn = new TestName();
+  private final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
 
   @Parameters
   public static Collection<Object[]> parameters() {
@@ -106,6 +109,7 @@ public class TestReplicationWALReaderManager {
     this.walEditKVs = walEditKVs;
     TEST_UTIL.getConfiguration().setBoolean(HConstants.ENABLE_WAL_COMPRESSION,
       enableCompression);
+    mvcc.advanceTo(1);
   }
   
   @BeforeClass
@@ -114,7 +118,6 @@ public class TestReplicationWALReaderManager {
     conf = TEST_UTIL.getConfiguration();
     TEST_UTIL.startMiniDFSCluster(3);
 
-    hbaseDir = TEST_UTIL.createRootDir();
     cluster = TEST_UTIL.getDFSCluster();
     fs = cluster.getFileSystem();
   }
@@ -198,8 +201,9 @@ public class TestReplicationWALReaderManager {
   }
 
   private void appendToLogPlus(int count) throws IOException {
-    final long txid = log.append(htd, info, new WALKey(info.getEncodedNameAsBytes(), tableName,
-        System.currentTimeMillis()), getWALEdits(count), sequenceId, true, null);
+    final long txid = log.append(htd, info,
+        new WALKey(info.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc),
+        getWALEdits(count), true);
     log.sync(txid);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java
deleted file mode 100644
index ad6e45a..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.security.visibility;
-
-import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
-
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.security.User;
-import org.junit.BeforeClass;
-import org.junit.experimental.categories.Category;
-
-/**
- * Test class that tests the visibility labels with distributed log replay feature ON.
- */
-@Category(MediumTests.class)
-public class TestVisibilityLabelsWithDistributedLogReplay extends
-    TestVisibilityLabelsWithDefaultVisLabelService {
-
-  @BeforeClass
-  public static void setupBeforeClass() throws Exception {
-    // setup configuration
-    conf = TEST_UTIL.getConfiguration();
-    conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
-    VisibilityTestUtil.enableVisiblityLabels(conf);
-
-    conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
-        ScanLabelGenerator.class);
-    conf.set("hbase.superuser", "admin");
-    // Put meta on master to avoid meta server shutdown handling
-    conf.set("hbase.balancer.tablesOnMaster", "hbase:meta");
-    TEST_UTIL.startMiniCluster(2);
-    SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" });
-    USER1 = User.createUserForTesting(conf, "user1", new String[] {});
-
-    // Wait for the labels table to become available
-    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
-    addLabels();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java
index 3212822..2954096 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/FaultyFSLog.java
@@ -66,11 +66,11 @@ public class FaultyFSLog extends FSHLog {
 
   @Override
   public long append(HTableDescriptor htd, HRegionInfo info, WALKey key, WALEdit edits,
-      AtomicLong sequenceId, boolean isInMemstore, List<Cell> cells) throws IOException
{
+      boolean inMemstore) throws IOException {
     if (this.ft == FailureType.APPEND) {
       throw new IOException("append");
     }
-    return super.append(htd, info, key, edits, sequenceId, isInMemstore, cells);
+    return super.append(htd, info, key, edits, inMemstore);
   }
 }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
index edf03d6..e928a4d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProvider.java
@@ -44,6 +44,8 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.junit.After;
@@ -65,12 +67,14 @@ public class TestDefaultWALProvider {
   protected static Configuration conf;
   protected static FileSystem fs;
   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  protected MultiVersionConcurrencyControl mvcc;
 
   @Rule
   public final TestName currentTest = new TestName();
 
   @Before
   public void setUp() throws Exception {
+    mvcc = new MultiVersionConcurrencyControl();
     FileStatus[] entries = fs.listStatus(new Path("/"));
     for (FileStatus dir : entries) {
       fs.delete(dir.getPath(), true);
@@ -147,14 +151,14 @@ public class TestDefaultWALProvider {
 
 
   protected void addEdits(WAL log, HRegionInfo hri, HTableDescriptor htd,
-                        int times, AtomicLong sequenceId) throws IOException {
+                        int times) throws IOException {
     final byte[] row = Bytes.toBytes("row");
     for (int i = 0; i < times; i++) {
       long timestamp = System.currentTimeMillis();
       WALEdit cols = new WALEdit();
       cols.add(new KeyValue(row, row, row, timestamp, row));
       log.append(htd, hri, getWalKey(hri.getEncodedNameAsBytes(), htd.getTableName(), timestamp),
-        cols, sequenceId, true, null);
+        cols, true);
     }
     log.sync();
   }
@@ -163,7 +167,7 @@ public class TestDefaultWALProvider {
    * used by TestDefaultWALProviderWithHLogKey
    */
   WALKey getWalKey(final byte[] info, final TableName tableName, final long timestamp) {
-    return new WALKey(info, tableName, timestamp);
+    return new WALKey(info, tableName, timestamp, mvcc);
   }
 
   /**
@@ -201,26 +205,26 @@ public class TestDefaultWALProvider {
 
       // Add a single edit and make sure that rolling won't remove the file
       // Before HBASE-3198 it used to delete it
-      addEdits(log, hri, htd, 1, sequenceId);
+      addEdits(log, hri, htd, 1);
       log.rollWriter();
       assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(log));
 
       // See if there's anything wrong with more than 1 edit
-      addEdits(log, hri, htd, 2, sequenceId);
+      addEdits(log, hri, htd, 2);
       log.rollWriter();
       assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(log));
 
       // Now mix edits from 2 regions, still no flushing
-      addEdits(log, hri, htd, 1, sequenceId);
-      addEdits(log, hri2, htd2, 1, sequenceId);
-      addEdits(log, hri, htd, 1, sequenceId);
-      addEdits(log, hri2, htd2, 1, sequenceId);
+      addEdits(log, hri, htd, 1);
+      addEdits(log, hri2, htd2, 1);
+      addEdits(log, hri, htd, 1);
+      addEdits(log, hri2, htd2, 1);
       log.rollWriter();
       assertEquals(3, DefaultWALProvider.getNumRolledLogFiles(log));
 
       // Flush the first region, we expect to see the first two files getting
       // archived. We need to append something or writer won't be rolled.
-      addEdits(log, hri2, htd2, 1, sequenceId);
+      addEdits(log, hri2, htd2, 1);
       log.startCacheFlush(hri.getEncodedNameAsBytes(), htd.getFamiliesKeys());
       log.completeCacheFlush(hri.getEncodedNameAsBytes());
       log.rollWriter();
@@ -229,7 +233,7 @@ public class TestDefaultWALProvider {
       // Flush the second region, which removes all the remaining output files
       // since the oldest was completely flushed and the two others only contain
       // flush information
-      addEdits(log, hri2, htd2, 1, sequenceId);
+      addEdits(log, hri2, htd2, 1);
       log.startCacheFlush(hri2.getEncodedNameAsBytes(), htd2.getFamiliesKeys());
       log.completeCacheFlush(hri2.getEncodedNameAsBytes());
       log.rollWriter();
@@ -276,34 +280,32 @@ public class TestDefaultWALProvider {
       hri1.setSplit(false);
       hri2.setSplit(false);
       // variables to mock region sequenceIds.
-      final AtomicLong sequenceId1 = new AtomicLong(1);
-      final AtomicLong sequenceId2 = new AtomicLong(1);
       // start with the testing logic: insert a waledit, and roll writer
-      addEdits(wal, hri1, table1, 1, sequenceId1);
+      addEdits(wal, hri1, table1, 1);
       wal.rollWriter();
       // assert that the wal is rolled
       assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(wal));
       // add edits in the second wal file, and roll writer.
-      addEdits(wal, hri1, table1, 1, sequenceId1);
+      addEdits(wal, hri1, table1, 1);
       wal.rollWriter();
       // assert that the wal is rolled
       assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
       // add a waledit to table1, and flush the region.
-      addEdits(wal, hri1, table1, 3, sequenceId1);
+      addEdits(wal, hri1, table1, 3);
       flushRegion(wal, hri1.getEncodedNameAsBytes(), table1.getFamiliesKeys());
       // roll log; all old logs should be archived.
       wal.rollWriter();
       assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));
       // add an edit to table2, and roll writer
-      addEdits(wal, hri2, table2, 1, sequenceId2);
+      addEdits(wal, hri2, table2, 1);
       wal.rollWriter();
       assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(wal));
       // add edits for table1, and roll writer
-      addEdits(wal, hri1, table1, 2, sequenceId1);
+      addEdits(wal, hri1, table1, 2);
       wal.rollWriter();
       assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
       // add edits for table2, and flush hri1.
-      addEdits(wal, hri2, table2, 2, sequenceId2);
+      addEdits(wal, hri2, table2, 2);
       flushRegion(wal, hri1.getEncodedNameAsBytes(), table2.getFamiliesKeys());
       // the log : region-sequenceId map is
       // log1: region2 (unflushed)
@@ -313,7 +315,7 @@ public class TestDefaultWALProvider {
       wal.rollWriter();
       assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
       // flush region2, and all logs should be archived.
-      addEdits(wal, hri2, table2, 2, sequenceId2);
+      addEdits(wal, hri2, table2, 2);
       flushRegion(wal, hri2.getEncodedNameAsBytes(), table2.getFamiliesKeys());
       wal.rollWriter();
       assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));

http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java
index 0bc8c80..21f28fc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestDefaultWALProviderWithHLogKey.java
@@ -27,6 +27,6 @@ import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 public class TestDefaultWALProviderWithHLogKey extends TestDefaultWALProvider {
   @Override
   WALKey getWalKey(final byte[] info, final TableName tableName, final long timestamp) {
-    return new HLogKey(info, tableName, timestamp);
+    return new HLogKey(info, tableName, timestamp, mvcc);
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java
index 41384aa..d9454bb 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java
@@ -90,7 +90,6 @@ public class TestSecureWAL {
     final byte[] value = Bytes.toBytes("Test value");
     FileSystem fs = TEST_UTIL.getTestFileSystem();
     final WALFactory wals = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestSecureWAL");
-    final AtomicLong sequenceId = new AtomicLong(1);
 
     // Write the WAL
     final WAL wal =
@@ -100,7 +99,7 @@ public class TestSecureWAL {
       WALEdit kvs = new WALEdit();
       kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
       wal.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
-          System.currentTimeMillis()), kvs, sequenceId, true, null);
+          System.currentTimeMillis()), kvs, true);
     }
     wal.sync();
     final Path walPath = DefaultWALProvider.getCurrentFileName(wal);

http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
index bc2716a..878bb32 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALFactory.java
@@ -30,7 +30,6 @@ import java.io.IOException;
 import java.lang.reflect.Method;
 import java.net.BindException;
 import java.util.List;
-import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -48,10 +47,17 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
 import org.apache.hadoop.hbase.coprocessor.SampleRegionWALObserver;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader;
+import org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter;
+import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
+import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
@@ -67,14 +73,6 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestName;
 
-// imports for things that haven't moved from regionserver.wal yet.
-import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
-import org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader;
-import org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter;
-import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
-import org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-
 /**
  * WAL tests that can be reused across providers.
  */
@@ -165,6 +163,7 @@ public class TestWALFactory {
   public void testSplit() throws IOException {
     final TableName tableName = TableName.valueOf(currentTest.getMethodName());
     final byte [] rowName = tableName.getName();
+    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
     final Path logdir = new Path(hbaseDir,
         DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName()));
     Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
@@ -182,7 +181,6 @@ public class TestWALFactory {
     htd.addFamily(new HColumnDescriptor("column"));
 
     // Add edits for three regions.
-    final AtomicLong sequenceId = new AtomicLong(1);
     for (int ii = 0; ii < howmany; ii++) {
       for (int i = 0; i < howmany; i++) {
         final WAL log =
@@ -195,11 +193,13 @@ public class TestWALFactory {
           edit.add(new KeyValue(rowName, family, qualifier,
               System.currentTimeMillis(), column));
           LOG.info("Region " + i + ": " + edit);
-          log.append(htd, infos[i], new WALKey(infos[i].getEncodedNameAsBytes(), tableName,
-              System.currentTimeMillis()), edit, sequenceId, true, null);
+          WALKey walKey =  new WALKey(infos[i].getEncodedNameAsBytes(), tableName,
+              System.currentTimeMillis(), mvcc);
+          log.append(htd, infos[i], walKey, edit, true);
+          walKey.getWriteEntry();
         }
         log.sync();
-        log.rollWriter();
+        log.rollWriter(true);
       }
     }
     wals.shutdown();
@@ -214,6 +214,7 @@ public class TestWALFactory {
   @Test
   public void Broken_testSync() throws Exception {
     TableName tableName = TableName.valueOf(currentTest.getMethodName());
+    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
     // First verify that using streams all works.
     Path p = new Path(dir, currentTest.getMethodName() + ".fsdos");
     FSDataOutputStream out = fs.create(p);
@@ -238,7 +239,6 @@ public class TestWALFactory {
     out.close();
     in.close();
 
-    final AtomicLong sequenceId = new AtomicLong(1);
     final int total = 20;
     WAL.Reader reader = null;
 
@@ -253,7 +253,7 @@ public class TestWALFactory {
         WALEdit kvs = new WALEdit();
         kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
         wal.append(htd, info, new WALKey(info.getEncodedNameAsBytes(), tableName,
-            System.currentTimeMillis()), kvs, sequenceId, true, null);
+            System.currentTimeMillis(), mvcc), kvs, true);
       }
       // Now call sync and try reading.  Opening a Reader before you sync just
       // gives you EOFE.
@@ -272,7 +272,7 @@ public class TestWALFactory {
         WALEdit kvs = new WALEdit();
         kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
         wal.append(htd, info, new WALKey(info.getEncodedNameAsBytes(), tableName,
-            System.currentTimeMillis()), kvs, sequenceId, true, null);
+            System.currentTimeMillis(), mvcc), kvs, true);
       }
       wal.sync();
       reader = wals.createReader(fs, walPath);
@@ -294,7 +294,7 @@ public class TestWALFactory {
         WALEdit kvs = new WALEdit();
         kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), value));
         wal.append(htd, info, new WALKey(info.getEncodedNameAsBytes(), tableName,
-            System.currentTimeMillis()), kvs, sequenceId, true, null);
+            System.currentTimeMillis(), mvcc), kvs,  true);
       }
       // Now I should have written out lots of blocks.  Sync then read.
       wal.sync();
@@ -364,7 +364,6 @@ public class TestWALFactory {
 
     final WAL wal =
         wals.getWAL(regioninfo.getEncodedNameAsBytes(), regioninfo.getTable().getNamespace());
-    final AtomicLong sequenceId = new AtomicLong(1);
     final int total = 20;
 
     HTableDescriptor htd = new HTableDescriptor();
@@ -374,7 +373,7 @@ public class TestWALFactory {
       WALEdit kvs = new WALEdit();
       kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
       wal.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
-          System.currentTimeMillis()), kvs, sequenceId, true, null);
+          System.currentTimeMillis()), kvs,  true);
     }
     // Now call sync to send the data to HDFS datanodes
     wal.sync();
@@ -487,7 +486,7 @@ public class TestWALFactory {
     final byte [] row = Bytes.toBytes("row");
     WAL.Reader reader = null;
     try {
-      final AtomicLong sequenceId = new AtomicLong(1);
+      final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
 
       // Write columns named 1, 2, 3, etc. and then values of single byte
       // 1, 2, 3...
@@ -503,8 +502,9 @@ public class TestWALFactory {
       final WAL log = wals.getWAL(info.getEncodedNameAsBytes(), info.getTable().getNamespace());
 
       final long txid = log.append(htd, info,
-        new WALKey(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis()),
-        cols, sequenceId, true, null);
+        new WALKey(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(),
+            mvcc),
+        cols, true);
       log.sync(txid);
       log.startCacheFlush(info.getEncodedNameAsBytes(), htd.getFamiliesKeys());
       log.completeCacheFlush(info.getEncodedNameAsBytes());
@@ -544,7 +544,7 @@ public class TestWALFactory {
             "column"));
     final byte [] row = Bytes.toBytes("row");
     WAL.Reader reader = null;
-    final AtomicLong sequenceId = new AtomicLong(1);
+    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
     try {
       // Write columns named 1, 2, 3, etc. and then values of single byte
       // 1, 2, 3...
@@ -559,8 +559,9 @@ public class TestWALFactory {
           HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
       final WAL log = wals.getWAL(hri.getEncodedNameAsBytes(), hri.getTable().getNamespace());
       final long txid = log.append(htd, hri,
-        new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis()),
-        cols, sequenceId, true, null);
+        new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(),
+            mvcc),
+        cols, true);
       log.sync(txid);
       log.startCacheFlush(hri.getEncodedNameAsBytes(), htd.getFamiliesKeys());
       log.completeCacheFlush(hri.getEncodedNameAsBytes());
@@ -598,7 +599,7 @@ public class TestWALFactory {
         TableName.valueOf("tablename");
     final byte [] row = Bytes.toBytes("row");
     final DumbWALActionsListener visitor = new DumbWALActionsListener();
-    final AtomicLong sequenceId = new AtomicLong(1);
+    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
     long timestamp = System.currentTimeMillis();
     HTableDescriptor htd = new HTableDescriptor();
     htd.addFamily(new HColumnDescriptor("column"));
@@ -613,7 +614,7 @@ public class TestWALFactory {
           Bytes.toBytes(Integer.toString(i)),
           timestamp, new byte[]{(byte) (i + '0')}));
       log.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,
-          System.currentTimeMillis()), cols, sequenceId, true, null);
+          System.currentTimeMillis(), mvcc), cols, true);
     }
     log.sync();
     assertEquals(COL_COUNT, visitor.increments);
@@ -623,7 +624,7 @@ public class TestWALFactory {
         Bytes.toBytes(Integer.toString(11)),
         timestamp, new byte[]{(byte) (11 + '0')}));
     log.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,
-        System.currentTimeMillis()), cols, sequenceId, true, null);
+        System.currentTimeMillis(), mvcc), cols, true);
     log.sync();
     assertEquals(COL_COUNT, visitor.increments);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
index ca9c5d6..613dea3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java
@@ -21,10 +21,8 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.commons.logging.impl.Log4JLogger;
 import org.apache.hadoop.conf.Configuration;
@@ -38,10 +36,19 @@ import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
+// imports for things that haven't moved from regionserver.wal yet.
+import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
+import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
+import org.apache.hadoop.hbase.regionserver.wal.SecureProtobufLogReader;
+import org.apache.hadoop.hbase.regionserver.wal.SecureProtobufLogWriter;
+import org.apache.hadoop.hbase.regionserver.wal.SecureWALCellCodec;
+import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.zookeeper.ZKSplitLog;
@@ -52,21 +59,11 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.TestName;
 
-// imports for things that haven't moved from regionserver.wal yet.
-import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
-import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
-import org.apache.hadoop.hbase.regionserver.wal.SecureProtobufLogReader;
-import org.apache.hadoop.hbase.regionserver.wal.SecureProtobufLogWriter;
-import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
-import org.apache.hadoop.hbase.regionserver.wal.WALCellCodec;
-import org.apache.hadoop.hbase.regionserver.wal.SecureWALCellCodec;
-
 /*
  * Test that verifies WAL written by SecureProtobufLogWriter is not readable by ProtobufLogReader
  */
 @Category(MediumTests.class)
 public class TestWALReaderOnSecureWAL {
-  private static final Log LOG = LogFactory.getLog(TestWALReaderOnSecureWAL.class);
   static {
     ((Log4JLogger)LogFactory.getLog("org.apache.hadoop.hbase.regionserver.wal"))
       .getLogger().setLevel(Level.ALL);
@@ -103,9 +100,7 @@ public class TestWALReaderOnSecureWAL {
       final int total = 10;
       final byte[] row = Bytes.toBytes("row");
       final byte[] family = Bytes.toBytes("family");
-      FileSystem fs = TEST_UTIL.getTestFileSystem();
-      Path logDir = TEST_UTIL.getDataTestDir(tblName);
-      final AtomicLong sequenceId = new AtomicLong(1);
+      final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
 
       // Write the WAL
       WAL wal =
@@ -114,7 +109,7 @@ public class TestWALReaderOnSecureWAL {
         WALEdit kvs = new WALEdit();
         kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
         wal.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
-            System.currentTimeMillis()), kvs, sequenceId, true, null);
+            System.currentTimeMillis(), mvcc), kvs, true);
       }
       wal.sync();
       final Path walPath = DefaultWALProvider.getCurrentFileName(wal);
@@ -149,7 +144,7 @@ public class TestWALReaderOnSecureWAL {
 
     // Confirm the WAL cannot be read back by ProtobufLogReader
     try {
-      WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
+      wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
       assertFalse(true);
     } catch (IOException ioe) {
       // expected IOE

http://git-wip-us.apache.org/repos/asf/hbase/blob/4812d9a1/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
index 64bf319..3af853b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/WALPerformanceEvaluation.java
@@ -19,14 +19,12 @@
 package org.apache.hadoop.hbase.wal;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
-import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.commons.logging.Log;
@@ -50,16 +48,15 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.LogRoller;
+import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
 import org.apache.hadoop.hbase.trace.HBaseHTraceConfiguration;
 import org.apache.hadoop.hbase.trace.SpanReceiverHost;
 import org.apache.hadoop.hbase.wal.WALProvider.Writer;
-import org.apache.hadoop.hbase.wal.WAL;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
-import org.apache.htrace.HTraceConfiguration;
 import org.apache.htrace.Sampler;
 import org.apache.htrace.Trace;
 import org.apache.htrace.TraceScope;
@@ -99,6 +96,8 @@ public final class WALPerformanceEvaluation extends Configured implements
Tool {
   private final Histogram latencyHistogram =
     metrics.newHistogram(WALPerformanceEvaluation.class, "latencyHistogram", "nanos", true);
 
+  private final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
+
   private HBaseTestingUtility TEST_UTIL;
 
   static final String TABLE_NAME = "WALPerformanceEvaluation";
@@ -179,8 +178,9 @@ public final class WALPerformanceEvaluation extends Configured implements
Tool {
             WALEdit walEdit = new WALEdit();
             addFamilyMapToWALEdit(put.getFamilyCellMap(), walEdit);
             HRegionInfo hri = region.getRegionInfo();
-            final WALKey logkey = new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(),
now);
-            wal.append(htd, hri, logkey, walEdit, region.getSequenceId(), true, null);
+            final WALKey logkey =
+                new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc);
+            wal.append(htd, hri, logkey, walEdit, true);
             if (!this.noSync) {
               if (++lastSync >= this.syncInterval) {
                 wal.sync();


Mime
View raw message