hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nspiegelb...@apache.org
Subject svn commit: r1181509 - in /hbase/branches/0.89/src: main/java/org/apache/hadoop/hbase/regionserver/MemStore.java main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Date Tue, 11 Oct 2011 02:14:52 GMT
Author: nspiegelberg
Date: Tue Oct 11 02:14:51 2011
New Revision: 1181509

URL: http://svn.apache.org/viewvc?rev=1181509&view=rev
Log:
Importing HBASE-3235.

Added:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java?rev=1181509&r1=1181508&r2=1181509&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java Tue
Oct 11 02:14:51 2011
@@ -376,7 +376,11 @@ public class MemStore implements HeapSiz
       long addedSize = add(newKv);
 
       // now find and RM the old one(s) to prevent version explosion:
-      SortedSet<KeyValue> ss = kvset.tailSet(newKv);
+	  KeyValue firstKv = KeyValue.createFirstOnRow(
+			  newKv.getBuffer(), newKv.getRowOffset(), newKv.getRowLength(),
+			  newKv.getBuffer(), newKv.getFamilyOffset(), newKv.getFamilyLength(),
+			  newKv.getBuffer(), newKv.getQualifierOffset(), newKv.getQualifierLength());
+      SortedSet<KeyValue> ss = kvset.tailSet(firstKv);
       Iterator<KeyValue> it = ss.iterator();
       while ( it.hasNext() ) {
         KeyValue kv = it.next();
@@ -397,9 +401,7 @@ public class MemStore implements HeapSiz
             newKv.getBuffer(), newKv.getQualifierOffset(), newKv.getQualifierLength(),
             kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength())) {
 
-          // to be extra safe we only remove Puts that have a memstoreTS==0
-          if (kv.getType() == KeyValue.Type.Put.getCode() &&
-              kv.getMemstoreTS() == 0) {
+          if (kv.getType() == KeyValue.Type.Put.getCode()) {
             // false means there was a change, so give us the size.
             addedSize -= heapSizeChange(kv, true);
 

Added: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java?rev=1181509&view=auto
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
(added)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/util/ManualEnvironmentEdge.java
Tue Oct 11 02:14:51 2011
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+/**
+ * An environment edge that uses a manually set value. This is useful for testing events
that are supposed to
+ * happen in the same millisecond.
+ */
+public class ManualEnvironmentEdge implements EnvironmentEdge {
+
+  // Sometimes 0 ts might have a special value, so lets start with 1
+  protected long value = 1L;
+
+  public void setValue(long newValue) {
+    value = newValue;
+  }
+
+  @Override
+  public long currentTimeMillis() {
+    return this.value;
+  }
+}

Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1181509&r1=1181508&r2=1181509&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
(original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Tue Oct 11 02:14:51 2011
@@ -49,8 +49,11 @@ import org.apache.hadoop.hbase.filter.Si
 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScanner;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdge;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
 import org.apache.hadoop.hbase.util.IncrementingEnvironmentEdge;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Threads;
 
@@ -1859,8 +1862,8 @@ public class TestHRegion extends HBaseTe
     assertEquals(value+amount, result);
 
     Store store = region.getStore(fam1);
-    // we will have the original Put, and also the ICV'ed Put as well.
-    assertEquals(2, store.memstore.kvset.size());
+    // ICV removes any extra values floating around in there.
+    assertEquals(1, store.memstore.kvset.size());
     assertTrue(store.memstore.snapshot.isEmpty());
 
     assertICV(row, fam1, qual1, value+amount);
@@ -2022,6 +2025,59 @@ public class TestHRegion extends HBaseTe
     assertICV(row, fam1, qual3, amount);
   }
 
+  /**
+   * Added for HBASE-3235.
+   *
+   * When the initial put and an ICV update were arriving with the same timestamp,
+   * the initial Put KV was being skipped during {@link MemStore#upsert(KeyValue)}
+   * causing the iteration for matching KVs, causing the update-in-place to not
+   * happen and the ICV put to effectively disappear.
+   * @throws IOException
+   */
+  public void testIncrementColumnValue_UpdatingInPlace_TimestampClobber() throws IOException
{
+    initHRegion(tableName, getName(), fam1);
+
+    long value = 1L;
+    long amount = 3L;
+    long now = EnvironmentEdgeManager.currentTimeMillis();
+    ManualEnvironmentEdge mock = new ManualEnvironmentEdge();
+    mock.setValue(now);
+    EnvironmentEdgeManagerTestHelper.injectEdge(mock);
+
+    // verify we catch an ICV on a put with the same timestamp
+    Put put = new Put(row);
+    put.add(fam1, qual1, now, Bytes.toBytes(value));
+    region.put(put);
+
+    long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
+
+    assertEquals(value+amount, result);
+
+    Store store = region.getStore(fam1);
+    // ICV should update the existing Put with the same timestamp
+    assertEquals(1, store.memstore.kvset.size());
+    assertTrue(store.memstore.snapshot.isEmpty());
+
+    assertICV(row, fam1, qual1, value+amount);
+
+    // verify we catch an ICV even when the put ts > now
+    put = new Put(row);
+    put.add(fam1, qual2, now+1, Bytes.toBytes(value));
+    region.put(put);
+
+    result = region.incrementColumnValue(row, fam1, qual2, amount, true);
+
+    assertEquals(value+amount, result);
+
+    store = region.getStore(fam1);
+    // ICV should update the existing Put with the same timestamp
+    assertEquals(2, store.memstore.kvset.size());
+    assertTrue(store.memstore.snapshot.isEmpty());
+
+    assertICV(row, fam1, qual2, value+amount);
+    EnvironmentEdgeManagerTestHelper.reset();
+  }
+
   private void assertICV(byte [] row,
                          byte [] familiy,
                          byte[] qualifier,



Mime
View raw message