hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1167407 - in /hbase/trunk: CHANGES.txt src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
Date Fri, 09 Sep 2011 23:55:07 GMT
Author: garyh
Date: Fri Sep  9 23:55:06 2011
New Revision: 1167407

URL: http://svn.apache.org/viewvc?rev=1167407&view=rev
Log:
HBASE-4331  Bypassing default actions in prePut fails sometimes with HTable client

Added:
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1167407&r1=1167406&r2=1167407&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Fri Sep  9 23:55:06 2011
@@ -256,6 +256,8 @@ Release 0.91.0 - Unreleased
    HBASE-4007  distributed log splitting can get indefinitely stuck
                (Prakash Khemani)
    HBASE-4301  META migration from 0.90 to trunk fails (Subbu Iyer)
+   HBASE-4331  Bypassing default actions in prePut fails sometimes with
+               HTable client (Lars Hofhansl via garyh)
                
 
   IMPROVEMENTS

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1167407&r1=1167406&r2=1167407&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Fri Sep  9
23:55:06 2011
@@ -1641,23 +1641,17 @@ public class HRegion implements HeapSize
       BatchOperationInProgress<Pair<Put, Integer>> batchOp) throws IOException
{
     /* Run coprocessor pre hook outside of locks to avoid deadlock */
     if (coprocessorHost != null) {
-      List<Pair<Put, Integer>> ops =
-        new ArrayList<Pair<Put, Integer>>(batchOp.operations.length);
       for (int i = 0; i < batchOp.operations.length; i++) {
         Pair<Put, Integer> nextPair = batchOp.operations[i];
         Put put = nextPair.getFirst();
         Map<byte[], List<KeyValue>> familyMap = put.getFamilyMap();
         if (coprocessorHost.prePut(familyMap, put.getWriteToWAL())) {
           // pre hook says skip this Put
-          // adjust nextIndexToProcess if we skipped before it
-          if (batchOp.nextIndexToProcess > i) {
-            batchOp.nextIndexToProcess--;
-          }
-          continue;
+          // mark as success and skip below
+          batchOp.retCodeDetails[i] = new OperationStatus(
+              OperationStatusCode.SUCCESS);
         }
-        ops.add(nextPair);
       }
-      batchOp.operations = ops.toArray(new Pair[ops.size()]);
     }
 
     long now = EnvironmentEdgeManager.currentTimeMillis();
@@ -1687,6 +1681,13 @@ public class HRegion implements HeapSize
         // store the family map reference to allow for mutations
         familyMaps[lastIndexExclusive] = familyMap;
 
+        // skip anything that "ran" already
+        if (batchOp.retCodeDetails[lastIndexExclusive].getOperationStatusCode()
+            != OperationStatusCode.NOT_RUN) {
+          lastIndexExclusive++;
+          continue;
+        }
+
         // Check the families in the put. If bad, skip this one.
         try {
           checkFamilies(familyMap.keySet());

Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java?rev=1167407&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
(added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
Fri Sep  9 23:55:06 2011
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.coprocessor;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestRegionObserverBypass {
+  private static HBaseTestingUtility util;
+  private static final byte[] tableName = Bytes.toBytes("test");
+  private static final byte[] dummy = Bytes.toBytes("dummy");
+  private static final byte[] row1 = Bytes.toBytes("r1");
+  private static final byte[] row2 = Bytes.toBytes("r2");
+  private static final byte[] row3 = Bytes.toBytes("r3");
+  private static final byte[] test = Bytes.toBytes("test");
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
+        TestCoprocessor.class.getName());
+    util = new HBaseTestingUtility(conf);
+    util.startMiniCluster();
+    util.createTable(tableName, new byte[][] {dummy, test});
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    util.shutdownMiniCluster();
+  }
+
+  /**
+   * do a single put that is bypassed by a RegionObserver
+   * @throws Exception
+   */
+  @Test
+  public void testSimple() throws Exception {
+    HTable t = new HTable(util.getConfiguration(), tableName);
+    Put p = new Put(row1);
+    p.add(test,dummy,dummy);
+    // before HBASE-4331, this would throw an exception
+    t.put(p);
+    checkRowAndDelete(t,row1,0);
+  }
+
+  /**
+   * Test various multiput operations.
+   * @throws Exception
+   */
+  @Test
+  public void testMulti() throws Exception {
+    HTable t = new HTable(util.getConfiguration(), tableName);
+    List<Put> puts = new ArrayList<Put>();
+    Put p = new Put(row1);
+    p.add(dummy,dummy,dummy);
+    puts.add(p);
+    p = new Put(row2);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    p = new Put(row3);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    // before HBASE-4331, this would throw an exception
+    t.put(puts);
+    checkRowAndDelete(t,row1,1);
+    checkRowAndDelete(t,row2,0);
+    checkRowAndDelete(t,row3,0);
+
+    puts.clear();
+    p = new Put(row1);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    p = new Put(row2);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    p = new Put(row3);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    // before HBASE-4331, this would throw an exception
+    t.put(puts);
+    checkRowAndDelete(t,row1,0);
+    checkRowAndDelete(t,row2,0);
+    checkRowAndDelete(t,row3,0);
+
+    puts.clear();
+    p = new Put(row1);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    p = new Put(row2);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    p = new Put(row3);
+    p.add(dummy,dummy,dummy);
+    puts.add(p);
+    // this worked fine even before HBASE-4331
+    t.put(puts);
+    checkRowAndDelete(t,row1,0);
+    checkRowAndDelete(t,row2,0);
+    checkRowAndDelete(t,row3,1);
+
+    puts.clear();
+    p = new Put(row1);
+    p.add(dummy,dummy,dummy);
+    puts.add(p);
+    p = new Put(row2);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    p = new Put(row3);
+    p.add(dummy,dummy,dummy);
+    puts.add(p);
+    // this worked fine even before HBASE-4331
+    t.put(puts);
+    checkRowAndDelete(t,row1,1);
+    checkRowAndDelete(t,row2,0);
+    checkRowAndDelete(t,row3,1);
+
+    puts.clear();
+    p = new Put(row1);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    p = new Put(row2);
+    p.add(dummy,dummy,dummy);
+    puts.add(p);
+    p = new Put(row3);
+    p.add(test,dummy,dummy);
+    puts.add(p);
+    // before HBASE-4331, this would throw an exception
+    t.put(puts);
+    checkRowAndDelete(t,row1,0);
+    checkRowAndDelete(t,row2,1);
+    checkRowAndDelete(t,row3,0);
+  }
+
+  private void checkRowAndDelete(HTable t, byte[] row, int count) throws IOException {
+    Get g = new Get(row);
+    Result r = t.get(g);
+    assertEquals(count, r.size());
+    Delete d = new Delete(row);
+    t.delete(d);
+  }
+
+  public static class TestCoprocessor extends BaseRegionObserver {
+    @Override
+    public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
+        final Map<byte[], List<KeyValue>> familyMap, final boolean writeToWAL)
+        throws IOException {
+      if (familyMap.containsKey(test)) {
+        e.bypass();
+      }
+    }
+  }
+}



Mime
View raw message