hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r721410 - in /hadoop/hbase/trunk/src: java/org/apache/hadoop/hbase/client/ java/org/apache/hadoop/hbase/master/ java/org/apache/hadoop/hbase/regionserver/ test/org/apache/hadoop/hbase/client/
Date Fri, 28 Nov 2008 10:01:39 GMT
Author: apurtell
Date: Fri Nov 28 02:01:39 2008
New Revision: 721410

URL: http://svn.apache.org/viewvc?rev=721410&view=rev
Log:
HBASE-902 Add force compaction and force split operations to UI and Admin; force compactions
before split, added test case

Added:
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java   (with
props)
Modified:
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Fri Nov 28
02:01:39 2008
@@ -529,6 +529,7 @@
         arr[0] = (HTableDescriptor)args[0];
         this.master.modifyTable(tableName, op, arr);
       } break;
+      case HConstants.MODIFY_TABLE_COMPACT:
       case HConstants.MODIFY_TABLE_SPLIT: {
         Writable[] arr = null;
         if (args != null && args.length > 0) {
@@ -538,8 +539,7 @@
           } else if (args[0] instanceof ImmutableBytesWritable) {
             arr[0] = (ImmutableBytesWritable)args[0];
           } else {
-            throw new IOException(
-              "SPLIT with arg requires byte[] or ImmutableBytesWritable");
+            throw new IOException("SPLIT or COMPACT with arg requires byte[] or ImmutableBytesWritable");
           }
         }
         this.master.modifyTable(tableName, op, arr);

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/master/RegionManager.java Fri Nov
28 02:01:39 2008
@@ -996,6 +996,10 @@
     while (i.hasNext()) {
       Pair<HRegionInfo,HServerAddress> pair = i.next();
       if (addr.equals(pair.getSecond())) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("sending MSG_REGION_COMPACT " + pair.getFirst() + " to " +
+              addr);
+        }
         returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_COMPACT, pair.getFirst()));
         i.remove();
       }
@@ -1004,6 +1008,10 @@
     while (i.hasNext()) {
       Pair<HRegionInfo,HServerAddress> pair = i.next();
       if (addr.equals(pair.getSecond())) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("sending MSG_REGION_SPLIT " + pair.getFirst() + " to " +
+              addr);
+        }
         returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_SPLIT, pair.getFirst()));
         i.remove();
       }

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Fri Nov
28 02:01:39 2008
@@ -534,8 +534,7 @@
       byte [] startKey = this.regionInfo.getStartKey();
       byte [] endKey = this.regionInfo.getEndKey();
       if (HStoreKey.equalsTwoRowKeys(this.regionInfo,startKey, midKey)) {
-        LOG.debug("Startkey (" + startKey + ") and midkey + (" + 
-          midKey + ") are same, not splitting");
+        LOG.debug("Startkey and midkey are same, not splitting");
         return null;
       }
       if (HStoreKey.equalsTwoRowKeys(this.regionInfo,midKey, endKey)) {

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Fri
Nov 28 02:01:39 2008
@@ -1018,7 +1018,11 @@
             case MSG_REGION_SPLIT: {
               // Force split a region
               HRegion region = getRegion(info.getRegionName());
+              // flush the memcache for the region
+              region.flushcache();
+              // flag that the region should be split
               region.regionInfo.shouldSplit(true);
+              // force a compaction
               compactSplitThread.compactionRequested(region,
                 "MSG_REGION_SPLIT");
             } break;
@@ -1026,6 +1030,9 @@
             case MSG_REGION_COMPACT: {
               // Compact a region
               HRegion region = getRegion(info.getRegionName());
+              // flush the memcache for the region
+              region.flushcache();
+              // force a compaction
               compactSplitThread.compactionRequested(region,
                 "MSG_REGION_COMPACT");
             } break;

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=721410&r1=721409&r2=721410&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HStore.java Fri Nov 28
02:01:39 2008
@@ -823,6 +823,7 @@
       List<HStoreFile> filesToCompact = null;
       synchronized (storefiles) {
         if (this.storefiles.size() <= 0) {
+          LOG.debug("no store files to compact");
           return null;
         }
         // filesToCompact are sorted oldest to newest.
@@ -839,8 +840,8 @@
         doMajorCompaction = isMajorCompaction(filesToCompact);
       }
       boolean references = hasReferences(filesToCompact);
-      if (!doMajorCompaction && !references &&
-          filesToCompact.size() < compactionThreshold) {
+      if (!doMajorCompaction && !references && 
+          (forceSplit || (filesToCompact.size() < compactionThreshold))) {
         return checkSplit(forceSplit);
       }
       if (!fs.exists(compactionDir) && !fs.mkdirs(compactionDir)) {
@@ -2036,6 +2037,9 @@
             splitable = !curHSF.isReference();
             if (!splitable) {
               // RETURN IN MIDDLE OF FUNCTION!!! If not splitable, just return.
+              if (LOG.isDebugEnabled()) {
+                LOG.debug(curHSF +  " is not splittable");
+              }
               return null;
             }
           }
@@ -2063,6 +2067,9 @@
         // (ever) split this region. 
         if (HStoreKey.equalsTwoRowKeys(info, mk.getRow(), firstKey.getRow()) && 
             HStoreKey.equalsTwoRowKeys(info, mk.getRow(), lastKey.getRow())) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("cannot split because midkey is the same as first or last row");
+          }
           return null;
         }
         return new StoreSize(maxSize, mk.getRow());

Added: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java?rev=721410&view=auto
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java (added)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java Fri Nov
28 02:01:39 2008
@@ -0,0 +1,119 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Tests forced splitting of HTable
+ */
+public class TestForceSplit extends HBaseClusterTestCase {
+  private static final byte[] tableName = Bytes.toBytes("test");
+  private static final byte[] columnName = Bytes.toBytes("a:");
+  private static final byte[] key_mmi = Bytes.toBytes("mmi");
+  private static final byte[] key_ssm = Bytes.toBytes("ssm");
+
+  /**
+   * the test
+   * @throws IOException
+   */
+  public void testHTable() throws Exception {
+    // create the test table
+    HTableDescriptor htd = new HTableDescriptor(tableName);
+    htd.addFamily(new HColumnDescriptor(columnName));
+    HBaseAdmin admin = new HBaseAdmin(conf);
+    admin.createTable(htd);
+    HTable table = new HTable(conf, tableName);
+    byte[] k = new byte[3];
+    for (byte b1 = 'a'; b1 < 'z'; b1++) {
+      for (byte b2 = 'a'; b2 < 'z'; b2++) {
+        for (byte b3 = 'a'; b3 < 'z'; b3++) {
+          k[0] = b1;
+          k[1] = b2;
+          k[2] = b3;
+          BatchUpdate update = new BatchUpdate(k);
+          update.put(columnName, k);
+          table.commit(update);
+        }
+      }
+    }
+
+    // get the initial layout (should just be one region)
+    Map<HRegionInfo,HServerAddress> m = table.getRegionsInfo();
+    System.out.println("Initial regions (" + m.size() + "): " + m);
+    assertTrue(m.size() == 1);
+
+    // tell the master to split the table
+    admin.modifyTable(tableName, HConstants.MODIFY_TABLE_SPLIT);
+
+    // give some time for the split to happen
+    Thread.sleep(15 * 1000);
+
+    // check again
+    table = new HTable(conf, tableName);
+    m = table.getRegionsInfo();
+    System.out.println("Regions after split (" + m.size() + "): " + m);
+    // should have two regions now
+    assertTrue(m.size() == 2);
+    // and "mmi" should be the midpoint
+    for (HRegionInfo hri: m.keySet()) {
+      byte[] start = hri.getStartKey();
+      byte[] end = hri.getEndKey();
+      if (Bytes.equals(start, HConstants.EMPTY_BYTE_ARRAY))
+        assertTrue(Bytes.equals(end, key_mmi));
+      if (Bytes.equals(end, key_mmi))
+        assertTrue(Bytes.equals(start, HConstants.EMPTY_BYTE_ARRAY));
+    }
+
+    // tell the master to split the table again, the second half
+    admin.modifyTable(tableName, HConstants.MODIFY_TABLE_SPLIT, key_mmi);
+
+    // give some time for the split to happen
+    Thread.sleep(15 * 1000);
+
+    // check again
+    table = new HTable(conf, tableName);
+    m = table.getRegionsInfo();
+    System.out.println("Regions after split (" + m.size() + "): " + m);
+    // should have three regions now
+    assertTrue(m.size() == 3);
+    // and "mmi" and "ssm" should be the midpoints
+    for (HRegionInfo hri: m.keySet()) {
+      byte[] start = hri.getStartKey();
+      byte[] end = hri.getEndKey();
+      if (Bytes.equals(start, HConstants.EMPTY_BYTE_ARRAY))
+        assertTrue(Bytes.equals(end, key_mmi));
+      if (Bytes.equals(start, key_mmi))
+        assertTrue(Bytes.equals(end, key_ssm));
+      if (Bytes.equals(start, key_ssm))
+        assertTrue(Bytes.equals(end, HConstants.EMPTY_BYTE_ARRAY));
+    }
+  }
+}

Propchange: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java
------------------------------------------------------------------------------
    svn:executable = *



Mime
View raw message