hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r632663 - in /hadoop/hbase: branches/0.1/CHANGES.txt branches/0.1/src/java/org/apache/hadoop/hbase/HMerge.java trunk/CHANGES.txt trunk/src/java/org/apache/hadoop/hbase/HMerge.java
Date Sat, 01 Mar 2008 22:07:13 GMT
Author: stack
Date: Sat Mar  1 14:07:12 2008
New Revision: 632663

URL: http://svn.apache.org/viewvc?rev=632663&view=rev
Log:
HBASE-480 Tool to manually merge two regions
Commit to TRUNK and branch.

Modified:
    hadoop/hbase/branches/0.1/CHANGES.txt
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HMerge.java
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java

Modified: hadoop/hbase/branches/0.1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/CHANGES.txt?rev=632663&r1=632662&r2=632663&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.1/CHANGES.txt Sat Mar  1 14:07:12 2008
@@ -34,6 +34,7 @@
                repetition of retry-on-failure logic (thanks to Peter Dolan and
                Bryan Duxbury)
    HBASE-281   Shell should allow deletions in .META. and -ROOT- tables
+   HBASE-480   Tool to manually merge two regions
 
 Release 0.16.0
 

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HMerge.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HMerge.java?rev=632663&r1=632662&r2=632663&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HMerge.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HMerge.java Sat Mar  1 14:07:12
2008
@@ -25,6 +25,8 @@
 import java.util.NoSuchElementException;
 import java.util.Random;
 import java.util.TreeMap;
+import java.util.SortedMap;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -32,19 +34,28 @@
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.util.Writables;
 import org.apache.hadoop.io.Text;
-
+import org.apache.hadoop.conf.Configuration;
+ 
 import org.apache.hadoop.hbase.io.BatchUpdate;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /** 
  * A non-instantiable class that has a static method capable of compacting
  * a table by merging adjacent regions that have grown too small.
  */
-class HMerge implements HConstants {
+class HMerge implements HConstants, Tool {
   static final Log LOG = LogFactory.getLog(HMerge.class);
   static final Random rand = new Random();
+  private Configuration conf;
   
+  /*
+   * Not instantiable
+   */
   private HMerge() {
-    // Not instantiable
+    super();
   }
   
   /**
@@ -61,8 +72,8 @@
    * @throws IOException
    */
   public static void merge(HBaseConfiguration conf, FileSystem fs,
-      Text tableName) throws IOException {
-    
+    Text tableName)
+  throws IOException {
     HConnection connection = HConnectionManager.getConnection(conf);
     boolean masterIsRunning = connection.isMasterRunning();
     HConnectionManager.deleteConnection(conf);
@@ -72,7 +83,6 @@
             "Can not compact META table if instance is on-line");
       }
       new OfflineMerger(conf, fs).process();
-
     } else {
       if(!masterIsRunning) {
         throw new IllegalStateException(
@@ -111,9 +121,8 @@
     void process() throws IOException {
       try {
         for(HRegionInfo[] regionsToMerge = next();
-        regionsToMerge != null;
-        regionsToMerge = next()) {
-
+            regionsToMerge != null;
+            regionsToMerge = next()) {
           if (!merge(regionsToMerge)) {
             return;
           }
@@ -128,7 +137,12 @@
       }
     }
     
-    private boolean merge(HRegionInfo[] info) throws IOException {
+    protected boolean merge(final HRegionInfo[] info) throws IOException {
+      return merge(info, false);
+    }
+    
+    protected boolean merge(final HRegionInfo[] info, final boolean force)
+    throws IOException {
       if(info.length < 2) {
         LOG.info("only one region - nothing to merge");
         return false;
@@ -150,23 +164,19 @@
 
         nextSize = nextRegion.largestHStore(midKey).getAggregate();
 
-        if ((currentSize + nextSize) <= (maxFilesize / 2)) {
+        if (force || (currentSize + nextSize) <= (maxFilesize / 2)) {
           // We merge two adjacent regions if their total size is less than
           // one half of the desired maximum size
-
           LOG.info("merging regions " + currentRegion.getRegionName()
               + " and " + nextRegion.getRegionName());
-
-          HRegion mergedRegion = HRegion.closeAndMerge(currentRegion, nextRegion);
-
+          HRegion mergedRegion =
+            HRegion.closeAndMerge(currentRegion, nextRegion);
           updateMeta(currentRegion.getRegionName(), nextRegion.getRegionName(),
               mergedRegion);
-
           break;
         }
         LOG.info("not merging regions " + currentRegion.getRegionName()
             + " and " + nextRegion.getRegionName());
-
         currentRegion.close();
         currentRegion = nextRegion;
         currentSize = nextSize;
@@ -193,7 +203,6 @@
     
     OnlineMerger(HBaseConfiguration conf, FileSystem fs, Text tableName)
     throws IOException {
-      
       super(conf, fs, tableName);
       this.tableName = tableName;
       this.table = new HTable(conf, META_TABLE_NAME);
@@ -203,35 +212,26 @@
     
     private HRegionInfo nextRegion() throws IOException {
       try {
-        HStoreKey key = new HStoreKey();
-        TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
-        if (! metaScanner.next(key, results)) {
+        Map<Text, byte[]> results = getMetaRow();
+        if (results == null) {
           return null;
         }
         byte[] bytes = results.get(COL_REGIONINFO);
         if (bytes == null || bytes.length == 0) {
-          throw new NoSuchElementException("meta region entry missing "
-              + COL_REGIONINFO);
+          throw new NoSuchElementException("meta region entry missing " +
+            COL_REGIONINFO);
         }
-        HRegionInfo region =
-          (HRegionInfo) Writables.getWritable(bytes, new HRegionInfo());
-
-        if (!region.getTableDesc().getName().equals(tableName)) {
+        HRegionInfo region = Writables.getHRegionInfo(bytes);
+        if (!region.getTableDesc().getName().equals(this.tableName)) {
           return null;
         }
-        
-        if (!region.isOffline()) {
-          throw new TableNotDisabledException("region " + region.getRegionName()
-              + " is not disabled");
-        }
+        checkOfflined(region);
         return region;
-        
       } catch (IOException e) {
         e = RemoteExceptionHandler.checkIOException(e);
         LOG.error("meta scanner error", e);
         try {
           metaScanner.close();
-          
         } catch (IOException ex) {
           ex = RemoteExceptionHandler.checkIOException(ex);
           LOG.error("error closing scanner", ex);
@@ -239,6 +239,35 @@
         throw e;
       }
     }
+    
+    protected void checkOfflined(final HRegionInfo hri)
+    throws TableNotDisabledException {
+      if (!hri.isOffline()) {
+        throw new TableNotDisabledException("region " +
+          hri.getRegionName() + " is not disabled");
+      }
+    }
+    
+    /*
+     * Check current row has a HRegionInfo.  Skip to next row if HRI is empty.
+     * @return A Map of the row content else null if we are off the end.
+     * @throws IOException
+     */
+    private Map<Text, byte[]> getMetaRow() throws IOException {
+      HStoreKey key = new HStoreKey();
+      SortedMap<Text, byte[]> value = new TreeMap<Text, byte[]>();
+      boolean foundResult = false;
+      while (metaScanner.next(key, value)) {
+        LOG.info("Row: <" + key.getRow() + ">");
+        byte[] bytes = value.get(COL_REGIONINFO);
+        if (bytes == null || bytes.length == 0) {
+          continue;
+        }
+        foundResult = true;
+        break;
+      }
+      return foundResult? value: null;
+    }
 
     @Override
     protected HRegionInfo[] next() throws IOException {
@@ -397,5 +426,95 @@
         LOG.debug("updated columns in row: " + newRegion.getRegionName());
       }
     }
+  }
+
+  public int run(String[] args) throws Exception {
+    if (args.length == 0 || args.length > 4) {
+      printUsage();
+      return 1;
+    }
+    final String masterPrefix = "--master=";
+    String tableName = null;
+    String loRegion = null;
+    String hiRegion = null;
+    for (int i = 0; i < args.length; i++) {
+      String arg = args[i];
+      if (arg.startsWith(masterPrefix)) {
+        this.conf.set("hbase.master", arg.substring(masterPrefix.length()));
+      } else if (tableName == null) {
+        tableName = arg;
+        continue;
+      } else if (loRegion == null) {
+        loRegion = arg;
+        continue;
+      } else if (hiRegion == null) {
+        hiRegion = arg;
+        continue;
+      } else {
+        throw new IllegalArgumentException("Unsupported argument: " + arg);
+      }
+    }
+    // Make finals of the region names so can be refererred to by anonymous
+    // class.
+    final Text lo = new Text(loRegion);
+    final Text hi = new Text(hiRegion);
+    // Make a version of OnlineMerger that does two regions only.
+    Merger m = new OnlineMerger((HBaseConfiguration)this.conf,
+        FileSystem.get(this.conf), new Text(tableName)) {
+      @Override
+      void process() throws IOException {
+        try {
+          for (HRegionInfo[] regionsToMerge = next(); regionsToMerge != null;
+                regionsToMerge = next()) {
+            if (regionsToMerge[0].getRegionName().equals(lo) &&
+                  regionsToMerge[1].getRegionName().equals(hi)) {
+              merge(regionsToMerge, true);
+              // Done
+              break;
+            }
+          }
+        } finally {
+          try {
+            this.hlog.closeAndDelete();
+          } catch(IOException e) {
+            LOG.error(e);
+          }
+        }
+      }
+      
+      @Override
+      protected void checkOfflined(@SuppressWarnings("unused") HRegionInfo hri)
+      throws TableNotDisabledException {
+        // Disabling does not work reliably.  Just go ahead and merge.
+        return;
+      }
+    };
+    m.process();
+    return 0;
+  }
+
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  public void setConf(final Configuration c) {
+    this.conf = c;
+  }
+  
+  static int printUsage() {
+    System.out.println("merge [--master=MASTER:PORT] <tableName> " +
+      "<lo-region> <hi-region>");
+    System.out.println("Presumes quiescent/offlined table -- does not check");
+    return -1;
+  }
+
+  /**
+   * Run merging of two regions.
+   * @param args
+   * @throws Exception 
+   */
+  public static void main(String[] args) throws Exception {
+    int errCode = ToolRunner.run(new HBaseConfiguration(), new HMerge(), args);
+    System.exit(errCode);
   }
 }

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=632663&r1=632662&r2=632663&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Sat Mar  1 14:07:12 2008
@@ -71,6 +71,7 @@
    HBASE-466   Move HMasterInterface, HRegionInterface, and 
                HMasterRegionInterface into o.a.h.h.ipc
    HBASE-479   Speed up TestLogRolling
+   HBASE-480   Tool to manually merge two regions
    
 Branch 0.1
 

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java?rev=632663&r1=632662&r2=632663&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/HMerge.java Sat Mar  1 14:07:12 2008
@@ -22,35 +22,42 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Random;
+import java.util.SortedMap;
 import java.util.TreeMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.io.Text;
-
-import org.apache.hadoop.hbase.io.BatchUpdate;
-import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
-
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.regionserver.HLog;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /** 
  * A non-instantiable class that has a static method capable of compacting
  * a table by merging adjacent regions that have grown too small.
  */
-class HMerge implements HConstants {
+class HMerge implements HConstants, Tool {
   static final Log LOG = LogFactory.getLog(HMerge.class);
   static final Random rand = new Random();
+  private Configuration conf;
   
+  /*
+   * Not instantiable
+   */
   private HMerge() {
-    // Not instantiable
+    super();
   }
   
   /**
@@ -67,8 +74,8 @@
    * @throws IOException
    */
   public static void merge(HBaseConfiguration conf, FileSystem fs,
-      Text tableName) throws IOException {
-    
+    Text tableName)
+  throws IOException {
     HConnection connection = HConnectionManager.getConnection(conf);
     boolean masterIsRunning = connection.isMasterRunning();
     HConnectionManager.deleteConnection(conf);
@@ -78,7 +85,6 @@
             "Can not compact META table if instance is on-line");
       }
       new OfflineMerger(conf, fs).process();
-
     } else {
       if(!masterIsRunning) {
         throw new IllegalStateException(
@@ -117,9 +123,8 @@
     void process() throws IOException {
       try {
         for(HRegionInfo[] regionsToMerge = next();
-        regionsToMerge != null;
-        regionsToMerge = next()) {
-
+            regionsToMerge != null;
+            regionsToMerge = next()) {
           if (!merge(regionsToMerge)) {
             return;
           }
@@ -134,7 +139,12 @@
       }
     }
     
-    private boolean merge(HRegionInfo[] info) throws IOException {
+    protected boolean merge(final HRegionInfo[] info) throws IOException {
+      return merge(info, false);
+    }
+    
+    protected boolean merge(final HRegionInfo[] info, final boolean force)
+    throws IOException {
       if(info.length < 2) {
         LOG.info("only one region - nothing to merge");
         return false;
@@ -156,23 +166,19 @@
 
         nextSize = nextRegion.largestHStore(midKey).getAggregate();
 
-        if ((currentSize + nextSize) <= (maxFilesize / 2)) {
+        if (force || (currentSize + nextSize) <= (maxFilesize / 2)) {
           // We merge two adjacent regions if their total size is less than
           // one half of the desired maximum size
-
           LOG.info("merging regions " + currentRegion.getRegionName()
               + " and " + nextRegion.getRegionName());
-
-          HRegion mergedRegion = HRegion.closeAndMerge(currentRegion, nextRegion);
-
+          HRegion mergedRegion =
+            HRegion.closeAndMerge(currentRegion, nextRegion);
           updateMeta(currentRegion.getRegionName(), nextRegion.getRegionName(),
               mergedRegion);
-
           break;
         }
         LOG.info("not merging regions " + currentRegion.getRegionName()
             + " and " + nextRegion.getRegionName());
-
         currentRegion.close();
         currentRegion = nextRegion;
         currentSize = nextSize;
@@ -199,7 +205,6 @@
     
     OnlineMerger(HBaseConfiguration conf, FileSystem fs, Text tableName)
     throws IOException {
-      
       super(conf, fs, tableName);
       this.tableName = tableName;
       this.table = new HTable(conf, META_TABLE_NAME);
@@ -209,35 +214,26 @@
     
     private HRegionInfo nextRegion() throws IOException {
       try {
-        HStoreKey key = new HStoreKey();
-        TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
-        if (! metaScanner.next(key, results)) {
+        Map<Text, byte[]> results = getMetaRow();
+        if (results == null) {
           return null;
         }
         byte[] bytes = results.get(COL_REGIONINFO);
         if (bytes == null || bytes.length == 0) {
-          throw new NoSuchElementException("meta region entry missing "
-              + COL_REGIONINFO);
+          throw new NoSuchElementException("meta region entry missing " +
+            COL_REGIONINFO);
         }
-        HRegionInfo region =
-          (HRegionInfo) Writables.getWritable(bytes, new HRegionInfo());
-
-        if (!region.getTableDesc().getName().equals(tableName)) {
+        HRegionInfo region = Writables.getHRegionInfo(bytes);
+        if (!region.getTableDesc().getName().equals(this.tableName)) {
           return null;
         }
-        
-        if (!region.isOffline()) {
-          throw new TableNotDisabledException("region " + region.getRegionName()
-              + " is not disabled");
-        }
+        checkOfflined(region);
         return region;
-        
       } catch (IOException e) {
         e = RemoteExceptionHandler.checkIOException(e);
         LOG.error("meta scanner error", e);
         try {
           metaScanner.close();
-          
         } catch (IOException ex) {
           ex = RemoteExceptionHandler.checkIOException(ex);
           LOG.error("error closing scanner", ex);
@@ -245,6 +241,35 @@
         throw e;
       }
     }
+    
+    protected void checkOfflined(final HRegionInfo hri)
+    throws TableNotDisabledException {
+      if (!hri.isOffline()) {
+        throw new TableNotDisabledException("region " +
+          hri.getRegionName() + " is not disabled");
+      }
+    }
+    
+    /*
+     * Check current row has a HRegionInfo.  Skip to next row if HRI is empty.
+     * @return A Map of the row content else null if we are off the end.
+     * @throws IOException
+     */
+    private Map<Text, byte[]> getMetaRow() throws IOException {
+      HStoreKey key = new HStoreKey();
+      SortedMap<Text, byte[]> value = new TreeMap<Text, byte[]>();
+      boolean foundResult = false;
+      while (metaScanner.next(key, value)) {
+        LOG.info("Row: <" + key.getRow() + ">");
+        byte[] bytes = value.get(COL_REGIONINFO);
+        if (bytes == null || bytes.length == 0) {
+          continue;
+        }
+        foundResult = true;
+        break;
+      }
+      return foundResult? value: null;
+    }
 
     @Override
     protected HRegionInfo[] next() throws IOException {
@@ -376,4 +401,94 @@
       }
     }
   }
-}
+
+  public int run(String[] args) throws Exception {
+    if (args.length == 0 || args.length > 4) {
+      printUsage();
+      return 1;
+    }
+    final String masterPrefix = "--master=";
+    String tableName = null;
+    String loRegion = null;
+    String hiRegion = null;
+    for (int i = 0; i < args.length; i++) {
+      String arg = args[i];
+      if (arg.startsWith(masterPrefix)) {
+        this.conf.set("hbase.master", arg.substring(masterPrefix.length()));
+      } else if (tableName == null) {
+        tableName = arg;
+        continue;
+      } else if (loRegion == null) {
+        loRegion = arg;
+        continue;
+      } else if (hiRegion == null) {
+        hiRegion = arg;
+        continue;
+      } else {
+        throw new IllegalArgumentException("Unsupported argument: " + arg);
+      }
+    }
+    // Make finals of the region names so can be refererred to by anonymous
+    // class.
+    final Text lo = new Text(loRegion);
+    final Text hi = new Text(hiRegion);
+    // Make a version of OnlineMerger that does two regions only.
+    Merger m = new OnlineMerger((HBaseConfiguration)this.conf,
+        FileSystem.get(this.conf), new Text(tableName)) {
+      @Override
+      void process() throws IOException {
+        try {
+          for (HRegionInfo[] regionsToMerge = next(); regionsToMerge != null;
+                regionsToMerge = next()) {
+            if (regionsToMerge[0].getRegionName().equals(lo) &&
+                  regionsToMerge[1].getRegionName().equals(hi)) {
+              merge(regionsToMerge, true);
+              // Done
+              break;
+            }
+          }
+        } finally {
+          try {
+            this.hlog.closeAndDelete();
+          } catch(IOException e) {
+            LOG.error(e);
+          }
+        }
+      }
+      
+      @Override
+      protected void checkOfflined(@SuppressWarnings("unused") HRegionInfo hri)
+      throws TableNotDisabledException {
+        // Disabling does not work reliably.  Just go ahead and merge.
+        return;
+      }
+    };
+    m.process();
+    return 0;
+  }
+
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  public void setConf(final Configuration c) {
+    this.conf = c;
+  }
+  
+  static int printUsage() {
+    System.out.println("merge [--master=MASTER:PORT] <tableName> " +
+      "<lo-region> <hi-region>");
+    System.out.println("Presumes quiescent/offlined table -- does not check");
+    return -1;
+  }
+
+  /**
+   * Run merging of two regions.
+   * @param args
+   * @throws Exception 
+   */
+  public static void main(String[] args) throws Exception {
+    int errCode = ToolRunner.run(new HBaseConfiguration(), new HMerge(), args);
+    System.exit(errCode);
+  }
+}
\ No newline at end of file



Mime
View raw message