Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hbase.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1138318 - in
 /hbase/trunk/src/main/java/org/apache/hadoop/hbase: catalog/MetaEditor.java
 master/HMaster.java regionserver/HRegion.java
Date: Wed, 22 Jun 2011 06:48:29 -0000
To: commits@hbase.apache.org
From: tedyu@apache.org
Message-Id: <20110622064829.2D535238890D@eris.apache.org>

Author: tedyu
Date: Wed Jun 22 06:48:28 2011
New Revision: 1138318

URL: http://svn.apache.org/viewvc?rev=1138318&view=rev
Log:
HBASE-4010 HMaster.createTable could be heavily optimized


Modified:
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1138318&r1=1138317&r2=1138318&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java Wed Jun 22 06:48:28 2011
@@ -47,6 +47,13 @@ import org.apache.hadoop.hbase.catalog.M
 public class MetaEditor {
   private static final Log LOG = LogFactory.getLog(MetaEditor.class);
 
+  private static Put makePutFromRegionInfo(HRegionInfo regionInfo) throws IOException {
+    Put put = new Put(regionInfo.getRegionName());
+    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+        Writables.getBytes(regionInfo));
+    return put;
+  }
+  
   /**
    * Adds a META row for the specified new region.
    * @param regionInfo region information
@@ -55,15 +62,31 @@ public class MetaEditor {
   public static void addRegionToMeta(CatalogTracker catalogTracker,
       HRegionInfo regionInfo)
   throws IOException {
-    Put put = new Put(regionInfo.getRegionName());
-    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
-        Writables.getBytes(regionInfo));
     catalogTracker.waitForMetaServerConnectionDefault().put(
-        CatalogTracker.META_REGION, put);
+        CatalogTracker.META_REGION, makePutFromRegionInfo(regionInfo));
     LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to META");
   }
 
   /**
+   * Adds a META row for each of the specified new regions.
+   * @param catalogTracker CatalogTracker
+   * @param regionInfos region information list
+   * @throws IOException if problem connecting or updating meta
+   */
+  public static void addRegionsToMeta(CatalogTracker catalogTracker,
+      List<HRegionInfo> regionInfos)
+  throws IOException {
+    List<Put> puts = new ArrayList<Put>();
+    for (HRegionInfo regionInfo : regionInfos) { 
+      puts.add(makePutFromRegionInfo(regionInfo));
+      LOG.debug("Added region " + regionInfo.getRegionNameAsString() + " to META");
+    }
+    catalogTracker.waitForMetaServerConnectionDefault().put(
+        CatalogTracker.META_REGION, puts);
+    LOG.info("Added " + puts.size() + " regions to META");
+  }
+
+  /**
    * Offline parent in meta.
    * Used when splitting.
    * @param catalogTracker

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1138318&r1=1138317&r2=1138318&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Wed Jun 22 06:48:28 2011
@@ -1,5 +1,5 @@
 /**
- * Copyright 2010 The Apache Software Foundation
+ * Copyright 2011 The Apache Software Foundation
  *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.HColumnDe
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HServerLoad;
-
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.MasterNotRunningException;
 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
@@ -55,8 +54,8 @@ import org.apache.hadoop.hbase.catalog.M
 import org.apache.hadoop.hbase.catalog.MetaReader;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.MetaScanner;
-import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.executor.ExecutorService;
 import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorType;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
@@ -76,16 +75,17 @@ import org.apache.hadoop.hbase.master.me
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.replication.regionserver.Replication;
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSTableDescriptors;
+import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.util.InfoServer;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Sleeper;
 import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.hbase.util.VersionInfo;
-import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.zookeeper.ClusterId;
 import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker;
 import org.apache.hadoop.hbase.zookeeper.RegionServerTracker;
@@ -975,25 +975,39 @@ implements HMasterInterface, HMasterRegi
     // do rename to move it into place?
     FSUtils.createTableDescriptor(hTableDescriptor, conf);
 
-    for (HRegionInfo newRegion : newRegions) {
-      // 1. Set table enabling flag up in zk.
-      try {
-        assignmentManager.getZKTable().setEnabledTable(tableName);
-      } catch (KeeperException e) {
-        throw new IOException("Unable to ensure that the table will be" +
+    // 1. Set table enabling flag up in zk.
+    try {
+      assignmentManager.getZKTable().setEnabledTable(tableName);
+    } catch (KeeperException e) {
+      throw new IOException("Unable to ensure that the table will be" +
           " enabled because of a ZooKeeper issue", e);
-      }
+    }
 
+    List<HRegionInfo> regionInfos = new ArrayList<HRegionInfo>();
+    final int batchSize = this.conf.getInt("hbase.master.createtable.batchsize", 100);
+    HLog hlog = null;
+    for (int regionIdx = 0; regionIdx < newRegions.length; regionIdx++) {
+      HRegionInfo newRegion = newRegions[regionIdx];
       // 2. Create HRegion
       HRegion region = HRegion.createHRegion(newRegion,
-        fileSystemManager.getRootDir(), conf, hTableDescriptor);
+          fileSystemManager.getRootDir(), conf, hTableDescriptor, hlog);
+      if (hlog == null) {
+        hlog = region.getLog();
+      }
 
-      // 3. Insert into META
-      MetaEditor.addRegionToMeta(catalogTracker, region.getRegionInfo());
+      regionInfos.add(region.getRegionInfo());
+      if (regionIdx % batchSize == 0) {
+        // 3. Insert into META
+        MetaEditor.addRegionsToMeta(catalogTracker, regionInfos);
+        regionInfos.clear();
+      }
 
       // 4. Close the new region to flush to disk.  Close log file too.
       region.close();
-      region.getLog().closeAndDelete();
+    }
+    hlog.closeAndDelete();
+    if (regionInfos.size() > 0) {
+      MetaEditor.addRegionsToMeta(catalogTracker, regionInfos);
     }
 
     // 5. Trigger immediate assignment of the regions in round-robin fashion

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1138318&r1=1138317&r2=1138318&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed Jun 22 06:48:28 2011
@@ -2778,23 +2778,48 @@ public class HRegion implements HeapSize
   public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
       final Configuration conf, final HTableDescriptor hTableDescriptor)
   throws IOException {
+    return createHRegion(info, rootDir, conf, hTableDescriptor, null);
+  }
+
+  /**
+   * Convenience method creating new HRegions. Used by createTable.
+   * The {@link HLog} for the created region needs to be closed explicitly. 
+   * Use {@link HRegion#getLog()} to get access.
+   * 
+   * @param info Info for region to create.
+   * @param rootDir Root directory for HBase instance
+   * @param conf
+   * @param hTableDescriptor
+   * @param hlog shared HLog
+   * @return new HRegion
+   *
+   * @throws IOException
+   */
+  public static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
+                                      final Configuration conf,
+                                      final HTableDescriptor hTableDescriptor,
+                                      final HLog hlog)
+      throws IOException {
     LOG.info("creating HRegion " + info.getTableNameAsString()
-    + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
-    " Table name == " + info.getTableNameAsString());
+        + " HTD == " + hTableDescriptor + " RootDir = " + rootDir +
+        " Table name == " + info.getTableNameAsString());
 
     Path tableDir =
         HTableDescriptor.getTableDir(rootDir, info.getTableName());
     Path regionDir = HRegion.getRegionDir(tableDir, info.getEncodedName());
     FileSystem fs = FileSystem.get(conf);
     fs.mkdirs(regionDir);
+    HLog effectiveHLog = hlog;
+    if (hlog == null) {
+      effectiveHLog = new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
+          new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf);     
+    }
     HRegion region = HRegion.newHRegion(tableDir,
-        new HLog(fs, new Path(regionDir, HConstants.HREGION_LOGDIR_NAME),
-            new Path(regionDir, HConstants.HREGION_OLDLOGDIR_NAME), conf),
-        fs, conf, info, hTableDescriptor, null);
+        effectiveHLog, fs, conf, info, hTableDescriptor, null);
     region.initialize();
     return region;
   }
-
+  
   /**
    * Open a Region.
    * @param info Info for region to be opened.