hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1183158 [1/3] - in /hbase/trunk: ./ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/catalog/ src/main/java/org/apache/hadoop/hbase/client/ src/main/java/org/apache/hadoop/hbase/ipc/ src/main/java/org/apache/had...
Date Thu, 13 Oct 2011 23:12:31 GMT
Author: stack
Date: Thu Oct 13 23:12:30 2011
New Revision: 1183158

URL: http://svn.apache.org/viewvc?rev=1183158&view=rev
Log:
HBASE-3446 ProcessServerShutdown fails if META moves, orphaning lots of regions

Added:
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/HConnectionTestingUtility.java
Modified:
    hbase/trunk/CHANGES.txt
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/HTable.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/MetaScanner.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/thrift/ThriftServer.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaNodeTracker.java
    hbase/trunk/src/main/ruby/hbase/admin.rb
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditor.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigration.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMaster.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestMergeTable.java
    hbase/trunk/src/test/ruby/hbase/admin_test.rb
    hbase/trunk/src/test/ruby/shell/shell_test.rb

Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu Oct 13 23:12:30 2011
@@ -361,6 +361,8 @@ Release 0.92.0 - Unreleased
                names, need to use a more consistent block naming scheme (jgray)
    HBASE-4551  Fix pom and some test cases to compile and run against
                Hadoop 0.23 (todd)
+   HBASE-3446  ProcessServerShutdown fails if META moves, orphaning lots of
+               regions
 
   TESTS
    HBASE-4450  test for number of blocks read: to serve as baseline for expected

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HConstants.java Thu Oct 13 23:12:30 2011
@@ -185,10 +185,6 @@ public final class HConstants {
   /** The file name used to store HTD in HDFS  */
   public static final String TABLEINFO_NAME = ".tableinfo";
 
-  /** The metaupdated column qualifier */
-  public static final byte [] META_MIGRATION_QUALIFIER = Bytes.toBytes("metamigrated");
-
-
   /** Default maximum file size */
   public static final long DEFAULT_MAX_FILE_SIZE = 256 * 1024 * 1024;
 

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java Thu Oct 13 23:12:30 2011
@@ -1043,6 +1043,7 @@ public class KeyValue implements Writabl
    * @return True if this KV is a {@link KeyValue.Type#Delete} type.
    */
   public boolean isDeleteType() {
+    // TODO: Fix this method name vis-a-vis isDelete!
     return getType() == Type.Delete.getCode();
   }
 

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/LocalHBaseCluster.java Thu Oct 13 23:12:30 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.security.User;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
+import org.apache.hadoop.hbase.util.Threads;
 
 import java.util.concurrent.CopyOnWriteArrayList;
 import org.apache.hadoop.hbase.master.HMaster;
@@ -238,6 +239,7 @@ public class LocalHBaseCluster {
     List<RegionServerThread> list = getRegionServers();
     for (JVMClusterUtil.RegionServerThread rst: list) {
       if (rst.isAlive()) liveServers.add(rst);
+      else LOG.info("Not alive " + rst.getName());
     }
     return liveServers;
   }
@@ -386,12 +388,12 @@ public class LocalHBaseCluster {
    */
   public void join() {
     if (this.regionThreads != null) {
-        for(Thread t: this.regionThreads) {
-          if (t.isAlive()) {
-            try {
-              t.join();
+      for(Thread t: this.regionThreads) {
+        if (t.isAlive()) {
+          try {
+            Threads.threadDumpingIsAlive(t);
           } catch (InterruptedException e) {
-            // continue
+            LOG.debug("Interrupted", e);
           }
         }
       }
@@ -400,9 +402,9 @@ public class LocalHBaseCluster {
       for (Thread t : this.masterThreads) {
         if (t.isAlive()) {
           try {
-            t.join();
+            Threads.threadDumpingIsAlive(t);
           } catch (InterruptedException e) {
-            // continue
+            LOG.debug("Interrupted", e);
           }
         }
       }

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java Thu Oct 13 23:12:30 2011
@@ -1,6 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -33,7 +31,6 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
-import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.HConnection;
 import org.apache.hadoop.hbase.client.HConnectionManager;
@@ -60,30 +57,76 @@ import org.apache.hadoop.ipc.RemoteExcep
  * interrupt waits and close up shop.
  */
 public class CatalogTracker {
+  // TODO: This class needs a rethink.  The original intent was that it would be
+  // the one-stop-shop for root and meta locations and that it would get this
+  // info from reading and watching zk state.  The class was to be used by
+  // servers when they needed to know of root and meta movement but also by
+  // client-side (inside in HTable) so rather than figure root and meta
+  // locations on fault, the client would instead get notifications out of zk.
+  // 
+  // But this original intent is frustrated by the fact that this class has to
+  // read an hbase table, the -ROOT- table, to figure out the .META. region
+  // location which means we depend on an HConnection.  HConnection will do
+  // retrying but also, it has its own mechanism for finding root and meta
+  // locations (and for 'verifying'; it tries the location and if it fails, does
+  // new lookup, etc.).  So, at least for now, HConnection (or HTable) can't
+  // have a CT since CT needs a HConnection (Even then, do want HT to have a CT?
+  // For HT keep up a session with ZK?  Rather, shouldn't we do like asynchbase
+  // where we'd open a connection to zk, read what we need then let the
+  // connection go?).  The 'fix' is make it so both root and meta addresses
+  // are wholey up in zk -- not in zk (root) -- and in an hbase table (meta).
+  //
+  // But even then, this class does 'verification' of the location and it does
+  // this by making a call over an HConnection (which will do its own root
+  // and meta lookups).  Isn't this verification 'useless' since when we
+  // return, whatever is dependent on the result of this call then needs to
+  // use HConnection; what we have verified may change in meantime (HConnection
+  // uses the CT primitives, the root and meta trackers finding root locations).
+  //
+  // When meta is moved to zk, this class may make more sense.  In the
+  // meantime, it does not cohere.  It should just watch meta and root and not
+  // NOT do verification -- let that be out in HConnection since its going to
+  // be done there ultimately anyways.
+  //
+  // This class has spread throughout the codebase.  It needs to be reigned in.
+  // This class should be used server-side only, even if we move meta location
+  // up into zk.  Currently its used over in the client package. Its used in
+  // MetaReader and MetaEditor classes usually just to get the Configuration
+  // its using (It does this indirectly by asking its HConnection for its
+  // Configuration and even then this is just used to get an HConnection out on
+  // the other end). I made https://issues.apache.org/jira/browse/HBASE-4495 for
+  // doing CT fixup. St.Ack 09/30/2011.
+  //
   private static final Log LOG = LogFactory.getLog(CatalogTracker.class);
-  private final Configuration conf;
   private final HConnection connection;
   private final ZooKeeperWatcher zookeeper;
   private final RootRegionTracker rootRegionTracker;
   private final MetaNodeTracker metaNodeTracker;
   private final AtomicBoolean metaAvailable = new AtomicBoolean(false);
-  /**
+  private boolean instantiatedzkw = false;
+
+  /*
    * Do not clear this address once set.  Its needed when we do
    * server shutdown processing -- we need to know who had .META. last.  If you
    * want to know if the address is good, rely on {@link #metaAvailable} value.
    */
   private ServerName metaLocation;
+
+  /*
+   * Timeout waiting on root or meta to be set.
+   */
   private final int defaultTimeout;
+
   private boolean stopped = false;
 
-  public static final byte [] ROOT_REGION =
+  static final byte [] ROOT_REGION_NAME =
     HRegionInfo.ROOT_REGIONINFO.getRegionName();
-  public static final byte [] META_REGION =
+  static final byte [] META_REGION_NAME =
     HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
 
   /**
-   * Constructs a catalog tracker. Find current state of catalog tables and
-   * begin active tracking by executing {@link #start()} post construction. Does
+   * Constructs a catalog tracker. Find current state of catalog tables.
+   * Begin active tracking by executing {@link #start()} post construction. Does
    * not timeout.
    *
    * @param conf
@@ -97,29 +140,36 @@ public class CatalogTracker {
   }
 
   /**
-   * Constructs the catalog tracker.  Find current state of catalog tables and
-   * begin active tracking by executing {@link #start()} post construction.
+   * Constructs the catalog tracker.  Find current state of catalog tables.
+   * Begin active tracking by executing {@link #start()} post construction.
    * Does not timeout.
-   * @param zk
+   * @param zk If zk is null, we'll create an instance (and shut it down
+   * when {@link #stop()} is called) else we'll use what is passed.
    * @param connection server connection
-   * @param abortable if fatal exception
+   * @param abortable If fatal exception we'll call abort on this.  May be null.
+   * If it is we'll use the Connection associated with the passed
+   * {@link Configuration} as our Abortable.
    * @throws IOException 
    */
   public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
       final Abortable abortable)
   throws IOException {
-    this(zk, conf, abortable, 0);
+    this(zk, conf, abortable,
+      conf.getInt("hbase.catalogtracker.default.timeout", 1000));
   }
 
   /**
-   * Constructs the catalog tracker.  Find current state of catalog tables and
-   * begin active tracking by executing {@link #start()} post construction.
-   * @param zk
-   * @param connection server connection
-   * @param abortable if fatal exception
+   * Constructs the catalog tracker.  Find current state of catalog tables.
+   * Begin active tracking by executing {@link #start()} post construction.
+   * @param zk If zk is null, we'll create an instance (and shut it down
+   * when {@link #stop()} is called) else we'll use what is passed.
+   * @param conf
+   * @param abortable If fatal exception we'll call abort on this.  May be null.
+   * If it is we'll use the Connection associated with the passed
+   * {@link Configuration} as our Abortable.
    * @param defaultTimeout Timeout to use.  Pass zero for no timeout
    * ({@link Object#wait(long)} when passed a <code>0</code> waits for ever).
-   * @throws IOException 
+   * @throws IOException
    */
   public CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
       Abortable abortable, final int defaultTimeout)
@@ -130,14 +180,29 @@ public class CatalogTracker {
   CatalogTracker(final ZooKeeperWatcher zk, final Configuration conf,
       HConnection connection, Abortable abortable, final int defaultTimeout)
   throws IOException {
-    this.conf = conf;
     this.connection = connection;
-    this.zookeeper = (zk == null) ? this.connection.getZooKeeperWatcher() : zk;
     if (abortable == null) {
+      // A connection is abortable.
       abortable = this.connection;
     }
+    if (zk == null) {
+      // Create our own.  Set flag so we tear it down on stop.
+      this.zookeeper =
+        new ZooKeeperWatcher(conf, "catalogtracker-on-" + connection.toString(),
+          abortable);
+      instantiatedzkw = true;
+    } else {
+      this.zookeeper = zk;
+    }
     this.rootRegionTracker = new RootRegionTracker(zookeeper, abortable);
-    this.metaNodeTracker = new MetaNodeTracker(zookeeper, this, abortable);
+    final CatalogTracker ct = this;
+    // Override nodeDeleted so we get notified when meta node deleted
+    this.metaNodeTracker = new MetaNodeTracker(zookeeper, abortable) {
+      public void nodeDeleted(String path) {
+        if (!path.equals(node)) return;
+        ct.resetMetaLocation();
+      }
+    };
     this.defaultTimeout = defaultTimeout;
   }
 
@@ -149,9 +214,9 @@ public class CatalogTracker {
    * @throws InterruptedException 
    */
   public void start() throws IOException, InterruptedException {
+    LOG.debug("Starting catalog tracker " + this);
     this.rootRegionTracker.start();
     this.metaNodeTracker.start();
-    LOG.debug("Starting catalog tracker " + this);
   }
 
   /**
@@ -173,6 +238,9 @@ public class CatalogTracker {
         // IOException}, in reality, the implementation would never do that.
         LOG.error("Attempt to close catalog tracker's connection failed.", e);
       }
+      if (this.instantiatedzkw) {
+        this.zookeeper.close();
+      }
       // Call this and it will interrupt any ongoing waits on meta.
       synchronized (this.metaAvailable) {
         this.metaAvailable.notifyAll();
@@ -183,7 +251,8 @@ public class CatalogTracker {
   /**
    * Gets the current location for <code>-ROOT-</code> or null if location is
    * not currently available.
-   * @return server name
+   * @return {@link ServerName} for server hosting <code>-ROOT-</code> or null
+   * if none available
    * @throws InterruptedException 
    */
   public ServerName getRootLocation() throws InterruptedException {
@@ -191,8 +260,8 @@ public class CatalogTracker {
   }
 
   /**
-   * @return Location of server hosting meta region formatted as per
-   * {@link ServerName}, or null if none available
+   * @return {@link ServerName} for server hosting <code>.META.</code> or null
+   * if none available
    */
   public ServerName getMetaLocation() {
     return this.metaLocation;
@@ -213,7 +282,8 @@ public class CatalogTracker {
    * for up to the specified timeout if not immediately available.  Returns null
    * if the timeout elapses before root is available.
    * @param timeout maximum time to wait for root availability, in milliseconds
-   * @return Location of server hosting root region or null if none available
+   * @return {@link ServerName} for server hosting <code>-ROOT-</code> or null
+   * if none available
    * @throws InterruptedException if interrupted while waiting
    * @throws NotAllMetaRegionsOnlineException if root not available before
    * timeout
@@ -230,14 +300,32 @@ public class CatalogTracker {
   /**
    * Gets a connection to the server hosting root, as reported by ZooKeeper,
    * waiting up to the specified timeout for availability.
+   * @param timeout How long to wait on root location
    * @see #waitForRoot(long) for additional information
    * @return connection to server hosting root
    * @throws InterruptedException
    * @throws NotAllMetaRegionsOnlineException if timed out waiting
    * @throws IOException
+   * @deprecated Use {@link #getRootServerConnection(long)}
    */
   public HRegionInterface waitForRootServerConnection(long timeout)
   throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
+    return getRootServerConnection(timeout);
+  }
+
+  /**
+   * Gets a connection to the server hosting root, as reported by ZooKeeper,
+   * waiting up to the specified timeout for availability.
+   * <p>WARNING: Does not retry.  Use an {@link HTable} instead.
+   * @param timeout How long to wait on root location
+   * @see #waitForRoot(long) for additional information
+   * @return connection to server hosting root
+   * @throws InterruptedException
+   * @throws NotAllMetaRegionsOnlineException if timed out waiting
+   * @throws IOException
+   */
+  HRegionInterface getRootServerConnection(long timeout)
+  throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
     return getCachedConnection(waitForRoot(timeout));
   }
 
@@ -248,30 +336,18 @@ public class CatalogTracker {
    * @return connection to server hosting root
    * @throws NotAllMetaRegionsOnlineException if timed out waiting
    * @throws IOException
+   * @deprecated Use {@link #getRootServerConnection(long)}
    */
   public HRegionInterface waitForRootServerConnectionDefault()
   throws NotAllMetaRegionsOnlineException, IOException {
     try {
-      return getCachedConnection(waitForRoot(defaultTimeout));
+      return getRootServerConnection(this.defaultTimeout);
     } catch (InterruptedException e) {
       throw new NotAllMetaRegionsOnlineException("Interrupted");
     }
   }
 
   /**
-   * Gets a connection to the server hosting root, as reported by ZooKeeper,
-   * if available.  Returns null if no location is immediately available.
-   * @return connection to server hosting root, null if not available
-   * @throws IOException
-   * @throws InterruptedException
-   */
-  private HRegionInterface getRootServerConnection()
-  throws IOException, InterruptedException {
-    ServerName sn = this.rootRegionTracker.getRootRegionLocation();
-    return getCachedConnection(sn);
-  }
-
-  /**
    * Gets a connection to the server currently hosting <code>.META.</code> or
    * null if location is not currently available.
    * <p>
@@ -292,24 +368,28 @@ public class CatalogTracker {
     synchronized (metaAvailable) {
       if (metaAvailable.get()) {
         HRegionInterface current = getCachedConnection(this.metaLocation);
-        if (verifyRegionLocation(current, this.metaLocation, META_REGION)) {
+        // If we are to refresh, verify we have a good connection by making
+        // an invocation on it.
+        if (verifyRegionLocation(current, this.metaLocation, META_REGION_NAME)) {
           return current;
         }
         resetMetaLocation();
       }
-      HRegionInterface rootConnection = getRootServerConnection();
-      if (rootConnection == null) {
-        LOG.debug("-ROOT- server unavailable.");
-        return null;
-      }
-      ServerName newLocation = MetaReader.readMetaLocation(rootConnection);
+      // We got here because there is no meta available or because whats
+      // available is bad.
+
+      // Now read the current .META. content from -ROOT-.  Note: This goes via
+      // an HConnection.  It has its own way of figuring root and meta locations
+      // which we have to wait on.
+      ServerName newLocation =
+        MetaReader.readRegionLocation(this, META_REGION_NAME);
       if (newLocation == null) {
         LOG.debug(".META. server unavailable.");
         return null;
       }
 
       HRegionInterface newConnection = getCachedConnection(newLocation);
-      if (verifyRegionLocation(newConnection, newLocation, META_REGION)) {
+      if (verifyRegionLocation(newConnection, newLocation, META_REGION_NAME)) {
         setMetaLocation(newLocation);
         return newConnection;
       } else {
@@ -322,13 +402,19 @@ public class CatalogTracker {
 
   /**
    * Waits indefinitely for availability of <code>.META.</code>.  Used during
-   * cluster startup.
+   * cluster startup.  Does not verify meta, just that something has been
+   * set up in zk.
+   * @see #waitForMeta(long)
    * @throws InterruptedException if interrupted while waiting
    */
   public void waitForMeta() throws InterruptedException {
-    synchronized (metaAvailable) {
-      while (!stopped && !metaAvailable.get()) {
-        metaAvailable.wait();
+    while (!this.stopped) {
+      try {
+        if (waitForMeta(100) != null) break;
+      } catch (NotAllMetaRegionsOnlineException e) {
+        LOG.info("Retrying", e);
+      } catch (IOException e) {
+        LOG.info("Retrying", e);
       }
     }
   }
@@ -340,7 +426,8 @@ public class CatalogTracker {
    * in that it will go ahead and verify the location gotten from ZooKeeper and
    * -ROOT- region by trying to use returned connection.
    * @param timeout maximum time to wait for meta availability, in milliseconds
-   * @return location of meta
+   * @return {@link ServerName} for server hosting <code>.META.</code> or null
+   * if none available
    * @throws InterruptedException if interrupted while waiting
    * @throws IOException unexpected exception connecting to meta server
    * @throws NotAllMetaRegionsOnlineException if meta not available before
@@ -359,8 +446,7 @@ public class CatalogTracker {
         metaAvailable.wait(waitTime);
       }
       if (getMetaServerConnection() == null) {
-        throw new NotAllMetaRegionsOnlineException(
-            "Timed out (" + timeout + "ms)");
+        throw new NotAllMetaRegionsOnlineException("Timed out (" + timeout + "ms)");
       }
       return metaLocation;
     }
@@ -374,6 +460,7 @@ public class CatalogTracker {
    * @throws InterruptedException
    * @throws NotAllMetaRegionsOnlineException if timed out waiting
    * @throws IOException
+   * @deprecated Does not retry; use an HTable instance instead.
    */
   public HRegionInterface waitForMetaServerConnection(long timeout)
   throws InterruptedException, NotAllMetaRegionsOnlineException, IOException {
@@ -383,10 +470,12 @@ public class CatalogTracker {
   /**
    * Gets a connection to the server hosting meta, as reported by ZooKeeper,
    * waiting up to the specified timeout for availability.
+   * Used in tests.
    * @see #waitForMeta(long) for additional information
    * @return connection to server hosting meta
    * @throws NotAllMetaRegionsOnlineException if timed out or interrupted
    * @throws IOException
+   * @deprecated Does not retry; use an HTable instance instead.
    */
   public HRegionInterface waitForMetaServerConnectionDefault()
   throws NotAllMetaRegionsOnlineException, IOException {
@@ -397,12 +486,22 @@ public class CatalogTracker {
     }
   }
 
-  private void resetMetaLocation() {
-    LOG.debug("Current cached META location: " + metaLocation +
-      " is not valid, resetting");
-    this.metaAvailable.set(false);
+  /**
+   * Called when we figure current meta is off (called from zk callback).
+   */
+  public void resetMetaLocation() {
+    LOG.debug("Current cached META location, " + metaLocation +
+      ", is not valid, resetting");
+    synchronized(this.metaAvailable) {
+      this.metaAvailable.set(false);
+      this.metaAvailable.notifyAll();
+    }
   }
 
+  /**
+   * Caller must be synchronized on this.metaAvailable
+   * @param metaLocation
+   */
   private void setMetaLocation(final ServerName metaLocation) {
     LOG.debug("set new cached META location: " + metaLocation);
     metaAvailable.set(true);
@@ -411,6 +510,13 @@ public class CatalogTracker {
     this.metaAvailable.notifyAll();
   }
 
+  /**
+   * @param sn ServerName to get a connection against.
+   * @return The HRegionInterface we got when we connected to <code>sn</code>
+   * May have come from cache, may not be good, may have been setup by this
+   * invocation, or may be null.
+   * @throws IOException
+   */
   private HRegionInterface getCachedConnection(ServerName sn)
   throws IOException {
     if (sn == null) {
@@ -446,17 +552,32 @@ public class CatalogTracker {
     return protocol;
   }
 
-  private boolean verifyRegionLocation(HRegionInterface metaServer,
-      final ServerName address,
-      byte [] regionName)
+  /**
+   * Verify we can connect to <code>hostingServer</code> and that its carrying
+   * <code>regionName</code>.
+   * @param hostingServer Interface to the server hosting <code>regionName</code>
+   * @param serverName The servername that goes with the <code>metaServer</code>
+   * Interface.  Used logging.
+   * @param regionName The regionname we are interested in.
+   * @return True if we were able to verify the region located at other side of
+   * the Interface.
+   * @throws IOException
+   */
+  // TODO: We should be able to get the ServerName from the HRegionInterface
+  // rather than have to pass it in.  Its made awkward by the fact that the
+  // HRI is likely a proxy against remote server so the getServerName needs
+  // to be fixed to go to a local method or to a cache before we can do this.
+  private boolean verifyRegionLocation(HRegionInterface hostingServer,
+      final ServerName address, final byte [] regionName)
   throws IOException {
-    if (metaServer == null) {
-      LOG.info("Passed metaserver is null");
+    if (hostingServer == null) {
+      LOG.info("Passed hostingServer is null");
       return false;
     }
     Throwable t = null;
     try {
-      return metaServer.getRegionInfo(regionName) != null;
+      // Try and get regioninfo from the hosting server.
+      return hostingServer.getRegionInfo(regionName) != null;
     } catch (ConnectException e) {
       t = e;
     } catch (RemoteException e) {
@@ -498,8 +619,7 @@ public class CatalogTracker {
     }
     return (connection == null)? false:
       verifyRegionLocation(connection,
-        this.rootRegionTracker.getRootRegionLocation(),
-        HRegionInfo.ROOT_REGIONINFO.getRegionName());
+        this.rootRegionTracker.getRootRegionLocation(), ROOT_REGION_NAME);
   }
 
   /**
@@ -523,6 +643,7 @@ public class CatalogTracker {
     return connection != null;
   }
 
+  // Used by tests.
   MetaNodeTracker getMetaNodeTracker() {
     return this.metaNodeTracker;
   }

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java Thu Oct 13 23:12:30 2011
@@ -1,6 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -19,40 +17,121 @@
  */
 package org.apache.hadoop.hbase.catalog;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import java.io.IOException;
 import java.net.ConnectException;
+import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.migration.HRegionInfo090x;
+import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
-import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.catalog.MetaReader.Visitor;
 
 /**
  * Writes region and assignment information to <code>.META.</code>.
- * <p>
- * Uses the {@link CatalogTracker} to obtain locations and connections to
- * catalogs.
  */
 public class MetaEditor {
+  // TODO: Strip CatalogTracker from this class.  Its all over and in the end
+  // its only used to get its Configuration so we can get associated
+  // Connection.
   private static final Log LOG = LogFactory.getLog(MetaEditor.class);
 
-  private static Put makePutFromRegionInfo(HRegionInfo regionInfo) throws IOException {
+  private static Put makePutFromRegionInfo(HRegionInfo regionInfo)
+  throws IOException {
     Put put = new Put(regionInfo.getRegionName());
     put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
         Writables.getBytes(regionInfo));
     return put;
   }
+
+  /**
+   * Put the passed <code>p</code> to the <code>.META.</code> table.
+   * @param ct CatalogTracker on whose back we will ride the edit.
+   * @param p Put to add to .META.
+   * @throws IOException
+   */
+  static void putToMetaTable(final CatalogTracker ct, final Put p)
+  throws IOException {
+    put(MetaReader.getMetaHTable(ct), p);
+  }
+
+  /**
+   * Put the passed <code>p</code> to the <code>.META.</code> table.
+   * @param ct CatalogTracker on whose back we will ride the edit.
+   * @param p Put to add to .META.
+   * @throws IOException
+   */
+  static void putToRootTable(final CatalogTracker ct, final Put p)
+  throws IOException {
+    put(MetaReader.getRootHTable(ct), p);
+  }
+
+  /**
+   * Put the passed <code>p</code> to a catalog table.
+   * @param ct CatalogTracker on whose back we will ride the edit.
+   * @param regionName Name of the catalog table to put too.
+   * @param p Put to add
+   * @throws IOException
+   */
+  static void putToCatalogTable(final CatalogTracker ct,
+      final byte [] regionName, final Put p)
+  throws IOException {
+    HTable t = MetaReader.getCatalogHTable(ct, regionName);
+    put(t, p);
+  }
+
+  /**
+   * @param t Table to use (will be closed when done).
+   * @param p
+   * @throws IOException
+   */
+  private static void put(final HTable t, final Put p) throws IOException {
+    try {
+      t.put(p);
+    } finally {
+      t.close();
+    }
+  }
+
+  /**
+   * Put the passed <code>ps</code> to the <code>.META.</code> table.
+   * @param ct CatalogTracker on whose back we will ride the edit.
+   * @param ps Put to add to .META.
+   * @throws IOException
+   */
+  static void putsToMetaTable(final CatalogTracker ct, final List<Put> ps)
+  throws IOException {
+    HTable t = MetaReader.getMetaHTable(ct);
+    try {
+      t.put(ps);
+    } finally {
+      t.close();
+    }
+  }
+
+  /**
+   * Delete the passed <code>d</code> from the <code>.META.</code> table.
+   * @param ct CatalogTracker on whose back we will ride the edit.
+   * @param d Delete to add to .META.
+   * @throws IOException
+   */
+  static void deleteMetaTable(final CatalogTracker ct, final Delete d)
+  throws IOException {
+    HTable t = MetaReader.getMetaHTable(ct);
+    try {
+      t.delete(d);
+    } finally {
+      t.close();
+    }
+  }
   
   /**
    * Adds a META row for the specified new region.
@@ -62,8 +141,7 @@ public class MetaEditor {
   public static void addRegionToMeta(CatalogTracker catalogTracker,
       HRegionInfo regionInfo)
   throws IOException {
-    catalogTracker.waitForMetaServerConnectionDefault().put(
-        CatalogTracker.META_REGION, makePutFromRegionInfo(regionInfo));
+    putToMetaTable(catalogTracker, makePutFromRegionInfo(regionInfo));
     LOG.info("Added region " + regionInfo.getRegionNameAsString() + " to META");
   }
 
@@ -79,11 +157,9 @@ public class MetaEditor {
     List<Put> puts = new ArrayList<Put>();
     for (HRegionInfo regionInfo : regionInfos) { 
       puts.add(makePutFromRegionInfo(regionInfo));
-      LOG.debug("Added region " + regionInfo.getRegionNameAsString() + " to META");
     }
-    catalogTracker.waitForMetaServerConnectionDefault().put(
-        CatalogTracker.META_REGION, puts);
-    LOG.info("Added " + puts.size() + " regions to META");
+    putsToMetaTable(catalogTracker, puts);
+    LOG.info("Added " + puts.size() + " regions in META");
   }
 
   /**
@@ -108,7 +184,7 @@ public class MetaEditor {
       Writables.getBytes(a));
     put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
       Writables.getBytes(b));
-    catalogTracker.waitForMetaServerConnectionDefault().put(CatalogTracker.META_REGION, put);
+    putToMetaTable(catalogTracker, put);
     LOG.info("Offlined parent region " + parent.getRegionNameAsString() +
       " in META");
   }
@@ -116,14 +192,11 @@ public class MetaEditor {
   public static void addDaughter(final CatalogTracker catalogTracker,
       final HRegionInfo regionInfo, final ServerName sn)
   throws NotAllMetaRegionsOnlineException, IOException {
-    HRegionInterface server = catalogTracker.waitForMetaServerConnectionDefault();
-    byte [] catalogRegionName = CatalogTracker.META_REGION;
     Put put = new Put(regionInfo.getRegionName());
     addRegionInfo(put, regionInfo);
     if (sn != null) addLocation(put, sn);
-    server.put(catalogRegionName, put);
+    putToMetaTable(catalogTracker, put);
     LOG.info("Added daughter " + regionInfo.getRegionNameAsString() +
-      " in region " + Bytes.toString(catalogRegionName) +
       (sn == null? ", serverName=null": ", serverName=" + sn.toString()));
   }
 
@@ -145,9 +218,7 @@ public class MetaEditor {
   public static void updateMetaLocation(CatalogTracker catalogTracker,
       HRegionInfo regionInfo, ServerName sn)
   throws IOException, ConnectException {
-    HRegionInterface server = catalogTracker.waitForRootServerConnectionDefault();
-    if (server == null) throw new IOException("No server for -ROOT-");
-    updateLocation(server, CatalogTracker.ROOT_REGION, regionInfo, sn);
+    updateLocation(catalogTracker, regionInfo, sn);
   }
 
   /**
@@ -165,8 +236,7 @@ public class MetaEditor {
   public static void updateRegionLocation(CatalogTracker catalogTracker,
       HRegionInfo regionInfo, ServerName sn)
   throws IOException {
-    updateLocation(catalogTracker.waitForMetaServerConnectionDefault(),
-        CatalogTracker.META_REGION, regionInfo, sn);
+    updateLocation(catalogTracker, regionInfo, sn);
   }
 
   /**
@@ -175,22 +245,21 @@ public class MetaEditor {
    * Connects to the specified server which should be hosting the specified
    * catalog region name to perform the edit.
    *
-   * @param server connection to server hosting catalog region
-   * @param catalogRegionName name of catalog region being updated
+   * @param catalogTracker
    * @param regionInfo region to update location of
    * @param sn Server name
    * @throws IOException In particular could throw {@link java.net.ConnectException}
    * if the server is down on other end.
    */
-  private static void updateLocation(HRegionInterface server,
-      byte [] catalogRegionName, HRegionInfo regionInfo, ServerName sn)
+  private static void updateLocation(final CatalogTracker catalogTracker,
+      HRegionInfo regionInfo, ServerName sn)
   throws IOException {
+    final byte [] regionName = regionInfo.getRegionName();
     Put put = new Put(regionInfo.getRegionName());
     addLocation(put, sn);
-    server.put(catalogRegionName, put);
+    putToCatalogTable(catalogTracker, regionName, put);
     LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
-      " in region " + Bytes.toStringBinary(catalogRegionName) + " with " +
-      "serverName=" + sn.toString());
+      " with server=" + sn);
   }
 
   /**
@@ -203,8 +272,7 @@ public class MetaEditor {
       HRegionInfo regionInfo)
   throws IOException {
     Delete delete = new Delete(regionInfo.getRegionName());
-    catalogTracker.waitForMetaServerConnectionDefault().
-      delete(CatalogTracker.META_REGION, delete);
+    deleteMetaTable(catalogTracker, delete);
     LOG.info("Deleted region " + regionInfo.getRegionNameAsString() + " from META");
   }
 
@@ -223,142 +291,12 @@ public class MetaEditor {
   throws NotAllMetaRegionsOnlineException, IOException {
     Delete delete = new Delete(parent.getRegionName());
     delete.deleteColumns(HConstants.CATALOG_FAMILY, qualifier);
-    catalogTracker.waitForMetaServerConnectionDefault().
-      delete(CatalogTracker.META_REGION, delete);
+    deleteMetaTable(catalogTracker, delete);
     LOG.info("Deleted daughter reference " + daughter.getRegionNameAsString() +
       ", qualifier=" + Bytes.toStringBinary(qualifier) + ", from parent " +
       parent.getRegionNameAsString());
   }
 
-  /**
-   * Update the metamigrated flag in -ROOT-.
-   * @param catalogTracker
-   * @throws IOException
-   */
-  public static void updateRootWithMetaMigrationStatus(
-      CatalogTracker catalogTracker) throws IOException {
-    updateRootWithMetaMigrationStatus(catalogTracker, true);
-  }
-
-  /**
-   * Update the metamigrated flag in -ROOT-.
-   * @param catalogTracker
-   * @param metaUpdated
-   * @throws IOException
-   */
-  public static void updateRootWithMetaMigrationStatus(
-      CatalogTracker catalogTracker, boolean metaUpdated)
-      throws IOException {
-    Put put = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
-    addMetaUpdateStatus(put, metaUpdated);
-    catalogTracker.waitForRootServerConnectionDefault().put(
-        CatalogTracker.ROOT_REGION, put);
-    LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
-  }
-
-  /**
-   * Update legacy META rows, removing HTD from HRI.
-   * @param masterServices
-   * @return
-   * @throws IOException
-   */
-  public static List<HTableDescriptor> updateMetaWithNewRegionInfo(
-      final MasterServices masterServices)
-  throws IOException {
-    final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
-    Visitor v = new Visitor() {
-      @Override
-      public boolean visit(Result r) throws IOException {
-        if (r ==  null || r.isEmpty()) return true;
-        HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
-        if (hrfm == null) return true;
-        htds.add(hrfm.getTableDesc());
-        masterServices.getMasterFileSystem()
-          .createTableDescriptor(hrfm.getTableDesc());
-        updateHRI(masterServices.getCatalogTracker()
-            .waitForMetaServerConnectionDefault(),
-            hrfm, CatalogTracker.META_REGION);
-        return true;
-      }
-    };
-    MetaReader.fullScan(masterServices.getCatalogTracker(), v);
-    updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker());
-    return htds;
-  }
-
-  /**
-   * Migrate root and meta to newer version. This updates the META and ROOT
-   * and removes the HTD from HRI.
-   * @param masterServices
-   * @throws IOException
-   */
-  public static void migrateRootAndMeta(final MasterServices masterServices)
-      throws IOException {
-    updateRootWithNewRegionInfo(masterServices);
-    updateMetaWithNewRegionInfo(masterServices);
-  }
-
-  /**
-   * Update the ROOT with new HRI. (HRI with no HTD)
-   * @param masterServices
-   * @return
-   * @throws IOException
-   */
-  public static List<HTableDescriptor> updateRootWithNewRegionInfo(
-      final MasterServices masterServices)
-  throws IOException {
-    final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
-    Visitor v = new Visitor() {
-      @Override
-      public boolean visit(Result r) throws IOException {
-        if (r ==  null || r.isEmpty()) return true;
-        HRegionInfo090x hrfm = getHRegionInfoForMigration(r);
-        if (hrfm == null) return true;
-        htds.add(hrfm.getTableDesc());
-        masterServices.getMasterFileSystem().createTableDescriptor(
-            hrfm.getTableDesc());
-        updateHRI(masterServices.getCatalogTracker()
-            .waitForRootServerConnectionDefault(),
-            hrfm, CatalogTracker.ROOT_REGION);
-        return true;
-      }
-    };
-    MetaReader.fullScan(
-        masterServices.getCatalogTracker().waitForRootServerConnectionDefault(),
-        v, HRegionInfo.ROOT_REGIONINFO.getRegionName(), null);
-    return htds;
-  }
-
-  private static void updateHRI(HRegionInterface hRegionInterface,
-                                HRegionInfo090x hRegionInfo090x, byte[] regionName)
-    throws IOException {
-    HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x);
-    Put put = new Put(regionInfo.getRegionName());
-    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
-        Writables.getBytes(regionInfo));
-    hRegionInterface.put(regionName, put);
-    LOG.info("Updated region " + regionInfo + " to " + Bytes.toString(regionName));
-  }
-
-  public static HRegionInfo090x getHRegionInfoForMigration(
-      Result data) throws IOException {
-    HRegionInfo090x info = null;
-    byte [] bytes =
-      data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-    if (bytes == null) return null;
-    try {
-      info = Writables.getHRegionInfoForMigration(bytes);
-    } catch(IOException ioe) {
-      if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) {
-         return null;
-      } else {
-        throw ioe;
-      }
-    }
-    LOG.info("Current INFO from scan results = " + info);
-    return info;
-  }
-
   public static HRegionInfo getHRegionInfo(
       Result data) throws IOException {
     byte [] bytes =
@@ -369,20 +307,6 @@ public class MetaEditor {
     return info;
   }
 
-  private static Put addMetaUpdateStatus(final Put p) {
-    p.add(HConstants.CATALOG_FAMILY, HConstants.META_MIGRATION_QUALIFIER,
-      Bytes.toBytes("true"));
-    return p;
-  }
-
-
-  private static Put addMetaUpdateStatus(final Put p, final boolean metaUpdated) {
-    p.add(HConstants.CATALOG_FAMILY, HConstants.META_MIGRATION_QUALIFIER,
-      Bytes.toBytes(metaUpdated));
-    return p;
-  }
-
-
   private static Put addRegionInfo(final Put p, final HRegionInfo hri)
   throws IOException {
     p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
@@ -397,4 +321,4 @@ public class MetaEditor {
       Bytes.toBytes(sn.getStartcode()));
     return p;
   }
-}
+}
\ No newline at end of file

Added: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java?rev=1183158&view=auto
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java (added)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationRemovingHTD.java Thu Oct 13 23:12:30 2011
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.catalog;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.catalog.MetaReader.Visitor;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.migration.HRegionInfo090x;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Writables;
+
+/**
+ * Tools to help with migration of meta tables so they no longer host
+ * instances of HTableDescriptor.
+ * @deprecated Used migration from 0.90 to 0.92 so will be going away in next
+ * release
+ */
+public class MetaMigrationRemovingHTD {
+  private static final Log LOG = LogFactory.getLog(MetaMigrationRemovingHTD.class);
+
+  /** The metaupdated column qualifier */
+  public static final byte [] META_MIGRATION_QUALIFIER =
+    Bytes.toBytes("metamigrated");
+
+  /**
+   * Update legacy META rows, removing HTD from HRI.
+   * @param masterServices
+   * @return
+   * @throws IOException
+   */
+  public static List<HTableDescriptor> updateMetaWithNewRegionInfo(
+      final MasterServices masterServices)
+  throws IOException {
+    final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
+    Visitor v = new Visitor() {
+      @Override
+      public boolean visit(Result r) throws IOException {
+        if (r ==  null || r.isEmpty()) return true;
+        HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r);
+        if (hrfm == null) return true;
+        htds.add(hrfm.getTableDesc());
+        masterServices.getMasterFileSystem()
+          .createTableDescriptor(hrfm.getTableDesc());
+        updateHRI(masterServices.getCatalogTracker(), false, hrfm);
+        return true;
+      }
+    };
+    MetaReader.fullScan(masterServices.getCatalogTracker(), v);
+    MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker(), true);
+    return htds;
+  }
+
+  /**
+   * Update the ROOT with new HRI. (HRI with no HTD)
+   * @param masterServices
+   * @return
+   * @throws IOException
+   */
+  public static List<HTableDescriptor> updateRootWithNewRegionInfo(
+      final MasterServices masterServices)
+  throws IOException {
+    final List<HTableDescriptor> htds = new ArrayList<HTableDescriptor>();
+    Visitor v = new Visitor() {
+      @Override
+      public boolean visit(Result r) throws IOException {
+        if (r ==  null || r.isEmpty()) return true;
+        HRegionInfo090x hrfm = MetaMigrationRemovingHTD.getHRegionInfoForMigration(r);
+        if (hrfm == null) return true;
+        htds.add(hrfm.getTableDesc());
+        masterServices.getMasterFileSystem().createTableDescriptor(
+            hrfm.getTableDesc());
+        updateHRI(masterServices.getCatalogTracker(), true, hrfm);
+        return true;
+      }
+    };
+    MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true);
+    return htds;
+  }
+
+  /**
+   * Migrate root and meta to newer version. This updates the META and ROOT
+   * and removes the HTD from HRI.
+   * @param masterServices
+   * @throws IOException
+   */
+  public static void migrateRootAndMeta(final MasterServices masterServices)
+      throws IOException {
+    updateRootWithNewRegionInfo(masterServices);
+    updateMetaWithNewRegionInfo(masterServices);
+  }
+
+  /**
+   * Update the metamigrated flag in -ROOT-.
+   * @param catalogTracker
+   * @param metaUpdated
+   * @throws IOException
+   */
+  public static void updateRootWithMetaMigrationStatus(
+      CatalogTracker catalogTracker, boolean metaUpdated)
+  throws IOException {
+    Put p = new Put(HRegionInfo.ROOT_REGIONINFO.getRegionName());
+    MetaMigrationRemovingHTD.addMetaUpdateStatus(p, metaUpdated);
+    MetaEditor.putToRootTable(catalogTracker, p);
+    LOG.info("Updated -ROOT- row with metaMigrated status = " + metaUpdated);
+  }
+
+  static void updateHRI(final CatalogTracker ct, final boolean rootTable,
+    final HRegionInfo090x hRegionInfo090x)
+  throws IOException {
+    HRegionInfo regionInfo = new HRegionInfo(hRegionInfo090x);
+    Put p = new Put(regionInfo.getRegionName());
+    p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
+      Writables.getBytes(regionInfo));
+    if (rootTable) {
+      MetaEditor.putToRootTable(ct, p);
+    } else {
+      MetaEditor.putToMetaTable(ct, p);
+    }
+    LOG.info("Updated region " + regionInfo + " to " +
+      (rootTable? "-ROOT-": ".META."));
+  }
+
+  /**
+   * @deprecated Going away in 0.94; used for migrating to 0.92 only.
+   */
+  public static HRegionInfo090x getHRegionInfoForMigration(
+      Result data) throws IOException {
+    HRegionInfo090x info = null;
+    byte [] bytes =
+      data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+    if (bytes == null) return null;
+    try {
+      info = Writables.getHRegionInfoForMigration(bytes);
+    } catch(IOException ioe) {
+      if (ioe.getMessage().equalsIgnoreCase("HTD not found in input buffer")) {
+         return null;
+      } else {
+        throw ioe;
+      }
+    }
+    LOG.info("Current INFO from scan results = " + info);
+    return info;
+  }
+
+  public static List<HRegionInfo090x> fullScanMetaAndPrintHRIM(
+      CatalogTracker catalogTracker)
+  throws IOException {
+    final List<HRegionInfo090x> regions =
+      new ArrayList<HRegionInfo090x>();
+    Visitor v = new Visitor() {
+      @Override
+      public boolean visit(Result r) throws IOException {
+        if (r ==  null || r.isEmpty()) return true;
+        LOG.info("fullScanMetaAndPrint1.Current Meta Result: " + r);
+        HRegionInfo090x hrim = getHRegionInfoForMigration(r);
+        LOG.info("fullScanMetaAndPrint.HRIM Print= " + hrim);
+        regions.add(hrim);
+        return true;
+      }
+    };
+    MetaReader.fullScan(catalogTracker, v);
+    return regions;
+  }
+
+  static Put addMetaUpdateStatus(final Put p, final boolean metaUpdated) {
+    p.add(HConstants.CATALOG_FAMILY,
+      MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER,
+      Bytes.toBytes(metaUpdated));
+    return p;
+  }
+
+  /**
+   * @return True if the meta table has been migrated.
+   * @throws IOException
+   */
+  // Public because used in tests
+  public static boolean isMetaHRIUpdated(final MasterServices services)
+      throws IOException {
+    boolean metaUpdated = false;
+    List<Result> results =
+      MetaReader.fullScanOfRoot(services.getCatalogTracker());
+    if (results == null || results.isEmpty()) {
+      LOG.info("metaUpdated = NULL.");
+      return metaUpdated;
+    }
+    // Presume only the one result.
+    Result r = results.get(0);
+    byte [] metaMigrated = r.getValue(HConstants.CATALOG_FAMILY,
+      MetaMigrationRemovingHTD.META_MIGRATION_QUALIFIER);
+    if (metaMigrated != null && metaMigrated.length > 0) {
+      metaUpdated = Bytes.toBoolean(metaMigrated);
+    }
+    LOG.info("Meta updated status = " + metaUpdated);
+    return metaUpdated;
+  }
+
+  /**
+   * @return True if migrated.
+   * @throws IOException
+   */
+  public static boolean updateMetaWithNewHRI(final MasterServices services)
+  throws IOException {
+    if (isMetaHRIUpdated(services)) {
+      LOG.info("ROOT/Meta already up-to date with new HRI.");
+      return true;
+    }
+    LOG.info("Meta has HRI with HTDs. Updating meta now.");
+    try {
+      migrateRootAndMeta(services);
+      LOG.info("ROOT and Meta updated with new HRI.");
+      return true;
+    } catch (IOException e) {
+      throw new RuntimeException("Update ROOT/Meta with new HRI failed." +
+        "Master startup aborted.");
+    }
+  }
+}
\ No newline at end of file

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1183158&r1=1183157&r2=1183158&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Thu Oct 13 23:12:30 2011
@@ -1,6 +1,4 @@
 /**
- * Copyright 2010 The Apache Software Foundation
- *
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -26,16 +24,20 @@ import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Set;
 import java.util.TreeMap;
-import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NotServingRegionException;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
-import org.apache.hadoop.hbase.migration.HRegionInfo090x;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.hadoop.hbase.util.Writables;
@@ -43,14 +45,14 @@ import org.apache.hadoop.ipc.RemoteExcep
 
 /**
  * Reads region and assignment information from <code>.META.</code>.
- * <p>
- * Uses the {@link CatalogTracker} to obtain locations and connections to
- * catalogs.
  */
 public class MetaReader {
+  // TODO: Strip CatalogTracker from this class.  Its all over and in the end
+  // its only used to get its Configuration so we can get associated
+  // Connection.
   private static final Log LOG = LogFactory.getLog(MetaReader.class);
 
-  public static final byte [] META_REGION_PREFIX;
+  static final byte [] META_REGION_PREFIX;
   static {
     // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
     // FIRST_META_REGIONINFO == '.META.,,1'.  META_REGION_PREFIX == '.META.,'
@@ -61,48 +63,6 @@ public class MetaReader {
   }
 
   /**
-   * @param ct
-   * @param tableName A user tablename or a .META. table name.
-   * @return Interface on to server hosting the <code>-ROOT-</code> or
-   * <code>.META.</code> regions.
-   * @throws NotAllMetaRegionsOnlineException
-   * @throws IOException
-   */
-  private static HRegionInterface getCatalogRegionInterface(final CatalogTracker ct,
-      final byte [] tableName)
-  throws NotAllMetaRegionsOnlineException, IOException {
-    return Bytes.equals(HConstants.META_TABLE_NAME, tableName)?
-      ct.waitForRootServerConnectionDefault():
-      ct.waitForMetaServerConnectionDefault();
-  }
-
-  /**
-   * @param tableName
-   * @return Returns region name to look in for regions for <code>tableName</code>;
-   * e.g. if we are looking for <code>.META.</code> regions, we need to look
-   * in the <code>-ROOT-</code> region, else if a user table, we need to look
-   * in the <code>.META.</code> region.
-   */
-  private static byte [] getCatalogRegionNameForTable(final byte [] tableName) {
-    return Bytes.equals(HConstants.META_TABLE_NAME, tableName)?
-      HRegionInfo.ROOT_REGIONINFO.getRegionName():
-      HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
-  }
-
-  /**
-   * @param regionName
-   * @return Returns region name to look in for <code>regionName</code>;
-   * e.g. if we are looking for <code>.META.,,1</code> region, we need to look
-   * in <code>-ROOT-</code> region, else if a user region, we need to look
-   * in the <code>.META.,,1</code> region.
-   */
-  private static byte [] getCatalogRegionNameForRegion(final byte [] regionName) {
-    return isMetaRegion(regionName)?
-      HRegionInfo.ROOT_REGIONINFO.getRegionName():
-      HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
-  }
-
-  /**
    * @param regionName
    * @return True if <code>regionName</code> is from <code>.META.</code> table.
    */
@@ -118,33 +78,13 @@ public class MetaReader {
   }
 
   /**
-   * Performs a full scan of <code>.META.</code>.
-   * <p>
-   * Returns a map of every region to it's currently assigned server, according
-   * to META.  If the region does not have an assignment it will have a null
-   * value in the map.
-   *
-   * @return map of regions to their currently assigned server where server is
-   * a String of &lt;host> ':' &lt;port>
-   * @throws IOException
-   */
-  public static Map<HRegionInfo, ServerName> fullScan(
-      CatalogTracker catalogTracker)
-  throws IOException {
-    return fullScan(catalogTracker, new TreeSet<String>());
-  }
-
-  /**
    * Performs a full scan of <code>.META.</code>, skipping regions from any
    * tables in the specified set of disabled tables.
-   * <p>
-   * Returns a map of every region to it's currently assigned server, according
-   * to META.  If the region does not have an assignment it will have a null
-   * value in the map.
-   *
    * @param catalogTracker
    * @param disabledTables set of disabled tables that will not be returned
-   * @return map of regions to their currently assigned server
+   * @return Returns a map of every region to it's currently assigned server,
+   * according to META.  If the region does not have an assignment it will have
+   * a null value in the map.
    * @throws IOException
    */
   public static Map<HRegionInfo, ServerName> fullScan(
@@ -156,16 +96,13 @@ public class MetaReader {
   /**
    * Performs a full scan of <code>.META.</code>, skipping regions from any
    * tables in the specified set of disabled tables.
-   * <p>
-   * Returns a map of every region to it's currently assigned server, according
-   * to META.  If the region does not have an assignment it will have a null
-   * value in the map.
-   *
    * @param catalogTracker
    * @param disabledTables set of disabled tables that will not be returned
    * @param excludeOfflinedSplitParents If true, do not include offlined split
    * parents in the return.
-   * @return map of regions to their currently assigned server
+   * @return Returns a map of every region to it's currently assigned server,
+   * according to META.  If the region does not have an assignment it will have
+   * a null value in the map.
    * @throws IOException
    */
   public static Map<HRegionInfo, ServerName> fullScan(
@@ -178,7 +115,7 @@ public class MetaReader {
       @Override
       public boolean visit(Result r) throws IOException {
         if (r ==  null || r.isEmpty()) return true;
-        Pair<HRegionInfo, ServerName> region = metaRowToRegionPair(r);
+        Pair<HRegionInfo, ServerName> region = parseCatalogResult(r);
         if (region == null) return true;
         HRegionInfo hri = region.getFirst();
         if (disabledTables.contains(
@@ -195,38 +132,32 @@ public class MetaReader {
 
   /**
    * Performs a full scan of <code>.META.</code>.
-   * <p>
-   * Returns a map of every region to it's currently assigned server, according
-   * to META.  If the region does not have an assignment it will have a null
-   * value in the map.
-   *
-   * @return map of regions to their currently assigned server
+   * @return List of {@link Result}
    * @throws IOException
    */
-  public static List<Result> fullScanOfResults(
-      CatalogTracker catalogTracker)
+  public static List<Result> fullScan(CatalogTracker catalogTracker)
   throws IOException {
-    final List<Result> regions = new ArrayList<Result>();
-    Visitor v = new Visitor() {
-      @Override
-      public boolean visit(Result r) throws IOException {
-        if (r ==  null || r.isEmpty()) return true;
-        regions.add(r);
-        return true;
-      }
-    };
-    fullScan(catalogTracker, v);
-    return regions;
+    CollectAllVisitor v = new CollectAllVisitor();
+    fullScan(catalogTracker, v, null);
+    return v.getResults();
+  }
+
+  /**
+   * Performs a full scan of a <code>-ROOT-</code> table.
+   * @return List of {@link Result}
+   * @throws IOException
+   */
+  public static List<Result> fullScanOfRoot(CatalogTracker catalogTracker)
+  throws IOException {
+    CollectAllVisitor v = new CollectAllVisitor();
+    fullScan(catalogTracker, v, null, true);
+    return v.getResults();
   }
 
   /**
    * Performs a full scan of <code>.META.</code>.
-   * <p>
-   * Returns a map of every region to it's currently assigned server, according
-   * to META.  If the region does not have an assignment it will have a null
-   * value in the map.
    * @param catalogTracker
-   * @param visitor
+   * @param visitor Visitor invoked against each row.
    * @throws IOException
    */
   public static void fullScan(CatalogTracker catalogTracker,
@@ -237,24 +168,82 @@ public class MetaReader {
 
   /**
    * Performs a full scan of <code>.META.</code>.
-   * <p>
-   * Returns a map of every region to it's currently assigned server, according
-   * to META.  If the region does not have an assignment it will have a null
-   * value in the map.
    * @param catalogTracker
-   * @param visitor
+   * @param visitor Visitor invoked against each row.
    * @param startrow Where to start the scan. Pass null if want to begin scan
-   * at first row.
+   * at first row (The visitor will stop the Scan when its done so no need to
+   * pass a stoprow).
    * @throws IOException
    */
   public static void fullScan(CatalogTracker catalogTracker,
       final Visitor visitor, final byte [] startrow)
   throws IOException {
-    HRegionInterface metaServer =
-      catalogTracker.waitForMetaServerConnectionDefault();
-    fullScan(metaServer, visitor,
-        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), startrow);
-    return;
+    fullScan(catalogTracker, visitor, startrow, false);
+  }
+
+  /**
+   * Callers should call close on the returned {@link HTable} instance.
+   * @param catalogTracker We'll use this catalogtracker's connection
+   * @param tableName Table to get an {@link HTable} against.
+   * @return An {@link HTable} for <code>tableName</code>
+   * @throws IOException
+   */
+  private static HTable getHTable(final CatalogTracker catalogTracker,
+      final byte [] tableName)
+  throws IOException {
+    // Passing the CatalogTracker's connection configuration ensures this
+    // HTable instance uses the CatalogTracker's connection.
+    return new HTable(catalogTracker.getConnection().getConfiguration(), tableName);
+  }
+
+  /**
+   * Callers should call close on the returned {@link HTable} instance.
+   * @param catalogTracker
+   * @param regionName
+   * @return
+   * @throws IOException
+   */
+  static HTable getCatalogHTable(final CatalogTracker catalogTracker,
+      final byte [] regionName)
+  throws IOException {
+    return isMetaRegion(regionName)?
+      getRootHTable(catalogTracker):
+      getMetaHTable(catalogTracker);
+  }
+
+  /**
+   * Callers should call close on the returned {@link HTable} instance.
+   * @param ct
+   * @return An {@link HTable} for <code>.META.</code>
+   * @throws IOException
+   */
+  static HTable getMetaHTable(final CatalogTracker ct)
+  throws IOException {
+    return getHTable(ct, HConstants.META_TABLE_NAME);
+  }
+
+  /**
+   * Callers should call close on the returned {@link HTable} instance.
+   * @param ct
+   * @return An {@link HTable} for <code>-ROOT-</code>
+   * @throws IOException
+   */
+  static HTable getRootHTable(final CatalogTracker ct)
+  throws IOException {
+    return getHTable(ct, HConstants.ROOT_TABLE_NAME);
+  }
+
+  /**
+   * @param t Table to use (will be closed when done).
+   * @param g Get to run
+   * @throws IOException
+   */
+  private static Result get(final HTable t, final Get g) throws IOException {
+    try {
+      return t.get(g);
+    } finally {
+      t.close();
+    }
   }
 
   /**
@@ -262,28 +251,29 @@ public class MetaReader {
    * @param metaServer connection to server hosting ROOT
    * @return location of META in ROOT where location, or null if not available
    * @throws IOException
+   * @deprecated Does not retry; use {@link #readRegionLocation(CatalogTracker, byte[])
    */
   public static ServerName readMetaLocation(HRegionInterface metaServer)
   throws IOException {
-    return readLocation(metaServer, CatalogTracker.ROOT_REGION,
-        CatalogTracker.META_REGION);
+    return readLocation(metaServer, CatalogTracker.ROOT_REGION_NAME,
+        CatalogTracker.META_REGION_NAME);
   }
 
   /**
-   * Reads the location of the specified region from META.
+   * Reads the location of the specified region
    * @param catalogTracker
-   * @param regionName region to read location of
-   * @return location of META in ROOT where location is, or null if not available
+   * @param regionName region whose location we are after
+   * @return location of region as a {@link ServerName} or null if not found
    * @throws IOException
    */
   public static ServerName readRegionLocation(CatalogTracker catalogTracker,
       byte [] regionName)
   throws IOException {
-    if (isMetaRegion(regionName)) throw new IllegalArgumentException("See readMetaLocation");
-    return readLocation(catalogTracker.waitForMetaServerConnectionDefault(),
-        CatalogTracker.META_REGION, regionName);
+    Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
+    return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
   }
 
+  // TODO: Remove when deprecated dependencies are removed.
   private static ServerName readLocation(HRegionInterface metaServer,
       byte [] catalogRegionName, byte [] regionName)
   throws IOException {
@@ -324,15 +314,14 @@ public class MetaReader {
     if (r == null || r.isEmpty()) {
       return null;
     }
-    return getServerNameFromResult(r);
+    return getServerNameFromCatalogResult(r);
   }
 
   /**
-   * Gets the region info and assignment for the specified region from META.
+   * Gets the region info and assignment for the specified region.
    * @param catalogTracker
-   * @param regionName
-   * @return location of META in ROOT where location is
-   * a String of &lt;host> ':' &lt;port>, or null if not available
+   * @param regionName Region to lookup.
+   * @return Location and HRegionInfo for <code>regionName</code>
    * @throws IOException
    */
   public static Pair<HRegionInfo, ServerName> getRegion(
@@ -340,44 +329,63 @@ public class MetaReader {
   throws IOException {
     Get get = new Get(regionName);
     get.addFamily(HConstants.CATALOG_FAMILY);
-    byte [] meta = getCatalogRegionNameForRegion(regionName);
-    Result r = catalogTracker.waitForMetaServerConnectionDefault().get(meta, get);
-    return (r == null || r.isEmpty())? null: metaRowToRegionPair(r);
+    Result r = get(getCatalogHTable(catalogTracker, regionName), get);
+    return (r == null || r.isEmpty())? null: parseCatalogResult(r);
   }
 
   /**
-   * @param data A .META. table row.
-   * @return A pair of the regioninfo and the ServerName
-   * (or null for server address if no address set in .META.).
-   * @throws IOException
-   */
-  public static Pair<HRegionInfo, ServerName> metaRowToRegionPair(Result data)
-  throws IOException {
-    byte [] bytes = data.getValue(HConstants.CATALOG_FAMILY,
-      HConstants.REGIONINFO_QUALIFIER);
-    if (bytes == null) return null;
-    HRegionInfo info = Writables.getHRegionInfo(bytes);
-    ServerName sn = getServerNameFromResult(data);
-    // sn can be null in case where no server inof.
-    return new Pair<HRegionInfo, ServerName>(info, sn);
-  }
-
-  /**
-   * @param data Result to interrogate.
+   * Extract a {@link ServerName}
+   * For use on catalog table {@link Result}.
+   * @param r Result to pull from
    * @return A ServerName instance or null if necessary fields not found or empty.
    */
-  private static ServerName getServerNameFromResult(final Result data) {
-    byte[] value = data.getValue(HConstants.CATALOG_FAMILY,
+  public static ServerName getServerNameFromCatalogResult(final Result r) {
+    byte[] value = r.getValue(HConstants.CATALOG_FAMILY,
       HConstants.SERVER_QUALIFIER);
     if (value == null || value.length == 0) return null;
     String hostAndPort = Bytes.toString(value);
-    value = data.getValue(HConstants.CATALOG_FAMILY,
+    value = r.getValue(HConstants.CATALOG_FAMILY,
       HConstants.STARTCODE_QUALIFIER);
     if (value == null || value.length == 0) return null;
     return new ServerName(hostAndPort, Bytes.toLong(value));
   }
 
   /**
+   * Extract a HRegionInfo and ServerName.
+   * For use on catalog table {@link Result}.
+   * @param r Result to pull from
+   * @return A pair of the {@link HRegionInfo} and the {@link ServerName}
+   * (or null for server address if no address set in .META.).
+   * @throws IOException
+   */
+  public static Pair<HRegionInfo, ServerName> parseCatalogResult(final Result r)
+  throws IOException {
+    HRegionInfo info =
+      parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
+    ServerName sn = getServerNameFromCatalogResult(r);
+    return new Pair<HRegionInfo, ServerName>(info, sn);
+  }
+
+  /**
+   * Parse the content of the cell at {@link HConstants#CATALOG_FAMILY} and
+   * <code>qualifier</code> as an HRegionInfo and return it, or null.
+   * For use on catalog table {@link Result}.
+   * @param r Result instance to pull from.
+   * @param qualifier Column family qualifier -- either
+   * {@link HConstants#SPLITA_QUALIFIER}, {@link HConstants#SPLITB_QUALIFIER} or
+   * {@link HConstants#REGIONINFO_QUALIFIER}.
+   * @return An HRegionInfo instance or null.
+   * @throws IOException
+   */
+  public static HRegionInfo parseHRegionInfoFromCatalogResult(final Result r,
+      byte [] qualifier)
+  throws IOException {
+    byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
+    if (bytes == null || bytes.length <= 0) return null;
+    return Writables.getHRegionInfoOrNull(bytes);
+  }
+
+  /**
    * Checks if the specified table exists.  Looks at the META table hosted on
    * the specified server.
    * @param catalogTracker
@@ -393,21 +401,36 @@ public class MetaReader {
       // Catalog tables always exist.
       return true;
     }
-    HRegionInterface metaServer =
-      catalogTracker.waitForMetaServerConnectionDefault();
-    Scan scan = getScanForTableName(Bytes.toBytes(tableName));
-    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-    long scannerid = metaServer.openScanner(
-        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
-    try {
-      Result data = metaServer.next(scannerid);
-      if (data != null && data.size() > 0) {
+    final byte [] tableNameBytes = Bytes.toBytes(tableName);
+    // Make a version of ResultCollectingVisitor that only collects the first
+    CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
+      private HRegionInfo current = null;
+
+      @Override
+      public boolean visit(Result r) throws IOException {
+        this.current =
+          parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
+        if (this.current == null) {
+          LOG.warn("No serialized HRegionInfo in " + r);
           return true;
+        }
+        if (!isInsideTable(this.current, tableNameBytes)) return false;
+        if (this.current.isSplitParent()) return true;
+        // Else call super and add this Result to the collection.
+        super.visit(r);
+        // Stop collecting regions from table after we get one.
+        return false;
       }
-      return false;
-    } finally {
-      metaServer.close(scannerid);
-    }
+
+      @Override
+      void add(Result r) {
+        // Add the current HRI.
+        this.results.add(this.current);
+      }
+    };
+    fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableNameBytes));
+    // If visitor has results >= 1 then table exists.
+    return visitor.getResults().size() >= 1;
   }
 
   /**
@@ -435,42 +458,46 @@ public class MetaReader {
   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
       byte [] tableName, final boolean excludeOfflinedSplitParents)
   throws IOException {
-    if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
-      // If root, do a bit of special handling.
-      List<HRegionInfo> list = new ArrayList<HRegionInfo>();
-      list.add(HRegionInfo.ROOT_REGIONINFO);
-      return list;
-    } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
-      // Same for .META. table
-      List<HRegionInfo> list = new ArrayList<HRegionInfo>();
-      list.add(HRegionInfo.FIRST_META_REGIONINFO);
-      return list;
+    List<Pair<HRegionInfo, ServerName>> result = null;
+    try {
+      result = getTableRegionsAndLocations(catalogTracker, tableName,
+        excludeOfflinedSplitParents);
+    } catch (InterruptedException e) {
+      throw new RuntimeException(e);
     }
+    return getListOfHRegionInfos(result);
+  }
 
-    // Its a user table.
-    HRegionInterface metaServer =
-      getCatalogRegionInterface(catalogTracker, tableName);
-    List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
-
-    Scan scan = getScanForTableName(tableName);
-    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
-    long scannerid =
-      metaServer.openScanner(getCatalogRegionNameForTable(tableName), scan);
-    try {
-      Result data;
-      while((data = metaServer.next(scannerid)) != null) {
-        if (data != null && data.size() > 0) {
-          HRegionInfo info = Writables.getHRegionInfo(
-              data.getValue(HConstants.CATALOG_FAMILY,
-                  HConstants.REGIONINFO_QUALIFIER));
-          if (excludeOfflinedSplitParents && info.isSplitParent()) continue;
-          regions.add(info);
-        }
-      }
-      return regions;
-    } finally {
-      metaServer.close(scannerid);
+  static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
+    if (pairs == null || pairs.isEmpty()) return null;
+    List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
+    for (Pair<HRegionInfo, ServerName> pair: pairs) {
+      result.add(pair.getFirst());
     }
+    return result;
+  }
+
+  /**
+   * @param current
+   * @param tableName
+   * @return True if <code>current</code> tablename is equal to
+   * <code>tableName</code>
+   */
+  static boolean isInsideTable(final HRegionInfo current, final byte [] tableName) {
+    return Bytes.equals(tableName, current.getTableName());
+  }
+
+  /**
+   * @param tableName
+   * @return Place to start Scan in <code>.META.</code> when passed a
+   * <code>tableName</code>; returns &lt;tableName&rt; &lt;,&rt; &lt;,&rt;
+   */
+  static byte [] getTableStartRowForMeta(final byte [] tableName) {
+    byte [] startRow = new byte[tableName.length + 2];
+    System.arraycopy(tableName, 0, startRow, 0, tableName.length);
+    startRow[startRow.length - 2] = HRegionInfo.DELIMITER;
+    startRow[startRow.length - 1] = HRegionInfo.DELIMITER;
+    return startRow;
   }
 
   /**
@@ -504,8 +531,22 @@ public class MetaReader {
   public static List<Pair<HRegionInfo, ServerName>>
   getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName)
   throws IOException, InterruptedException {
-    byte [] tableNameBytes = Bytes.toBytes(tableName);
-    if (Bytes.equals(tableNameBytes, HConstants.ROOT_TABLE_NAME)) {
+    return getTableRegionsAndLocations(catalogTracker, Bytes.toBytes(tableName),
+      true);
+  }
+
+  /**
+   * @param catalogTracker
+   * @param tableName
+   * @return Return list of regioninfos and server addresses.
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public static List<Pair<HRegionInfo, ServerName>>
+  getTableRegionsAndLocations(final CatalogTracker catalogTracker,
+      final byte [] tableName, final boolean excludeOfflinedSplitParents)
+  throws IOException, InterruptedException {
+    if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
       // If root, do a bit of special handling.
       ServerName serverName = catalogTracker.getRootLocation();
       List<Pair<HRegionInfo, ServerName>> list =
@@ -514,27 +555,36 @@ public class MetaReader {
         serverName));
       return list;
     }
-    HRegionInterface metaServer =
-      getCatalogRegionInterface(catalogTracker, tableNameBytes);
-    List<Pair<HRegionInfo, ServerName>> regions =
-      new ArrayList<Pair<HRegionInfo, ServerName>>();
-    Scan scan = getScanForTableName(tableNameBytes);
-    scan.addFamily(HConstants.CATALOG_FAMILY);
-    long scannerid =
-      metaServer.openScanner(getCatalogRegionNameForTable(tableNameBytes), scan);
-    try {
-      Result data;
-      while((data = metaServer.next(scannerid)) != null) {
-        if (data != null && data.size() > 0) {
-          Pair<HRegionInfo, ServerName> region = metaRowToRegionPair(data);
-          if (region == null) continue;
-          regions.add(region);
+    // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
+    CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
+        new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
+      private Pair<HRegionInfo, ServerName> current = null;
+
+      @Override
+      public boolean visit(Result r) throws IOException {
+        HRegionInfo hri =
+          parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
+        if (hri == null) {
+          LOG.warn("No serialized HRegionInfo in " + r);
+          return true;
         }
+        if (!isInsideTable(hri, tableName)) return false;
+        if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
+        ServerName sn = getServerNameFromCatalogResult(r);
+        // Populate this.current so available when we call #add
+        this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
+        // Else call super and add this Result to the collection.
+        return super.visit(r);
       }
-      return regions;
-    } finally {
-      metaServer.close(scannerid);
-    }
+
+      @Override
+      void add(Result r) {
+        this.results.add(this.current);
+      }
+    };
+    fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName),
+      Bytes.equals(tableName, HConstants.META_TABLE_NAME));
+    return visitor.getResults();
   }
 
   /**
@@ -547,36 +597,31 @@ public class MetaReader {
   public static NavigableMap<HRegionInfo, Result>
   getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
   throws IOException {
-    HRegionInterface metaServer =
-      catalogTracker.waitForMetaServerConnectionDefault();
-    NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
-    Scan scan = new Scan();
-    scan.addFamily(HConstants.CATALOG_FAMILY);
-    long scannerid = metaServer.openScanner(
-        HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
-    try {
-      Result result;
-      while((result = metaServer.next(scannerid)) != null) {
-        if (result != null && result.size() > 0) {
-          Pair<HRegionInfo, ServerName> pair = metaRowToRegionPair(result);
-          if (pair == null) continue;
-          if (pair.getSecond() == null || !serverName.equals(pair.getSecond())) {
-            continue;
-          }
-          hris.put(pair.getFirst(), result);
-        }
+    final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
+    // Fill the above hris map with entries from .META. that have the passed
+    // servername.
+    CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
+      @Override
+      void add(Result r) {
+        if (r == null || r.isEmpty()) return;
+        ServerName sn = getServerNameFromCatalogResult(r);
+        if (sn != null && sn.equals(serverName)) this.results.add(r); 
+      }
+    };
+    fullScan(catalogTracker, v);
+    List<Result> results = v.getResults();
+    if (results != null && !results.isEmpty()) {
+      // Convert results to Map keyed by HRI
+      for (Result r: results) {
+        Pair<HRegionInfo, ServerName> p = parseCatalogResult(r);
+        if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
       }
-      return hris;
-    } finally {
-      metaServer.close(scannerid);
     }
+    return hris;
   }
 
-  public static void fullScanMetaAndPrint(
-      CatalogTracker catalogTracker)
+  public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
   throws IOException {
-    final List<HRegionInfo090x> regions =
-      new ArrayList<HRegionInfo090x>();
     Visitor v = new Visitor() {
       @Override
       public boolean visit(Result r) throws IOException {
@@ -590,27 +635,6 @@ public class MetaReader {
     fullScan(catalogTracker, v);
   }
 
-
-  public static List<HRegionInfo090x> fullScanMetaAndPrintHRIM(
-      CatalogTracker catalogTracker)
-  throws IOException {
-    final List<HRegionInfo090x> regions =
-      new ArrayList<HRegionInfo090x>();
-    Visitor v = new Visitor() {
-      @Override
-      public boolean visit(Result r) throws IOException {
-        if (r ==  null || r.isEmpty()) return true;
-        LOG.info("fullScanMetaAndPrint1.Current Meta Result: " + r);
-        HRegionInfo090x hrim = MetaEditor.getHRegionInfoForMigration(r);
-        LOG.info("fullScanMetaAndPrint.HRIM Print= " + hrim);
-        regions.add(hrim);
-        return true;
-      }
-    };
-    fullScan(catalogTracker, v);
-    return regions;
-  }
-
   /**
    * Fully scan a given region, on a given server starting with given row.
    * @param hRegionInterface region server
@@ -618,6 +642,8 @@ public class MetaReader {
    * @param regionName name of region
    * @param startrow start row
    * @throws IOException
+   * @deprecated Does not retry; use fullScan xxx instead.
+   x
    */
   public static void fullScan(HRegionInterface hRegionInterface,
                               Visitor visitor, final byte[] regionName,
@@ -638,6 +664,38 @@ public class MetaReader {
     return;
   }
 
+  /**
+   * Performs a full scan of a catalog table.
+   * @param catalogTracker
+   * @param visitor Visitor invoked against each row.
+   * @param startrow Where to start the scan. Pass null if want to begin scan
+   * at first row.
+   * @param scanRoot True if we are to scan <code>-ROOT-</code> rather than
+   * <code>.META.</code>, the default (pass false to scan .META.)
+   * @throws IOException
+   */
+  static void fullScan(CatalogTracker catalogTracker,
+    final Visitor visitor, final byte [] startrow, final boolean scanRoot)
+  throws IOException {
+    Scan scan = new Scan();
+    if (startrow != null) scan.setStartRow(startrow);
+    scan.addFamily(HConstants.CATALOG_FAMILY);
+    HTable metaTable = scanRoot?
+      getRootHTable(catalogTracker): getMetaHTable(catalogTracker);
+    ResultScanner scanner = metaTable.getScanner(scan);
+    try {
+      Result data;
+      while((data = scanner.next()) != null) {
+        if (data.isEmpty()) continue;
+        // Break if visit returns false.
+        if (!visitor.visit(data)) break;
+      }
+    } finally {
+      scanner.close();
+      metaTable.close();
+    }
+    return;
+  }
 
   /**
    * Implementations 'visit' a catalog table row.
@@ -651,4 +709,37 @@ public class MetaReader {
      */
     public boolean visit(final Result r) throws IOException;
   }
-}
+
+  /**
+   * A {@link Visitor} that collects content out of passed {@link Result}.
+   */
+  static abstract class CollectingVisitor<T> implements Visitor {
+    final List<T> results = new ArrayList<T>();
+    @Override
+    public boolean visit(Result r) throws IOException {
+      if (r ==  null || r.isEmpty()) return true;
+      add(r);
+      return true;
+    }
+
+    abstract void add(Result r);
+
+    /**
+     * @return Collected results; wait till visits complete to collect all
+     * possible results
+     */
+    List<T> getResults() {
+      return this.results;
+    }
+  }
+
+  /**
+   * Collects all returned.
+   */
+  static class CollectAllVisitor extends CollectingVisitor<Result> {
+    @Override
+    void add(Result r) {
+      this.results.add(r);
+    }
+  }
+}
\ No newline at end of file



Mime
View raw message