hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r732909 [1/2] - in /hadoop/hbase/branches/0.19_on_hadoop_0.18: ./ bin/ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/ src/java/org/apache/hadoop/hbase/ipc/ src/java/org/apache/hadoop/hbase/master/ src/java/org/ap...
Date Fri, 09 Jan 2009 01:55:35 GMT
Author: apurtell
Date: Thu Jan  8 17:55:34 2009
New Revision: 732909

URL: http://svn.apache.org/viewvc?rev=732909&view=rev
Log:
merge up to trunk (revision 732908)

Modified:
    hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt
    hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/HBase.rb
    hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/hirb.rb
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HConstants.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HMsg.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HRegionInfo.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/Leases.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HTable.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/HMaster.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/MetaScanner.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RootScanner.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ServerManager.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStore.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/client/TestForceSplit.java
    hadoop/hbase/branches/0.19_on_hadoop_0.18/src/test/org/apache/hadoop/hbase/client/TestHTable.java

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/CHANGES.txt Thu Jan  8 17:55:34 2009
@@ -127,6 +127,16 @@
    HBASE-1093  NPE in HStore#compact
    HBASE-1097  SequenceFile.Reader keeps around buffer whose size is that of
                largest item read -> results in lots of dead heap
+   HBASE-1107  NPE in HStoreScanner.updateReaders
+   HBASE-1083  Will keep scheduling major compactions if last time one ran, we
+               didn't.
+   HBASE-1101  NPE in HConnectionManager$TableServers.processBatchOfRows
+   HBASE-1099  Regions assigned while master is splitting logs of recently
+               crashed server; regionserver tries to execute incomplete log
+   HBASE-1104, HBASE-1098, HBASE-1096: Doubly-assigned regions redux,
+               IllegalStateException: Cannot set a region to be closed it it was
+               not already marked as closing, Does not recover if HRS carrying 
+               -ROOT- goes down
 
   IMPROVEMENTS
    HBASE-901   Add a limit to key length, check key and value length on client side
@@ -202,6 +212,10 @@
    HBASE-1062  Compactions at (re)start on a large table can overwhelm DFS
    HBASE-1102  boolean HTable.exists()
    HBASE-1105  Remove duplicated code in HCM, add javadoc to RegionState, etc.
+   HBASE-1106  Expose getClosestRowBefore in HTable
+               (Michael Gottesman via Stack)
+   HBASE-1082  Administrative functions for table/region maintenance
+   HBASE-1090  Atomic Check And Save in HTable (Michael Gottesman via Stack)
 
   NEW FEATURES
    HBASE-875   Use MurmurHash instead of JenkinsHash [in bloomfilters]

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/HBase.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/HBase.rb?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/HBase.rb (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/HBase.rb Thu Jan  8 17:55:34 2009
@@ -78,6 +78,34 @@
       @formatter.footer(now)
     end
 
+    def flush(tableNameOrRegionName)
+      now = Time.now 
+      @formatter.header()
+      @admin.flush(tableNameOrRegionName)
+      @formatter.footer(now)
+    end
+
+    def compact(tableNameOrRegionName)
+      now = Time.now 
+      @formatter.header()
+      @admin.compact(tableNameOrRegionName)
+      @formatter.footer(now)
+    end
+
+    def major_compact(tableNameOrRegionName)
+      now = Time.now 
+      @formatter.header()
+      @admin.majorCompact(tableNameOrRegionName)
+      @formatter.footer(now)
+    end
+
+    def split(tableNameOrRegionName)
+      now = Time.now 
+      @formatter.header()
+      @admin.split(tableNameOrRegionName)
+      @formatter.footer(now)
+    end
+
     def enable(tableName)
       # TODO: Need an isEnabled method
       now = Time.now 

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/hirb.rb
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/hirb.rb?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/hirb.rb (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/bin/hirb.rb Thu Jan  8 17:55:34 2009
@@ -105,18 +105,32 @@
   # Help for hbase shell surgery tools
   h  = <<HERE
 HBASE SURGERY TOOLS:
- enable_region   Enable a single region. For example,
-  
-                 hbase> enable_region 'REGIONNAME'
- 
- disable_region  Disable a single region
-
  close_region    Close a single region. Optionally specify regionserver.
                  Examples:
                  
                  hbase> close_region 'REGIONNAME'
                  hbase> close_region 'REGIONNAME', 'REGIONSERVER_IP:PORT'
 
+ compact         Compact all regions in passed table or pass a region row
+                 to compact an individual region
+
+ disable_region  Disable a single region
+
+ enable_region   Enable a single region. For example:
+  
+                 hbase> enable_region 'REGIONNAME'
+ 
+ flush           Flush all regions in passed table or pass a region row to
+                 flush an individual region.  For example:
+
+                 hbase> flush 'TABLENAME'
+                 hbase> flush 'REGIONNAME'
+
+ major_compact   Run major compaction on passed table or pass a region row
+                 to major compact an individual region
+
+ split           Split table or pass a region row to split individual region
+
 Above commands are for 'experts'-only as misuse can damage an install
 HERE
   puts h
@@ -334,6 +348,22 @@
   table(table).count(interval)
 end
 
+def flush(tableNameOrRegionName)
+  admin().flush(tableNameOrRegionName)
+end
+
+def compact(tableNameOrRegionName)
+  admin().compact(tableNameOrRegionName)
+end
+
+def major_compact(tableNameOrRegionName)
+  admin().major_compact(tableNameOrRegionName)
+end
+
+def split(tableNameOrRegionName)
+  admin().split(tableNameOrRegionName)
+end
+
 # Output a banner message that tells users where to go for help
 puts <<HERE
 HBase Shell; enter 'help<RETURN>' for list of supported commands.

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HConstants.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HConstants.java Thu Jan  8 17:55:34 2009
@@ -257,7 +257,7 @@
    * This is a retry backoff multiplier table similar to the BSD TCP syn
    * backoff table, a bit more aggressive than simple exponential backoff.
    */ 
-  public static int RETRY_BACKOFF[] = { 1, 2, 2, 4, 4, 4, 8, 8, 16, 32 };
+  public static int RETRY_BACKOFF[] = { 1, 1, 1, 2, 2, 4, 4, 8, 16, 32 };
 
   /** modifyTable op for replacing the table descriptor */
   public static final int MODIFY_TABLE_SET_HTD = 1;
@@ -268,4 +268,7 @@
   
   // Messages client can send master.
   public static final int MODIFY_CLOSE_REGION = MODIFY_TABLE_COMPACT + 1;
-}
+  
+  public static final int MODIFY_TABLE_FLUSH = MODIFY_CLOSE_REGION + 1;
+  public static final int MODIFY_TABLE_MAJOR_COMPACT = MODIFY_TABLE_FLUSH + 1;
+}
\ No newline at end of file

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HMsg.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HMsg.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HMsg.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HMsg.java Thu Jan  8 17:55:34 2009
@@ -34,7 +34,6 @@
  * Most of the time the messages are simple but some messages are accompanied
  * by the region affected.  HMsg may also carry optional message.
  */
-@SuppressWarnings("serial")
 public class HMsg implements Writable {
   /**
    * Message types sent between master and regionservers
@@ -100,6 +99,16 @@
      * regions
      */
     MSG_REPORT_QUIESCED,
+    
+    /**
+     * Flush
+     */
+    MSG_REGION_FLUSH,
+    
+    /**
+     * Run Major Compaction
+     */
+    MSG_REGION_MAJOR_COMPACT,
   }
 
   private Type type = null;

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/HRegionInfo.java Thu Jan  8 17:55:34 2009
@@ -404,5 +404,4 @@
     this.splitRequest = b;
     return old;
   }
-
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/Leases.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/Leases.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/Leases.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/Leases.java Thu Jan  8 17:55:34 2009
@@ -55,7 +55,6 @@
   private final int leasePeriod;
   private final int leaseCheckFrequency;
   private volatile DelayQueue<Lease> leaseQueue = new DelayQueue<Lease>();
-
   protected final Map<String, Lease> leases = new HashMap<String, Lease>();
   private volatile boolean stopRequested = false;
 
@@ -88,15 +87,16 @@
       if (lease == null) {
         continue;
       }
-      // A lease expired
+      // A lease expired.  Run the expired code before removing from queue
+      // since its presence in queue is used to see if lease exists still.
+      if (lease.getListener() == null) {
+        LOG.error("lease listener is null for lease " + lease.getLeaseName());
+      } else {
+        lease.getListener().leaseExpired();
+      }
       synchronized (leaseQueue) {
         leases.remove(lease.getLeaseName());
-        if (lease.getListener() == null) {
-          LOG.error("lease listener is null for lease " + lease.getLeaseName());
-          continue;
-        }
       }
-      lease.getListener().leaseExpired();
     }
     close();
   }

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Thu Jan  8 17:55:34 2009
@@ -45,6 +45,7 @@
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.util.Shell.ExitCodeException;
 
 /**
  * Provides administrative functions for HBase
@@ -539,11 +540,105 @@
     modifyTable(HConstants.META_TABLE_NAME, HConstants.MODIFY_CLOSE_REGION,
       newargs);
   }
+  
+  /**
+   * Flush a table or an individual region
+   * @param tableNameOrRegionName
+   * @throws IOException
+   */
+  public void flush(final String tableNameOrRegionName) throws IOException {
+    flush(Bytes.toBytes(tableNameOrRegionName));
+  }
+
+  /**
+   * Flush a table or an individual region
+   * @param tableNameOrRegionName
+   * @throws IOException
+   */
+  public void flush(final byte [] tableNameOrRegionName) throws IOException {
+    modifyTable(tableNameOrRegionName, HConstants.MODIFY_TABLE_FLUSH);
+  }
+
+  /**
+   * Compact a table or an individual region
+   * @param tableNameOrRegionName
+   * @throws IOException
+   */
+  public void compact(final String tableNameOrRegionName) throws IOException {
+    compact(Bytes.toBytes(tableNameOrRegionName));
+  }
+
+  /**
+   * Compact a table or an individual region
+   * @param tableNameOrRegionName
+   * @throws IOException
+   */
+  public void compact(final byte [] tableNameOrRegionName) throws IOException {
+    modifyTable(tableNameOrRegionName, HConstants.MODIFY_TABLE_COMPACT);
+  }
+  
+  /**
+   * Major compact a table or an individual region
+   * @param tableNameOrRegionName
+   * @throws IOException
+   */
+  public void majorCompact(final String tableNameOrRegionName)
+  throws IOException {
+    majorCompact(Bytes.toBytes(tableNameOrRegionName));
+  }
+
+  /**
+   * Major compact a table or an individual region
+   * @param tableNameOrRegionName
+   * @throws IOException
+   */
+  public void majorCompact(final byte [] tableNameOrRegionName)
+  throws IOException {
+    modifyTable(tableNameOrRegionName, HConstants.MODIFY_TABLE_MAJOR_COMPACT);
+  }
+
+  /**
+   * Split a table or an individual region
+   * @param tableNameOrRegionName
+   * @throws IOException
+   */
+  public void split(final String tableNameOrRegionName) throws IOException {
+    split(Bytes.toBytes(tableNameOrRegionName));
+  }
+
+  /**
+   * Split a table or an individual region
+   * @param tableNameOrRegionName
+   * @throws IOException
+   */
+  public void split(final byte [] tableNameOrRegionName) throws IOException {
+    modifyTable(tableNameOrRegionName, HConstants.MODIFY_TABLE_SPLIT);
+  }
+
+  /*
+   * Call modifyTable using passed tableName or region name String.  If no
+   * such table, presume we have been passed a region name.
+   * @param tableNameOrRegionName
+   * @param op
+   * @throws IOException
+   */
+  private void modifyTable(final byte [] tableNameOrRegionName, final int op)
+  throws IOException {
+    if (tableNameOrRegionName == null) {
+      throw new IllegalArgumentException("Pass a table name or region name");
+    }
+    byte [] tableName = tableExists(tableNameOrRegionName)?
+      tableNameOrRegionName: null;
+    byte [] regionName = tableName == null? tableNameOrRegionName: null;
+    Object [] args = regionName == null? null: new byte [][] {regionName};
+    modifyTable(tableName == null? null: tableName, op, args);
+  }
 
   /**
    * Modify an existing table
    * 
-   * @param tableName name of table
+   * @param tableName name of table.  May be null if we are operating on a
+   * region.
    * @param op table modification operation
    * @param args operation specific arguments
    * @throws IOException
@@ -554,7 +649,7 @@
       throw new MasterNotRunningException("master has been shut down");
     }
     // Let pass if its a catalog table.  Used by admins.
-    if (!MetaUtils.isMetaTableName(tableName)) {
+    if (tableName != null && !MetaUtils.isMetaTableName(tableName)) {
       // This will throw exception
       HTableDescriptor.isLegalTableName(tableName);
     }
@@ -573,15 +668,19 @@
 
       case HConstants.MODIFY_TABLE_COMPACT:
       case HConstants.MODIFY_TABLE_SPLIT:
+      case HConstants.MODIFY_TABLE_MAJOR_COMPACT:
+      case HConstants.MODIFY_TABLE_FLUSH:
         if (args != null && args.length > 0) {
           arr = new Writable[1];
           if (args[0] instanceof byte[]) {
             arr[0] = new ImmutableBytesWritable((byte[])args[0]);
           } else if (args[0] instanceof ImmutableBytesWritable) {
             arr[0] = (ImmutableBytesWritable)args[0];
+          } else if (args[0] instanceof String) {
+            arr[0] = new ImmutableBytesWritable(Bytes.toBytes((String)args[0]));
           } else {
-            throw new IllegalArgumentException("SPLIT or COMPACT with arg " +
-              "requires byte[] or ImmutableBytesWritable");
+            throw new IllegalArgumentException("Requires byte[], String, or" +
+              "ImmutableBytesWritable");
           }
         }
         this.master.modifyTable(tableName, op, arr);

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Thu Jan  8 17:55:34 2009
@@ -872,32 +872,65 @@
       }
       return null;    
     }
-    
+
+    private HRegionLocation
+    getRegionLocationForRowWithRetries(byte[] tableName, byte[] rowKey, 
+        boolean reload) throws IOException {
+      getMaster();
+      List<Throwable> exceptions = new ArrayList<Throwable>();
+      HRegionLocation location = null;
+      int tries = 0;
+      while (tries < numRetries) {
+        try {
+          location = getRegionLocation(tableName, rowKey, reload);
+        } catch (Throwable t) {
+          exceptions.add(t);
+        }
+        if (location != null) {
+          break;
+        }
+        reload = true;
+        tries++;
+        try {
+          Thread.sleep(getPauseTime(tries));
+        } catch (InterruptedException e) {
+          // continue
+        }
+      }
+      if (location == null) {
+        throw new RetriesExhaustedException("Some server",
+          HConstants.EMPTY_BYTE_ARRAY, rowKey, tries, exceptions);
+      }
+      return location;
+    }
+
     public void processBatchOfRows(ArrayList<BatchUpdate> list, byte[] tableName)
         throws IOException {
-      // See HBASE-748 for pseudo code of this method
       if (list.isEmpty()) {
         return;
       }
       boolean retryOnlyOne = false;
+      int tries = 0;
       Collections.sort(list);
       List<BatchUpdate> tempUpdates = new ArrayList<BatchUpdate>();
-      byte [] currentRegion = getRegionLocation(tableName, list.get(0).getRow(),
-        false).getRegionInfo().getRegionName();
+      HRegionLocation location =
+        getRegionLocationForRowWithRetries(tableName, list.get(0).getRow(),
+            false);
+      byte [] currentRegion = location.getRegionInfo().getRegionName();
       byte [] region = currentRegion;
       boolean isLastRow = false;
-      int tries = 0;
       for (int i = 0; i < list.size() && tries < numRetries; i++) {
         BatchUpdate batchUpdate = list.get(i);
         tempUpdates.add(batchUpdate);
         isLastRow = (i + 1) == list.size();
         if (!isLastRow) {
-          region = getRegionLocation(tableName, list.get(i + 1).getRow(), false)
-              .getRegionInfo().getRegionName();
+          location = getRegionLocationForRowWithRetries(tableName,
+            list.get(i+1).getRow(), false);
+          region = location.getRegionInfo().getRegionName();
         }
         if (!Bytes.equals(currentRegion, region) || isLastRow || retryOnlyOne) {
           final BatchUpdate[] updates = tempUpdates.toArray(new BatchUpdate[0]);
-          int index = getRegionServerForWithoutRetries(new ServerCallable<Integer>(
+          int index = getRegionServerWithRetries(new ServerCallable<Integer>(
               this, tableName, batchUpdate.getRow()) {
             public Integer call() throws IOException {
               int i = server.batchUpdates(location.getRegionInfo()
@@ -926,8 +959,9 @@
             }
             i = i - updates.length + index;
             retryOnlyOne = true;
-            region = getRegionLocation(tableName, list.get(i + 1).getRow(),
-                true).getRegionInfo().getRegionName();
+            location = getRegionLocationForRowWithRetries(tableName, 
+              list.get(i + 1).getRow(), true);
+            region = location.getRegionInfo().getRegionName();
           }
           else {
             retryOnlyOne = false;

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/client/HTable.java Thu Jan  8 17:55:34 2009
@@ -45,6 +45,7 @@
 import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.io.HbaseMapWritable;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
 
@@ -618,6 +619,19 @@
     );
   }
 
+  public RowResult getClosestRowBefore(final byte[] row, final byte[] columnFamily)
+  throws IOException {
+    return connection.getRegionServerWithRetries(
+      new ServerCallable<RowResult>(connection,tableName,row) {
+        public RowResult call() throws IOException {
+          return server.getClosestRowBefore(
+              location.getRegionInfo().getRegionName(), row, columnFamily
+            );
+          }
+        }
+      );
+    }
+
   /** 
    * Get a scanner on the current table starting at first row.
    * Return the specified columns.
@@ -1322,6 +1336,33 @@
   }
   
   /**
+   * Atomically checks if a row's values match
+   * the expectedValues. If it does, it uses the
+   * batchUpdate to update the row.
+   * @param batchUpdate batchupdate to apply if check is successful
+   * @param expectedValues values to check
+   * @param rl rowlock
+   * @throws IOException
+   */
+  public synchronized boolean checkAndSave(final BatchUpdate batchUpdate,
+    final HbaseMapWritable<byte[],byte[]> expectedValues, final RowLock rl)
+  throws IOException {
+    checkRowAndColumns(batchUpdate);
+    if(rl != null) {
+      batchUpdate.setRowLock(rl.getLockId());
+    }
+    return connection.getRegionServerWithRetries(
+      new ServerCallable<Boolean>(connection, tableName, batchUpdate.getRow()) {
+        public Boolean call() throws IOException {
+          return server.checkAndSave(location.getRegionInfo().getRegionName(),
+            batchUpdate, expectedValues)?
+              Boolean.TRUE: Boolean.FALSE;
+        }
+      }
+    ).booleanValue();
+  }
+  
+  /**
    * Commit to the table the buffer of BatchUpdate.
    * Called automaticaly in the commit methods when autoFlush is true.
    * @throws IOException

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java Thu Jan  8 17:55:34 2009
@@ -25,6 +25,7 @@
 import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.io.RowResult;
+import org.apache.hadoop.hbase.io.HbaseMapWritable;
 
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.NotServingRegionException;
@@ -118,6 +119,22 @@
   throws IOException;
   
   /**
+   * Applies a batch of updates to one row atomically via one RPC
+   * if the columns specified in expectedValues match
+   * the given values in expectedValues
+   * 
+   * @param regionName name of the region to update
+   * @param b BatchUpdate
+   * @param expectedValues map of column names to expected data values.
+   * @return true if update was applied
+   * @throws IOException
+   */
+  public boolean checkAndSave(final byte [] regionName, final BatchUpdate b,
+      final HbaseMapWritable<byte[],byte[]> expectedValues)
+  throws IOException;
+  
+
+  /**
    * Delete all cells that match the passed row and column and whose timestamp
    * is equal-to or older than the passed timestamp.
    * 
@@ -198,7 +215,7 @@
    * @param row The row
    * @param column The column, or null for any
    * @param timestamp The timestamp, or LATEST_TIMESTAMP for any
-   * @param lockId lock id
+   * @param lockID lock id
    * @return true if the row exists, false otherwise
    * @throws IOException
    */

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/BaseScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/BaseScanner.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/BaseScanner.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/BaseScanner.java Thu Jan  8 17:55:34 2009
@@ -104,7 +104,6 @@
     
   private final boolean rootRegion;
   protected final HMaster master;
-  protected final RegionManager regionManager;
   
   protected boolean initialScanComplete;
   
@@ -115,12 +114,11 @@
   // mid-scan
   final Integer scannerLock = new Integer(0);
   
-  BaseScanner(final HMaster master, final RegionManager regionManager, 
-    final boolean rootRegion, final int period, final AtomicBoolean stop) {
+  BaseScanner(final HMaster master, final boolean rootRegion, final int period,
+      final AtomicBoolean stop) {
     super(period, stop);
     this.rootRegion = rootRegion;
     this.master = master;
-    this.regionManager = regionManager;
     this.initialScanComplete = false;
   }
   
@@ -180,7 +178,7 @@
         rows += 1;
       }
       if (rootRegion) {
-        regionManager.setNumMetaRegions(rows);
+        this.master.regionManager.setNumMetaRegions(rows);
       }
     } catch (IOException e) {
       if (e instanceof RemoteException) {
@@ -210,7 +208,7 @@
     if (emptyRows.size() > 0) {
       LOG.warn("Found " + emptyRows.size() + " rows with empty HRegionInfo " +
         "while scanning meta region " + Bytes.toString(region.getRegionName()));
-      master.deleteEmptyMetaRows(regionServer, region.getRegionName(),
+      this.master.deleteEmptyMetaRows(regionServer, region.getRegionName(),
           emptyRows);
     }
 
@@ -264,7 +262,7 @@
     if (!hasReferencesA && !hasReferencesB) {
       LOG.info("Deleting region " + parent.getRegionNameAsString() +
         " because daughter splits no longer hold references");
-      HRegion.deleteRegion(master.fs, master.rootdir, parent);
+      HRegion.deleteRegion(this.master.fs, this.master.rootdir, parent);
       HRegion.removeRegionFromMETA(srvr, metaRegionName,
         parent.getRegionName());
       result = true;
@@ -294,8 +292,8 @@
     if (split == null) {
       return result;
     }
-    Path tabledir =
-      HTableDescriptor.getTableDir(master.rootdir, split.getTableDesc().getName());
+    Path tabledir = HTableDescriptor.getTableDir(this.master.rootdir,
+        split.getTableDesc().getName());
     for (HColumnDescriptor family: split.getTableDesc().getFamilies()) {
       Path p = HStoreFile.getMapDir(tabledir, split.getEncodedName(),
         family.getName());
@@ -303,7 +301,7 @@
       // Look for reference files.  Call listStatus with an anonymous
       // instance of PathFilter.
 
-      FileStatus [] ps = master.fs.listStatus(p,
+      FileStatus [] ps = this.master.fs.listStatus(p,
           new PathFilter () {
             public boolean accept(Path path) {
               return HStore.isReference(path);
@@ -337,72 +335,56 @@
     final String serverName, final long startCode) 
   throws IOException {
     
-    synchronized (regionManager) {
-      // Skip region - if ...
-      if(info.isOffline()                                 // offline
-          || regionManager.isOfflined(info.getRegionName())) { // queued for offline
+    synchronized (this.master.regionManager) {
+      /*
+       * We don't assign regions that are offline, in transition or were on
+       * a dead server. Regions that were on a dead server will get reassigned
+       * by ProcessServerShutdown
+       */
+      if(info.isOffline() ||
+          this.master.regionManager.regionIsInTransition(info.getRegionName()) ||
+          this.master.serverManager.isDead(serverName)) {
 
-        regionManager.removeRegion(info);
         return;
       }
       HServerInfo storedInfo = null;
-      boolean deadServer = false;
       if (serverName.length() != 0) {
-
-        if (regionManager.isOfflined(info.getRegionName())) {
-          // Skip if region is on kill list
-          if(LOG.isDebugEnabled()) {
-            LOG.debug("not assigning region (on kill list): " +
-                info.getRegionNameAsString());
-          }
-          return;
-        }
-
-        storedInfo = master.serverManager.getServerInfo(serverName);
-        deadServer = master.serverManager.isDead(serverName);
+        storedInfo = this.master.serverManager.getServerInfo(serverName);
       }
 
       /*
-       * If the server is a dead server or its startcode is off -- either null
-       * or doesn't match the start code for the address -- then add it to the
-       * list of unassigned regions IF not already there (or pending open).
+       * If the startcode is off -- either null or doesn't match the start code
+       * for the address -- then add it to the list of unassigned regions.
        */ 
-      if ((deadServer || 
-          (storedInfo == null || storedInfo.getStartCode() != startCode)) &&
-          (!regionManager.isUnassigned(info) &&
-              !regionManager.isPending(info.getRegionName()) &&
-              !regionManager.isAssigned(info.getRegionName()))) {
+      if (storedInfo == null || storedInfo.getStartCode() != startCode) {
 
         // The current assignment is invalid
-
         if (LOG.isDebugEnabled()) {
-          LOG.debug("Current assignment of " +
-              info.getRegionNameAsString() +
-              " is not valid." +
-              (storedInfo == null ? " Server '" + serverName + "' unknown." :
+          LOG.debug("Current assignment of " + info.getRegionNameAsString() +
+            " is not valid; " +
+            (storedInfo == null ? " Server '" + serverName + "' unknown." :
                 " serverInfo: " + storedInfo + ", passed startCode: " +
-                startCode + ", storedInfo.startCode: " + storedInfo.getStartCode()) +
-          " Region is not unassigned, assigned or pending");
+                startCode + ", storedInfo.startCode: " +
+                storedInfo.getStartCode()));
         }
 
         // Recover the region server's log if there is one.
         // This is only done from here if we are restarting and there is stale
         // data in the meta region. Once we are on-line, dead server log
         // recovery is handled by lease expiration and ProcessServerShutdown
-
-        if (!regionManager.isInitialMetaScanComplete() &&
+        if (!this.master.regionManager.isInitialMetaScanComplete() &&
             serverName.length() != 0) {
           StringBuilder dirName = new StringBuilder("log_");
           dirName.append(serverName.replace(":", "_"));
-          Path logDir = new Path(master.rootdir, dirName.toString());
+          Path logDir = new Path(this.master.rootdir, dirName.toString());
           try {
             if (master.fs.exists(logDir)) {
-              regionManager.splitLogLock.lock();
+              this.master.regionManager.splitLogLock.lock();
               try {
                 HLog.splitLog(master.rootdir, logDir, master.fs,
                     master.getConfiguration());
               } finally {
-                regionManager.splitLogLock.unlock();
+                this.master.regionManager.splitLogLock.unlock();
               }
             }
             if (LOG.isDebugEnabled()) {
@@ -414,11 +396,11 @@
           }
         }
         // Now get the region assigned
-        regionManager.setUnassigned(info, true);
+        this.master.regionManager.setUnassigned(info, true);
       }
     }
   }
-  
+
   /**
    * Notify the thread to die at the end of its next run
    */

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ChangeTableState.java Thu Jan  8 17:55:34 2009
@@ -91,9 +91,7 @@
       synchronized (master.regionManager) {
         if (online) {
           // Bring offline regions on-line
-          if (!master.regionManager.isUnassigned(i) &&
-              !master.regionManager.isAssigned(i.getRegionName()) &&
-              !master.regionManager.isPending(i.getRegionName())) {
+          if (!master.regionManager.regionIsOpening(i.getRegionName())) {
             master.regionManager.setUnassigned(i, false);
           }
         } else {

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/HMaster.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/HMaster.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/HMaster.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/HMaster.java Thu Jan  8 17:55:34 2009
@@ -532,15 +532,14 @@
   /*
    * HMasterRegionInterface
    */
-  public MapWritable regionServerStartup(HServerInfo serverInfo) {
-    // Set the address for now even tho it will not be persisted on
-    // the HRS side.
+  public MapWritable regionServerStartup(final HServerInfo serverInfo) {
+    // Set the address for now even tho it will not be persisted on HRS side.
     String rsAddress = HBaseServer.getRemoteAddress();
-    serverInfo.setServerAddress(new HServerAddress
-        (rsAddress, serverInfo.getServerAddress().getPort()));
-    // register with server manager
-    serverManager.regionServerStartup(serverInfo);
-    // send back some config info
+    serverInfo.setServerAddress(new HServerAddress(rsAddress,
+      serverInfo.getServerAddress().getPort()));
+    // Register with server manager
+    this.serverManager.regionServerStartup(serverInfo);
+    // Send back some config info
     return createConfigurationSubset();
   }
   
@@ -795,20 +794,22 @@
 
     case MODIFY_TABLE_SPLIT:
     case MODIFY_TABLE_COMPACT:
+    case MODIFY_TABLE_MAJOR_COMPACT:
+    case MODIFY_TABLE_FLUSH:
       if (args != null && args.length > 0) {
         if (!(args[0] instanceof ImmutableBytesWritable))
           throw new IOException(
             "request argument must be ImmutableBytesWritable");
-        byte[] rowKey = ((ImmutableBytesWritable)args[0]).get();
+        byte [] rowKey = ((ImmutableBytesWritable)args[0]).get();
         Pair<HRegionInfo,HServerAddress> pair =
           getTableRegionClosest(tableName, rowKey);
         if (pair != null) {
-          regionManager.startAction(pair.getFirst().getRegionName(),
+          this.regionManager.startAction(pair.getFirst().getRegionName(),
             pair.getFirst(), pair.getSecond(), op);
         }
       } else {
         for (Pair<HRegionInfo,HServerAddress> pair: getTableRegions(tableName))
-          regionManager.startAction(pair.getFirst().getRegionName(),
+          this.regionManager.startAction(pair.getFirst().getRegionName(),
             pair.getFirst(), pair.getSecond(), op);
       }
       break;
@@ -834,7 +835,7 @@
       LOG.info("Marking " + hri.getRegionNameAsString() +
         " as closed on " + servername + "; cleaning SERVER + STARTCODE; " +
           "master will tell regionserver to close region on next heartbeat");
-      this.regionManager.setClosing(servername, hri, hri.isOffline(), false);
+      this.regionManager.setClosing(servername, hri, hri.isOffline());
       MetaRegion meta = this.regionManager.getMetaRegionForRow(regionname);
       HRegionInterface srvr = getMETAServer(meta);
       HRegion.cleanRegionInMETA(srvr, meta.getRegionName(), hri);

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/MetaScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/MetaScanner.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/MetaScanner.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/MetaScanner.java Thu Jan  8 17:55:34 2009
@@ -49,21 +49,21 @@
    * Constructor
    * 
    * @param master
-   * @param regionManager
    */
-  public MetaScanner(HMaster master, RegionManager regionManager) {
-    super(master, regionManager, false, master.metaRescanInterval, master.closed);
+  public MetaScanner(HMaster master) {
+    super(master, false, master.metaRescanInterval, master.closed);
   }
 
   // Don't retry if we get an error while scanning. Errors are most often
   // caused by the server going away. Wait until next rescan interval when
   // things should be back to normal.
   private boolean scanOneMetaRegion(MetaRegion region) {
-    while (!master.closed.get() && !regionManager.isInitialRootScanComplete() &&
-      regionManager.getRootRegionLocation() == null) {
+    while (!this.master.closed.get() &&
+        !this.master.regionManager.isInitialRootScanComplete() &&
+        this.master.regionManager.getRootRegionLocation() == null) {
       sleep();
     }
-    if (master.closed.get()) {
+    if (this.master.closed.get()) {
       return false;
     }
 
@@ -71,7 +71,7 @@
       // Don't interrupt us while we're working
       synchronized (scannerLock) {
         scanRegion(region);
-        regionManager.putMetaRegionOnline(region);
+        this.master.regionManager.putMetaRegionOnline(region);
       }
     } catch (IOException e) {
       e = RemoteExceptionHandler.checkIOException(e);
@@ -80,13 +80,13 @@
       // so, either it won't be in the onlineMetaRegions list or its host
       // address has changed and the containsValue will fail. If not
       // found, best thing to do here is probably return.
-      if (!regionManager.isMetaRegionOnline(region.getStartKey())) {
+      if (!this.master.regionManager.isMetaRegionOnline(region.getStartKey())) {
         LOG.debug("Scanned region is no longer in map of online " +
         "regions or its value has changed");
         return false;
       }
       // Make sure the file system is still available
-      master.checkFileSystem();
+      this.master.checkFileSystem();
     } catch (Exception e) {
       // If for some reason we get some other kind of exception, 
       // at least log it rather than go out silently.
@@ -98,11 +98,11 @@
   @Override
   protected boolean initialScan() {
     MetaRegion region = null;
-    while (!master.closed.get() &&
+    while (!this.master.closed.get() &&
         (region == null && metaRegionsToScan.size() > 0) &&
           !metaRegionsScanned()) {
       try {
-        region = metaRegionsToScan.poll(master.threadWakeFrequency, 
+        region = metaRegionsToScan.poll(this.master.threadWakeFrequency, 
           TimeUnit.MILLISECONDS);
       } catch (InterruptedException e) {
         // continue
@@ -122,7 +122,8 @@
 
   @Override
   protected void maintenanceScan() {
-    List<MetaRegion> regions = regionManager.getListOfOnlineMetaRegions();
+    List<MetaRegion> regions =
+      this.master.regionManager.getListOfOnlineMetaRegions();
     int regionCount = 0;
     for (MetaRegion r: regions) {
       scanOneMetaRegion(r);
@@ -140,8 +141,9 @@
    * @return False if number of meta regions matches count of online regions.
    */
   private synchronized boolean metaRegionsScanned() {
-    if (!regionManager.isInitialRootScanComplete() ||
-      regionManager.numMetaRegions() != regionManager.numOnlineMetaRegions()) {
+    if (!this.master.regionManager.isInitialRootScanComplete() ||
+        this.master.regionManager.numMetaRegions() !=
+          this.master.regionManager.numOnlineMetaRegions()) {
       return false;
     }
     notifyAll();
@@ -153,21 +155,21 @@
    * been scanned.
    */
   synchronized boolean waitForMetaRegionsOrClose() {
-    while (!master.closed.get()) {
+    while (!this.master.closed.get()) {
       synchronized (master.regionManager) {
-        if (regionManager.isInitialRootScanComplete() &&
-            regionManager.numMetaRegions() ==
-              regionManager.numOnlineMetaRegions()) {
+        if (this.master.regionManager.isInitialRootScanComplete() &&
+            this.master.regionManager.numMetaRegions() ==
+              this.master.regionManager.numOnlineMetaRegions()) {
           break;
         }
       }
       try {
-        wait(master.threadWakeFrequency);
+        wait(this.master.threadWakeFrequency);
       } catch (InterruptedException e) {
         // continue
       }
     }
-    return master.closed.get();
+    return this.master.closed.get();
   }
   
   /**

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java Thu Jan  8 17:55:34 2009
@@ -45,7 +45,10 @@
  */
 class ProcessServerShutdown extends RegionServerOperation {
   private final HServerAddress deadServer;
-  private final String deadServerName;
+  /*
+   * Cache of the server name.
+   */
+  private final String deadServerStr;
   private final boolean rootRegionServer;
   private boolean rootRegionReassigned = false;
   private Path oldLogDir;
@@ -74,7 +77,7 @@
       boolean rootRegionServer) {
     super(master);
     this.deadServer = serverInfo.getServerAddress();
-    this.deadServerName = this.deadServer.toString();
+    this.deadServerStr = this.deadServer.toString();
     this.rootRegionServer = rootRegionServer;
     this.logSplit = false;
     this.rootRescanned = false;
@@ -84,13 +87,14 @@
 
   @Override
   public String toString() {
-    return "ProcessServerShutdown of " + this.deadServer.toString();
+    return "ProcessServerShutdown of " + this.deadServerStr;
   }
 
-  /** Finds regions that the dead region server was serving */
+  /** Finds regions that the dead region server was serving
+   */
   protected void scanMetaRegion(HRegionInterface server, long scannerId,
-      byte [] regionName) throws IOException {
-
+    byte [] regionName)
+  throws IOException {
     List<ToDoEntry> toDoList = new ArrayList<ToDoEntry>();
     Set<HRegionInfo> regions = new HashSet<HRegionInfo>();
     List<byte []> emptyRows = new ArrayList<byte []>();
@@ -107,14 +111,13 @@
         if (values == null || values.size() == 0) {
           break;
         }
-        
         byte [] row = values.getRow();
-        // Check server name.  If null, be conservative and treat as though
-        // region had been on shutdown server (could be null because we
-        // missed edits in hlog because hdfs does not do write-append).
+        // Check server name.  If null, skip (We used to consider it was on
+        // shutdown server but that would mean that we'd reassign regions that
+        // were already out being assigned, ones that were product of a split
+        // that happened while the shutdown was being processed.
         String serverName = Writables.cellToString(values.get(COL_SERVER));
-        if (serverName != null && serverName.length() > 0 &&
-            deadServerName.compareTo(serverName) != 0) {
+        if (serverName == null || !deadServerStr.equals(serverName)) {
           // This isn't the server you're looking for - move along
           continue;
         }
@@ -159,7 +162,7 @@
         }
       }
     } finally {
-      if(scannerId != -1L) {
+      if (scannerId != -1L) {
         try {
           server.close(scannerId);
         } catch (IOException e) {
@@ -222,21 +225,20 @@
       long scannerId =
         server.openScanner(m.getRegionName(), COLUMN_FAMILY_ARRAY,
         EMPTY_START_ROW, HConstants.LATEST_TIMESTAMP, null);
-
-        scanMetaRegion(server, scannerId, m.getRegionName());
+      scanMetaRegion(server, scannerId, m.getRegionName());
       return true;
     }
   }
 
   @Override
   protected boolean process() throws IOException {
-    LOG.info("process shutdown of server " + deadServer + ": logSplit: " +
-      this.logSplit + ", rootRescanned: " + rootRescanned +
+    LOG.info("process shutdown of server " + this.deadServerStr +
+      ": logSplit: " +
+      logSplit + ", rootRescanned: " + rootRescanned +
       ", numberOfMetaRegions: " + 
       master.regionManager.numMetaRegions() +
       ", onlineMetaRegions.size(): " + 
       master.regionManager.numOnlineMetaRegions());
-
     if (!logSplit) {
       // Process the old log file
       if (master.fs.exists(oldLogDir)) {
@@ -252,7 +254,7 @@
       }
       logSplit = true;
     }
-    
+
     if (this.rootRegionServer && !this.rootRegionReassigned) {
       // avoid multiple root region reassignment 
       this.rootRegionReassigned = true;
@@ -277,7 +279,6 @@
           new MetaRegion(master.getRootRegionLocation(),
               HRegionInfo.ROOT_REGIONINFO.getRegionName(),
               HConstants.EMPTY_START_ROW), this.master).doWithRetries();
-        
       if (result == null) {
         // Master is closing - give up
         return true;
@@ -290,7 +291,6 @@
       }
       rootRescanned = true;
     }
-    
     if (!metaTableAvailable()) {
       // We can't proceed because not all meta regions are online.
       // metaAvailable() has put this request on the delayedToDoQueue
@@ -309,7 +309,11 @@
           Bytes.toString(r.getRegionName()) + " on " + r.getServer());
       }
     }
-    master.serverManager.removeDeadServer(deadServerName);
+    // Remove this server from dead servers list.  Finished splitting logs.
+    this.master.serverManager.removeDeadServer(deadServerStr);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Removed " + deadServerStr + " from deadservers Map");
+    }
     return true;
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RegionManager.java Thu Jan  8 17:55:34 2009
@@ -78,12 +78,11 @@
 
   private static final byte[] OVERLOADED = Bytes.toBytes("Overloaded");
 
-  /*
+  /**
    * Map of region name to RegionState for regions that are in transition such as
    * 
-   * unassigned -> assigned -> pending -> open
-   * closing -> closed -> offline
-   * closing -> closed -> unassigned -> assigned -> pending -> open
+   * unassigned -> pendingOpen -> open
+   * closing -> pendingClose -> closed; if (closed && !offline) -> unassigned
    * 
    * At the end of a transition, removeRegion is used to remove the region from
    * the map (since it is no longer in transition)
@@ -104,12 +103,22 @@
   private final float slop;
 
   /** Set of regions to split. */
-  private final SortedMap<byte[],Pair<HRegionInfo,HServerAddress>> regionsToSplit = 
+  private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> regionsToSplit = 
     Collections.synchronizedSortedMap(
       new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
       (Bytes.BYTES_COMPARATOR));
   /** Set of regions to compact. */
-  private final SortedMap<byte[],Pair<HRegionInfo,HServerAddress>> regionsToCompact =
+  private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> regionsToCompact =
+    Collections.synchronizedSortedMap(
+      new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
+      (Bytes.BYTES_COMPARATOR));
+  /** Set of regions to major compact. */
+  private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> regionsToMajorCompact =
+    Collections.synchronizedSortedMap(
+      new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
+      (Bytes.BYTES_COMPARATOR));
+  /** Set of regions to flush. */
+  private final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> regionsToFlush =
     Collections.synchronizedSortedMap(
       new TreeMap<byte[],Pair<HRegionInfo,HServerAddress>>
       (Bytes.BYTES_COMPARATOR));
@@ -123,10 +132,10 @@
       (float)0.1);
 
     // The root region
-    rootScannerThread = new RootScanner(master, this);
+    rootScannerThread = new RootScanner(master);
 
     // Scans the meta table
-    metaScannerThread = new MetaScanner(master, this);
+    metaScannerThread = new MetaScanner(master);
     
     reassignRootRegion();
   }
@@ -262,7 +271,7 @@
       for (RegionState s: regionsToAssign) {
         LOG.info("assigning region " + Bytes.toString(s.getRegionName())+
           " to server " + serverName);
-        s.setAssigned(serverName);
+        s.setPendingOpen(serverName);
         this.historian.addRegionAssignment(s.getRegionInfo(), serverName);
         returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_OPEN, s.getRegionInfo()));
         if (--nregions <= 0) {
@@ -313,7 +322,8 @@
    * Get the set of regions that should be assignable in this pass.
    * 
    * Note that no synchronization on regionsInTransition is needed because the
-   * only caller (assignRegions) whose caller owns the monitor for RegionManager
+   * only caller (assignRegions, whose caller is ServerManager.processMsgs) owns
+   * the monitor for RegionManager
    */ 
   private Set<RegionState> regionsAwaitingAssignment() {
     // set of regions we want to assign to this server
@@ -332,8 +342,7 @@
         // and are on-line
         continue;
       }
-      if (!s.isAssigned() && !s.isClosing() && !s.isPending()) {
-        s.setUnassigned();
+      if (s.isUnassigned()) {
         regionsToAssign.add(s);
       }
     }
@@ -389,7 +398,7 @@
     for (RegionState s: regionsToAssign) {
       LOG.info("assigning region " + Bytes.toString(s.getRegionName()) +
           " to the only server " + serverName);
-      s.setAssigned(serverName);
+      s.setPendingOpen(serverName);
       this.historian.addRegionAssignment(s.getRegionInfo(), serverName);
       returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_OPEN, s.getRegionInfo()));
     }
@@ -422,8 +431,7 @@
         continue;
       }
       byte[] regionName = currentRegion.getRegionName();
-      if (isClosing(regionName) || isUnassigned(currentRegion) ||
-          isAssigned(regionName) || isPending(regionName)) {
+      if (regionIsInTransition(regionName)) {
         skipped++;
         continue;
       }
@@ -434,6 +442,7 @@
         OVERLOADED));
       // mark the region as closing
       setClosing(serverName, currentRegion, false);
+      setPendingClose(currentRegion.getRegionName());
       // increment the count of regions we've marked
       regionsClosed++;
     }
@@ -554,15 +563,6 @@
   }
 
   /**
-   * Remove a region from the region state map.
-   * 
-   * @param info
-   */
-  public void removeRegion(HRegionInfo info) {
-    regionsInTransition.remove(info.getRegionName());
-  }
-  
-  /**
    * Get metaregion that would host passed in row.
    * @param row Row need to know all the meta regions for
    * @return set of MetaRegion objects that contain the table
@@ -653,6 +653,53 @@
     onlineMetaRegions.remove(startKey); 
   }
   
+  /**
+   * Remove a region from the region state map.
+   * 
+   * @param info
+   */
+  public void removeRegion(HRegionInfo info) {
+    regionsInTransition.remove(info.getRegionName());
+  }
+  
+  /**
+   * @param regionName
+   * @return true if the named region is in a transition state
+   */
+  public boolean regionIsInTransition(byte[] regionName) {
+    return regionsInTransition.containsKey(regionName);
+  }
+
+  /**
+   * @param regionName
+   * @return true if the region is unassigned, pendingOpen or open
+   */
+  public boolean regionIsOpening(byte[] regionName) {
+    RegionState state = regionsInTransition.get(regionName);
+    if (state != null) {
+      return state.isOpening();
+    }
+    return false;
+  }
+
+  /** 
+   * Set a region to unassigned 
+   * @param info Region to set unassigned
+   * @param force if true mark region unassigned whatever its current state
+   */
+  public void setUnassigned(HRegionInfo info, boolean force) {
+    synchronized(this.regionsInTransition) {
+      RegionState s = regionsInTransition.get(info.getRegionName());
+      if (s == null) {
+        s = new RegionState(info);
+        regionsInTransition.put(info.getRegionName(), s);
+      }
+      if (force || (!s.isPendingOpen() && !s.isOpen())) {
+        s.setUnassigned();
+      }
+    }
+  }
+  
   /** 
    * Check if a region is on the unassigned list
    * @param info HRegionInfo to check for
@@ -671,34 +718,35 @@
   }
   
   /**
-   * Check if a region is pending 
+   * Check if a region has been assigned and we're waiting for a response from
+   * the region server.
+   * 
    * @param regionName name of the region
-   * @return true if pending, false otherwise
+   * @return true if open, false otherwise
    */
-  public boolean isPending(byte [] regionName) {
+  public boolean isPendingOpen(byte [] regionName) {
     synchronized (regionsInTransition) {
       RegionState s = regionsInTransition.get(regionName);
       if (s != null) {
-        return s.isPending();
+        return s.isPendingOpen();
       }
     }
     return false;
   }
-  
+
   /**
+   * Region has been assigned to a server and the server has told us it is open
    * @param regionName
-   * @return true if region has been assigned
    */
-  public boolean isAssigned(byte[] regionName) {
+  public void setOpen(byte [] regionName) {
     synchronized (regionsInTransition) {
       RegionState s = regionsInTransition.get(regionName);
       if (s != null) {
-        return s.isAssigned() || s.isPending();
+        s.setOpen();
       }
     }
-    return false;
   }
-
+  
   /**
    * @param regionName
    * @return true if region is marked to be offlined.
@@ -714,33 +762,20 @@
   }
 
   /** 
-   * Set a region to unassigned 
-   * @param info Region to set unassigned
-   * @param force if true mark region unassigned whatever its current state
+   * Mark a region as closing 
+   * @param serverName
+   * @param regionInfo
+   * @param setOffline
    */
-  public void setUnassigned(HRegionInfo info, boolean force) {
-    synchronized(this.regionsInTransition) {
-      RegionState s = regionsInTransition.get(info.getRegionName());
+  public void setClosing(final String serverName, final HRegionInfo regionInfo,
+      final boolean setOffline) {
+    synchronized (this.regionsInTransition) {
+      RegionState s = this.regionsInTransition.get(regionInfo.getRegionName());
       if (s == null) {
-        s = new RegionState(info);
-        regionsInTransition.put(info.getRegionName(), s);
-      }
-      if (force || (!s.isAssigned() && !s.isPending())) {
-        s.setUnassigned();
-      }
-    }
-  }
-  
-  /**
-   * Set a region to pending assignment 
-   * @param regionName
-   */
-  public void setPending(byte [] regionName) {
-    synchronized (regionsInTransition) {
-      RegionState s = regionsInTransition.get(regionName);
-      if (s != null) {
-        s.setPending();
+        s = new RegionState(regionInfo);
       }
+      s.setClosing(serverName, setOffline);
+      this.regionsInTransition.put(regionInfo.getRegionName(), s);
     }
   }
   
@@ -755,7 +790,7 @@
     Set<HRegionInfo> result = new HashSet<HRegionInfo>();
     synchronized (regionsInTransition) {
       for (RegionState s: regionsInTransition.values()) {
-        if (s.isClosing() && !s.isClosed() &&
+        if (s.isClosing() && !s.isPendingClose() && !s.isClosed() &&
             s.getServerName().compareTo(serverName) == 0) {
           result.add(s.getRegionInfo());
         }
@@ -764,55 +799,18 @@
     return result;
   }
   
-  /** 
-   * Check if a region is closing 
-   * @param regionName 
-   * @return true if the region is marked as closing, false otherwise
+  /**
+   * Called when we have told a region server to close the region
+   * 
+   * @param regionName
    */
-  public boolean isClosing(byte [] regionName) {
+  public void setPendingClose(byte[] regionName) {
     synchronized (regionsInTransition) {
       RegionState s = regionsInTransition.get(regionName);
       if (s != null) {
-        return s.isClosing();
+        s.setPendingClose();
       }
     }
-    return false;
-  }
-
-  /** 
-   * Mark a region as closing 
-   * @param serverName
-   * @param regionInfo
-   * @param setOffline
-   */
-  public void setClosing(final String serverName, final HRegionInfo regionInfo,
-      final boolean setOffline) {
-    setClosing(serverName, regionInfo, setOffline, true);
-  }
-
-  /**
-   * Mark a region as closing 
-   * @param serverName
-   * @param regionInfo
-   * @param setOffline
-   * @param check False if we are to skip state transition check.
-   */
-  void setClosing(final String serverName, final HRegionInfo regionInfo,
-      final boolean setOffline, final boolean check) {
-    synchronized (this.regionsInTransition) {
-      RegionState s = this.regionsInTransition.get(regionInfo.getRegionName());
-      if (check && s != null) {
-        if (!s.isClosing()) {
-          throw new IllegalStateException(
-            "Cannot transition to closing from any other state. Region: " +
-            Bytes.toString(regionInfo.getRegionName()));
-        }
-        return;
-      }
-      s = new RegionState(regionInfo);
-      s.setClosing(serverName, setOffline);
-      this.regionsInTransition.put(regionInfo.getRegionName(), s);
-    }
   }
   
   /**
@@ -937,18 +935,28 @@
       HServerAddress server, int op) {
     switch (op) {
       case HConstants.MODIFY_TABLE_SPLIT:
-        regionsToSplit.put(regionName, 
-          new Pair<HRegionInfo,HServerAddress>(info, server));
+        startAction(regionName, info, server, this.regionsToSplit);
         break;
       case HConstants.MODIFY_TABLE_COMPACT:
-        regionsToCompact.put(regionName,
-          new Pair<HRegionInfo,HServerAddress>(info, server));
+        startAction(regionName, info, server, this.regionsToCompact);
+        break;
+      case HConstants.MODIFY_TABLE_MAJOR_COMPACT:
+        startAction(regionName, info, server, this.regionsToMajorCompact);
+        break;
+      case HConstants.MODIFY_TABLE_FLUSH:
+        startAction(regionName, info, server, this.regionsToFlush);
         break;
       default:
         throw new IllegalArgumentException("illegal table action " + op);
     }
   }
 
+  private void startAction(final byte[] regionName, final HRegionInfo info,
+      final HServerAddress server,
+      final SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> map) {
+    map.put(regionName, new Pair<HRegionInfo,HServerAddress>(info, server));
+  }
+
   /**
    * @param regionName
    * @param op
@@ -956,10 +964,16 @@
   public void endAction(byte[] regionName, int op) {
     switch (op) {
     case HConstants.MODIFY_TABLE_SPLIT:
-      regionsToSplit.remove(regionName);
+      this.regionsToSplit.remove(regionName);
       break;
     case HConstants.MODIFY_TABLE_COMPACT:
-      regionsToCompact.remove(regionName);
+      this.regionsToCompact.remove(regionName);
+      break;
+    case HConstants.MODIFY_TABLE_MAJOR_COMPACT:
+      this.regionsToMajorCompact.remove(regionName);
+      break;
+    case HConstants.MODIFY_TABLE_FLUSH:
+      this.regionsToFlush.remove(regionName);
       break;
     default:
       throw new IllegalArgumentException("illegal table action " + op);
@@ -976,43 +990,42 @@
 
   /**
    * Send messages to the given region server asking it to split any
-   * regions in 'regionsToSplit'
+   * regions in 'regionsToSplit', etc.
    * @param serverInfo
    * @param returnMsgs
    */
   public void applyActions(HServerInfo serverInfo, ArrayList<HMsg> returnMsgs) {
+    applyActions(serverInfo, returnMsgs, this.regionsToCompact,
+        HMsg.Type.MSG_REGION_COMPACT);
+    applyActions(serverInfo, returnMsgs, this.regionsToSplit,
+      HMsg.Type.MSG_REGION_SPLIT);
+    applyActions(serverInfo, returnMsgs, this.regionsToFlush,
+        HMsg.Type.MSG_REGION_FLUSH);
+    applyActions(serverInfo, returnMsgs, this.regionsToMajorCompact,
+        HMsg.Type.MSG_REGION_MAJOR_COMPACT);
+  }
+  
+  private void applyActions(final HServerInfo serverInfo,
+      final ArrayList<HMsg> returnMsgs,
+      SortedMap<byte[], Pair<HRegionInfo,HServerAddress>> map,
+      final HMsg.Type msg) {
     HServerAddress addr = serverInfo.getServerAddress();
     Iterator<Pair<HRegionInfo, HServerAddress>> i =
-      regionsToCompact.values().iterator();
-    synchronized (regionsToCompact) {
-      while (i.hasNext()) {
-        Pair<HRegionInfo,HServerAddress> pair = i.next();
-        if (addr.equals(pair.getSecond())) {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("sending MSG_REGION_COMPACT " + pair.getFirst() + " to " +
-                addr);
-          }
-          returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_COMPACT, pair.getFirst()));
-          i.remove();
-        }
-      }
-    }
-    i = regionsToSplit.values().iterator();
-    synchronized (regionsToSplit) {
+      map.values().iterator();
+    synchronized (map) {
       while (i.hasNext()) {
         Pair<HRegionInfo,HServerAddress> pair = i.next();
         if (addr.equals(pair.getSecond())) {
           if (LOG.isDebugEnabled()) {
-            LOG.debug("sending MSG_REGION_SPLIT " + pair.getFirst() + " to " +
-                addr);
+            LOG.debug("Sending " + msg + " " + pair.getFirst() + " to " + addr);
           }
-          returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_SPLIT, pair.getFirst()));
+          returnMsgs.add(new HMsg(msg, pair.getFirst()));
           i.remove();
         }
       }
     }
   }
-  
+
   /*
    * State of a Region as it transitions from closed to open, etc.  See
    * note on regionsInTransition data member above for listing of state
@@ -1021,27 +1034,28 @@
   private static class RegionState implements Comparable<RegionState> {
     private final HRegionInfo regionInfo;
     private volatile boolean unassigned = false;
-    private volatile boolean assigned = false;
-    private volatile boolean pending = false;
+    private volatile boolean pendingOpen = false;
+    private volatile boolean open = false;
     private volatile boolean closing = false;
+    private volatile boolean pendingClose = false;
     private volatile boolean closed = false;
     private volatile boolean offlined = false;
     
-    /* Set when region is assigned.
-     */
-    private String serverName = null;
+    /* Set when region is assigned or closing */
+    private volatile String serverName = null;
 
+    /* Constructor */
     RegionState(HRegionInfo info) {
       this.regionInfo = info;
     }
     
-    byte [] getRegionName() {
-      return this.regionInfo.getRegionName();
-    }
-
     synchronized HRegionInfo getRegionInfo() {
       return this.regionInfo;
     }
+    
+    synchronized byte [] getRegionName() {
+      return this.regionInfo.getRegionName();
+    }
 
     /*
      * @return Server this region was assigned to
@@ -1049,7 +1063,17 @@
     synchronized String getServerName() {
       return this.serverName;
     }
-    
+
+    /*
+     * @return true if the region is being opened
+     */
+    synchronized boolean isOpening() {
+      return this.unassigned || this.pendingOpen || this.open;
+    }
+
+    /*
+     * @return true if region is unassigned
+     */
     synchronized boolean isUnassigned() {
       return unassigned;
     }
@@ -1061,46 +1085,55 @@
      */
     synchronized void setUnassigned() {
       this.unassigned = true;
-      this.assigned = false;
-      this.pending = false;
+      this.pendingOpen = false;
+      this.open = false;
       this.closing = false;
+      this.pendingClose = false;
+      this.closed = false;
+      this.offlined = false;
       this.serverName = null;
     }
 
-    synchronized boolean isAssigned() {
-      return assigned;
+    synchronized boolean isPendingOpen() {
+      return pendingOpen;
     }
 
     /*
      * @param serverName Server region was assigned to.
      */
-    synchronized void setAssigned(final String serverName) {
+    synchronized void setPendingOpen(final String serverName) {
       if (!this.unassigned) {
         throw new IllegalStateException(
-            "Cannot assign a region that is not currently unassigned. " +
-            "State: " + toString());
+            "Cannot assign a region that is not currently unassigned. State: " +
+            toString());
       }
       this.unassigned = false;
-      this.assigned = true;
-      this.pending = false;
+      this.pendingOpen = true;
+      this.open = false;
       this.closing = false;
+      this.pendingClose = false;
+      this.closed = false;
+      this.offlined = false;
       this.serverName = serverName;
     }
 
-    synchronized boolean isPending() {
-      return pending;
+    synchronized boolean isOpen() {
+      return open;
     }
 
-    synchronized void setPending() {
-      if (!assigned) {
+    synchronized void setOpen() {
+      if (!pendingOpen) {
         throw new IllegalStateException(
-            "Cannot set a region as pending if it has not been assigned. " +
-            "State: " + toString());
+            "Cannot set a region as open if it has not been pending. State: " +
+            toString());
       }
       this.unassigned = false;
-      this.assigned = false;
-      this.pending = true;
+      this.pendingOpen = false;
+      this.open = true;
       this.closing = false;
+      this.pendingClose = false;
+      this.closed = false;
+      this.offlined = false;
     }
 
     synchronized boolean isClosing() {
@@ -1109,24 +1142,48 @@
 
     synchronized void setClosing(String serverName, boolean setOffline) {
       this.unassigned = false;
-      this.assigned = false;
-      this.pending = false;
+      this.pendingOpen = false;
+      this.open = false;
       this.closing = true;
+      this.pendingClose = false;
+      this.closed = false;
       this.offlined = setOffline;
       this.serverName = serverName;
     }
     
+    synchronized boolean isPendingClose() {
+      return this.pendingClose;
+    }
+
+    synchronized void setPendingClose() {
+      if (!closing) {
+        throw new IllegalStateException(
+            "Cannot set a region as pending close if it has not been closing. " +
+            "State: " + toString());
+      }
+      this.unassigned = false;
+      this.pendingOpen = false;
+      this.open = false;
+      this.closing = false;
+      this.pendingClose = true;
+      this.closed = false;
+    }
+
     synchronized boolean isClosed() {
       return this.closed;
     }
     
     synchronized void setClosed() {
-      if (!closing) {
+      if (!pendingClose) {
         throw new IllegalStateException(
             "Cannot set a region to be closed if it was not already marked as" +
-            " closing. State: " + toString());
+            " pending close. State: " + toString());
       }
+      this.unassigned = false;
+      this.pendingOpen = false;
+      this.open = false;
       this.closing = false;
+      this.pendingClose = false;
       this.closed = true;
     }
     
@@ -1136,11 +1193,14 @@
 
     @Override
     public synchronized String toString() {
-      return "name=" + Bytes.toString(getRegionName()) +
-          ", isUnassigned=" + this.unassigned + ", isAssigned=" +
-          this.assigned + ", isPending=" + this.pending + ", isClosing=" +
-          this.closing + ", isClosed=" + this.closed + ", isOfflined=" +
-          this.offlined;
+      return ("name=" + Bytes.toString(getRegionName()) +
+          ", unassigned=" + this.unassigned +
+          ", pendingOpen=" + this.pendingOpen +
+          ", open=" + this.open +
+          ", closing=" + this.closing +
+          ", pendingClose=" + this.pendingClose +
+          ", closed=" + this.closed +
+          ", offlined=" + this.offlined);
     }
     
     @Override

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RootScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RootScanner.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RootScanner.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/RootScanner.java Thu Jan  8 17:55:34 2009
@@ -30,10 +30,9 @@
    * Constructor
    * 
    * @param master
-   * @param regionManager
    */
-  public RootScanner(HMaster master, RegionManager regionManager) {
-    super(master, regionManager, true, master.metaRescanInterval, master.closed);
+  public RootScanner(HMaster master) {
+    super(master, true, master.metaRescanInterval, master.closed);
   }
 
   /*

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ServerManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ServerManager.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/master/ServerManager.java Thu Jan  8 17:55:34 2009
@@ -59,9 +59,13 @@
   final Map<String, HServerInfo> serversToServerInfo =
     new ConcurrentHashMap<String, HServerInfo>();
   
-  /** Set of known dead servers */
-  final Set<String> deadServers =
-    Collections.synchronizedSet(new HashSet<String>());
+  /**
+   * Set of known dead servers.  On lease expiration, servers are added here.
+   * Boolean holds whether its logs have been split or not.  Initially set to
+   * false.
+   */
+  private final Map<String, Boolean> deadServers =
+    new ConcurrentHashMap<String, Boolean>();
 
   /** SortedMap server load -> Set of server names */
   final SortedMap<HServerLoad, Set<String>> loadToServers =
@@ -89,24 +93,67 @@
     this.loggingPeriodForAverageLoad = master.getConfiguration().
       getLong("hbase.master.avgload.logging.period", 60000);
   }
-  
+ 
+  /*
+   * Look to see if we have ghost references to this regionserver such as
+   * still-existing leases or if regionserver is on the dead servers list
+   * getting its logs processed.
+   * @param serverInfo
+   * @return True if still ghost references and we have not been able to clear
+   * them or the server is shutting down.
+   */
+  private boolean checkForGhostReferences(final HServerInfo serverInfo) {
+    String s = serverInfo.getServerAddress().toString().trim();
+    boolean result = false;
+    boolean lease = false;
+    for (long sleepTime = -1; !master.closed.get() && !result;) {
+      if (sleepTime != -1) {
+        try {
+          Thread.sleep(sleepTime);
+        } catch (InterruptedException e) {
+          // Continue
+        }
+      }
+      if (!lease) {
+        try {
+          this.serverLeases.createLease(s, new ServerExpirer(s));
+        } catch (Leases.LeaseStillHeldException e) {
+          LOG.debug("Waiting on current lease to expire for " + e.getName());
+          sleepTime = this.master.leaseTimeout / 4;
+          continue;
+        }
+        lease = true;
+      }
+      // May be on list of dead servers.  If so, wait till we've cleared it.
+      String addr = serverInfo.getServerAddress().toString();
+      if (isDead(addr)) {
+        LOG.debug("Waiting on " + addr + " removal from dead list before " +
+          "processing report-for-duty request");
+        sleepTime = this.master.threadWakeFrequency;
+        try {
+          // Keep up lease.  May be here > lease expiration.
+          this.serverLeases.renewLease(s);
+        } catch (LeaseException e) {
+          LOG.warn("Failed renewal. Retrying.", e);
+        }
+        continue;
+      }
+      result = true;
+    }
+    return result;
+  }
+
   /**
    * Let the server manager know a new regionserver has come online
    * @param serverInfo
    */
-  public void regionServerStartup(HServerInfo serverInfo) {
+  public void regionServerStartup(final HServerInfo serverInfo) {
     String s = serverInfo.getServerAddress().toString().trim();
     LOG.info("Received start message from: " + s);
-    // Do the lease check up here. There might already be one out on this
-    // server expecially if it just shutdown and came back up near-immediately.
-    if (!master.closed.get()) {
-      try {
-        serverLeases.createLease(s, new ServerExpirer(s));
-      } catch (Leases.LeaseStillHeldException e) {
-        LOG.debug("Lease still held on " + e.getName());
-        return;
-      }
+    if (!checkForGhostReferences(serverInfo)) {
+      return;
     }
+    // Go on to process the regionserver registration.
     HServerLoad load = serversToLoad.remove(s);
     if (load != null) {
       // The startup message was from a known server.
@@ -119,7 +166,6 @@
         }
       }
     }
-
     HServerInfo storedInfo = serversToServerInfo.remove(s);
     if (storedInfo != null && !master.closed.get()) {
       // The startup message was from a known server with the same name.
@@ -137,7 +183,6 @@
         LOG.error("Insertion into toDoQueue was interrupted", e);
       }
     }
-
     // record new server
     load = new HServerLoad();
     serverInfo.setLoad(load);
@@ -263,13 +308,15 @@
               synchronized (master.regionManager) {
                 if (info.isRootRegion()) {
                   master.regionManager.reassignRootRegion();
-                } else if (info.isMetaTable()) {
-                  master.regionManager.offlineMetaRegion(info.getStartKey());
-                }
-                if (!master.regionManager.isOfflined(info.getRegionName())) {
-                  master.regionManager.setUnassigned(info, true);
                 } else {
-                  master.regionManager.removeRegion(info);
+                  if (info.isMetaTable()) {
+                    master.regionManager.offlineMetaRegion(info.getStartKey());
+                  }
+                  if (!master.regionManager.isOfflined(info.getRegionName())) {
+                    master.regionManager.setUnassigned(info, true);
+                  } else {
+                    master.regionManager.removeRegion(info);
+                  }
                 }
               }
             }
@@ -374,7 +421,7 @@
       for (HRegionInfo i: master.regionManager.getMarkedToClose(serverName)) {
         returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_CLOSE, i));
         // Transition the region from toClose to closing state
-        master.regionManager.setClosed(i.getRegionName());
+        master.regionManager.setPendingClose(i.getRegionName());
       }
 
       // Figure out what the RegionServer ought to do, and write back.
@@ -427,7 +474,7 @@
     boolean duplicateAssignment = false;
     synchronized (master.regionManager) {
       if (!master.regionManager.isUnassigned(region) &&
-          !master.regionManager.isAssigned(region.getRegionName())) {
+          !master.regionManager.isPendingOpen(region.getRegionName())) {
         if (region.isRootRegion()) {
           // Root region
           HServerAddress rootServer = master.getRootRegionLocation();
@@ -444,7 +491,7 @@
           // Not root region. If it is not a pending region, then we are
           // going to treat it as a duplicate assignment, although we can't 
           // tell for certain that's the case.
-          if (master.regionManager.isPending(region.getRegionName())) {
+          if (master.regionManager.isPendingOpen(region.getRegionName())) {
             // A duplicate report from the correct server
             return;
           }
@@ -478,7 +525,7 @@
         } else {
           // Note that the table has been assigned and is waiting for the
           // meta table to be updated.
-          master.regionManager.setPending(region.getRegionName());
+          master.regionManager.setOpen(region.getRegionName());
           // Queue up an update to note the region location.
           try {
             master.toDoQueue.put(
@@ -519,9 +566,10 @@
       //       the messages we've received. In this case, a close could be
       //       processed before an open resulting in the master not agreeing on
       //       the region's state.
+      master.regionManager.setClosed(region.getRegionName());
       try {
-        master.toDoQueue.put(new ProcessRegionClose(master, region, offlineRegion,
-            reassignRegion));
+        master.toDoQueue.put(new ProcessRegionClose(master, region,
+            offlineRegion, reassignRegion));
       } catch (InterruptedException e) {
         throw new RuntimeException("Putting into toDoQueue was interrupted.", e);
       }
@@ -535,11 +583,11 @@
     // Only cancel lease and update load information once.
     // This method can be called a couple of times during shutdown.
     if (info != null) {
+      LOG.info("Cancelling lease for " + serverName);
       if (master.getRootRegionLocation() != null &&
         info.getServerAddress().equals(master.getRootRegionLocation())) {
-        master.regionManager.reassignRootRegion();
+        master.regionManager.unsetRootRegion();
       }
-      LOG.info("Cancelling lease for " + serverName);
       try {
         serverLeases.cancelLease(serverName);
       } catch (LeaseException e) {
@@ -703,7 +751,7 @@
             }
           }
         }
-        deadServers.add(server);
+        deadServers.put(server, Boolean.FALSE);
         try {
           master.toDoQueue.put(
               new ProcessServerShutdown(master, info, rootServer));
@@ -742,6 +790,6 @@
    * @return true if server is dead
    */
   public boolean isDead(String serverName) {
-    return deadServers.contains(serverName);
+    return deadServers.containsKey(serverName);
   }
-}
\ No newline at end of file
+}

Modified: hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=732909&r1=732908&r2=732909&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hadoop/hbase/branches/0.19_on_hadoop_0.18/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Thu Jan  8 17:55:34 2009
@@ -132,19 +132,33 @@
     compactionQueue.clear();
     LOG.info(getName() + " exiting");
   }
-  
+
   /**
    * @param r HRegion store belongs to
    * @param why Why compaction requested -- used in debug messages
    */
   public synchronized void compactionRequested(final HRegion r,
       final String why) {
+    compactionRequested(r, false, why);
+  }
+
+  /**
+   * @param r HRegion store belongs to
+   * @param force Whether next compaction should be major
+   * @param why Why compaction requested -- used in debug messages
+   */
+  public synchronized void compactionRequested(final HRegion r,
+      final boolean force, final String why) {
     if (this.server.stopRequested.get()) {
       return;
     }
-    LOG.debug("Compaction requested for region " +
-      Bytes.toString(r.getRegionName()) +
-      (why != null && !why.isEmpty()? " because: " + why: ""));
+    r.setForceMajorCompaction(force);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Compaction " + (force? "(major) ": "") +
+        "requested for region " + Bytes.toString(r.getRegionName()) +
+        "/" + r.getRegionInfo().getEncodedName() +
+        (why != null && !why.isEmpty()? " because: " + why: ""));
+    }
     synchronized (regionsInQueue) {
       if (!regionsInQueue.contains(r)) {
         compactionQueue.add(r);
@@ -222,6 +236,10 @@
     this.limit = limit;
   }
 
+  int getLimit() {
+    return this.limit;
+  }
+
   /**
    * Only interrupt once it's done with a run through the work loop.
    */ 



Mime
View raw message