hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r535970 [3/4] - in /lucene/hadoop/trunk: ./ src/contrib/hbase/bin/ src/contrib/hbase/conf/ src/contrib/hbase/src/java/org/apache/hadoop/hbase/ src/contrib/hbase/src/test/org/apache/hadoop/hbase/
Date Mon, 07 May 2007 19:58:57 GMT
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java?view=diff&rev=535970&r1=535969&r2=535970
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegion.java Mon May  7 12:58:53 2007
@@ -24,25 +24,37 @@
 import java.io.*;
 import java.util.*;
 
-/*******************************************************************************
- * HRegion stores data for a certain region of a table.  It stores all columns 
+/**
+ * HRegion stores data for a certain region of a table.  It stores all columns
  * for each row. A given table consists of one or more HRegions.
  *
- * We maintain multiple HStores for a single HRegion.
+ * <p>We maintain multiple HStores for a single HRegion.
  * 
- * An HStore is a set of rows with some column data; together, they make up all 
- * the data for the rows.  
+ * <p>An HStore is a set of rows with some column data; together,
+ * they make up all the data for the rows.  
  *
- * Each HRegion has a 'startKey' and 'endKey'.
+ * <p>Each HRegion has a 'startKey' and 'endKey'.
  *   
- * The first is inclusive, the second is exclusive (except for the final region)
- * The endKey of region 0 is the same as startKey for region 1 (if it exists).  
- * The startKey for the first region is null.
- * The endKey for the final region is null.
+ * <p>The first is inclusive, the second is exclusive (except for
+ * the final region)  The endKey of region 0 is the same as
+ * startKey for region 1 (if it exists).  The startKey for the
+ * first region is null. The endKey for the final region is null.
  *
- * The HStores have no locking built-in.  All row-level locking and row-level 
- * atomicity is provided by the HRegion.
- ******************************************************************************/
+ * <p>The HStores have no locking built-in.  All row-level locking
+ * and row-level atomicity is provided by the HRegion.
+ * 
+ * <p>An HRegion is defined by its table and its key extent.
+ * 
+ * <p>It consists of at least one HStore.  The number of HStores should be 
+ * configurable, so that data which is accessed together is stored in the same
+ * HStore.  Right now, we approximate that by building a single HStore for 
+ * each column family.  (This config info will be communicated via the 
+ * tabledesc.)
+ * 
+ * The HTableDescriptor contains metainfo about the HRegion's table.
+ * regionName is a unique identifier for this HRegion. (startKey, endKey]
+ * defines the keyspace for this HRegion.
+ */
 public class HRegion implements HConstants {
   static String SPLITDIR = "splits";
   static String MERGEDIR = "merges";
@@ -53,8 +65,9 @@
   private static final Log LOG = LogFactory.getLog(HRegion.class);
 
   /**
-   * Merge two HRegions.  They must be available on the current HRegionServer.
-   * Returns a brand-new active HRegion, also running on the current HRegionServer.
+   * Merge two HRegions.  They must be available on the current
+   * HRegionServer. Returns a brand-new active HRegion, also
+   * running on the current HRegionServer.
    */
   public static HRegion closeAndMerge(HRegion srcA, HRegion srcB) throws IOException {
 
@@ -110,7 +123,9 @@
     // Flush each of the sources, and merge their files into a single 
     // target for each column family.
 
-    LOG.debug("flushing and getting file names for region " + srcA.getRegionName());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("flushing and getting file names for region " + srcA.getRegionName());
+    }
     
     TreeSet<HStoreFile> alreadyMerged = new TreeSet<HStoreFile>();
     TreeMap<Text, Vector<HStoreFile>> filesToMerge = new TreeMap<Text, Vector<HStoreFile>>();
@@ -124,7 +139,9 @@
       v.add(src);
     }
     
-    LOG.debug("flushing and getting file names for region " + srcB.getRegionName());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("flushing and getting file names for region " + srcB.getRegionName());
+    }
     
     for(Iterator<HStoreFile> it = srcB.flushcache(true).iterator(); it.hasNext(); ) {
       HStoreFile src = it.next();
@@ -136,7 +153,9 @@
       v.add(src);
     }
     
-    LOG.debug("merging stores");
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("merging stores");
+    }
     
     for(Iterator<Text> it = filesToMerge.keySet().iterator(); it.hasNext(); ) {
       Text colFamily = it.next();
@@ -152,8 +171,10 @@
     // Now close the source HRegions for good, and repeat the above to take care
     // of any last-minute inserts
 
-    LOG.debug("flushing changes since start of merge for region " 
-        + srcA.getRegionName());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("flushing changes since start of merge for region " 
+          + srcA.getRegionName());
+    }
 
     filesToMerge.clear();
     for(Iterator<HStoreFile> it = srcA.close().iterator(); it.hasNext(); ) {
@@ -169,8 +190,10 @@
       }
     }
     
-    LOG.debug("flushing changes since start of merge for region " 
-        + srcB.getRegionName());
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("flushing changes since start of merge for region " 
+          + srcB.getRegionName());
+    }
     
     for(Iterator<HStoreFile> it = srcB.close().iterator(); it.hasNext(); ) {
       HStoreFile src = it.next();
@@ -185,7 +208,9 @@
       }
     }
     
-    LOG.debug("merging changes since start of merge");
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("merging changes since start of merge");
+    }
     
     for(Iterator<Text> it = filesToMerge.keySet().iterator(); it.hasNext(); ) {
       Text colFamily = it.next();
@@ -217,8 +242,8 @@
   TreeMap<Text, Long> rowsToLocks = new TreeMap<Text, Long>();
   TreeMap<Long, Text> locksToRows = new TreeMap<Long, Text>();
   TreeMap<Text, HStore> stores = new TreeMap<Text, HStore>();
-  TreeMap<Long, TreeMap<Text, byte[]>> targetColumns 
-      = new TreeMap<Long, TreeMap<Text, byte[]>>();
+  TreeMap<Long, TreeMap<Text, BytesWritable>> targetColumns 
+      = new TreeMap<Long, TreeMap<Text, BytesWritable>>();
   
   HMemcache memcache = new HMemcache();
 
@@ -251,40 +276,25 @@
   //////////////////////////////////////////////////////////////////////////////
 
   /**
-   * An HRegion is defined by its table and its key extent.
-   * 
-   * It consists of at least one HStore.  The number of HStores should be 
-   * configurable, so that data which is accessed together is stored in the same
-   * HStore.  Right now, we approximate that by building a single HStore for 
-   * each column family.  (This config info will be communicated via the 
-   * tabledesc.)
-   *
-   * The HLog is the outbound log for any updates to the HRegion.  (There's a 
-   * single HLog for all the HRegions on a single HRegionServer.)
-   *
-   * The HTableDescriptor contains metainfo about the HRegion's table.  
-   *
-   * regionName is a unique identifier for this HRegion.
-   *
-   * (startKey, endKey] defines the keyspace for this HRegion.  NULL values
-   * indicate we're at the start or end of the table.
-   *
-   * fs is the filesystem.  regiondir is where the HRegion is stored.
+   * HRegion constructor.
    *
-   * logfile is a logfile from the previous execution that's custom-computed for
-   * this HRegion.  The HRegionServer computes and sorts the appropriate log
-   * info for this HRegion.
-   *
-   * conf is global configuration settings.
-   *
-   * If there are initial files (implying that the HRegion is new), then read 
-   * them from the supplied path.
-   *
-   * If there is a previous log file (implying that the HRegion has been 
-   * written-to before), then read it from the supplied path.
+   * @param log The HLog is the outbound log for any updates to the HRegion
+   * (There's a single HLog for all the HRegions on a single HRegionServer.)
+   * The log file is a logfile from the previous execution that's
+   * custom-computed for this HRegion. The HRegionServer computes and sorts the
+   * appropriate log info for this HRegion. If there is a previous log file
+   * (implying that the HRegion has been written-to before), then read it from
+   * the supplied path.
+   * @param fs is the filesystem.  
+   * @param dir dir is where the HRegion is stored.
+   * @param conf is global configuration settings.
+   * @param initialFiles If there are initial files (implying that the HRegion
+   * is new), then read them from the supplied path.
+   * @throws IOException
    */
   public HRegion(Path dir, HLog log, FileSystem fs, Configuration conf, 
-      HRegionInfo regionInfo, Path initialFiles, Path oldLogFile) throws IOException {
+      HRegionInfo regionInfo, Path initialFiles, Path oldLogFile)
+  throws IOException {
     
     this.dir = dir;
     this.log = log;
@@ -308,33 +318,28 @@
     }
 
     // Load in all the HStores.
-    
     for(Iterator<Text> it = this.regionInfo.tableDesc.families().iterator();
-        it.hasNext(); ) {
-      
+        it.hasNext(); ) { 
       Text colFamily = HStoreKey.extractFamily(it.next());
-      stores.put(colFamily, new HStore(dir, this.regionInfo.regionName, colFamily, 
-          this.regionInfo.tableDesc.getMaxVersions(), fs, oldLogFile, conf));
+      stores.put(colFamily, new HStore(dir, this.regionInfo.regionName,
+          colFamily, this.regionInfo.tableDesc.getMaxVersions(), fs,
+          oldLogFile, conf));
     }
 
     // Get rid of any splits or merges that were lost in-progress
-    
     Path splits = new Path(regiondir, SPLITDIR);
-    if(fs.exists(splits)) {
+    if (fs.exists(splits)) {
       fs.delete(splits);
     }
-    
     Path merges = new Path(regiondir, MERGEDIR);
-    if(fs.exists(merges)) {
+    if (fs.exists(merges)) {
       fs.delete(merges);
     }
 
     this.maxUnflushedEntries = conf.getInt("hbase.hregion.maxunflushed", 10000);
 
     // HRegion is ready to go!
-    
     this.writestate.writesOngoing = false;
-    
     LOG.info("region " + this.regionInfo.regionName + " available");
   }
 
@@ -445,8 +450,10 @@
     for(Iterator<HStoreFile> it = hstoreFilesToSplit.iterator(); it.hasNext(); ) {
       HStoreFile hsf = it.next();
       
-      LOG.debug("splitting HStore " + hsf.getRegionName() + "/" + hsf.getColFamily()
-          + "/" + hsf.fileId());
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("splitting HStore " + hsf.getRegionName() + "/"
+            + hsf.getColFamily() + "/" + hsf.fileId());
+      }
 
       HStoreFile dstA = new HStoreFile(conf, splits, regionAInfo.regionName, 
           hsf.getColFamily(), Math.abs(rand.nextLong()));
@@ -466,8 +473,10 @@
       HStoreFile hsf = it.next();
       
       if(! alreadySplit.contains(hsf)) {
-        LOG.debug("splitting HStore " + hsf.getRegionName() + "/" + hsf.getColFamily()
-            + "/" + hsf.fileId());
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("splitting HStore " + hsf.getRegionName() + "/"
+              + hsf.getColFamily() + "/" + hsf.fileId());
+        }
 
         HStoreFile dstA = new HStoreFile(conf, splits, regionAInfo.regionName, 
             hsf.getColFamily(), Math.abs(rand.nextLong()));
@@ -672,7 +681,9 @@
     }
     
     if(! shouldFlush) {
-      LOG.debug("not flushing cache for region " + this.regionInfo.regionName);
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("not flushing cache for region " + this.regionInfo.regionName);
+      }
       return null;
       
     } else {
@@ -715,7 +726,9 @@
   Vector<HStoreFile> internalFlushcache() throws IOException {
     Vector<HStoreFile> allHStoreFiles = new Vector<HStoreFile>();
     
-    LOG.debug("flushing cache for region " + this.regionInfo.regionName);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("flushing cache for region " + this.regionInfo.regionName);
+    }
 
     // We pass the log to the HMemcache, so we can lock down 
     // both simultaneously.  We only have to do this for a moment:
@@ -728,7 +741,9 @@
     // object stored that must be explicitly cleaned up using
     // a call to deleteSnapshot().
     
-    LOG.debug("starting memcache snapshot");
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("starting memcache snapshot");
+    }
     
     HMemcache.Snapshot retval = memcache.snapshotMemcacheForLog(log);
     TreeMap<HStoreKey, BytesWritable> memcacheSnapshot = retval.memcacheSnapshot;
@@ -745,7 +760,9 @@
 
     // A.  Flush memcache to all the HStores.
     
-    LOG.debug("flushing memcache to HStores");
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("flushing memcache to HStores");
+    }
     
     for(Iterator<HStore> it = stores.values().iterator(); it.hasNext(); ) {
       HStore hstore = it.next();
@@ -760,7 +777,9 @@
     //     and that all updates to the log for this regionName that have lower 
     //     log-sequence-ids can be safely ignored.
     
-    LOG.debug("writing flush cache complete to log");
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("writing flush cache complete to log");
+    }
     
     log.completeCacheFlush(this.regionInfo.regionName,
         regionInfo.tableDesc.getName(), logCacheFlushId);
@@ -768,11 +787,15 @@
     // C. Delete the now-irrelevant memcache snapshot; its contents have been 
     //    dumped to disk-based HStores.
     
-    LOG.debug("deleting memcache snapshot");
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("deleting memcache snapshot");
+    }
     
     memcache.deleteSnapshot();
 
-    LOG.debug("cache flush complete for region " + this.regionInfo.regionName);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("cache flush complete for region " + this.regionInfo.regionName);
+    }
     
     this.commitsSinceFlush = 0;
     return allHStoreFiles;
@@ -783,8 +806,8 @@
   //////////////////////////////////////////////////////////////////////////////
 
   /** Fetch a single data item. */
-  public byte[] get(Text row, Text column) throws IOException {
-    byte results[][] = get(row, column, Long.MAX_VALUE, 1);
+  public BytesWritable get(Text row, Text column) throws IOException {
+    BytesWritable[] results = get(row, column, Long.MAX_VALUE, 1);
     if(results == null) {
       return null;
       
@@ -794,12 +817,12 @@
   }
   
   /** Fetch multiple versions of a single data item */
-  public byte[][] get(Text row, Text column, int numVersions) throws IOException {
+  public BytesWritable[] get(Text row, Text column, int numVersions) throws IOException {
     return get(row, column, Long.MAX_VALUE, numVersions);
   }
 
   /** Fetch multiple versions of a single data item, with timestamp. */
-  public byte[][] get(Text row, Text column, long timestamp, int numVersions) 
+  public BytesWritable[] get(Text row, Text column, long timestamp, int numVersions) 
       throws IOException {
     
     if(writestate.closed) {
@@ -825,11 +848,11 @@
 
   // Private implementation: get the value for the indicated HStoreKey
 
-  private byte[][] get(HStoreKey key, int numVersions) throws IOException {
+  private BytesWritable[] get(HStoreKey key, int numVersions) throws IOException {
 
     // Check the memcache
 
-    byte[][] result = memcache.get(key, numVersions);
+    BytesWritable[] result = memcache.get(key, numVersions);
     if(result != null) {
       return result;
     }
@@ -855,10 +878,10 @@
    * determine which column groups are useful for that row.  That would let us 
    * avoid a bunch of disk activity.
    */
-  public TreeMap<Text, byte[]> getFull(Text row) throws IOException {
+  public TreeMap<Text, BytesWritable> getFull(Text row) throws IOException {
     HStoreKey key = new HStoreKey(row, System.currentTimeMillis());
 
-    TreeMap<Text, byte[]> memResult = memcache.getFull(key);
+    TreeMap<Text, BytesWritable> memResult = memcache.getFull(key);
     for(Iterator<Text> it = stores.keySet().iterator(); it.hasNext(); ) {
       Text colFamily = it.next();
       HStore targetStore = stores.get(colFamily);
@@ -871,13 +894,13 @@
    * Return an iterator that scans over the HRegion, returning the indicated 
    * columns.  This Iterator must be closed by the caller.
    */
-  public HScannerInterface getScanner(Text cols[], Text firstRow) throws IOException {
+  public HInternalScannerInterface getScanner(Text[] cols, Text firstRow) throws IOException {
     TreeSet<Text> families = new TreeSet<Text>();
     for(int i = 0; i < cols.length; i++) {
       families.add(HStoreKey.extractFamily(cols[i]));
     }
 
-    HStore storelist[] = new HStore[families.size()];
+    HStore[] storelist = new HStore[families.size()];
     int i = 0;
     for(Iterator<Text> it = families.iterator(); it.hasNext(); ) {
       Text family = it.next();
@@ -917,21 +940,11 @@
    * This method really just tests the input, then calls an internal localput() 
    * method.
    */
-  public void put(long lockid, Text targetCol, byte[] val) throws IOException {
-    if(val.length == HStoreKey.DELETE_BYTES.length) {
-      boolean matches = true;
-      for(int i = 0; i < val.length; i++) {
-        if(val[i] != HStoreKey.DELETE_BYTES[i]) {
-          matches = false;
-          break;
-        }
-      }
-      
-      if(matches) {
+  public void put(long lockid, Text targetCol, BytesWritable val) throws IOException {
+    if(val.getSize() == DELETE_BYTES.getSize()
+        && val.compareTo(DELETE_BYTES) == 0) {
         throw new IOException("Cannot insert value: " + val);
-      }
     }
-    
     localput(lockid, targetCol, val);
   }
 
@@ -939,7 +952,7 @@
    * Delete a value or write a value. This is a just a convenience method for put().
    */
   public void delete(long lockid, Text targetCol) throws IOException {
-    localput(lockid, targetCol, HStoreKey.DELETE_BYTES);
+    localput(lockid, targetCol, DELETE_BYTES);
   }
 
   /**
@@ -949,7 +962,7 @@
    * a per-row pending area, until a commit() or abort() call is received.
    * (Or until the user's write-lock expires.)
    */
-  void localput(long lockid, Text targetCol, byte[] val) throws IOException {
+  void localput(long lockid, Text targetCol, BytesWritable val) throws IOException {
     checkColumn(targetCol);
     
     Text row = getRowFromLock(lockid);
@@ -971,9 +984,9 @@
             + " unexpected aborted by another thread");
       }
       
-      TreeMap<Text, byte[]> targets = targetColumns.get(lockid);
+      TreeMap<Text, BytesWritable> targets = targetColumns.get(lockid);
       if(targets == null) {
-        targets = new TreeMap<Text, byte[]>();
+        targets = new TreeMap<Text, BytesWritable>();
         targetColumns.put(lockid, targets);
       }
       targets.put(targetCol, val);
@@ -1132,42 +1145,79 @@
   /*******************************************************************************
    * HScanner is an iterator through a bunch of rows in an HRegion.
    ******************************************************************************/
-  private class HScanner implements HScannerInterface {
-    HScannerInterface scanners[] = null;
-    TreeMap<Text, byte[]> resultSets[] = null;
-    HStoreKey keys[] = null;
+  private class HScanner implements HInternalScannerInterface {
+    private HInternalScannerInterface[] scanners;
+    private TreeMap<Text, BytesWritable>[] resultSets;
+    private HStoreKey[] keys;
+    private boolean wildcardMatch;
+    private boolean multipleMatchers;
 
     /** Create an HScanner with a handle on many HStores. */
     @SuppressWarnings("unchecked")
-    public HScanner(Text cols[], Text firstRow, HMemcache memcache, HStore stores[]) throws IOException {
+    public HScanner(Text[] cols, Text firstRow, HMemcache memcache, HStore[] stores) throws IOException {
       long scanTime = System.currentTimeMillis();
-      this.scanners = new HScannerInterface[stores.length + 1];
-      this.keys = new HStoreKey[scanners.length];
+      this.scanners = new HInternalScannerInterface[stores.length + 1];
       this.resultSets = new TreeMap[scanners.length];
+      this.keys = new HStoreKey[scanners.length];
+      this.wildcardMatch = false;
+      this.multipleMatchers = false;
 
       // Advance to the first key in each store.
       // All results will match the required column-set and scanTime.
+      
+      // NOTE: the memcache scanner should be the first scanner
 
+      HInternalScannerInterface scanner =
+        memcache.getScanner(scanTime, cols, firstRow);
+      
+      if(scanner.isWildcardScanner()) {
+        this.wildcardMatch = true;
+      }
+      if(scanner.isMultipleMatchScanner()) {
+        this.multipleMatchers = true;
+      }
+      scanners[0] = scanner;
+      
       for(int i = 0; i < stores.length; i++) {
-        scanners[i] = stores[i].getScanner(scanTime, cols, firstRow);
+        scanner = stores[i].getScanner(scanTime, cols, firstRow);
+        if(scanner.isWildcardScanner()) {
+          this.wildcardMatch = true;
+        }
+        if(scanner.isMultipleMatchScanner()) {
+          this.multipleMatchers = true;
+        }
+        scanners[i + 1] = scanner;
       }
-      scanners[scanners.length-1] = memcache.getScanner(scanTime, cols, firstRow);
 
       for(int i = 0; i < scanners.length; i++) {
         keys[i] = new HStoreKey();
-        resultSets[i] = new TreeMap<Text, byte[]>();
+        resultSets[i] = new TreeMap<Text, BytesWritable>();
 
-        if(! scanners[i].next(keys[i], resultSets[i])) {
+        if(scanners[i] != null && ! scanners[i].next(keys[i], resultSets[i])) {
           closeScanner(i);
         }
       }
     }
 
+    /* (non-Javadoc)
+     * @see org.apache.hadoop.hbase.HInternalScannerInterface#isWildcardScanner()
+     */
+    public boolean isWildcardScanner() {
+      return wildcardMatch;
+    }
+    
+    /* (non-Javadoc)
+     * @see org.apache.hadoop.hbase.HInternalScannerInterface#isMultipleMatchScanner()
+     */
+    public boolean isMultipleMatchScanner() {
+      return multipleMatchers;
+    }
+    
     /**
      * Grab the next row's worth of values.  The HScanner will return the most 
      * recent data value for each row that is not newer than the target time.
      */
-    public boolean next(HStoreKey key, TreeMap<Text, byte[]> results) throws IOException {
+    public boolean next(HStoreKey key, TreeMap<Text, BytesWritable> results) throws IOException {
       
       // Find the lowest-possible key.
       
@@ -1195,11 +1245,34 @@
 
         for(int i = 0; i < scanners.length; i++) {        
           while((scanners[i] != null)
-              && (keys[i].getRow().compareTo(chosenRow) == 0)
-              && (keys[i].getTimestamp() == chosenTimestamp)) {
+              && (keys[i].getRow().compareTo(chosenRow) == 0)) {
+            
+
+            // If we are doing a wild card match or there are multiple matchers
+            // per column, we need to scan all the older versions of this row
+            // to pick up the rest of the family members
+            
+            if(!wildcardMatch
+                && !multipleMatchers
+                && (keys[i].getTimestamp() != chosenTimestamp)) {
+              break;
+            }
+
+            // NOTE: We used to do results.putAll(resultSets[i]);
+            //       but this had the effect of overwriting newer
+            //       values with older ones. So now we only insert
+            //       a result if the map does not contain the key.
             
-            results.putAll(resultSets[i]);
-            insertedItem = true;
+            for(Iterator<Map.Entry<Text, BytesWritable>> it
+                = resultSets[i].entrySet().iterator();
+                it.hasNext(); ) {
+              
+              Map.Entry<Text, BytesWritable> e = it.next();
+              if(!results.containsKey(e.getKey())) {
+                results.put(e.getKey(), e.getValue());
+                insertedItem = true;
+              }
+            }
 
             resultSets[i].clear();
             if(! scanners[i].next(keys[i], resultSets[i])) {

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInfo.java?view=diff&rev=535970&r1=535969&r2=535970
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInfo.java Mon May  7 12:58:53 2007
@@ -15,13 +15,20 @@
  */
 package org.apache.hadoop.hbase;
 
+import java.io.ByteArrayInputStream;
 import java.io.DataInput;
+import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.IOException;
 
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 
+/**
+ * HRegion information.
+ * Contains HRegion id, start and end keys, a reference to this
+ * HRegions' table descriptor, etc.
+ */
 public class HRegionInfo implements Writable {
   public long regionId;
   public HTableDescriptor tableDesc;
@@ -36,9 +43,20 @@
     this.endKey = new Text();
     this.regionName = new Text();
   }
+  
+  public HRegionInfo(final byte [] serializedBytes) {
+    this();
+    try {
+      readFields(new DataInputStream(
+        new ByteArrayInputStream(serializedBytes)));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
 
-  public HRegionInfo(long regionId, HTableDescriptor tableDesc, Text startKey, 
-      Text endKey) throws IllegalArgumentException {
+  public HRegionInfo(long regionId, HTableDescriptor tableDesc,
+      Text startKey, Text endKey)
+  throws IllegalArgumentException {
     
     this.regionId = regionId;
     
@@ -58,8 +76,16 @@
       this.endKey.set(endKey);
     }
     
-    this.regionName = new Text(tableDesc.getName() + "_"
-        + (startKey == null ? "" : startKey.toString()) + "_" + regionId);
+    this.regionName = new Text(tableDesc.getName() + "_" +
+      (startKey == null ? "" : startKey.toString()) + "_" +
+      regionId);
+  }
+  
+  @Override
+  public String toString() {
+    return "regionname: " + this.regionName.toString() + ", startKey: <" +
+      this.startKey.toString() + ">, tableDesc: {" +
+      this.tableDesc.toString() + "}";
   }
     
   //////////////////////////////////////////////////////////////////////////////
@@ -81,4 +107,4 @@
     this.endKey.readFields(in);
     this.regionName.readFields(in);
   }
-}
+}
\ No newline at end of file

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java?view=diff&rev=535970&r1=535969&r2=535970
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionInterface.java Mon May  7 12:58:53 2007
@@ -69,7 +69,7 @@
    * @param columns     - columns to scan
    * @param startRow    - starting row to scan
    *
-   * @param scannerId   - scanner identifier used in other calls
+   * @return scannerId  - scanner identifier used in other calls
    * @throws IOException
    */
   public long openScanner(Text regionName, Text[] columns, Text startRow) throws IOException;

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java?view=diff&rev=535970&r1=535969&r2=535970
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegionServer.java Mon May  7 12:58:53 2007
@@ -20,6 +20,7 @@
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.fs.*;
 import org.apache.hadoop.ipc.*;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.conf.*;
 
 import java.io.*;
@@ -45,7 +46,7 @@
 
   private static final Log LOG = LogFactory.getLog(HRegionServer.class);
   
-  private boolean stopRequested;
+  private volatile boolean stopRequested;
   private Path regionDir;
   private HServerAddress address;
   private Configuration conf;
@@ -82,12 +83,10 @@
         long startTime = System.currentTimeMillis();
 
         // Grab a list of regions to check
-
         Vector<HRegion> checkSplit = new Vector<HRegion>();
         locker.readLock().lock();
         try {
           checkSplit.addAll(regions.values());
-          
         } finally {
           locker.readLock().unlock();
         }
@@ -132,13 +131,15 @@
               = (oldRegion.find(META_TABLE_NAME.toString()) == 0)
                 ? ROOT_TABLE_NAME : META_TABLE_NAME;
 
-            LOG.debug("region split complete. updating meta");
+            if(LOG.isDebugEnabled()) {
+              LOG.debug("region split complete. updating meta");
+            }
             
             client.openTable(tableToUpdate);
             long lockid = client.startUpdate(oldRegion);
-            client.delete(lockid, META_COL_REGIONINFO);
-            client.delete(lockid, META_COL_SERVER);
-            client.delete(lockid, META_COL_STARTCODE);
+            client.delete(lockid, COL_REGIONINFO);
+            client.delete(lockid, COL_SERVER);
+            client.delete(lockid, COL_STARTCODE);
             client.commit(lockid);
             
             for(int i = 0; i < newRegions.length; i++) {
@@ -147,13 +148,15 @@
               newRegions[i].getRegionInfo().write(out);
               
               lockid = client.startUpdate(newRegions[i].getRegionName());
-              client.put(lockid, META_COL_REGIONINFO, bytes.toByteArray());
+              client.put(lockid, COL_REGIONINFO, bytes.toByteArray());
               client.commit(lockid);
             }
             
             // Now tell the master about the new regions
             
-            LOG.debug("reporting region split to master");
+            if(LOG.isDebugEnabled()) {
+              LOG.debug("reporting region split to master");
+            }
             
             reportSplit(newRegions[0].getRegionInfo(), newRegions[1].getRegionInfo());
             
@@ -172,18 +175,24 @@
         }
         
         // Sleep
-
-        long waitTime =
-          splitCheckFrequency - (System.currentTimeMillis() - startTime);
-        
-        if(waitTime > 0) {
+        long waitTime = stopRequested ? 0
+            : splitCheckFrequency - (System.currentTimeMillis() - startTime);
+        if (waitTime > 0) {
           try {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Sleep splitChecker");
+            }
             Thread.sleep(waitTime);
-
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Wake splitChecker");
+            }
           } catch(InterruptedException iex) {
           }
         }
       }
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("splitChecker exiting");
+      }
     }
   }
   
@@ -221,18 +230,25 @@
         }
 
         // Sleep
-
-        long waitTime =
-          threadWakeFrequency - (System.currentTimeMillis() - startTime);
+        long waitTime = stopRequested ? 0
+            : threadWakeFrequency - (System.currentTimeMillis() - startTime);
         
         if(waitTime > 0) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Sleep cacheFlusher");
+          }
           try {
             Thread.sleep(waitTime);
-
           } catch(InterruptedException iex) {
           }
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Wake cacheFlusher");
+          }
         }
       }
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("cacheFlusher exiting");
+      }
     }
   }
   
@@ -249,24 +265,31 @@
   private class LogRoller implements Runnable {
     public void run() {
       while(! stopRequested) {
-
         // If the number of log entries is high enough, roll the log.  This is a
         // very fast operation, but should not be done too frequently.
-
         if(log.getNumEntries() > maxLogEntries) {
           try {
             log.rollWriter();
-            
           } catch(IOException iex) {
           }
         }
-        
-        try {
-          Thread.sleep(threadWakeFrequency);
-          
-        } catch(InterruptedException iex) {
+
+        if(!stopRequested) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Sleep logRoller");
+          }
+          try {
+            Thread.sleep(threadWakeFrequency);
+          } catch(InterruptedException iex) {
+          }
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Wake logRoller");
+          }
         }
       }
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("logRoller exiting");
+      }
     }
   }
   
@@ -290,11 +313,11 @@
   }
   
   /** Start a HRegionServer at an indicated location */
-  public HRegionServer(Path regionDir, HServerAddress address, Configuration conf) 
-      throws IOException {
+  public HRegionServer(Path regionDir, HServerAddress address,
+      Configuration conf) 
+  throws IOException {
     
     // Basic setup
-    
     this.stopRequested = false;
     this.regionDir = regionDir;
     this.conf = conf;
@@ -302,69 +325,70 @@
     this.regions = new TreeMap<Text, HRegion>();
     this.locker = new ReentrantReadWriteLock();
     this.outboundMsgs = new Vector<HMsg>();
-    this.scanners = Collections.synchronizedMap(new TreeMap<Text, HScannerInterface>());
+    this.scanners =
+      Collections.synchronizedMap(new TreeMap<Text, HInternalScannerInterface>());
 
     // Config'ed params
-    
-    this.threadWakeFrequency = conf.getLong("hbase.hregionserver.thread.wakefrequency", 10 * 1000);
-    this.maxLogEntries = conf.getInt("hbase.hregionserver.maxlogentries", 30 * 1000);
-    this.msgInterval = conf.getLong("hbase.hregionserver.msginterval", 15 * 1000);
-    this.splitCheckFrequency = conf.getLong("hbase.hregionserver.thread.splitcheckfrequency", 60 * 1000);
+    this.threadWakeFrequency = conf.getLong(THREAD_WAKE_FREQUENCY, 10 * 1000);
+    this.maxLogEntries = conf.getInt("hbase.regionserver.maxlogentries", 30 * 1000);
+    this.msgInterval = conf.getLong("hbase.regionserver.msginterval",
+        15 * 1000);
+    this.splitCheckFrequency =
+      conf.getLong("hbase.regionserver.thread.splitcheckfrequency", 60 * 1000);
     
     // Cache flushing
-    
     this.cacheFlusher = new Flusher();
-    this.cacheFlusherThread = new Thread(cacheFlusher, "HRegionServer.cacheFlusher");
+    this.cacheFlusherThread =
+      new Thread(cacheFlusher, "HRegionServer.cacheFlusher");
     
     // Check regions to see if they need to be split
-    
     this.splitChecker = new SplitChecker();
-    this.splitCheckerThread = new Thread(splitChecker, "HRegionServer.splitChecker");
+    this.splitCheckerThread =
+      new Thread(splitChecker, "HRegionServer.splitChecker");
     
     // Process requests from Master
-    
     this.toDo = new Vector<HMsg>();
     this.worker = new Worker();
     this.workerThread = new Thread(worker, "HRegionServer.worker");
 
     try {
-      
       // Server to handle client requests
-      
       this.server = RPC.getServer(this, address.getBindAddress().toString(), 
-          address.getPort(), conf.getInt("hbase.hregionserver.handler.count", 10), false, conf);
+        address.getPort(), conf.getInt("hbase.regionserver.handler.count", 10),
+        false, conf);
 
       this.address = new HServerAddress(server.getListenerAddress());
 
       // Local file paths
-
-      String serverName = this.address.getBindAddress() + "_" + this.address.getPort();
+      String serverName =
+        this.address.getBindAddress() + "_" + this.address.getPort();
       Path newlogdir = new Path(regionDir, "log" + "_" + serverName);
       this.oldlogfile = new Path(regionDir, "oldlogfile" + "_" + serverName);
 
       // Logging
-
       this.fs = FileSystem.get(conf);
       HLog.consolidateOldLog(newlogdir, oldlogfile, fs, conf);
+      // TODO: Now we have a consolidated log for all regions, sort and
+      // then split result by region passing the splits as reconstruction
+      // logs to HRegions on start. Or, rather than consolidate, split logs
+      // into per region files.
       this.log = new HLog(fs, newlogdir, conf);
       this.logRoller = new LogRoller();
       this.logRollerThread = new Thread(logRoller, "HRegionServer.logRoller");
 
       // Remote HMaster
-
-      this.hbaseMaster = (HMasterRegionInterface)
-      RPC.waitForProxy(HMasterRegionInterface.class,
-          HMasterRegionInterface.versionID,
-          new HServerAddress(conf.get(MASTER_ADDRESS)).getInetSocketAddress(),
-          conf);
+      this.hbaseMaster = (HMasterRegionInterface)RPC.
+        waitForProxy(HMasterRegionInterface.class,
+        HMasterRegionInterface.versionID,
+        new HServerAddress(conf.get(MASTER_ADDRESS)).getInetSocketAddress(),
+        conf);
 
       // Threads
-
       this.workerThread.start();
       this.cacheFlusherThread.start();
       this.splitCheckerThread.start();
       this.logRollerThread.start();
-      this.leases = new Leases(conf.getLong("hbase.hregionserver.lease.period", 
+      this.leases = new Leases(conf.getLong("hbase.regionserver.lease.period", 
           3 * 60 * 1000), threadWakeFrequency);
       
       // Server
@@ -380,56 +404,44 @@
   }
 
   /**
-   * Stop all the HRegionServer threads and close everything down. All ongoing 
-   * transactions will be aborted all threads will be shut down. This method
-   * will return immediately. The caller should call join to wait for all 
-   * processing to cease.
+   * Set a flag that will cause all the HRegionServer threads to shut down
+   * in an orderly fashion.
    */
-  public void stop() throws IOException {
-    if(! stopRequested) {
-      stopRequested = true;
- 
-      closeAllRegions();
-      log.close();
-      fs.close();
-      server.stop();
-    }
-    LOG.info("stopping server at: " + address.toString());
+  public synchronized void stop() throws IOException {
+    stopRequested = true;
+    notifyAll();                        // Wakes run() if it is sleeping
   }
 
-  /** Call join to wait for all the threads to finish */
+  /** Wait on all threads to finish.
+   * Presumption is that all closes and stops have already been called.
+   */
   public void join() {
     try {
       this.workerThread.join();
-      
     } catch(InterruptedException iex) {
     }
-
     try {
       this.logRollerThread.join();
-      
     } catch(InterruptedException iex) {
     }
-    
     try {
       this.cacheFlusherThread.join();
-      
     } catch(InterruptedException iex) {
     }
-    
-    this.leases.close();
-  
+    try {
+      this.splitCheckerThread.join();
+    } catch(InterruptedException iex) {
+    }
     try {
       this.server.join();
-      
     } catch(InterruptedException iex) {
     }
     LOG.info("server stopped at: " + address.toString());
   }
   
   /**
-   * The HRegionServer sticks in this loop until close. It repeatedly checks in 
-   * with the HMaster, sending heartbeats & reports, and receiving HRegion 
+   * The HRegionServer sticks in this loop until closed. It repeatedly checks
+   * in with the HMaster, sending heartbeats & reports, and receiving HRegion 
    * load/unload instructions.
    */
   public void run() {
@@ -439,27 +451,40 @@
       long waitTime;
 
       // Let the master know we're here
-      
       try {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Telling master we are up");
+        }
+        
         hbaseMaster.regionServerStartup(info);
         
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Done telling master we are up");
+        }
       } catch(IOException e) {
-        waitTime = msgInterval - (System.currentTimeMillis() - lastMsg);
-
+        waitTime = stopRequested ? 0
+            : msgInterval - (System.currentTimeMillis() - lastMsg);
+        
         if(waitTime > 0) {
-          try {
-            Thread.sleep(waitTime);
-
-          } catch(InterruptedException iex) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Sleep");
+          }
+          synchronized(this) {
+            try {
+              Thread.sleep(waitTime);
+            } catch(InterruptedException iex) {
+            }
+          }
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Wake");
           }
         }
         continue;
       }
       
-      // Now ask the master what it wants us to do and tell it what we have done.
-      
-      while(! stopRequested) {
-        if((System.currentTimeMillis() - lastMsg) >= msgInterval) {
+      // Now ask master what it wants us to do and tell it what we have done.
+      while (!stopRequested) {
+        if ((System.currentTimeMillis() - lastMsg) >= msgInterval) {
 
           HMsg outboundArray[] = null;
           synchronized(outboundMsgs) {
@@ -474,28 +499,43 @@
             // Queue up the HMaster's instruction stream for processing
 
             synchronized(toDo) {
-              boolean restartOrStop = false;
-              for(int i = 0; i < msgs.length; i++) {
+              boolean restart = false;
+              for(int i = 0; i < msgs.length && !stopRequested && !restart; i++) {
                 switch(msgs[i].getMsg()) {
                 
                 case HMsg.MSG_CALL_SERVER_STARTUP:
+                  if (LOG.isDebugEnabled()) {
+                    LOG.debug("Got call server startup message");
+                  }
+                  toDo.clear();
                   closeAllRegions();
-                  restartOrStop = true;
+                  restart = true;
                   break;
                 
-                case HMsg.MSG_REGIONSERVER_ALREADY_RUNNING:
-                  stop();
-                  restartOrStop = true;
+                case HMsg.MSG_REGIONSERVER_STOP:
+                  if (LOG.isDebugEnabled()) {
+                    LOG.debug("Got regionserver stop message");
+                  }
+                  toDo.clear();
+                  closeAllRegions();
+                  stopRequested = true;
                   break;
                   
                 default:
+                  if (LOG.isDebugEnabled()) {
+                    LOG.debug("Got default message");
+                  }
                   toDo.add(msgs[i]);
                 }
               }
+              
               if(toDo.size() > 0) {
+                if (LOG.isDebugEnabled()) {
+                  LOG.debug("notify on todo");
+                }
                 toDo.notifyAll();
               }
-              if(restartOrStop) {
+              if(restart || stopRequested) {
                 break;
               }
             }
@@ -505,16 +545,47 @@
           }
         }
 
-        waitTime = msgInterval - (System.currentTimeMillis() - lastMsg);
-
-        if(waitTime > 0) {
-          try {
-            Thread.sleep(waitTime);
-          } catch(InterruptedException iex) {
+        waitTime = stopRequested ? 0
+            : msgInterval - (System.currentTimeMillis() - lastMsg);
+        if (waitTime > 0) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Sleep");
+          }
+          synchronized(this) {
+            try {
+              Thread.sleep(waitTime);
+            } catch(InterruptedException iex) {
+            }
+          }
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Wake");
           }
         }
       }
     }
+    try {
+      LOG.info("stopping server at: " + address.toString());
+
+      // Send interrupts to wake up threads if sleeping so they notice shutdown.
+
+      this.logRollerThread.interrupt();
+      this.cacheFlusherThread.interrupt();
+      this.splitCheckerThread.interrupt();
+      
+      this.worker.stop();
+      this.server.stop();
+
+      closeAllRegions();
+      log.close();
+      leases.close();
+      join();
+      
+    } catch(IOException e) {
+      e.printStackTrace();
+    }
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("main thread exiting");
+    }
   }
 
   /** Add to the outbound message buffer */
@@ -556,48 +627,83 @@
   private Worker worker;
   private Thread workerThread;
   private class Worker implements Runnable {
+    public void stop() {
+      synchronized(toDo) {
+        toDo.notifyAll();
+      }
+    }
+    
     public void run() {
-      while(!stopRequested) {
-        HMsg msg = null;
+      for(HMsg msg = null; !stopRequested; ) {
         synchronized(toDo) {
-          while(toDo.size() == 0) {
+          while(toDo.size() == 0 && !stopRequested) {
             try {
+              if (LOG.isDebugEnabled()) {
+                LOG.debug("Wait on todo");
+              }
               toDo.wait();
-              
+              if (LOG.isDebugEnabled()) {
+                LOG.debug("Wake on todo");
+              }
             } catch(InterruptedException e) {
             }
           }
+          if(stopRequested) {
+            continue;
+          }
           msg = toDo.remove(0);
         }
+        
         try {
           switch(msg.getMsg()) {
 
           case HMsg.MSG_REGION_OPEN:                    // Open a region
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("MSG_REGION_OPEN");
+            }
             openRegion(msg.getRegionInfo());
             break;
 
           case HMsg.MSG_REGION_CLOSE:                   // Close a region
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("MSG_REGION_CLOSE");
+            }
             closeRegion(msg.getRegionInfo(), true);
             break;
 
           case HMsg.MSG_REGION_MERGE:                   // Merge two regions
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("MSG_REGION_MERGE");
+            }
             //TODO ???
             throw new IOException("TODO: need to figure out merge");
             //break;
 
           case HMsg.MSG_CALL_SERVER_STARTUP:            // Close regions, restart
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("MSG_CALL_SERVER_STARTUP");
+            }
             closeAllRegions();
             continue;
 
-          case HMsg.MSG_REGIONSERVER_ALREADY_RUNNING:   // Go away
-            stop();
+          case HMsg.MSG_REGIONSERVER_STOP:              // Go away
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("MSG_REGIONSERVER_STOP");
+            }
+            stopRequested = true;
             continue;
 
           case HMsg.MSG_REGION_CLOSE_WITHOUT_REPORT:    // Close a region, don't reply
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("MSG_REGION_CLOSE_WITHOUT_REPORT");
+            }
             closeRegion(msg.getRegionInfo(), false);
             break;
 
           case HMsg.MSG_REGION_CLOSE_AND_DELETE:
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("MSG_REGION_CLOSE_AND_DELETE");
+            }
             closeAndDeleteRegion(msg.getRegionInfo());
             break;
 
@@ -608,15 +714,17 @@
           e.printStackTrace();
         }
       }
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("worker thread exiting");
+      }
     }
   }
   
   private void openRegion(HRegionInfo regionInfo) throws IOException {
-    
     this.locker.writeLock().lock();
     try {
-      HRegion region = new HRegion(regionDir, log, fs, conf, regionInfo, null, oldlogfile);
-      
+      HRegion region =
+        new HRegion(regionDir, log, fs, conf, regionInfo, null, oldlogfile);
       regions.put(region.getRegionName(), region);
       reportOpen(region);
       
@@ -661,12 +769,20 @@
   }
 
   /** Called either when the master tells us to restart or from stop() */
-  private void closeAllRegions() throws IOException {
+  private void closeAllRegions() {
     this.locker.writeLock().lock();
     try {
       for(Iterator<HRegion> it = regions.values().iterator(); it.hasNext(); ) {
         HRegion region = it.next();
-        region.close();
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("closing region " + region.getRegionName());
+        }
+        try {
+          region.close();
+          
+        } catch(IOException e) {
+          e.printStackTrace();
+        }
       }
       regions.clear();
       
@@ -722,9 +838,12 @@
       throw new IOException("Not serving region " + regionName);
     }
     
-    byte results[] = region.get(row, column);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("get " + row.toString() + ", " + column.toString());
+    }
+    BytesWritable results = region.get(row, column);
     if(results != null) {
-      return new BytesWritable(results);
+      return results;
     }
     return null;
   }
@@ -738,15 +857,9 @@
       throw new IOException("Not serving region " + regionName);
     }
     
-    byte results[][] = region.get(row, column, numVersions);
+    BytesWritable[] results = region.get(row, column, numVersions);
     if(results != null) {
-      BytesWritable realResults[] = new BytesWritable[results.length];
-      for(int i = 0; i < realResults.length; i++) {
-        if(results[i] != null) {
-          realResults[i] = new BytesWritable(results[i]);
-        }
-      }
-      return realResults;
+      return results;
     }
     return null;
   }
@@ -760,15 +873,9 @@
       throw new IOException("Not serving region " + regionName);
     }
     
-    byte results[][] = region.get(row, column, timestamp, numVersions);
+    BytesWritable[] results = region.get(row, column, timestamp, numVersions);
     if(results != null) {
-      BytesWritable realResults[] = new BytesWritable[results.length];
-      for(int i = 0; i < realResults.length; i++) {
-        if(results[i] != null) {
-          realResults[i] = new BytesWritable(results[i]);
-        }
-      }
-      return realResults;
+      return results;
     }
     return null;
   }
@@ -780,12 +887,12 @@
       throw new IOException("Not serving region " + regionName);
     }
     
-    TreeMap<Text, byte[]> map = region.getFull(row);
+    TreeMap<Text, BytesWritable> map = region.getFull(row);
     LabelledData result[] = new LabelledData[map.size()];
     int counter = 0;
     for(Iterator<Text> it = map.keySet().iterator(); it.hasNext(); ) {
       Text colname = it.next();
-      byte val[] = map.get(colname);
+      BytesWritable val = map.get(colname);
       result[counter++] = new LabelledData(colname, val);
     }
     return result;
@@ -842,7 +949,7 @@
     leases.renewLease(new Text(String.valueOf(clientid)), 
         new Text(String.valueOf(lockid)));
     
-    region.put(lockid, column, val.get());
+    region.put(lockid, column, val);
   }
 
   /** Remove a cell from the HBase. */
@@ -911,7 +1018,7 @@
   // remote scanner interface
   //////////////////////////////////////////////////////////////////////////////
 
-  private Map<Text, HScannerInterface> scanners;
+  private Map<Text, HInternalScannerInterface> scanners;
   private class ScannerListener extends LeaseListener {
     private Text scannerName;
     
@@ -920,7 +1027,7 @@
     }
     
     public void leaseExpired() {
-      HScannerInterface s = scanners.remove(scannerName);
+      HInternalScannerInterface s = scanners.remove(scannerName);
       if(s != null) {
         try {
           s.close();
@@ -943,7 +1050,7 @@
 
     long scannerId = -1L;
     try {
-      HScannerInterface s = r.getScanner(cols, firstRow);
+      HInternalScannerInterface s = r.getScanner(cols, firstRow);
       scannerId = rand.nextLong();
       Text scannerName = new Text(String.valueOf(scannerId));
       scanners.put(scannerName, s);
@@ -959,18 +1066,30 @@
   public LabelledData[] next(long scannerId, HStoreKey key) throws IOException {
     
     Text scannerName = new Text(String.valueOf(scannerId));
-    HScannerInterface s = scanners.get(scannerName);
+    HInternalScannerInterface s = scanners.get(scannerName);
     if(s == null) {
       throw new IOException("unknown scanner");
     }
     leases.renewLease(scannerName, scannerName);
-    TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
+    TreeMap<Text, BytesWritable> results = new TreeMap<Text, BytesWritable>();
     ArrayList<LabelledData> values = new ArrayList<LabelledData>();
     if(s.next(key, results)) {
-      for(Iterator<Map.Entry<Text, byte[]>> it = results.entrySet().iterator();
+      for(Iterator<Map.Entry<Text, BytesWritable>> it
+          = results.entrySet().iterator();
           it.hasNext(); ) {
-        Map.Entry<Text, byte[]> e = it.next();
-        values.add(new LabelledData(e.getKey(), e.getValue()));
+        
+        Map.Entry<Text, BytesWritable> e = it.next();
+        BytesWritable val = e.getValue();
+        if(val.getSize() == DELETE_BYTES.getSize()
+            && val.compareTo(DELETE_BYTES) == 0) {
+            
+          // Value is deleted. Don't return a value
+          
+          continue;
+
+        } else {
+          values.add(new LabelledData(e.getKey(), val));
+        }
       }
     }
     return values.toArray(new LabelledData[values.size()]);
@@ -978,7 +1097,7 @@
   
   public void close(long scannerId) throws IOException {
     Text scannerName = new Text(String.valueOf(scannerId));
-    HScannerInterface s = scanners.remove(scannerName);
+    HInternalScannerInterface s = scanners.remove(scannerName);
     if(s == null) {
       throw new IOException("unknown scanner");
     }
@@ -991,33 +1110,53 @@
     leases.cancelLease(scannerName, scannerName);
   }
 
-  //////////////////////////////////////////////////////////////////////////////
-  // Main program
-  //////////////////////////////////////////////////////////////////////////////
-
-  private static void printUsage() {
+  private static void printUsageAndExit() {
+    printUsageAndExit(null);
+  }
+  
+  private static void printUsageAndExit(final String message) {
+    if (message != null) {
+      System.err.println(message);
+    }
     System.err.println("Usage: java " +
-        "org.apache.hbase.HRegionServer [--bind=hostname:port]");
+        "org.apache.hbase.HRegionServer [--bind=hostname:port] start");
+    System.exit(0);
   }
   
   public static void main(String [] args) throws IOException {
+    if (args.length < 1) {
+      printUsageAndExit();
+    }
+    
     Configuration conf = new HBaseConfiguration();
     
     // Process command-line args. TODO: Better cmd-line processing
     // (but hopefully something not as painful as cli options).
+    final String addressArgKey = "--bind=";
     for (String cmd: args) {
-      if (cmd.equals("-h") || cmd.startsWith("--h")) {
-        printUsage();
-        return;
+      if (cmd.startsWith(addressArgKey)) {
+        conf.set(REGIONSERVER_ADDRESS, cmd.substring(addressArgKey.length()));
+        continue;
       }
       
-      final String addressArgKey = "--bind=";
-      if (cmd.startsWith(addressArgKey)) {
-        conf.set(REGIONSERVER_ADDRESS,
-            cmd.substring(addressArgKey.length()));
+      if (cmd.equals("start")) {
+        try {
+          (new Thread(new HRegionServer(conf))).start();
+        } catch (Throwable t) {
+          LOG.error( "Can not start master because "+
+              StringUtils.stringifyException(t) );
+          System.exit(-1);
+        }
+        break;
       }
+      
+      if (cmd.equals("stop")) {
+        printUsageAndExit("There is no regionserver stop mechanism. To stop " +
+          "regionservers, shutdown the hbase master");
+      }
+      
+      // Print out usage if we get to here.
+      printUsageAndExit();
     }
-    
-    new HRegionServer(conf);
   }
-}
+}
\ No newline at end of file

Added: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegiondirReader.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegiondirReader.java?view=auto&rev=535970
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegiondirReader.java (added)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HRegiondirReader.java Mon May  7 12:58:53 2007
@@ -0,0 +1,219 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+/**
+ * A standalone HRegion directory reader.  Currently reads content on
+ * file system only.
+ * TODO: Add dumping of HStoreFile content and HLog.
+ */
+public class HRegiondirReader {
+  private final Configuration conf;
+  private final Path parentdir;
+  
+  private static final Pattern REGION_NAME_PARSER =
+    Pattern.compile(HGlobals.HREGIONDIR_PREFIX +
+        "([^_]+)_([^_]*)_([^_]*)");
+  
+  private static final String USAGE = "Usage: " +
+      "java org.apache.hadoop.hbase.HRegionDirReader <regiondir> " +
+      "[<tablename>]";
+  
+  private final List<HRegionInfo> infos;
+  
+  public HRegiondirReader(final HBaseConfiguration conf,
+      final String parentdirName)
+  throws IOException {
+    this.conf = conf;
+    FileSystem fs = FileSystem.get(conf);
+    this.parentdir = new Path(parentdirName);
+    if (!fs.exists(parentdir)) {
+      throw new FileNotFoundException(parentdirName);
+    }
+    if (!fs.isDirectory(parentdir)) {
+      throw new IOException(parentdirName + " not a directory");
+    }
+    // Look for regions in parentdir.
+    Path [] regiondirs =
+      fs.listPaths(parentdir, new PathFilter() {
+        public boolean accept(Path path) {
+          Matcher m = REGION_NAME_PARSER.matcher(path.getName());
+          return m != null && m.matches();
+        }
+    });
+    // Create list of HRegionInfos for all regions found in
+    // parentdir.
+    this.infos = new ArrayList<HRegionInfo>();
+    for (Path d: regiondirs) {
+      Matcher m = REGION_NAME_PARSER.matcher(d.getName());
+      if (m == null || !m.matches()) {
+        throw new IOException("Unparseable region dir name");
+      }
+      String tableName = m.group(1);
+      String endKey = m.group(2);
+      long regionid = Long.parseLong(m.group(3));
+      HTableDescriptor desc = getTableDescriptor(fs, d, tableName);
+      HRegionInfo info = new HRegionInfo(regionid, desc,
+          new Text(), (endKey == null || endKey.length() == 0)?
+              new Text(): new Text(endKey));
+      infos.add(info);
+    }
+  }
+  
+  /**
+   * Returns a populated table descriptor.
+   * @param fs Current filesystem.
+   * @param d The regiondir for <code>tableName</code>
+   * @param tableName Name of this table.
+   * @return A HTableDescriptor populated with all known column
+   * families.
+   * @throws IOException
+   */
+  private HTableDescriptor getTableDescriptor(final FileSystem fs,
+      final Path d, final String tableName)
+  throws IOException {
+    HTableDescriptor desc = new HTableDescriptor(tableName, 1);
+    Text [] families = getFamilies(fs, d);
+    for (Text f: families) {
+      desc.addFamily(f);
+    }
+    return desc;
+  }
+  
+  /**
+   * Get column families for this region by looking at
+   * directory names under this region.
+   * This is a hack. HRegions only know what columns they have
+   * because they are told by passed-in metadata.
+   * @param regiondir
+   * @return Array of family names.
+   * @throws IOException
+   */
+  private Text [] getFamilies(final FileSystem fs,
+      final Path regiondir)
+  throws IOException {
+    Path [] subdirs = fs.listPaths(regiondir, new PathFilter() {
+      public boolean accept(Path path) {
+        return !path.getName().equals("log");
+      }
+    });
+    List<Text> families = new ArrayList<Text>();
+    for (Path d: subdirs) {
+      // Convert names of subdirectories into column family names
+      // by adding the colon.
+      Text family = new Text(d.getName() + ":");
+      families.add(family);
+    }
+    return families.toArray(new Text [] {});
+  }
+  
+  public List <HRegionInfo> getRegions() {
+    return this.infos;
+  }
+  
+  public HRegionInfo getRegionInfo(final String tableName)
+  throws IOException {
+    HRegionInfo result = null;
+    for(HRegionInfo i: getRegions()) {
+      if(i.tableDesc.getName().equals(tableName)) {
+        result = i;
+        break;
+      }
+    }
+    if (result == null) {
+      throw new NullPointerException("No such table: " +
+          tableName);
+    }
+    return result;
+  }
+  
+  private void dump(final String tableName) throws IOException {
+    dump(getRegionInfo(tableName));
+  }
+  
+  private void dump(final HRegionInfo info) throws IOException {
+    HRegion r = new HRegion(this.parentdir, null,
+        FileSystem.get(this.conf), conf, info, null, null);
+    Text [] families = info.tableDesc.families().toArray(new Text [] {});
+    HInternalScannerInterface scanner = r.getScanner(families, new Text());
+    HStoreKey key = new HStoreKey();
+    TreeMap<Text, BytesWritable> results = new TreeMap<Text, BytesWritable>();
+    // Print out table header line.
+    String s = info.startKey.toString();
+    String startKey = (s == null || s.length() <= 0)? "<>": s;
+    s = info.endKey.toString();
+    String endKey = (s == null || s.length() <= 0)? "<>": s;
+    String tableName = info.tableDesc.getName().toString();
+    System.out.println("table: " + tableName +
+      ", regionid: " + info.regionId +
+      ", startkey: " +  startKey +
+      ", endkey: " + endKey);
+    // Now print rows.  Offset by a space to distingush rows from
+    // table headers. TODO: Add in better formatting of output.
+    // Every line starts with row name followed by column name
+    // followed by cell content.
+    while(scanner.next(key, results)) {
+      for (Text colname: results.keySet()) {
+        Object value = null;
+        byte[] bytes = new byte[results.get(colname).getSize()];
+        if (colname.toString().equals("info:regioninfo")) {
+          // Then bytes are instance of an HRegionInfo.
+          System.arraycopy(results.get(colname).get(), 0, bytes, 0, bytes.length);
+          value = new HRegionInfo(bytes);
+        } else {
+          value = new String(bytes, HConstants.UTF8_ENCODING);
+        }
+   
+        System.out.println(" " + key + ", " + colname.toString() + ": \"" +
+            value.toString() + "\"");
+      }
+    }
+  }
+  
+  public static void main(String[] args) throws IOException {
+    if (args.length < 1) {
+      System.err.println(USAGE);
+      System.exit(-1);
+    }
+    HBaseConfiguration c = new HBaseConfiguration();
+    HRegiondirReader reader = new HRegiondirReader(c, args[0]);
+    if (args.length == 1) {
+      // Do all regions.
+      for(HRegionInfo info: reader.getRegions()) {
+        reader.dump(info);
+      }
+    } else {
+      for (int i = 1; i < args.length; i++) {
+        reader.dump(args[i]);
+      }
+    }
+  }
+}
\ No newline at end of file

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerAddress.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerAddress.java?view=diff&rev=535970&r1=535969&r2=535970
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerAddress.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerAddress.java Mon May  7 12:58:53 2007
@@ -92,7 +92,7 @@
       
     } else {
       address = new InetSocketAddress(bindAddress, port);
-      stringValue = bindAddress + "_" + port;
+      stringValue = bindAddress + ":" + port;
     }
   }
 

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerInfo.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerInfo.java?view=diff&rev=535970&r1=535969&r2=535970
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerInfo.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HServerInfo.java Mon May  7 12:58:53 2007
@@ -49,6 +49,11 @@
   public long getStartCode() {
     return startCode;
   }
+  
+  @Override
+  public String toString() {
+    return "address: " + this.serverAddress + ", startcode: " + this.startCode;
+  }
 
   //////////////////////////////////////////////////////////////////////////////
   // Writable
@@ -63,4 +68,4 @@
     this.serverAddress.write(out);
     out.writeLong(this.startCode);
   }
-}
+}
\ No newline at end of file

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java?view=diff&rev=535970&r1=535969&r2=535970
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStore.java Mon May  7 12:58:53 2007
@@ -78,30 +78,31 @@
 
   /**
    * An HStore is a set of zero or more MapFiles, which stretch backwards over 
-   * time.  A given HStore is responsible for a certain set of columns for a row
-   * in the HRegion.
+   * time.  A given HStore is responsible for a certain set of columns for a
+   * row in the HRegion.
    *
-   * The HRegion starts writing to its set of HStores when the HRegion's 
+   * <p>The HRegion starts writing to its set of HStores when the HRegion's 
    * memcache is flushed.  This results in a round of new MapFiles, one for
    * each HStore.
    *
-   * There's no reason to consider append-logging at this level; all logging and 
-   * locking is handled at the HRegion level.  HStore just provides services to 
-   * manage sets of MapFiles.  One of the most important of those services is 
-   * MapFile-compaction services.
+   * <p>There's no reason to consider append-logging at this level; all logging 
+   * and locking is handled at the HRegion level.  HStore just provides
+   * services to manage sets of MapFiles.  One of the most important of those
+   * services is MapFile-compaction services.
    *
-   * The only thing having to do with logs that HStore needs to deal with is
+   * <p>The only thing having to do with logs that HStore needs to deal with is
    * the reconstructionLog.  This is a segment of an HRegion's log that might
-   * be present upon startup.  If the param is NULL, there's nothing to do.
+   * NOT be present upon startup.  If the param is NULL, there's nothing to do.
    * If the param is non-NULL, we need to process the log to reconstruct
-   * a TreeMap that might not have been written to disk before the process died.
+   * a TreeMap that might not have been written to disk before the process
+   * died.
    *
-   * It's assumed that after this constructor returns, the reconstructionLog file
-   * will be deleted (by whoever has instantiated the HStore).
+   * <p>It's assumed that after this constructor returns, the reconstructionLog
+   * file will be deleted (by whoever has instantiated the HStore).
    */
   public HStore(Path dir, Text regionName, Text colFamily, int maxVersions, 
-      FileSystem fs, Path reconstructionLog, Configuration conf) throws IOException {
-    
+      FileSystem fs, Path reconstructionLog, Configuration conf)
+  throws IOException {  
     this.dir = dir;
     this.regionName = regionName;
     this.colFamily = colFamily;
@@ -114,14 +115,17 @@
     this.loginfodir = HStoreFile.getInfoDir(dir, regionName, colFamily);
     fs.mkdirs(loginfodir);
 
-    LOG.debug("starting HStore for " + regionName + "/"+ colFamily);
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("starting HStore for " + regionName + "/"+ colFamily);
+    }
     
     // Either restart or get rid of any leftover compaction work.  Either way, 
     // by the time processReadyCompaction() returns, we can get rid of the 
     // existing compaction-dir.
 
     this.compactdir = new Path(dir, COMPACTION_DIR);
-    Path curCompactStore = HStoreFile.getHStoreDir(compactdir, regionName, colFamily);
+    Path curCompactStore =
+      HStoreFile.getHStoreDir(compactdir, regionName, colFamily);
     if(fs.exists(curCompactStore)) {
       processReadyCompaction();
       fs.delete(curCompactStore);
@@ -129,28 +133,25 @@
 
     // Go through the 'mapdir' and 'loginfodir' together, make sure that all 
     // MapFiles are in a reliable state.  Every entry in 'mapdir' must have a 
-    // corresponding one in 'loginfodir'. Without a corresponding log info file,
-    // the entry in 'mapdir'must be deleted.
-
+    // corresponding one in 'loginfodir'. Without a corresponding log info
+    // file, the entry in 'mapdir' must be deleted.
     Vector<HStoreFile> hstoreFiles 
       = HStoreFile.loadHStoreFiles(conf, dir, regionName, colFamily, fs);
-    
     for(Iterator<HStoreFile> it = hstoreFiles.iterator(); it.hasNext(); ) {
       HStoreFile hsf = it.next();
       mapFiles.put(hsf.loadInfo(fs), hsf);
     }
 
-    // Now go through all the HSTORE_LOGINFOFILEs and figure out the most-recent
-    // log-seq-ID that's present.  The most-recent such ID means we can ignore 
-    // all log messages up to and including that ID (because they're already 
-    // reflected in the TreeMaps).
+    // Now go through all the HSTORE_LOGINFOFILEs and figure out the
+    // most-recent log-seq-ID that's present.  The most-recent such ID means we
+    // can ignore all log messages up to and including that ID (because they're
+    // already reflected in the TreeMaps).
     //
-    // If the HSTORE_LOGINFOFILE doesn't contain a number, just ignore it.  That
+    // If the HSTORE_LOGINFOFILE doesn't contain a number, just ignore it. That
     // means it was built prior to the previous run of HStore, and so it cannot 
     // contain any updates also contained in the log.
-
     long maxSeqID = -1;
-    for(Iterator<HStoreFile> it = hstoreFiles.iterator(); it.hasNext(); ) {
+    for (Iterator<HStoreFile> it = hstoreFiles.iterator(); it.hasNext(); ) {
       HStoreFile hsf = it.next();
       long seqid = hsf.loadInfo(fs);
       if(seqid > 0) {
@@ -166,63 +167,72 @@
     // We can ignore any log message that has a sequence ID that's equal to or 
     // lower than maxSeqID.  (Because we know such log messages are already 
     // reflected in the MapFiles.)
-
-    LOG.debug("reading reconstructionLog");
-    
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("reading reconstructionLog");
+    }
     if(reconstructionLog != null && fs.exists(reconstructionLog)) {
       long maxSeqIdInLog = -1;
       TreeMap<HStoreKey, BytesWritable> reconstructedCache 
         = new TreeMap<HStoreKey, BytesWritable>();
-      
       SequenceFile.Reader login 
         = new SequenceFile.Reader(fs, reconstructionLog, conf);
-      
       try {
         HLogKey key = new HLogKey();
         HLogEdit val = new HLogEdit();
         while(login.next(key, val)) {
           maxSeqIdInLog = Math.max(maxSeqIdInLog, key.getLogSeqNum());
-          if(key.getLogSeqNum() <= maxSeqID) {
+          if (key.getLogSeqNum() <= maxSeqID) {
+            continue;
+          }
+          // Check this edit is for me.  Also, guard against writing
+          // METACOLUMN info such as HBASE::CACHEFLUSH entries
+          Text column = val.getColumn();
+          if (!key.getRegionName().equals(this.regionName) ||
+              column.equals(HLog.METACOLUMN) ||
+              HStoreKey.extractFamily(column).equals(this.colFamily)) {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Passing on edit " + key.getRegionName() + ", "
+                  + key.getRegionName() + ", " + column.toString() + ": "
+                  + new String(val.getVal().get()));
+            }
             continue;
           }
           reconstructedCache.put(new HStoreKey(key.getRow(), val.getColumn(), 
               val.getTimestamp()), val.getVal());
         }
-        
       } finally {
         login.close();
       }
 
       if(reconstructedCache.size() > 0) {
-        
         // We create a "virtual flush" at maxSeqIdInLog+1.
-        
-        LOG.debug("flushing reconstructionCache");
-        
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("flushing reconstructionCache");
+        }
         flushCacheHelper(reconstructedCache, maxSeqIdInLog+1, true);
       }
     }
 
     // Compact all the MapFiles into a single file.  The resulting MapFile 
     // should be "timeless"; that is, it should not have an associated seq-ID, 
-    // because all log messages have been reflected in the TreeMaps at this point.
-    
+    // because all log messages have been reflected in the TreeMaps at this
+    // point.
     if(mapFiles.size() >= 1) {
       compactHelper(true);
     }
 
     // Finally, start up all the map readers! (There should be just one at this 
     // point, as we've compacted them all.)
-
-    LOG.debug("starting map readers");
-    
+    if(LOG.isDebugEnabled()) {
+      LOG.debug("starting map readers");
+    }
     for(Iterator<Long> it = mapFiles.keySet().iterator(); it.hasNext(); ) {
       Long key = it.next().longValue();
       HStoreFile hsf = mapFiles.get(key);
 
-      //TODO - is this really necessary?  Don't I do this inside compact()?
-      
-      maps.put(key, new MapFile.Reader(fs, hsf.getMapFilePath().toString(), conf));
+      // TODO - is this really necessary?  Don't I do this inside compact()?
+      maps.put(key,
+        new MapFile.Reader(fs, hsf.getMapFilePath().toString(), conf));
     }
     
     LOG.info("HStore online for " + this.regionName + "/" + this.colFamily);
@@ -273,7 +283,9 @@
       long logCacheFlushId, boolean addToAvailableMaps) throws IOException {
     
     synchronized(flushLock) {
-      LOG.debug("flushing HStore " + this.regionName + "/" + this.colFamily);
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("flushing HStore " + this.regionName + "/" + this.colFamily);
+      }
       
       // A. Write the TreeMap out to the disk
 
@@ -281,18 +293,23 @@
         = HStoreFile.obtainNewHStoreFile(conf, dir, regionName, colFamily, fs);
       
       Path mapfile = flushedFile.getMapFilePath();
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("map file is: " + mapfile.toString());
+      }
+      
       MapFile.Writer out = new MapFile.Writer(conf, fs, mapfile.toString(), 
           HStoreKey.class, BytesWritable.class);
       
       try {
-        for(Iterator<HStoreKey> it = inputCache.keySet().iterator(); it.hasNext(); ) {
-          HStoreKey curkey = it.next();
+        for (HStoreKey curkey: inputCache.keySet()) {
           if(this.colFamily.equals(HStoreKey.extractFamily(curkey.getColumn()))) {
             BytesWritable val = inputCache.get(curkey);
             out.append(curkey, val);
           }
         }
-        LOG.debug("HStore " + this.regionName + "/" + this.colFamily + " flushed");
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("HStore " + this.regionName + "/" + this.colFamily + " flushed");
+        }
         
       } finally {
         out.close();
@@ -301,7 +318,9 @@
       // B. Write out the log sequence number that corresponds to this output
       // MapFile.  The MapFile is current up to and including the log seq num.
 
-      LOG.debug("writing log cache flush id");
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("writing log cache flush id");
+      }
       flushedFile.writeInfo(fs, logCacheFlushId);
 
       // C. Finally, make the new MapFile available.
@@ -312,7 +331,10 @@
         try {
           maps.put(logCacheFlushId, new MapFile.Reader(fs, mapfile.toString(), conf));
           mapFiles.put(logCacheFlushId, flushedFile);
-          LOG.debug("HStore available for " + this.regionName + "/" + this.colFamily);
+          if(LOG.isDebugEnabled()) {
+            LOG.debug("HStore available for " + this.regionName + "/"
+                + this.colFamily + " flush id=" + logCacheFlushId);
+          }
         
         } finally {
           this.locker.writeLock().unlock();
@@ -357,7 +379,9 @@
   
   void compactHelper(boolean deleteSequenceInfo) throws IOException {
     synchronized(compactLock) {
-      LOG.debug("started compaction of " + this.regionName + "/" + this.colFamily);
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("started compaction of " + this.regionName + "/" + this.colFamily);
+      }
       
       Path curCompactStore = HStoreFile.getHStoreDir(compactdir, regionName, colFamily);
       fs.mkdirs(curCompactStore);
@@ -387,13 +411,17 @@
             }
           }
         }
-        LOG.debug("max sequence id =" + maxSeenSeqID);
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("max sequence id =" + maxSeenSeqID);
+        }
         
         HStoreFile compactedOutputFile 
           = new HStoreFile(conf, compactdir, regionName, colFamily, -1);
         
         if(toCompactFiles.size() == 1) {
-          LOG.debug("nothing to compact for " + this.regionName + "/" + this.colFamily);
+          if(LOG.isDebugEnabled()) {
+            LOG.debug("nothing to compact for " + this.regionName + "/" + this.colFamily);
+          }
           
           HStoreFile hsf = toCompactFiles.elementAt(0);
           if(hsf.loadInfo(fs) == -1) {
@@ -438,7 +466,9 @@
           // Now, advance through the readers in order.  This will have the
           // effect of a run-time sort of the entire dataset.
 
-          LOG.debug("processing HStoreFile readers");
+          if(LOG.isDebugEnabled()) {
+            LOG.debug("processing HStoreFile readers");
+          }
           
           int numDone = 0;
           for(int i = 0; i < readers.length; i++) {
@@ -518,13 +548,17 @@
             }
           }
           
-          LOG.debug("all HStores processed");
+          if(LOG.isDebugEnabled()) {
+            LOG.debug("all HStores processed");
+          }
           
         } finally {
           compactedOut.close();
         }
 
-        LOG.debug("writing new compacted HStore");
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("writing new compacted HStore");
+        }
 
         // Now, write out an HSTORE_LOGINFOFILE for the brand-new TreeMap.
 
@@ -564,7 +598,9 @@
 
         processReadyCompaction();
         
-        LOG.debug("compaction complete for " + this.regionName + "/" + this.colFamily);
+        if(LOG.isDebugEnabled()) {
+          LOG.debug("compaction complete for " + this.regionName + "/" + this.colFamily);
+        }
 
       } finally {
         fs.delete(compactdir);
@@ -609,7 +645,9 @@
 
       // OK, there's actually compaction work that needs to be put into place.
 
-      LOG.debug("compaction starting");
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("compaction starting");
+      }
       
       // 2. Load in the files to be deleted.
       //    (Figuring out what MapFiles are going to be replaced)
@@ -629,7 +667,9 @@
         in.close();
       }
 
-      LOG.debug("loaded files to be deleted");
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("loaded files to be deleted");
+      }
       
       // 3. Unload all the replaced MapFiles.
       
@@ -650,7 +690,9 @@
         }
       }
 
-      LOG.debug("unloaded existing MapFiles");
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("unloaded existing MapFiles");
+      }
       
       // What if we crash at this point?  No big deal; we will restart
       // processReadyCompaction(), and nothing has been lost.
@@ -663,7 +705,9 @@
         fs.delete(hsf.getInfoFilePath());
       }
 
-      LOG.debug("old files deleted");
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("old files deleted");
+      }
       
       // What if we fail now?  The above deletes will fail silently. We'd better
       // make sure not to write out any new files with the same names as 
@@ -671,7 +715,9 @@
 
       // 5. Moving the new MapFile into place
       
-      LOG.debug("moving new MapFile into place");
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("moving new MapFile into place");
+      }
       
       HStoreFile compactedFile 
         = new HStoreFile(conf, compactdir, regionName, colFamily, -1);
@@ -691,7 +737,9 @@
 
       // 6. Loading the new TreeMap.
       
-      LOG.debug("loading new TreeMap");
+      if(LOG.isDebugEnabled()) {
+        LOG.debug("loading new TreeMap");
+      }
       
       mapFiles.put(orderVal, finalCompactedFile);
       maps.put(orderVal, new MapFile.Reader(fs, 
@@ -716,7 +764,7 @@
    *
    * The returned object should map column names to byte arrays (byte[]).
    */
-  public void getFull(HStoreKey key, TreeMap<Text, byte[]> results) throws IOException {
+  public void getFull(HStoreKey key, TreeMap<Text, BytesWritable> results) throws IOException {
     this.locker.readLock().lock();
     try {
       MapFile.Reader[] maparray 
@@ -734,7 +782,7 @@
             Text readcol = readkey.getColumn();
             if(results.get(readcol) == null
                 && key.matchesWithoutColumn(readkey)) {
-              results.put(new Text(readcol), readval.get());
+              results.put(new Text(readcol), readval);
               readval = new BytesWritable();
               
             } else if(key.getRow().compareTo(readkey.getRow()) > 0) {
@@ -756,12 +804,12 @@
    *
    * If 'numVersions' is negative, the method returns all available versions.
    */
-  public byte[][] get(HStoreKey key, int numVersions) throws IOException {
-    if(numVersions == 0) {
-      throw new IllegalArgumentException("Must request at least one value.");
+  public BytesWritable[] get(HStoreKey key, int numVersions) throws IOException {
+    if(numVersions <= 0) {
+      throw new IllegalArgumentException("Number of versions must be > 0");
     }
     
-    Vector<byte[]> results = new Vector<byte[]>();
+    Vector<BytesWritable> results = new Vector<BytesWritable>();
     this.locker.readLock().lock();
     try {
       MapFile.Reader[] maparray 
@@ -776,7 +824,7 @@
           HStoreKey readkey = (HStoreKey)map.getClosest(key, readval);
           
           if(readkey.matchesRowCol(key)) {
-            results.add(readval.get());
+            results.add(readval);
             readval = new BytesWritable();
 
             while(map.next(readkey, readval) && readkey.matchesRowCol(key)) {
@@ -784,7 +832,7 @@
                 break;
                 
               } else {
-                results.add(readval.get());
+                results.add(readval);
                 readval = new BytesWritable();
               }
             }
@@ -799,14 +847,14 @@
         return null;
         
       } else {
-        return (byte[][]) results.toArray(new byte[results.size()][]);
+        return results.toArray(new BytesWritable[results.size()]);
       }
       
     } finally {
       this.locker.readLock().unlock();
     }
   }
-
+  
   /**
    * Gets the size of the largest MapFile and its mid key.
    * 
@@ -866,31 +914,54 @@
    * Return a set of MapFile.Readers, one for each HStore file.
    * These should be closed after the user is done with them.
    */
-  public HScannerInterface getScanner(long timestamp, Text targetCols[],
+  public HInternalScannerInterface getScanner(long timestamp, Text targetCols[],
       Text firstRow) throws IOException {
     
     return new HStoreScanner(timestamp, targetCols, firstRow);
   }
 
+  /** For debuging purposes. Dumps the keys from all the MapFiles */
+  void dumpMaps() throws IOException {
+    this.locker.readLock().lock();
+    try {
+      for(Iterator<MapFile.Reader> i = maps.values().iterator(); i.hasNext(); ) {
+        MapFile.Reader r = i.next();
+        synchronized(r) {
+          r.reset();
+          HStoreKey key = new HStoreKey();
+          BytesWritable val = new BytesWritable();
+          while(r.next(key, val)) {
+            System.out.println(key);
+          }
+        }
+      }
+      
+    } finally {
+      this.locker.readLock().unlock();
+    }
+  }
+
   //////////////////////////////////////////////////////////////////////////////
   // This class implements the HScannerInterface.
   // It lets the caller scan the contents of this HStore.
   //////////////////////////////////////////////////////////////////////////////
   
   class HStoreScanner extends HAbstractScanner {
-    MapFile.Reader readers[];
-    Text lastRow = null;
+    private MapFile.Reader[] readers;
     
-    public HStoreScanner(long timestamp, Text targetCols[], Text firstRow) throws IOException {
+    public HStoreScanner(long timestamp, Text[] targetCols, Text firstRow) throws IOException {
       super(timestamp, targetCols);
 
       locker.readLock().lock();
       try {
         this.readers = new MapFile.Reader[mapFiles.size()];
-        int i = 0;
+        
+        // Most recent map file should be first
+        
+        int i = readers.length - 1;
         for(Iterator<HStoreFile> it = mapFiles.values().iterator(); it.hasNext(); ) {
           HStoreFile curHSF = it.next();
-          readers[i++] = new MapFile.Reader(fs, curHSF.getMapFilePath().toString(), conf);
+          readers[i--] = new MapFile.Reader(fs, curHSF.getMapFilePath().toString(), conf);
         }
         
         this.keys = new HStoreKey[readers.length];
@@ -952,6 +1023,7 @@
      * @return - true if there is more data available
      */
     boolean getNext(int i) throws IOException {
+      vals[i] = new BytesWritable();
       if(! readers[i].next(keys[i], vals[i])) {
         closeSubScanner(i);
         return false;



Mime
View raw message