hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r564012 [3/4] - in /lucene/hadoop/trunk/src/contrib/hbase: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/util/ src/test/org/apache/hadoop/hbase/
Date Wed, 08 Aug 2007 20:30:20 GMT
Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HStoreFile.java Wed Aug  8 13:30:13 2007
@@ -19,54 +19,103 @@
  */
 package org.apache.hadoop.hbase;
 
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Random;
+import java.util.Vector;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.conf.*;
+import org.apache.hadoop.hbase.util.Writables;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
+import org.onelab.filter.Filter;
+import org.onelab.filter.Key;
 
-import java.io.*;
-import java.util.*;
 
 /**
- * Each HStore maintains a bunch of different data files.
- *
- * An HStoreFile tracks 4 things: its parent dir, the region identifier, the 
- * column family, and the file identifier.  If you know those four things, you
- * know how to obtain the right HStoreFile.
+ * A HStore data file.  HStores usually have one or more of these files.  They
+ * are produced by flushing the memcache to disk.
  *
- * When merging or splitting HRegions, we might want to modify one of the 
- * params for an HStoreFile (effectively moving it elsewhere).
+ * <p>Each HStore maintains a bunch of different data files. The filename is a
+ * mix of the parent dir, the region name, the column name, and a file
+ * identifier. The name may also be a reference to a store file located
+ * elsewhere. This class handles all that path-building stuff for you.
+ * 
+ * <p>An HStoreFile usually tracks 4 things: its parent dir, the region
+ * identifier, the column family, and the file identifier.  If you know those
+ * four things, you know how to obtain the right HStoreFile.  HStoreFiles may
+ * also refernce store files in another region serving either from
+ * the top-half of the remote file or from the bottom-half.  Such references
+ * are made fast splitting regions.
  * 
- * The filename is a mix of the parent dir, the region name, the column name, 
- * and the file identifier.
+ * <p>Plain HStoreFiles are named for a randomly generated id as in:
+ * <code>1278437856009925445</code>  A file by this name is made in both the
+ * <code>mapfiles</code> and <code>info</code> subdirectories of a
+ * HStore columnfamily directoy: E.g. If the column family is 'anchor:', then
+ * under the region directory there is a subdirectory named 'anchor' within
+ * which is a 'mapfiles' and 'info' subdirectory.  In each will be found a
+ * file named something like <code>1278437856009925445</code>, one to hold the
+ * data in 'mapfiles' and one under 'info' that holds the sequence id for this
+ * store file.
  * 
- * This class handles all that path-building stuff for you.
+ * <p>References to store files located over in some other region look like
+ * this:
+ * <code>1278437856009925445.hbaserepository,qAReLZD-OyQORZWq_vqR1k==,959247014679548184</code>:
+ * i.e. an id followed by the name of the referenced region.  The data
+ * ('mapfiles') of HStoreFile references are empty. The accompanying
+ * <code>info</code> file contains the
+ * midkey, the id of the remote store we're referencing and whether we're
+ * to serve the top or bottom region of the remote store file.  Note, a region
+ * is not splitable if it has instances of store file references (References
+ * are cleaned up by compactions).
+ * 
+ * <p>When merging or splitting HRegions, we might want to modify one of the 
+ * params for an HStoreFile (effectively moving it elsewhere).
  */
 public class HStoreFile implements HConstants, WritableComparable {
-  private static final Log LOG = LogFactory.getLog(HStoreFile.class.getName());
+  static final Log LOG = LogFactory.getLog(HStoreFile.class.getName());
   static final byte INFO_SEQ_NUM = 0;
-  static final String HSTORE_DATFILE_PREFIX = "mapfile.dat.";
-  static final String HSTORE_INFOFILE_PREFIX = "mapfile.info.";
   static final String HSTORE_DATFILE_DIR = "mapfiles";
   static final String HSTORE_INFO_DIR = "info";
   static final String HSTORE_FILTER_DIR = "filter";
+  public static enum Range {top, bottom}
+  
+  /*
+   * Regex that will work for straight filenames and for reference names.
+   * If reference, then the regex has more than just one group.  Group 1 is
+   * this files id.  Group 2 the referenced region name, etc.
+   */
+  private static Pattern REF_NAME_PARSER =
+    Pattern.compile("^(\\d+)(?:\\.(.+))?$");
+  
   private static Random rand = new Random();
 
-  Path dir;
-  Text regionName;
-  Text colFamily;
-  long fileId;
-  Configuration conf;
+  private Path dir;
+  private Text regionName;
+  private Text colFamily;
+  private long fileId;
+  private final Configuration conf;
+  private Reference reference;
 
-  /** Constructor used by Writable */
+  /** Shutdown constructor used by Writable */
   HStoreFile(Configuration conf) {
-    this.conf = conf;
-    this.dir = new Path(Path.CUR_DIR);
-    this.regionName = new Text();
-    this.colFamily = new Text();
-    this.fileId = 0;
+    this(conf, new Path(Path.CUR_DIR), new Text(), new Text(), 0);
   }
   
   /**
@@ -77,54 +126,181 @@
    * @param colFamily name of the column family
    * @param fileId file identifier
    */
+  HStoreFile(final Configuration conf, final Path dir, final Text regionName, 
+      final Text colFamily, final long fileId) {
+    this(conf, dir, regionName, colFamily, fileId, null);
+  }
+
+  /**
+   * Constructor that fully initializes the object
+   * @param conf Configuration object
+   * @param dir directory path
+   * @param regionName name of the region
+   * @param colFamily name of the column family
+   * @param fileId file identifier
+   * @param ref Reference to another HStoreFile.
+   */
   HStoreFile(Configuration conf, Path dir, Text regionName, 
-      Text colFamily, long fileId) {
-    
+      Text colFamily, long fileId, final Reference ref) {
     this.conf = conf;
     this.dir = dir;
     this.regionName = new Text(regionName);
     this.colFamily = new Text(colFamily);
     this.fileId = fileId;
+    // If a reference, construction does not write the pointer files.  Thats
+    // done by invocations of writeReferenceFiles(hsf, fs).  Happens at fast
+    // split time.
+    this.reference = ref;
   }
 
-  /** @return the directory path */
-  Path getDir() {
-    return dir;
+  /*
+   * Data structure to hold reference to a store file over in another region.
+   */
+  static class Reference {
+    Text regionName;
+    long fileid;
+    Range region;
+    HStoreKey midkey;
+    
+    Reference(final Text rn, final long fid, final HStoreKey m,
+        final Range fr) {
+      this.regionName = rn;
+      this.fileid = fid;
+      this.region = fr;
+      this.midkey = m;
+    }
+    
+    Reference() {
+      this(null, -1, null, Range.bottom);
+    }
+
+    long getFileId() {
+      return this.fileid;
+    }
+
+    Range getFileRegion() {
+      return this.region;
+    }
+    
+    HStoreKey getMidkey() {
+      return this.midkey;
+    }
+    
+    Text getRegionName() {
+      return this.regionName;
+    }
+   
+    public String toString() {
+      return this.regionName + "/" + this.fileid + "/" + this.region;
+    }
+
+    // Make it serializable.
+    public void write(DataOutput out) throws IOException {
+      this.regionName.write(out);
+      out.writeLong(this.fileid);
+      // Write true if we're doing top of the file.
+      out.writeBoolean(isTopFileRegion(this.region));
+      this.midkey.write(out);
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      this.regionName = new Text();
+      this.regionName.readFields(in);
+      this.fileid = in.readLong();
+      boolean tmp = in.readBoolean();
+      // If true, set region to top.
+      this.region = tmp? Range.top: Range.bottom;
+      this.midkey = new HStoreKey();
+      this.midkey.readFields(in);
+    }
+  }
+
+  static boolean isTopFileRegion(final Range r) {
+    return r.equals(Range.top);
   }
 
   /** @return the region name */
+  boolean isReference() {
+    return this.reference != null;
+  }
+  
+  Reference getReference() {
+    return this.reference;
+  }
+
   Text getRegionName() {
-    return regionName;
+    return this.regionName;
   }
 
   /** @return the column family */
   Text getColFamily() {
-    return colFamily;
+    return this.colFamily;
   }
 
   /** @return the file identifier */
-  long fileId() {
-    return fileId;
+  long getFileId() {
+    return this.fileId;
   }
 
   // Build full filenames from those components
-  
   /** @return path for MapFile */
   Path getMapFilePath() {
-    return new Path(HStoreFile.getMapDir(dir, regionName, colFamily), 
-        HSTORE_DATFILE_PREFIX + fileId);
+    return isReference()?
+      getMapFilePath(this.regionName, this.fileId,
+        this.reference.getRegionName()):
+      getMapFilePath(this.regionName, this.fileId);
+  }
+
+  private Path getMapFilePath(final Reference r) {
+    return r == null?
+      getMapFilePath():
+      getMapFilePath(r.getRegionName(), r.getFileId());
+  }
+
+  private Path getMapFilePath(final Text name, final long fid) {
+    return new Path(HStoreFile.getMapDir(dir, name, colFamily), 
+      createHStoreFilename(fid, null));
   }
   
+  private Path getMapFilePath(final Text name, final long fid, final Text rn) {
+    return new Path(HStoreFile.getMapDir(dir, name, colFamily), 
+      createHStoreFilename(fid, rn));
+  }
+
   /** @return path for info file */
   Path getInfoFilePath() {
-    return new Path(HStoreFile.getInfoDir(dir, regionName, colFamily), 
-        HSTORE_INFOFILE_PREFIX + fileId);
+    return isReference()?
+      getInfoFilePath(this.regionName, this.fileId,
+        this.reference.getRegionName()):
+      getInfoFilePath(this.regionName, this.fileId);
+  }
+  
+  private Path getInfoFilePath(final Text name, final long fid) {
+    return new Path(HStoreFile.getInfoDir(dir, name, colFamily), 
+      createHStoreFilename(fid, null));
+  }
+  
+  private Path getInfoFilePath(final Text name, final long fid, final Text rn) {
+    return new Path(HStoreFile.getInfoDir(dir, name, colFamily), 
+      createHStoreFilename(fid, rn));
   }
 
   // Static methods to build partial paths to internal directories.  Useful for 
   // HStore construction and log-rebuilding.
-
-  /** @return the map directory path */
+  private static String createHStoreFilename(final long fid) {
+    return createHStoreFilename(fid, null);
+  }
+  
+  private static String createHStoreFilename(final long fid,
+      final Text regionName) {
+    return Long.toString(fid) +
+      ((regionName != null)? "." + regionName.toString(): "");
+  }
+  
+  private static String createHStoreInfoFilename(final long fid) {
+    return createHStoreFilename(fid, null);
+  }
+  
   static Path getMapDir(Path dir, Text regionName, Text colFamily) {
     return new Path(dir, new Path(HREGIONDIR_PREFIX + regionName, 
         new Path(colFamily.toString(), HSTORE_DATFILE_DIR)));
@@ -148,11 +324,6 @@
         colFamily.toString()));
   }
 
-  /** @return the HRegion directory path */
-  static Path getHRegionDir(Path dir, Text regionName) {
-    return new Path(dir, new Path(HREGIONDIR_PREFIX + regionName));
-  }
-
   /**
    * @return a brand-new randomly-named HStoreFile.
    * 
@@ -165,19 +336,18 @@
     Path mapdir = HStoreFile.getMapDir(dir, regionName, colFamily);
     long fileId = Math.abs(rand.nextLong());
 
-    Path testpath1 = new Path(mapdir, HSTORE_DATFILE_PREFIX + fileId);
-    Path testpath2 = new Path(mapdir, HSTORE_INFOFILE_PREFIX + fileId);
+    Path testpath1 = new Path(mapdir, createHStoreFilename(fileId));
+    Path testpath2 = new Path(mapdir, createHStoreInfoFilename(fileId));
     while(fs.exists(testpath1) || fs.exists(testpath2)) {
       fileId = Math.abs(rand.nextLong());
-      testpath1 = new Path(mapdir, HSTORE_DATFILE_PREFIX + fileId);
-      testpath2 = new Path(mapdir, HSTORE_INFOFILE_PREFIX + fileId);
+      testpath1 = new Path(mapdir, createHStoreFilename(fileId));
+      testpath2 = new Path(mapdir, createHStoreInfoFilename(fileId));
     }
     return new HStoreFile(conf, dir, regionName, colFamily, fileId);
   }
 
-  /**
+  /*
    * Creates a series of HStoreFiles loaded from the given directory.
-   * 
    * There must be a matching 'mapdir' and 'loginfo' pair of files.
    * If only one exists, we'll delete it.
    *
@@ -186,65 +356,82 @@
    * @param regionName region name
    * @param colFamily column family
    * @param fs file system
-   * @return Vector of HStoreFiles
+   * @return List of store file instances loaded from passed dir.
    * @throws IOException
    */
   static Vector<HStoreFile> loadHStoreFiles(Configuration conf, Path dir, 
-      Text regionName, Text colFamily, FileSystem fs) throws IOException {
+      Text regionName, Text colFamily, FileSystem fs)
+  throws IOException {
+    // Look first at info files.  If a reference, these contain info we need
+    // to create the HStoreFile.
+    Path infodir = HStoreFile.getInfoDir(dir, regionName, colFamily);
+    Path infofiles[] = fs.listPaths(infodir);
+    Vector<HStoreFile> results = new Vector<HStoreFile>(infofiles.length);
+    Vector<Path> mapfiles = new Vector<Path>(infofiles.length);
+    for (int i = 0; i < infofiles.length; i++) {
+      Path p = infofiles[i];
+      Matcher m = REF_NAME_PARSER.matcher(p.getName());
+      boolean isReference =  isReference(p, m);
+      long fid = Long.parseLong(m.group(1));
+      HStoreFile curfile = null;
+      if (isReference) {
+        Reference reference = readSplitInfo(infofiles[i], fs);
+        curfile = new HStoreFile(conf, dir, regionName, colFamily, fid,
+          reference);
+      } else {
+        curfile = new HStoreFile(conf, dir, regionName, colFamily, fid);
+      }
+      Path mapfile = curfile.getMapFilePath();
+      if (!fs.exists(mapfile)) {
+        fs.delete(curfile.getInfoFilePath());
+      }
+      // Found map and sympathetic info file.  Add this hstorefile to result.
+      results.add(curfile);
+      // Keep list of sympathetic data mapfiles for cleaning info dir in next
+      // section.  Make sure path is fully qualified for compare.
+      Path qualified = fs.makeQualified(mapfile);
+      mapfiles.add(qualified);
+    }
     
-    Vector<HStoreFile> results = new Vector<HStoreFile>();
     Path mapdir = HStoreFile.getMapDir(dir, regionName, colFamily);
-
+    // List paths by experience returns fully qualified names -- at least when
+    // running on a mini hdfs cluster.
     Path datfiles[] = fs.listPaths(mapdir);
-    for(int i = 0; i < datfiles.length; i++) {
-      String name = datfiles[i].getName();
-      
-      if(name.startsWith(HSTORE_DATFILE_PREFIX)) {
-        Long fileId =
-          Long.parseLong(name.substring(HSTORE_DATFILE_PREFIX.length()));
-
-        HStoreFile curfile =
-          new HStoreFile(conf, dir, regionName, colFamily, fileId);
-
-        Path mapfile = curfile.getMapFilePath();
-        Path infofile = curfile.getInfoFilePath();
-        
-        if(fs.exists(infofile)) {
-          results.add(curfile);
-          
-        } else {
-          fs.delete(mapfile);
-        }
-      }
-    }
-
-    Path infodir = HStoreFile.getInfoDir(dir, regionName, colFamily);
-    Path infofiles[] = fs.listPaths(infodir);
-    for(int i = 0; i < infofiles.length; i++) {
-      String name = infofiles[i].getName();
-      
-      if(name.startsWith(HSTORE_INFOFILE_PREFIX)) {
-        long fileId =
-          Long.parseLong(name.substring(HSTORE_INFOFILE_PREFIX.length()));
-
-        HStoreFile curfile =
-          new HStoreFile(conf, dir, regionName, colFamily, fileId);
-
-        Path mapfile = curfile.getMapFilePath();
-        
-        if(! fs.exists(mapfile)) {
-          fs.delete(curfile.getInfoFilePath());
-        }
+    for (int i = 0; i < datfiles.length; i++) {
+      // If does not have sympathetic info file, delete.
+      if (!mapfiles.contains(fs.makeQualified(datfiles[i]))) {
+        fs.delete(datfiles[i]);
       }
     }
     return results;
   }
+  
+  /**
+   * @param p Path to check.
+   * @return True if the path has format of a HStoreFile reference.
+   */
+  static boolean isReference(final Path p) {
+    return isReference(p, REF_NAME_PARSER.matcher(p.getName()));
+  }
+ 
+  private static boolean isReference(final Path p, final Matcher m) {
+    if (m == null || !m.matches()) {
+      LOG.warn("Failed match of store file name " + p.toString());
+    }
+    return m.groupCount() > 1 && m.group(2) != null;
+  }
 
   // File handling
 
-  /**
-   * Break this HStoreFile file into two new parts, which live in different 
-   * brand-new HRegions.
+  /*
+   * Split by making two new store files that reference top and bottom regions
+   * of original store file.
+   * @param midKey
+   * @param dstA
+   * @param dstB
+   * @param fs
+   * @param c
+   * @throws IOException
    *
    * @param midKey the key which will be the starting key of the second region
    * @param dstA the file which will contain keys from the start of the source
@@ -253,57 +440,66 @@
    * @param c configuration
    * @throws IOException
    */
-  void splitStoreFile(Text midKey, HStoreFile dstA, HStoreFile dstB,
-      FileSystem fs, Configuration c) throws IOException {
-    
-    // Copy the appropriate tuples to one MapFile or the other.
-    
-    MapFile.Reader in = new MapFile.Reader(fs, getMapFilePath().toString(), c);
-    try {
-      MapFile.Writer outA = new MapFile.Writer(c, fs, 
-        dstA.getMapFilePath().toString(), HStoreKey.class,
-        ImmutableBytesWritable.class);
-      
-      try {
-        MapFile.Writer outB = new MapFile.Writer(c, fs, 
-          dstB.getMapFilePath().toString(), HStoreKey.class,
-          ImmutableBytesWritable.class);
-        
-        try {
-          long count = 0;
-          HStoreKey readkey = new HStoreKey();
-          ImmutableBytesWritable readval = new ImmutableBytesWritable();
-          
-          while(in.next(readkey, readval)) {
-            if(readkey.getRow().compareTo(midKey) < 0) {
-              outA.append(readkey, readval);
-            } else {
-              outB.append(readkey, readval);
-            }
-            if (LOG.isDebugEnabled()) {
-              count++;
-              if ((count % 10000) == 0) {
-                LOG.debug("Write " + count + " records");
-              }
-            }
-          }
-          
-        } finally {
-          outB.close();
-        }
-        
-      } finally {
-        outA.close();
-      }
-      
-    } finally {
-      in.close();
+  void splitStoreFile(final HStoreFile dstA, final HStoreFile dstB,
+      final FileSystem fs)
+  throws IOException {
+    dstA.writeReferenceFiles(fs);
+    dstB.writeReferenceFiles(fs);
+  }
+  
+  void writeReferenceFiles(final FileSystem fs)
+  throws IOException {
+    createOrFail(fs, getMapFilePath());
+    writeSplitInfo(fs);
+  }
+  
+  /*
+   * If reference, create and write the remote store file id, the midkey and
+   * whether we're going against the top file region of the referent out to
+   * the info file. 
+   * @param p Path to info file.
+   * @param hsf
+   * @param fs
+   * @throws IOException
+   */
+  private void writeSplitInfo(final FileSystem fs)
+  throws IOException {
+    Path p = getInfoFilePath();
+    if (fs.exists(p)) {
+      throw new IOException("File already exists " + p.toString());
     }
+    FSDataOutputStream out = fs.create(p);
+    getReference().getRegionName().write(out);
+    getReference().getMidkey().write(out);
+    out.writeLong(getReference().getFileId());
+    out.writeBoolean(isTopFileRegion(getReference().getFileRegion()));
+    out.close();
+  }
+  
+  /*
+   * @see writeSplitInfo(Path p, HStoreFile hsf, FileSystem fs)
+   */
+  static Reference readSplitInfo(final Path p, final FileSystem fs)
+  throws IOException {
+    FSDataInputStream in = fs.open(p);
+    Text rn = new Text();
+    rn.readFields(in);
+    HStoreKey midkey = new HStoreKey();
+    midkey.readFields(in);
+    long fid = in.readLong();
+    boolean tmp = in.readBoolean();
+    return new Reference(rn, fid, midkey, tmp? Range.top: Range.bottom);
+    
+  }
 
-    // Build an InfoFile for each output
-    long seqid = loadInfo(fs);
-    dstA.writeInfo(fs, seqid);
-    dstB.writeInfo(fs, seqid);
+  private void createOrFail(final FileSystem fs, final Path p)
+  throws IOException {
+    if (fs.exists(p)) {
+      throw new IOException("File already exists " + p.toString());
+    }
+    if (!fs.createNewFile(p)) {
+      throw new IOException("Failed create of " + p);
+    }
   }
 
   /**
@@ -315,10 +511,9 @@
    * @throws IOException
    */
   void mergeStoreFiles(Vector<HStoreFile> srcFiles, FileSystem fs, 
-      Configuration conf) throws IOException {
-
+      @SuppressWarnings("hiding") Configuration conf)
+  throws IOException {
     // Copy all the source MapFile tuples into this HSF's MapFile
-
     MapFile.Writer out = new MapFile.Writer(conf, fs,
       getMapFilePath().toString(),
       HStoreKey.class, ImmutableBytesWritable.class);
@@ -327,7 +522,6 @@
       for(HStoreFile src: srcFiles) {
         MapFile.Reader in =
           new MapFile.Reader(fs, src.getMapFilePath().toString(), conf);
-        
         try {
           HStoreKey readkey = new HStoreKey();
           ImmutableBytesWritable readval = new ImmutableBytesWritable();
@@ -343,12 +537,10 @@
     } finally {
       out.close();
     }
-
     // Build a unified InfoFile from the source InfoFiles.
     
     long unifiedSeqId = -1;
-    for(Iterator<HStoreFile> it = srcFiles.iterator(); it.hasNext(); ) {
-      HStoreFile hsf = it.next();
+    for(HStoreFile hsf: srcFiles) {
       long curSeqId = hsf.loadInfo(fs);
       if(curSeqId > unifiedSeqId) {
         unifiedSeqId = curSeqId;
@@ -358,14 +550,17 @@
   }
 
   /** 
-   * Reads in an info file, and gives it a unique ID.
+   * Reads in an info file
    *
    * @param fs file system
-   * @return new unique id
+   * @return The sequence id contained in the info file
    * @throws IOException
    */
   long loadInfo(FileSystem fs) throws IOException {
-    Path p = getInfoFilePath();
+    Path p = isReference()?
+      getInfoFilePath(this.reference.getRegionName(),
+        this.reference.getFileId()):
+      getInfoFilePath();
     DataInputStream in = new DataInputStream(fs.open(p));
     try {
       byte flag = in.readByte();
@@ -388,7 +583,6 @@
   void writeInfo(FileSystem fs, long infonum) throws IOException {
     Path p = getInfoFilePath();
     DataOutputStream out = new DataOutputStream(fs.create(p));
-    
     try {
       out.writeByte(INFO_SEQ_NUM);
       out.writeLong(infonum);
@@ -397,8 +591,327 @@
       out.close();
     }
   }
+  
+  /**
+   * Delete store map files.
+   * @throws IOException 
+   */
+  public void delete() throws IOException {
+    delete(getMapFilePath());
+    delete(getInfoFilePath());
+  }
+  
+  private void delete(final Path p) throws IOException {
+    p.getFileSystem(this.conf).delete(p);
+  }
+  
+  /**
+   * Renames the mapfiles and info directories under the passed
+   * <code>hsf</code> directory.
+   * @param fs
+   * @param hsf
+   * @throws IOException
+   */
+  public void rename(final FileSystem fs, final HStoreFile hsf)
+  throws IOException {
+    fs.rename(getMapFilePath(), hsf.getMapFilePath());
+    fs.rename(getInfoFilePath(), hsf.getInfoFilePath());
+  }
+  
+  /**
+   * A facade for a {@link MapFile.Reader} that serves up either the top or
+   * bottom half of a MapFile (where 'bottom' is the first half of the file
+   * containing the keys that sort lowest and 'top' is the second half of the
+   * file with keys that sort greater than those of the bottom half).
+   * Subclasses BloomFilterMapFile.Reader in case 
+   * 
+   * <p>This file is not splitable.  Calls to {@link #midKey()} return null.
+   */
+  static class HalfMapFileReader extends BloomFilterMapFile.Reader {
+    private final boolean top;
+    private final WritableComparable midkey;
+    private boolean topFirstNextCall = true;
+    
+    HalfMapFileReader(final FileSystem fs, final String dirName, 
+        final Configuration conf, final Range r,
+        final WritableComparable midKey)
+    throws IOException {
+      this(fs, dirName, conf, r, midKey, null);
+    }
+    
+    HalfMapFileReader(final FileSystem fs, final String dirName, 
+        final Configuration conf, final Range r,
+        final WritableComparable midKey, final Filter filter)
+    throws IOException {
+      super(fs, dirName, conf, filter);
+      this.top = isTopFileRegion(r);
+      this.midkey = midKey;
+    }
+    
+    @SuppressWarnings("unchecked")
+    private void checkKey(final WritableComparable key)
+    throws IOException {
+      if (this.top) {
+        if (key.compareTo(this.midkey) < 0) {
+          throw new IOException("Illegal Access: Key is less than midKey of " +
+          "backing mapfile");
+        }
+      } else if (key.compareTo(this.midkey) >= 0) {
+        throw new IOException("Illegal Access: Key is greater than or equal " +
+        "to midKey of backing mapfile");
+      }
+    }
+
+    @SuppressWarnings({ "unused"})
+    @Override
+    public synchronized void finalKey(WritableComparable key)
+    throws IOException {
+      throw new UnsupportedOperationException("Unsupported");
+    }
+
+    @Override
+    public synchronized Writable get(WritableComparable key, Writable val)
+        throws IOException {
+      checkKey(key);
+      return super.get(key, val);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public synchronized WritableComparable getClosest(WritableComparable key,
+        Writable val)
+    throws IOException {
+      if (this.top) {
+        if (key.compareTo(this.midkey) < 0) {
+          return this.midkey;
+        }
+      } else if (key.compareTo(this.midkey) >= 0) {
+        // Contract says return null if EOF.
+        return null;
+      }
+      return super.getClosest(key, val);
+    }
+
+    @SuppressWarnings("unused")
+    @Override
+    public synchronized WritableComparable midKey() throws IOException {
+      // Returns null to indicate file is not splitable.
+      return null;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public synchronized boolean next(WritableComparable key, Writable val)
+    throws IOException {
+      if (this.top && this.topFirstNextCall) {
+        this.topFirstNextCall = false;
+        return doFirstNextProcessing(key, val);
+      }
+      boolean result = super.next(key, val);
+      if (!top && key.compareTo(this.midkey) >= 0) {
+        result = false;
+      }
+      return result;
+    }
+    
+    private boolean doFirstNextProcessing(WritableComparable key, Writable val)
+    throws IOException {
+      // Seek to midkey.  Midkey may not exist in this file.  That should be
+      // fine.  Then we'll either be positioned at end or start of file.
+      WritableComparable nearest = getClosest(this.midkey, val);
+      // Now copy the mid key into the passed key.
+      if (nearest != null) {
+        Writables.copyWritable(nearest, key);
+        return true;
+      }
+      return false;
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+      if (top) {
+        this.topFirstNextCall = true;
+        seek(this.midkey);
+        return;
+      }
+      super.reset();
+    }
+
+    @Override
+    public synchronized boolean seek(WritableComparable key)
+    throws IOException {
+      checkKey(key);
+      return super.seek(key);
+    }
+  }
+  
+  /**
+   * On write, all keys are added to a bloom filter.  On read, all keys are
+   * tested first against bloom filter. Keys are HStoreKey.  If passed bloom
+   * filter is null, just passes invocation to parent.
+   */
+  static class BloomFilterMapFile extends MapFile {
+    protected BloomFilterMapFile() {
+      super();
+    }
+    
+    static class Reader extends MapFile.Reader {
+      private final Filter bloomFilter;
+
+      public Reader(FileSystem fs, String dirName, Configuration conf,
+          final Filter filter)
+      throws IOException {
+        super(fs, dirName, conf);
+        this.bloomFilter = filter;
+      }
+      
+      /** {@inheritDoc} */
+      @Override
+      public Writable get(WritableComparable key, Writable val)
+      throws IOException {
+        if (this.bloomFilter == null) {
+          return super.get(key, val);
+        }
+        if(this.bloomFilter.membershipTest(getBloomFilterKey(key))) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("bloom filter reported that key exists");
+          }
+          return super.get(key, val);
+        }
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("bloom filter reported that key does not exist");
+        }
+        return null;
+      }
+
+      /** {@inheritDoc} */
+      @Override
+      public WritableComparable getClosest(WritableComparable key,
+          Writable val)
+      throws IOException {
+        if (this.bloomFilter == null) {
+          return super.getClosest(key, val);
+        }
+        // Note - the key being passed to us is always a HStoreKey
+        if(this.bloomFilter.membershipTest(getBloomFilterKey(key))) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("bloom filter reported that key exists");
+          }
+          return super.getClosest(key, val);
+        }
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("bloom filter reported that key does not exist");
+        }
+        return null;
+      }
+    }
+    
+    static class Writer extends MapFile.Writer {
+      private final Filter bloomFilter;
+      
+
+      @SuppressWarnings("unchecked")
+      public Writer(Configuration conf, FileSystem fs, String dirName,
+          Class keyClass, Class valClass,
+          SequenceFile.CompressionType compression, final Filter filter)
+      throws IOException {
+        super(conf, fs, dirName, keyClass, valClass, compression);
+        this.bloomFilter = filter;
+      }
+
+      /** {@inheritDoc} */
+      @Override
+      public void append(WritableComparable key, Writable val)
+      throws IOException {
+        if (this.bloomFilter != null) {
+          this.bloomFilter.add(getBloomFilterKey(key));
+        }
+        super.append(key, val);
+      }
+    }
+  }
+  
+  /**
+   * Custom bloom filter key maker.
+   * @param key
+   * @return Key made of bytes of row and column only.
+   * @throws IOException
+   */
+  static Key getBloomFilterKey(WritableComparable key)
+  throws IOException {
+    HStoreKey hsk = (HStoreKey)key;
+    byte [] bytes = null;
+    try {
+      bytes = (hsk.getRow().toString() + hsk.getColumn().toString()).
+        getBytes(UTF8_ENCODING);
+    } catch (UnsupportedEncodingException e) {
+      throw new IOException(e.toString());
+    }
+    return new Key(bytes);
+  }
+
+  /**
+   * Get reader for the store file map file.
+   * Client is responsible for closing file when done.
+   * @param fs
+   * @param bloomFilter If null, no filtering is done.
+   * @return MapFile.Reader
+   * @throws IOException
+   */
+  public synchronized MapFile.Reader getReader(final FileSystem fs,
+      final Filter bloomFilter)
+  throws IOException {
+    return isReference()?
+      new HStoreFile.HalfMapFileReader(fs,
+        getMapFilePath(getReference().getRegionName(),
+          getReference().getFileId()).toString(),
+        this.conf, getReference().getFileRegion(), getReference().getMidkey(),
+        bloomFilter):
+      new BloomFilterMapFile.Reader(fs, getMapFilePath().toString(),
+        this.conf, bloomFilter);
+  }
+
+  /**
+   * Get a store file writer.
+   * Client is responsible for closing file when done.
+   * @param fs
+   * @param compression Pass <code>SequenceFile.CompressionType.NONE</code>
+   * for none.
+   * @param bloomFilter If null, no filtering is done.
+   * @return MapFile.Writer
+   * @throws IOException
+   */
+  public MapFile.Writer getWriter(final FileSystem fs,
+      final SequenceFile.CompressionType compression,
+      final Filter bloomFilter)
+  throws IOException {
+    if (isReference()) {
+      throw new IOException("Illegal Access: Cannot get a writer on a" +
+        "HStoreFile reference");
+    }
+    return new BloomFilterMapFile.Writer(conf, fs,
+      getMapFilePath().toString(), HStoreKey.class,
+      ImmutableBytesWritable.class, compression, bloomFilter);
+  }
+
+  /**
+   * @return Length of the store map file.  If a reference, size is
+   * approximation.
+   * @throws IOException
+   */
+  public long length() throws IOException {
+    Path p = new Path(getMapFilePath(getReference()), MapFile.DATA_FILE_NAME);
+    long l = p.getFileSystem(this.conf).getFileStatus(p).getLen();
+    return (isReference())? l / 2: l;
+  }
 
-  /** {@inheritDoc} */
+  @Override
+  public String toString() {
+    return this.regionName.toString() + "/" + this.colFamily.toString() +
+      "/" + this.fileId +
+      (isReference()? "/" + this.reference.toString(): "");
+  }
+  
   @Override
   public boolean equals(Object o) {
     return this.compareTo(o) == 0;
@@ -419,9 +932,13 @@
   /** {@inheritDoc} */
   public void write(DataOutput out) throws IOException {
     out.writeUTF(dir.toString());
-    regionName.write(out);
-    colFamily.write(out);
+    this.regionName.write(out);
+    this.colFamily.write(out);
     out.writeLong(fileId);
+    out.writeBoolean(isReference());
+    if (isReference()) {
+      this.reference.write(out);
+    }
   }
   
   /** {@inheritDoc} */
@@ -430,6 +947,12 @@
     this.regionName.readFields(in);
     this.colFamily.readFields(in);
     this.fileId = in.readLong();
+    this.reference = null;
+    boolean isReferent = in.readBoolean();
+    this.reference = new HStoreFile.Reference();
+    if (isReferent) {
+      this.reference.readFields(in);
+    }
   }
 
   // Comparable
@@ -454,4 +977,4 @@
     }
     return result;
   }
-}
\ No newline at end of file
+}

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/HTable.java Wed Aug  8 13:30:13 2007
@@ -131,6 +131,11 @@
       throw new IllegalStateException("update in progress");
     }
   }
+  
+
+  public Text getTableName() {
+    return this.tableName;
+  }
 
   /**
    * Gets the starting row key for every region in the currently open table

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Keying.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Keying.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Keying.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Keying.java Wed Aug  8 13:30:13 2007
@@ -19,10 +19,18 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
 import java.util.StringTokenizer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HStoreKey;
+
 /**
  * Utility creating hbase friendly keys.
  * Use fabricating row names or column qualifiers.
@@ -110,5 +118,62 @@
       sb.insert(0, next);
     }
     return sb.toString();
+  }
+  
+  /**
+   * @param i
+   * @return <code>i</code> as byte array.
+   */
+  public static byte[] intToBytes(final int i){
+    ByteBuffer buffer = ByteBuffer.allocate(Integer.SIZE);
+    buffer.putInt(i);
+    return buffer.array();
+  }
+  
+  /**
+   * @param l
+   * @return <code>i</code> as byte array.
+   */
+  public static byte[] longToBytes(final long l){
+    ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE);
+    buffer.putLong(l);
+    return buffer.array();
+  }
+
+  /**
+   * Returns row and column bytes out of an HStoreKey.
+   * @param hsk Store key.
+   * @throws UnsupportedEncodingException
+   */
+  public static byte[] getBytes(final HStoreKey hsk)
+  throws UnsupportedEncodingException {
+    StringBuilder s = new StringBuilder(hsk.getRow().toString());
+    s.append(hsk.getColumn().toString());
+    return s.toString().getBytes(HConstants.UTF8_ENCODING);
+  }
+
+  /**
+   * @param bytes
+   * @return String made of the bytes or null if bytes are null.
+   * @throws UnsupportedEncodingException
+   */
+  public static String bytesToString(final byte [] bytes)
+  throws UnsupportedEncodingException {
+    if(bytes == null) {
+      return null;
+    }
+    return new String(bytes, HConstants.UTF8_ENCODING);
+  }
+  
+  public static long bytesToLong(final byte [] bytes) throws IOException {
+    long result = -1;
+    DataInputStream dis = null;
+    try {
+      dis = new DataInputStream(new ByteArrayInputStream(bytes));
+      result = dis.readLong();
+    } finally {
+      dis.close();
+    }
+    return result;
   }
 }

Added: lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java?view=auto&rev=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java (added)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/Writables.java Wed Aug  8 13:30:13 2007
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.Writable;
+
+
+public class Writables {
+  /**
+   * @param w
+   * @return The bytes of <code>w</code> gotten by running its 
+   * {@link Writable#write(java.io.DataOutput)} method.
+   * @throws IOException
+   * @see #getWritable(byte[], Writable)
+   */
+  public static byte [] getBytes(final Writable w) throws IOException {
+    ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+    DataOutputStream out = new DataOutputStream(byteStream);
+    try {
+      w.write(out);
+      out.close();
+      out = null;
+      return byteStream.toByteArray();
+    } finally {
+      if (out != null) {
+        out.close();
+      }
+    }
+  }
+
+  /**
+   * Set bytes into the passed Writable by calling its
+   * {@link Writable#readFields(java.io.DataInput)}.
+   * @param bytes
+   * @param w An empty Writable (usually made by calling the null-arg
+   * constructor).
+   * @return The passed Writable after its readFields has been called fed
+   * by the passed <code>bytes</code> array or null if passed null or
+   * empty <code>bytes</code>.
+   * @throws IOException
+   */
+  public static Writable getWritable(final byte [] bytes, final Writable w)
+  throws IOException {
+    if (bytes == null || bytes.length == 0) {
+      throw new IOException("Con't build a writable with empty bytes array");
+    }
+    DataInputBuffer in = new DataInputBuffer();
+    try {
+      in.reset(bytes, bytes.length);
+      w.readFields(in);
+      return w;
+    } finally {
+      in.close();
+    }
+  }
+
+  /**
+   * Copy one Writable to another.  Copies bytes using data streams.
+   * @param src Source Writable
+   * @param tgt Target Writable
+   * @return The target Writable.
+   * @throws IOException
+   */
+  public static Writable copyWritable(final Writable src, final Writable tgt)
+  throws IOException {
+    byte [] bytes = getBytes(src);
+    DataInputStream dis = null;
+    try {
+      dis = new DataInputStream(new ByteArrayInputStream(bytes));
+      tgt.readFields(dis);
+    } finally {
+      dis.close();
+    }
+    return tgt;
+  }
+}

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/HBaseTestCase.java Wed Aug  8 13:30:13 2007
@@ -56,7 +56,7 @@
     HTableDescriptor desc, long regionId, Text startKey, Text endKey)
   throws IOException {
     HRegionInfo info = new HRegionInfo(regionId, desc, startKey, endKey);
-    Path regionDir = HStoreFile.getHRegionDir(dir, info.regionName);
+    Path regionDir = HRegion.getRegionDir(dir, info.regionName);
     FileSystem fs = dir.getFileSystem(c);
     fs.mkdirs(regionDir);
     return new HRegion(dir,

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java Wed Aug  8 13:30:13 2007
@@ -33,18 +33,24 @@
 /**
  * This class creates a single process HBase cluster for junit testing.
  * One thread is created for each server.
+ * 
+ * <p>TestCases do not need to subclass to start a HBaseCluster.  Call
+ * {@link #startMaster(Configuration)} and
+ * {@link #startRegionServers(Configuration, int)} to startup master and
+ * region servers.  Save off the returned values and pass them to
+ * {@link #shutdown(org.apache.hadoop.hbase.MiniHBaseCluster.MasterThread, List)}
+ * to shut it all down when done.
+ * 
  */
 public class MiniHBaseCluster implements HConstants {
-  private static final Logger LOG =
+  static final Logger LOG =
     Logger.getLogger(MiniHBaseCluster.class.getName());
   private Configuration conf;
   private MiniDFSCluster cluster;
   private FileSystem fs;
   private Path parentdir;
-  private HMaster master = null;
-  private Thread masterThread = null;
-  List<HRegionServer> regionServers;
-  List<Thread> regionThreads;
+  private MasterThread masterThread = null;
+  ArrayList<RegionServerThread> regionThreads;
   private boolean deleteOnExit = true;
 
   /**
@@ -87,8 +93,6 @@
 
     this.conf = conf;
     this.cluster = dfsCluster;
-    this.regionServers = new ArrayList<HRegionServer>(nRegionNodes);
-    this.regionThreads = new ArrayList<Thread>(nRegionNodes);
     init(nRegionNodes);
   }
 
@@ -108,13 +112,9 @@
   throws IOException {
     this.conf = conf;
     this.deleteOnExit = deleteOnExit;
-    this.regionServers = new ArrayList<HRegionServer>(nRegionNodes);
-    this.regionThreads = new ArrayList<Thread>(nRegionNodes);
-
     if (miniHdfsFilesystem) {
       try {
         this.cluster = new MiniDFSCluster(this.conf, 2, format, (String[])null);
-
       } catch(Throwable t) {
         LOG.error("Failed setup of mini dfs cluster", t);
         t.printStackTrace();
@@ -124,45 +124,123 @@
     init(nRegionNodes);
   }
 
-  private void init(int nRegionNodes) throws IOException {
+  private void init(final int nRegionNodes)
+  throws IOException {
     try {
       try {
         this.fs = FileSystem.get(conf);
         this.parentdir = new Path(conf.get(HBASE_DIR, DEFAULT_HBASE_DIR));
         fs.mkdirs(parentdir);
-
       } catch(IOException e) {
         LOG.error("Failed setup of FileSystem", e);
         throw e;
       }
-
-      if(this.conf.get(MASTER_ADDRESS) == null) {
-        this.conf.set(MASTER_ADDRESS, "localhost:0");
-      }
-
-      // Create the master
-      this.master = new HMaster(conf);
-      this.masterThread = new Thread(this.master, "HMaster");
-
-      // Start up the master
-      LOG.info("Starting HMaster");
-      masterThread.start();
-
-      // Set the master's port for the HRegionServers
-      String address = master.getMasterAddress().toString();
-      this.conf.set(MASTER_ADDRESS, address);
-
-      // Start the HRegionServers.  Always have regionservers come up on
-      // port '0' so there won't be clashes over default port as unit tests
-      // start/stop ports at different times during the life of the test.
-      this.conf.set(REGIONSERVER_ADDRESS, DEFAULT_HOST + ":0");
-      LOG.info("Starting HRegionServers");
-      startRegionServers(nRegionNodes);
+      this.masterThread = startMaster(this.conf);
+      this.regionThreads = startRegionServers(this.conf, nRegionNodes);
     } catch(IOException e) {
       shutdown();
       throw e;
     }
   }
+  
+  public static class MasterThread extends Thread {
+    private final HMaster master;
+    MasterThread(final HMaster m) {
+      super(m, "Master:" + m.getMasterAddress().toString());
+      this.master = m;
+    }
+    @Override
+    public void run() {
+      LOG.info("Starting " + getName());
+      super.run();
+    }
+    public HMaster getMaster() {
+      return this.master;
+    }
+  }
+  
+  public static class RegionServerThread extends Thread {
+    private final HRegionServer regionServer;
+    RegionServerThread(final HRegionServer r, final int index) {
+      super(r, "RegionServer:" + index);
+      this.regionServer = r;
+    }
+    @Override
+    public void run() {
+      LOG.info("Starting " + getName());
+      super.run();
+    }
+    public HRegionServer getRegionServer() {
+      return this.regionServer;
+    }
+  }
+  
+  /**
+   * Use this method to start a master.
+   * If you want to start an hbase cluster
+   * without subclassing this test case, run this method and
+   * {@link #startRegionServers(Configuration, int)} to start servers.
+   * Call {@link #shutdown(org.apache.hadoop.hbase.MiniHBaseCluster.MasterThread, List)}
+   * to shut them down.
+   * @param c
+   * @return Thread running the master.
+   * @throws IOException
+   * @see #startRegionServers(Configuration, int)
+   * @see #shutdown(org.apache.hadoop.hbase.MiniHBaseCluster.MasterThread, List)
+   */
+  public static MasterThread startMaster(final Configuration c)
+  throws IOException {
+    if(c.get(MASTER_ADDRESS) == null) {
+      c.set(MASTER_ADDRESS, "localhost:0");
+    }
+    // Create the master
+    final HMaster m = new HMaster(c);
+    MasterThread masterThread = new MasterThread(m);
+    // Start up the master
+    masterThread.start();
+    // Set the master's port for the HRegionServers
+    c.set(MASTER_ADDRESS, m.getMasterAddress().toString());
+    return masterThread;
+  }
+
+  /**
+   * @param c
+   * @param count
+   * @return List of region server threads started.  Synchronize on the
+   * returned list when iterating to avoid ConcurrentModificationExceptions.
+   * @throws IOException
+   * @see #startMaster(Configuration)
+   */
+  public static ArrayList<RegionServerThread> startRegionServers(
+    final Configuration c, final int count)
+  throws IOException {
+    // Start the HRegionServers.  Always have regionservers come up on
+    // port '0' so there won't be clashes over default port as unit tests
+    // start/stop ports at different times during the life of the test.
+    c.set(REGIONSERVER_ADDRESS, DEFAULT_HOST + ":0");
+    LOG.info("Starting HRegionServers");
+    ArrayList<RegionServerThread> threads =
+      new ArrayList<RegionServerThread>();
+    for(int i = 0; i < count; i++) {
+      threads.add(startRegionServer(c, i));
+    }
+    return threads;
+  }
+  
+  public void startRegionServer() throws IOException {
+    RegionServerThread t =
+      startRegionServer(this.conf, this.regionThreads.size());
+    this.regionThreads.add(t);
+  }
+  
+  private static RegionServerThread startRegionServer(final Configuration c,
+    final int index)
+  throws IOException {
+    final HRegionServer hsr = new HRegionServer(c);
+    RegionServerThread t = new RegionServerThread(hsr, index);
+    t.start();
+    return t;
+  }
 
   /**
    * Get the cluster on which this HBase cluster is running
@@ -173,27 +251,12 @@
     return cluster;
   }
 
-  private void startRegionServers(final int nRegionNodes)
-  throws IOException {
-    for(int i = 0; i < nRegionNodes; i++) {
-      startRegionServer();
-    }
-  }
-
-  void startRegionServer() throws IOException {
-    HRegionServer hsr = new HRegionServer(this.conf);
-    this.regionServers.add(hsr);
-    Thread t = new Thread(hsr, "HRegionServer-" + this.regionServers.size());
-    t.start();
-    this.regionThreads.add(t);
-  }
-
   /** 
    * @return Returns the rpc address actually used by the master server, because
    * the supplied port is not necessarily the actual port used.
    */
   public HServerAddress getHMasterAddress() {
-    return master.getMasterAddress();
+    return this.masterThread.getMaster().getMasterAddress();
   }
 
   /**
@@ -202,7 +265,9 @@
    * @param serverNumber
    */
   public void abortRegionServer(int serverNumber) {
-    HRegionServer server = this.regionServers.remove(serverNumber);
+    HRegionServer server =
+      this.regionThreads.get(serverNumber).getRegionServer();
+    LOG.info("Aborting " + server.serverInfo.toString());
     server.abort();
   }
 
@@ -211,53 +276,77 @@
    * 
    * @param serverNumber
    */
-  public void stopRegionServer(int serverNumber) {
-    HRegionServer server = this.regionServers.remove(serverNumber);
+  public HRegionServer stopRegionServer(int serverNumber) {
+    HRegionServer server =
+      this.regionThreads.get(serverNumber).getRegionServer();
+    LOG.info("Stopping " + server.toString());
     server.stop();
+    return server;
   }
 
   /**
    * Wait for the specified region server to stop
-   * 
+   * Removes this thread from list of running threads.
    * @param serverNumber
    */
   public void waitOnRegionServer(int serverNumber) {
-    Thread regionServerThread = this.regionThreads.remove(serverNumber);
+    RegionServerThread regionServerThread =
+      this.regionThreads.remove(serverNumber);
     try {
+      LOG.info("Waiting on " +
+        regionServerThread.getRegionServer().serverInfo.toString());
       regionServerThread.join();
     } catch (InterruptedException e) {
       e.printStackTrace();
     }
   }
-
-  /** Shut down the HBase cluster */
-  public void shutdown() {
-    LOG.info("Shutting down the HBase Cluster");
-    for(HRegionServer hsr: this.regionServers) {
-      hsr.stop();
-    }
-    if(master != null) {
-      master.shutdown();
-    }
-    for(Thread t: this.regionThreads) {
-      if (t.isAlive()) {
-        try {
-          t.join();
-        } catch (InterruptedException e) {
-          // continue
+  
+  /**
+   * Shut down HBase cluster started by calling
+   * {@link #startMaster(Configuration)} and then
+   * {@link #startRegionServers(Configuration, int)};
+   * @param masterThread
+   * @param regionServerThreads
+   */
+  public static void shutdown(final MasterThread masterThread,
+      final List<RegionServerThread> regionServerThreads) {
+    LOG.info("Shutting down HBase Cluster");
+    /** This is not needed.  Remove.
+    for(RegionServerThread hsr: regionServerThreads) {
+      hsr.getRegionServer().stop();
+    }
+    */
+    if(masterThread != null) {
+      masterThread.getMaster().shutdown();
+    }
+    synchronized(regionServerThreads) {
+      if (regionServerThreads != null) {
+        for(Thread t: regionServerThreads) {
+          if (t.isAlive()) {
+            try {
+              t.join();
+            } catch (InterruptedException e) {
+              // continue
+            }
+          }
         }
       }
     }
     if (masterThread != null) {
       try {
         masterThread.join();
-
       } catch(InterruptedException e) {
         // continue
       }
     }
-    LOG.info("HBase Cluster shutdown complete");
-    
+    LOG.info("Shutdown " +
+      ((masterThread != null)? masterThread.getName(): "0 masters") + " " +
+      ((regionServerThreads == null)? 0: regionServerThreads.size()) +
+      " region server(s)");
+  }
+  
+  void shutdown() {
+    shutdown(this.masterThread, this.regionThreads);
     // Close the file system.  Will complain if files open so helps w/ leaks.
     try {
       this.cluster.getFileSystem().close();
@@ -285,4 +374,4 @@
     }
     f.delete();
   }
-}
+}
\ No newline at end of file

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java Wed Aug  8 13:30:13 2007
@@ -63,7 +63,7 @@
       debugging = true;
       
       Logger rootLogger = Logger.getRootLogger();
-      rootLogger.setLevel(Level.WARN);
+      // rootLogger.setLevel(Level.WARN);
 
       Level logLevel = Level.INFO;
       value = System.getenv("LOGGING_LEVEL");

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestBatchUpdate.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestBatchUpdate.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestBatchUpdate.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestBatchUpdate.java Wed Aug  8 13:30:13 2007
@@ -35,14 +35,10 @@
   private HTableDescriptor desc = null;
   private HTable table = null;
 
-  /** constructor */
-  public TestBatchUpdate() {
-    try {
-      value = "abcd".getBytes(HConstants.UTF8_ENCODING);
-      
-    } catch (UnsupportedEncodingException e) {
-      fail();
-    }
+  /** constructor 
+   * @throws UnsupportedEncodingException */
+  public TestBatchUpdate() throws UnsupportedEncodingException {
+    value = "abcd".getBytes(HConstants.UTF8_ENCODING);
   }
   
   /**

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestGet.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestGet.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestGet.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestGet.java Wed Aug  8 13:30:13 2007
@@ -88,7 +88,7 @@
       desc.addFamily(new HColumnDescriptor(HConstants.COLUMN_FAMILY.toString()));
       
       HRegionInfo info = new HRegionInfo(0L, desc, null, null);
-      Path regionDir = HStoreFile.getHRegionDir(dir, info.regionName);
+      Path regionDir = HRegion.getRegionDir(dir, info.regionName);
       fs.mkdirs(regionDir);
       
       HLog log = new HLog(fs, new Path(regionDir, "log"), conf);

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHRegion.java Wed Aug  8 13:30:13 2007
@@ -40,7 +40,8 @@
  * HRegions or in the HBaseMaster, so only basic testing is possible.
  */
 public class TestHRegion extends HBaseTestCase implements RegionUnavailableListener {
-  Logger LOG = Logger.getLogger(this.getClass().getName());
+  private static final Logger LOG =
+    Logger.getLogger(TestHRegion.class.getName());
   
   /** Constructor */
   public TestHRegion() {
@@ -51,8 +52,9 @@
    * Since all the "tests" depend on the results of the previous test, they are
    * not Junit tests that can stand alone. Consequently we have a single Junit
    * test that runs the "sub-tests" as private methods.
+   * @throws IOException 
    */
-  public void testHRegion() {
+  public void testHRegion() throws IOException {
     try {
       setup();
       locks();
@@ -63,13 +65,10 @@
       splitAndMerge();
       read();
       cleanup();
-      
-    } catch(Exception e) {
+    } finally {
       if(cluster != null) {
         cluster.shutdown();
       }
-      e.printStackTrace();
-      fail();
     }
   }
   
@@ -674,7 +673,7 @@
             anchorFetched++;
             
           } else {
-            System.out.println(col);
+            System.out.println("UNEXPECTED COLUMN " + col);
           }
         }
         curVals.clear();

Added: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java?view=auto&rev=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java (added)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestHStoreFile.java Wed Aug  8 13:30:13 2007
@@ -0,0 +1,382 @@
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+
+public class TestHStoreFile extends TestCase {
+  static final Log LOG = LogFactory.getLog(TestHStoreFile.class);
+  private static String DIR = System.getProperty("test.build.data", ".");
+  private static final char FIRST_CHAR = 'a';
+  private static final char LAST_CHAR = 'z';
+  private FileSystem fs;
+  private Configuration conf;
+  private Path dir = null;
+  
+  @Override
+  protected void setUp() throws Exception {
+    super.setUp();
+    this.conf = new HBaseConfiguration();
+    this.fs = FileSystem.getLocal(this.conf);
+    this.dir = new Path(DIR, getName());
+  }
+  
+  @Override
+  protected void tearDown() throws Exception {
+    if (this.fs.exists(this.dir)) {
+      this.fs.delete(this.dir);
+    }
+    super.tearDown();
+  }
+  
+  private Path writeMapFile(final String name)
+  throws IOException {
+    Path path = new Path(DIR, name);
+    MapFile.Writer writer = new MapFile.Writer(this.conf, fs, path.toString(),
+      HStoreKey.class, ImmutableBytesWritable.class);
+    writeStoreFile(writer);
+    return path;
+  }
+  
+  private Path writeSmallMapFile(final String name)
+  throws IOException {
+    Path path = new Path(DIR, name);
+    MapFile.Writer writer = new MapFile.Writer(this.conf, fs, path.toString(),
+      HStoreKey.class, ImmutableBytesWritable.class);
+    try {
+      for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {
+        byte[] b = new byte[] {(byte)d};
+        Text t = new Text(new String(b));
+        writer.append(new HStoreKey(t, t, System.currentTimeMillis()),
+            new ImmutableBytesWritable(t.getBytes()));
+      }
+    } finally {
+      writer.close();
+    }
+    return path;
+  }
+  
+  /*
+   * Writes HStoreKey and ImmutableBytes data to passed writer and
+   * then closes it.
+   * @param writer
+   * @throws IOException
+   */
+  private void writeStoreFile(final MapFile.Writer writer)
+  throws IOException {
+    try {
+      for (char d = FIRST_CHAR; d <= LAST_CHAR; d++) {
+        for (char e = FIRST_CHAR; e <= LAST_CHAR; e++) {
+          byte[] b = new byte[] { (byte) d, (byte) e };
+          Text t = new Text(new String(b));
+          writer.append(new HStoreKey(t, t, System.currentTimeMillis()),
+            new ImmutableBytesWritable(t.getBytes()));
+        }
+      }
+    } finally {
+      writer.close();
+    }
+  }
+  
+  /**
+   * Test that our mechanism of writing store files in one region to reference
+   * store files in other regions works.
+   * @throws IOException
+   */
+  public void testReference()
+  throws IOException {
+    // Make a store file and write data to it.
+    HStoreFile hsf = new HStoreFile(this.conf, this.dir, new Text(getName()),
+        new Text("colfamily"), 1234567890L);
+    MapFile.Writer writer =
+      hsf.getWriter(this.fs, SequenceFile.CompressionType.NONE, null);
+    writeStoreFile(writer);
+    MapFile.Reader reader = hsf.getReader(this.fs, null);
+    // Split on a row, not in middle of row.  Midkey returned by reader
+    // may be in middle of row.  Create new one with empty column and
+    // timestamp.
+    HStoreKey midkey = new HStoreKey(((HStoreKey)reader.midKey()).getRow());
+    HStoreKey hsk = new HStoreKey();
+    reader.finalKey(hsk);
+    Text finalKey = hsk.getRow();
+    // Make a reference for the bottom half of the just written file.
+    HStoreFile.Reference reference =
+      new HStoreFile.Reference(hsf.getRegionName(), hsf.getFileId(), midkey,
+          HStoreFile.Range.top);
+    HStoreFile refHsf = new HStoreFile(this.conf, new Path(DIR, getName()),
+        new Text(getName() + "_reference"), hsf.getColFamily(), 456,
+        reference);
+    // Assert that reference files are written and that we can write and
+    // read the info reference file at least.
+    refHsf.writeReferenceFiles(this.fs);
+    assertTrue(this.fs.exists(refHsf.getMapFilePath()));
+    assertTrue(this.fs.exists(refHsf.getInfoFilePath()));
+    HStoreFile.Reference otherReference =
+      HStoreFile.readSplitInfo(refHsf.getInfoFilePath(), this.fs);
+    assertEquals(reference.getRegionName().toString(),
+        otherReference.getRegionName().toString());
+    assertEquals(reference.getFileId(),
+        otherReference.getFileId());
+    assertEquals(reference.getMidkey().toString(),
+        otherReference.getMidkey().toString());
+    // Now confirm that I can read from the reference and that it only gets
+    // keys from top half of the file.
+    MapFile.Reader halfReader = refHsf.getReader(this.fs, null);
+    HStoreKey key = new HStoreKey();
+    ImmutableBytesWritable value = new ImmutableBytesWritable();
+    boolean first = true;
+    while(halfReader.next(key, value)) {
+      if (first) {
+        assertEquals(key.getRow().toString(), midkey.getRow().toString());
+        first = false;
+      }
+    }
+    assertEquals(key.getRow().toString(), finalKey.toString());
+  }
+
+  /**
+   * Write a file and then assert that we can read from top and bottom halves
+   * using two HalfMapFiles.
+   * @throws Exception
+   */
+  public void testBasicHalfMapFile() throws Exception {
+    Path p = writeMapFile(getName());
+    WritableComparable midkey = getMidkey(p);
+    checkHalfMapFile(p, midkey);
+  }
+  
+  /**
+   * Check HalfMapFile works even if file we're to go against is smaller than
+   * the default MapFile interval of 128: i.e. index gets entry every 128 
+   * keys.
+   * @throws Exception
+   */
+  public void testSmallHalfMapFile() throws Exception {
+    Path p = writeSmallMapFile(getName());
+    // I know keys are a-z.  Let the midkey we want to use be 'd'.  See if
+    // HalfMapFiles work even if size of file is < than default MapFile
+    // interval.
+    checkHalfMapFile(p, new HStoreKey(new Text("d")));
+  }
+  
+  private WritableComparable getMidkey(final Path p) throws IOException {
+    MapFile.Reader reader =
+      new MapFile.Reader(this.fs, p.toString(), this.conf);
+    HStoreKey key = new HStoreKey();
+    ImmutableBytesWritable value = new ImmutableBytesWritable();
+    reader.next(key, value);
+    String firstKey = key.toString();
+    WritableComparable midkey = reader.midKey();
+    reader.finalKey(key);
+    LOG.info("First key " + firstKey + ", midkey " + midkey.toString()
+        + ", last key " + key.toString());
+    reader.close();
+    return midkey;
+  }
+  
+  private void checkHalfMapFile(final Path p, WritableComparable midkey)
+  throws IOException {
+    MapFile.Reader top = null;
+    MapFile.Reader bottom = null;
+    HStoreKey key = new HStoreKey();
+    ImmutableBytesWritable value = new ImmutableBytesWritable();
+    String previous = null;
+    try {
+      // Now make two HalfMapFiles and assert they can read the full backing
+      // file, one from the top and the other from the bottom.
+      // Test bottom half first.
+      bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+          this.conf, HStoreFile.Range.bottom, midkey);
+      boolean first = true;
+      while (bottom.next(key, value)) {
+        previous = key.toString();
+        if (first) {
+          first = false;
+          LOG.info("First in bottom: " + previous);
+        }
+        assertTrue(key.compareTo(midkey) < 0);
+      }
+      LOG.info("Last in bottom: " + previous.toString());
+      // Now test reading from the top.
+      top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(), this.conf,
+          HStoreFile.Range.top, midkey);
+      first = true;
+      while (top.next(key, value)) {
+        assertTrue(key.compareTo(midkey) >= 0);
+        if (first) {
+          first = false;
+          assertEquals(((HStoreKey)midkey).getRow().toString(),
+            key.getRow().toString());
+          LOG.info("First in top: " + key.toString());
+        }
+      }
+      LOG.info("Last in top: " + key.toString());
+      top.getClosest(midkey, value);
+      // Assert value is same as key.
+      assertEquals(new String(value.get()),
+        ((HStoreKey) midkey).getRow().toString());
+
+      // Next test using a midkey that does not exist in the file.
+      // First, do a key that is < than first key. Ensure splits behave
+      // properly.
+      midkey = new HStoreKey(new Text("   "));
+      bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+          this.conf, HStoreFile.Range.bottom, midkey);
+      // When midkey is < than the bottom, should return no values.
+      assertFalse(bottom.next(key, value));
+      // Now read from the top.
+      top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(), this.conf,
+          HStoreFile.Range.top, midkey);
+      first = true;
+      while (top.next(key, value)) {
+        assertTrue(key.compareTo(midkey) >= 0);
+        if (first) {
+          first = false;
+          LOG.info("First top when key < bottom: " + key.toString());
+          String tmp = key.getRow().toString();
+          for (int i = 0; i < tmp.length(); i++) {
+            assertTrue(tmp.charAt(i) == 'a');
+          }
+        }
+      }
+      LOG.info("Last top when key < bottom: " + key.toString());
+      String tmp = key.getRow().toString();
+      for (int i = 0; i < tmp.length(); i++) {
+        assertTrue(tmp.charAt(i) == 'z');
+      }
+
+      // Test when midkey is > than last key in file ('||' > 'zz').
+      midkey = new HStoreKey(new Text("|||"));
+      bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+          this.conf, HStoreFile.Range.bottom, midkey);
+      first = true;
+      while (bottom.next(key, value)) {
+        if (first) {
+          first = false;
+          LOG.info("First bottom when key > top: " + key.toString());
+          tmp = key.getRow().toString();
+          for (int i = 0; i < tmp.length(); i++) {
+            assertTrue(tmp.charAt(i) == 'a');
+          }
+        }
+      }
+      LOG.info("Last bottom when key > top: " + key.toString());
+      tmp = key.getRow().toString();
+      for (int i = 0; i < tmp.length(); i++) {
+        assertTrue(tmp.charAt(i) == 'z');
+      }
+      // Now look at top. Should not return any values.
+      top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(), this.conf,
+          HStoreFile.Range.top, midkey);
+      assertFalse(top.next(key, value));
+      
+    } finally {
+      if (top != null) {
+        top.close();
+      }
+      if (bottom != null) {
+        bottom.close();
+      }
+      fs.delete(p);
+    }
+  }
+  
+  /**
+   * Assert HalFMapFile does right thing when midkey does not exist in the
+   * backing file (its larger or smaller than any of the backing mapfiles keys).
+   * 
+   * @throws Exception
+   */
+  public void testOutOfRangeMidkeyHalfMapFile() throws Exception {
+    MapFile.Reader top = null;
+    MapFile.Reader bottom = null;
+    HStoreKey key = new HStoreKey();
+    ImmutableBytesWritable value = new ImmutableBytesWritable();
+    Path p = writeMapFile(getName());
+    try {
+      try {
+        // Test using a midkey that does not exist in the file.
+        // First, do a key that is < than first key.  Ensure splits behave
+        // properly.
+        HStoreKey midkey = new HStoreKey(new Text("   "));
+        bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+          this.conf, HStoreFile.Range.bottom, midkey);
+        // When midkey is < than the bottom, should return no values.
+        assertFalse(bottom.next(key, value));
+        // Now read from the top.
+        top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+          this.conf, HStoreFile.Range.top, midkey);
+        boolean first = true;
+        while (top.next(key, value)) {
+          assertTrue(key.compareTo(midkey) >= 0);
+          if (first) {
+            first = false;
+            LOG.info("First top when key < bottom: " + key.toString());
+            assertEquals("aa", key.getRow().toString());
+          }
+        }
+        LOG.info("Last top when key < bottom: " + key.toString());
+        assertEquals("zz", key.getRow().toString());
+        
+        // Test when midkey is > than last key in file ('||' > 'zz').
+        midkey = new HStoreKey(new Text("|||"));
+        bottom = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+          this.conf, HStoreFile.Range.bottom, midkey);
+        first = true;
+        while (bottom.next(key, value)) {
+          if (first) {
+            first = false;
+            LOG.info("First bottom when key > top: " + key.toString());
+            assertEquals("aa", key.getRow().toString());
+          }
+        }
+        LOG.info("Last bottom when key > top: " + key.toString());
+        assertEquals("zz", key.getRow().toString());
+        // Now look at top.  Should not return any values.
+        top = new HStoreFile.HalfMapFileReader(this.fs, p.toString(),
+          this.conf, HStoreFile.Range.top, midkey);
+        assertFalse(top.next(key, value));
+      } finally {
+        if (top != null) {
+          top.close();
+        }
+        if (bottom != null) {
+          bottom.close();
+        }
+        fs.delete(p);
+      }
+    } finally {
+      this.fs.delete(p);
+    }
+  }
+}
\ No newline at end of file

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestRegionServerAbort.java Wed Aug  8 13:30:13 2007
@@ -77,10 +77,9 @@
     // Now shutdown the region server and wait for it to go down.
     this.cluster.abortRegionServer(0);
     this.cluster.waitOnRegionServer(0);
-
+    
     // Verify that the client can find the data after the region has been moved
     // to a different server
-
     HScannerInterface scanner =
       table.obtainScanner(HConstants.COLUMN_FAMILY_ARRAY, new Text());
 

Modified: lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner.java?view=diff&rev=564012&r1=564011&r2=564012
==============================================================================
--- lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner.java (original)
+++ lucene/hadoop/trunk/src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestScanner.java Wed Aug  8 13:30:13 2007
@@ -146,7 +146,7 @@
       Path dir = new Path("/hbase");
       fs.mkdirs(dir);
       
-      Path regionDir = HStoreFile.getHRegionDir(dir, REGION_INFO.regionName);
+      Path regionDir = HRegion.getRegionDir(dir, REGION_INFO.regionName);
       fs.mkdirs(regionDir);
       
       HLog log = new HLog(fs, new Path(regionDir, "log"), conf);



Mime
View raw message