Return-Path: 
 <hbase-commits-return-1464-apmail-hadoop-hbase-commits-archive=hadoop.apache.org@hadoop.apache.org>
Delivered-To: apmail-hadoop-hbase-commits-archive@minotaur.apache.org
Received: (qmail 77995 invoked from network); 7 Jun 2009 19:58:27 -0000
Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3)
  by minotaur.apache.org with SMTP; 7 Jun 2009 19:58:27 -0000
Received: (qmail 96795 invoked by uid 500); 7 Jun 2009 19:58:39 -0000
Delivered-To: apmail-hadoop-hbase-commits-archive@hadoop.apache.org
Received: (qmail 96710 invoked by uid 500); 7 Jun 2009 19:58:39 -0000
Mailing-List: contact hbase-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:hbase-commits-help@hadoop.apache.org>
List-Unsubscribe: <mailto:hbase-commits-unsubscribe@hadoop.apache.org>
List-Post: <mailto:hbase-commits@hadoop.apache.org>
List-Id: <hbase-commits.hadoop.apache.org>
Reply-To: hbase-dev@hadoop.apache.org
Delivered-To: mailing list hbase-commits@hadoop.apache.org
Received: (qmail 96681 invoked by uid 99); 7 Jun 2009 19:58:37 -0000
Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136)
    by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 07 Jun 2009 19:58:37 +0000
X-ASF-Spam-Status: No, hits=-2000.0 required=10.0
	tests=ALL_TRUSTED
X-Spam-Check-By: apache.org
Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4)
    by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 07 Jun 2009 19:58:35 +0000
Received: by eris.apache.org (Postfix, from userid 65534)
	id 4EF612388901; Sun,  7 Jun 2009 19:57:51 +0000 (UTC)
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r782445 [6/17] - in /hadoop/hbase/trunk_on_hadoop-0.18.3:
 ./
 bin/ src/java/org/apache/hadoop/hbase/
 src/java/org/apache/hadoop/hbase/client/
 src/java/org/apache/hadoop/hbase/client/tableindexed/
 src/java/org/apache/hadoop/hbase/client/trans...
Date: Sun, 07 Jun 2009 19:57:43 -0000
To: hbase-commits@hadoop.apache.org
From: apurtell@apache.org
X-Mailer: svnmailer-1.0.8
Message-Id: <20090607195751.4EF612388901@eris.apache.org>
X-Virus-Checked: Checked by ClamAV on apache.org

Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java?rev=782445&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ColumnTracker.java Sun Jun  7 19:57:37 2009
@@ -0,0 +1,78 @@
+/**
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
+
+/**
+ * Implementing classes of this interface will be used for the tracking
+ * and enforcement of columns and numbers of versions during the course of a 
+ * Get or Scan operation.
+ * <p>
+ * Currently there are two different types of Store/Family-level queries.
+ * <ul><li>{@link ExplicitColumnTracker} is used when the query specifies
+ * one or more column qualifiers to return in the family.
+ * <li>{@link WildcardColumnTracker} is used when the query asks for all
+ * qualifiers within the family.
+ * <p>
+ * This class is utilized by {@link QueryMatcher} through two methods:
+ * <ul><li>{@link checkColumn} is called when a Put satisfies all other
+ * conditions of the query.  This method returns a {@link MatchCode} to define
+ * what action should be taken.
+ * <li>{@link update} is called at the end of every StoreFile or Memcache.
+ * <p>
+ * This class is NOT thread-safe as queries are never multi-threaded 
+ */
+public interface ColumnTracker {
+  /**
+   * Keeps track of the number of versions for the columns asked for
+   * @param bytes
+   * @param offset
+   * @param length
+   * @return
+   */
+  public MatchCode checkColumn(byte [] bytes, int offset, int length);
+  /**
+   * Updates internal variables in between files
+   */
+  public void update();
+  /**
+   * Resets the Matcher
+   */
+  public void reset();
+  /**
+   * 
+   * @return
+   */
+  public boolean done();
+
+  /**
+   * Used by matcher and scan/get to get a hint of the next column
+   * to seek to after checkColumn() returns SKIP.  Returns the next interesting
+   * column we want, or NULL there is none (wildcard scanner).
+   *
+   * Implementations aren't required to return anything useful unless the most recent
+   * call was to checkColumn() and the return code was SKIP.  This is pretty implementation
+   * detail-y, but optimizations are like that.
+   *
+   * @return null, or a ColumnCount that we should seek to
+   */
+  public ColumnCount getColumnHint();
+}

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=782445&r1=782444&r2=782445&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Sun Jun  7 19:57:37 2009
@@ -30,11 +30,11 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.io.BatchUpdate;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.Writables;
 
@@ -114,14 +114,14 @@
         continue;
       } catch (IOException ex) {
         LOG.error("Compaction/Split failed" +
-            (r != null ? (" for region " + Bytes.toString(r.getRegionName())) : ""),
+            (r != null ? (" for region " + r.getRegionNameAsString()) : ""),
             RemoteExceptionHandler.checkIOException(ex));
         if (!server.checkFileSystem()) {
           break;
         }
       } catch (Exception ex) {
         LOG.error("Compaction failed" +
-            (r != null ? (" for region " + Bytes.toString(r.getRegionName())) : ""),
+            (r != null ? (" for region " + r.getRegionNameAsString()) : ""),
             ex);
         if (!server.checkFileSystem()) {
           break;
@@ -155,7 +155,7 @@
     r.setForceMajorCompaction(force);
     if (LOG.isDebugEnabled()) {
       LOG.debug("Compaction " + (force? "(major) ": "") +
-        "requested for region " + Bytes.toString(r.getRegionName()) +
+        "requested for region " + r.getRegionNameAsString() +
         "/" + r.getRegionInfo().getEncodedName() +
         (why != null && !why.isEmpty()? " because: " + why: ""));
     }
@@ -202,18 +202,21 @@
     // Inform the HRegionServer that the parent HRegion is no-longer online.
     this.server.removeFromOnlineRegions(oldRegionInfo);
     
-    BatchUpdate update = new BatchUpdate(oldRegionInfo.getRegionName());
-    update.put(COL_REGIONINFO, Writables.getBytes(oldRegionInfo));
-    update.put(COL_SPLITA, Writables.getBytes(newRegions[0].getRegionInfo()));
-    update.put(COL_SPLITB, Writables.getBytes(newRegions[1].getRegionInfo()));
-    t.commit(update);
+    Put put = new Put(oldRegionInfo.getRegionName());
+    put.add(CATALOG_FAMILY, REGIONINFO_QUALIFIER, 
+        Writables.getBytes(oldRegionInfo));
+    put.add(CATALOG_FAMILY, SPLITA_QUALIFIER,
+        Writables.getBytes(newRegions[0].getRegionInfo()));
+    put.add(CATALOG_FAMILY, SPLITB_QUALIFIER,
+        Writables.getBytes(newRegions[0].getRegionInfo()));
+    t.put(put);
     
     // Add new regions to META
     for (int i = 0; i < newRegions.length; i++) {
-      update = new BatchUpdate(newRegions[i].getRegionName());
-      update.put(COL_REGIONINFO, Writables.getBytes(
+      put = new Put(newRegions[i].getRegionName());
+      put.add(CATALOG_FAMILY, REGIONINFO_QUALIFIER, Writables.getBytes(
         newRegions[i].getRegionInfo()));
-      t.commit(update);
+      t.put(put);
     }
         
     // Now tell the master about the new regions

Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java?rev=782445&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/DeleteCompare.java Sun Jun  7 19:57:37 2009
@@ -0,0 +1,120 @@
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+
+
+/**
+ * Class that provides static method needed when putting deletes into memcache 
+ */
+public class DeleteCompare {
+  
+  /**
+   * Return codes from deleteCompare.
+   */
+  enum DeleteCode {
+    /**
+     * Do nothing.  Move to next KV in Memcache
+     */
+    SKIP,
+    
+    /**
+     * Add to the list of deletes.
+     */
+    DELETE,
+    
+    /**
+     * Stop looking at KVs in Memcache.  Finalize.
+     */
+    DONE
+  }
+
+  /**
+   * Method used when putting deletes into memcache to remove all the previous
+   * entries that are affected by this Delete
+   * @param mem
+   * @param deleteBuffer
+   * @param deleteRowOffset
+   * @param deleteRowLength
+   * @param deleteQualifierOffset
+   * @param deleteQualifierLength
+   * @param deleteTimeOffset
+   * @param deleteType
+   * @param comparator
+   * @return SKIP if current KeyValue should not be deleted, DELETE if
+   * current KeyValue should be deleted and DONE when the current KeyValue is
+   * out of the Deletes range
+   */
+  public static DeleteCode deleteCompare(KeyValue mem, byte [] deleteBuffer,
+      int deleteRowOffset, short deleteRowLength, int deleteQualifierOffset,
+      int deleteQualifierLength, int deleteTimeOffset, byte deleteType,
+      KeyValue.KeyComparator comparator) {
+
+    //Parsing new KeyValue
+    byte [] memBuffer = mem.getBuffer();
+    int memOffset = mem.getOffset();
+
+    //Getting key lengths
+    int memKeyLen = Bytes.toInt(memBuffer, memOffset);
+    memOffset += Bytes.SIZEOF_INT;
+
+    //Skipping value lengths
+    memOffset += Bytes.SIZEOF_INT;
+
+    //Getting row lengths
+    short memRowLen = Bytes.toShort(memBuffer, memOffset);
+    memOffset += Bytes.SIZEOF_SHORT;
+    int res = comparator.compareRows(memBuffer, memOffset, memRowLen,
+        deleteBuffer, deleteRowOffset, deleteRowLength);
+    if(res > 0) {
+      return DeleteCode.DONE;
+    } else if(res < 0){
+      System.out.println("SKIPPING ROW");
+      return DeleteCode.SKIP;
+    }
+
+    memOffset += memRowLen;
+
+    //Getting family lengths
+    byte memFamLen = memBuffer[memOffset];
+    memOffset += Bytes.SIZEOF_BYTE + memFamLen;
+
+    //Get column lengths
+    int memQualifierLen = memKeyLen - memRowLen - memFamLen -
+      Bytes.SIZEOF_SHORT - Bytes.SIZEOF_BYTE - Bytes.SIZEOF_LONG -
+      Bytes.SIZEOF_BYTE;
+
+    //Compare timestamp
+    int tsOffset = memOffset + memQualifierLen;
+    int timeRes = Bytes.compareTo(memBuffer, tsOffset, Bytes.SIZEOF_LONG,
+        deleteBuffer, deleteTimeOffset, Bytes.SIZEOF_LONG);
+
+    if(deleteType == KeyValue.Type.DeleteFamily.getCode()) {
+      if(timeRes <= 0){
+        return DeleteCode.DELETE;
+      }
+      return DeleteCode.SKIP;
+    }
+
+    //Compare columns
+    res = Bytes.compareTo(memBuffer, memOffset, memQualifierLen,
+        deleteBuffer, deleteQualifierOffset, deleteQualifierLength);
+    if(res < 0) {
+      return DeleteCode.SKIP;
+    } else if(res > 0) {
+      return DeleteCode.DONE;
+    }
+    // same column, compare the time.
+    if(timeRes == 0) {
+      return DeleteCode.DELETE;
+    } else if (timeRes < 0) {
+      if(deleteType == KeyValue.Type.DeleteColumn.getCode()) {
+        return DeleteCode.DELETE;
+      }
+      return DeleteCode.DONE;
+    } else {
+      System.out.println("SKIPPING TS");
+      return DeleteCode.SKIP;
+    }
+  } 
+}

Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java?rev=782445&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/DeleteTracker.java Sun Jun  7 19:57:37 2009
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+/**
+ * This interface is used for the tracking and enforcement of Deletes
+ * during the course of a Get or Scan operation.
+ * <p>
+ * This class is utilized through three methods:
+ * <ul><li>{@link add} when encountering a Delete
+ * <li>{@link isDeleted} when checking if a Put KeyValue has been deleted
+ * <li>{@link update} when reaching the end of a StoreFile 
+ */
+public interface DeleteTracker {
+  
+  /**
+   * Add the specified KeyValue to the list of deletes to check against for
+   * this row operation.
+   * <p>
+   * This is called when a Delete is encountered in a StoreFile.
+   * @param buffer KeyValue buffer
+   * @param qualifierOffset column qualifier offset
+   * @param qualifierLength column qualifier length
+   * @param timestamp timestamp
+   * @param type delete type as byte
+   */
+  public void add(byte [] buffer, int qualifierOffset, int qualifierLength,
+      long timestamp, byte type);
+  
+  /**
+   * Check if the specified KeyValue buffer has been deleted by a previously
+   * seen delete.
+   * @param buffer KeyValue buffer
+   * @param qualifierOffset column qualifier offset
+   * @param qualifierLength column qualifier length
+   * @param timestamp timestamp
+   * @return true is the specified KeyValue is deleted, false if not
+   */
+  public boolean isDeleted(byte [] buffer, int qualifierOffset,
+      int qualifierLength, long timestamp);
+  
+  /**
+   * @return true if there are no current delete, false otherwise
+   */
+  public boolean isEmpty();
+  
+  /**
+   * Called at the end of every StoreFile.
+   * <p>
+   * Many optimized implementations of Trackers will require an update at
+   * when the end of each StoreFile is reached.
+   */
+  public void update();
+  
+  /**
+   * Called between rows.
+   * <p>
+   * This clears everything as if a new DeleteTracker was instantiated.
+   */
+  public void reset();
+  
+
+  /**
+   * Return codes for comparison of two Deletes.
+   * <p>
+   * The codes tell the merging function what to do.
+   * <p>
+   * INCLUDE means add the specified Delete to the merged list.
+   * NEXT means move to the next element in the specified list(s).
+   */
+  enum DeleteCompare { 
+    INCLUDE_OLD_NEXT_OLD,
+    INCLUDE_OLD_NEXT_BOTH,
+    INCLUDE_NEW_NEXT_NEW,
+    INCLUDE_NEW_NEXT_BOTH,
+    NEXT_OLD,
+    NEXT_NEW
+  }
+  
+}

Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java?rev=782445&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java Sun Jun  7 19:57:37 2009
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NavigableSet;
+import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * This class is used for the tracking and enforcement of columns and numbers 
+ * of versions during the course of a Get or Scan operation, when explicit
+ * column qualifiers have been asked for in the query.
+ *
+ * With a little magic (see {@link ScanQueryMatcher}), we can use this matcher
+ * for both scans and gets.  The main difference is 'next' and 'done' collapse
+ * for the scan case (since we see all columns in order), and we only reset
+ * between rows.
+ * 
+ * <p>
+ * This class is utilized by {@link QueryMatcher} through two methods:
+ * <ul><li>{@link checkColumn} is called when a Put satisfies all other
+ * conditions of the query.  This method returns a {@link MatchCode} to define
+ * what action should be taken.
+ * <li>{@link update} is called at the end of every StoreFile or Memcache.
+ * <p>
+ * This class is NOT thread-safe as queries are never multi-threaded 
+ */
+public class ExplicitColumnTracker implements ColumnTracker {
+
+  private int maxVersions;
+  private List<ColumnCount> columns;
+  private int index;
+  private ColumnCount column;
+  private NavigableSet<byte[]> origColumns;
+  
+  /**
+   * Default constructor.
+   * @param columns columns specified user in query
+   * @param maxVersions maximum versions to return per column
+   */
+  public ExplicitColumnTracker(NavigableSet<byte[]> columns, int maxVersions) {
+    this.maxVersions = maxVersions;
+    this.origColumns = columns;
+    reset();
+  }
+  
+  /**
+   * Done when there are no more columns to match against.
+   */
+  public boolean done() {
+    return this.columns.size() == 0;
+  }
+
+  public ColumnCount getColumnHint() {
+    return this.column;
+  }
+  
+  /**
+   * Checks against the parameters of the query and the columns which have
+   * already been processed by this query.
+   * @param bytes KeyValue buffer
+   * @param offset offset to the start of the qualifier
+   * @param length length of the qualifier
+   * @return MatchCode telling QueryMatcher what action to take
+   */
+  public MatchCode checkColumn(byte [] bytes, int offset, int length) {
+    // No more columns left, we are done with this query
+    if(this.columns.size() == 0) {
+      return MatchCode.DONE; // done_row
+    }
+    
+    // No more columns to match against, done with storefile
+    if(this.column == null) {
+      return MatchCode.NEXT; // done_row
+    }
+    
+    // Compare specific column to current column
+    int ret = Bytes.compareTo(column.getBuffer(), column.getOffset(), 
+        column.getLength(), bytes, offset, length);
+    
+    // Matches, decrement versions left and include
+    if(ret == 0) {
+      if(this.column.decrement() == 0) {
+        // Done with versions for this column
+        this.columns.remove(this.index);
+        if(this.columns.size() == this.index) {
+          // Will not hit any more columns in this storefile
+          this.column = null;
+        } else {
+          this.column = this.columns.get(this.index);
+        }
+      }
+      return MatchCode.INCLUDE;
+    }
+
+    // Specified column is bigger than current column
+    // Move down current column and check again
+    if(ret <= -1) {
+      if(++this.index == this.columns.size()) {
+        // No more to match, do not include, done with storefile
+        return MatchCode.NEXT; // done_row
+      }
+      this.column = this.columns.get(this.index);
+      return checkColumn(bytes, offset, length);
+    }
+
+    // Specified column is smaller than current column
+    // Skip
+    return MatchCode.SKIP; // skip to next column, with hint?
+  }
+  
+  /**
+   * Called at the end of every StoreFile or Memcache.
+   */
+  public void update() {
+    if(this.columns.size() != 0) {
+      this.index = 0;
+      this.column = this.columns.get(this.index);
+    } else {
+      this.index = -1;
+      this.column = null;
+    }
+  }
+
+  // Called between every row.
+  public void reset() {
+    buildColumnList(this.origColumns);
+    this.index = 0;
+    this.column = this.columns.get(this.index);
+  }
+
+  private void buildColumnList(NavigableSet<byte[]> columns) {
+    this.columns = new ArrayList<ColumnCount>(columns.size());
+    for(byte [] column : columns) {
+      this.columns.add(new ColumnCount(column,maxVersions));
+    }
+  }
+}

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/FailedLogCloseException.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/FailedLogCloseException.java?rev=782445&r1=782444&r2=782445&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/FailedLogCloseException.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/FailedLogCloseException.java Sun Jun  7 19:57:37 2009
@@ -28,10 +28,16 @@
 class FailedLogCloseException extends IOException {
   private static final long serialVersionUID = 1759152841462990925L;
 
+  /**
+   * 
+   */
   public FailedLogCloseException() {
     super();
   }
 
+  /**
+   * @param arg0
+   */
   public FailedLogCloseException(String arg0) {
     super(arg0);
   }

Added: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java?rev=782445&view=auto
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java (added)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/GetDeleteTracker.java Sun Jun  7 19:57:37 2009
@@ -0,0 +1,405 @@
+/*
+ * Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * This class is responsible for the tracking and enforcement of Deletes
+ * during the course of a Get operation.
+ * <p>
+ * This class is utilized through three methods:
+ * <ul><li>{@link add} when encountering a Delete
+ * <li>{@link isDeleted} when checking if a Put KeyValue has been deleted
+ * <li>{@link update} when reaching the end of a StoreFile
+ * <p>
+ * This class is NOT thread-safe as queries are never multi-threaded 
+ */
+public class GetDeleteTracker implements DeleteTracker {
+
+  private long familyStamp = -1L;
+  protected List<Delete> deletes = null;
+  private List<Delete> newDeletes = new ArrayList<Delete>();
+  private Iterator<Delete> iterator;
+  private Delete delete = null;
+
+  private KeyValue.KeyComparator comparator;
+
+  /**
+   * Constructor
+   * @param comparator
+   */
+  public GetDeleteTracker(KeyValue.KeyComparator comparator) {
+    this.comparator = comparator;
+  }
+
+  /**
+   * Add the specified KeyValue to the list of deletes to check against for
+   * this row operation.
+   * <p>
+   * This is called when a Delete is encountered in a StoreFile.
+   * @param kv
+   * @param type
+   * @param timestamp
+   */
+  @Override
+  public void add(byte [] buffer, int qualifierOffset, int qualifierLength,
+      long timestamp, byte type) {
+    if(type == KeyValue.Type.DeleteFamily.getCode()) {
+      if(timestamp > familyStamp) {
+        familyStamp = timestamp;
+      }
+      return;
+    }
+    if(timestamp > familyStamp) {
+      this.newDeletes.add(new Delete(buffer, qualifierOffset, qualifierLength,
+          type, timestamp));
+    }
+  }
+
+  /** 
+   * Check if the specified KeyValue buffer has been deleted by a previously
+   * seen delete.
+   * @param buffer KeyValue buffer
+   * @param qualifierOffset column qualifier offset
+   * @param qualifierLength column qualifier length
+   * @param timestamp timestamp
+   * @return true is the specified KeyValue is deleted, false if not
+   */
+  @Override
+  public boolean isDeleted(byte [] buffer, int qualifierOffset,
+      int qualifierLength, long timestamp) {
+
+    // Check against DeleteFamily
+    if(timestamp <= familyStamp) {
+      return true;
+    }
+
+    // Check if there are other deletes
+    if(this.delete == null) {
+      return false;
+    }
+
+    // Check column
+    int ret = comparator.compareRows(buffer, qualifierOffset, qualifierLength,
+        this.delete.buffer, this.delete.qualifierOffset, 
+        this.delete.qualifierLength);
+    if(ret <= -1) {
+      // Have not reached the next delete yet
+      return false;
+    } else if(ret >= 1) {
+      // Deletes an earlier column, need to move down deletes
+      if(this.iterator.hasNext()) {
+        this.delete = this.iterator.next();
+      } else {
+        this.delete = null;
+        return false;
+      }
+      return isDeleted(buffer, qualifierOffset, qualifierLength, timestamp);
+    }
+
+    // Check Timestamp
+    if(timestamp > this.delete.timestamp) {
+      return false;
+    }
+
+    // Check Type
+    switch(KeyValue.Type.codeToType(this.delete.type)) {
+    case Delete:
+      boolean equal = timestamp == this.delete.timestamp;
+
+      if(this.iterator.hasNext()) {
+        this.delete = this.iterator.next();
+      } else {
+        this.delete = null;
+      }
+
+      if(equal){
+        return true;
+      }
+      // timestamp < this.delete.timestamp
+      // Delete of an explicit column newer than current
+      return isDeleted(buffer, qualifierOffset, qualifierLength, timestamp);
+    case DeleteColumn:
+      return true;
+    }
+
+    // should never reach this
+    return false;
+  }
+
+  @Override
+  public boolean isEmpty() {
+    if(this.familyStamp == 0L && this.delete == null) {
+      return true;
+    }
+    return false;
+  }
+
+  @Override
+  public void reset() {
+    this.deletes = null;
+    this.delete = null;
+    this.newDeletes = new ArrayList<Delete>();
+    this.familyStamp = 0L;
+    this.iterator = null;
+  }
+
+  /**
+   * Called at the end of every StoreFile.
+   * <p>
+   * Many optimized implementations of Trackers will require an update at
+   * when the end of each StoreFile is reached.
+   */
+  @Override
+  public void update() {
+    // If no previous deletes, use new deletes and return
+    if(this.deletes == null || this.deletes.size() == 0) {
+      finalize(this.newDeletes);
+      return;
+    }
+
+    // If no new delete, retain previous deletes and return
+    if(this.newDeletes.size() == 0) {
+      return;
+    }
+
+    // Merge previous deletes with new deletes
+    List<Delete> mergeDeletes = 
+      new ArrayList<Delete>(this.newDeletes.size());
+    int oldIndex = 0;
+    int newIndex = 0;
+
+    Delete newDelete = newDeletes.get(oldIndex);
+    Delete oldDelete = deletes.get(oldIndex);
+    while(true) {
+      switch(compareDeletes(oldDelete,newDelete)) {
+      case NEXT_NEW: {
+        if(++newIndex == newDeletes.size()) {
+          // Done with new, add the rest of old to merged and return
+          mergeDown(mergeDeletes, deletes, oldIndex);
+          finalize(mergeDeletes);
+          return;
+        }
+        newDelete = this.newDeletes.get(newIndex);
+        break;
+      }
+
+      case INCLUDE_NEW_NEXT_NEW: {
+        mergeDeletes.add(newDelete);
+        if(++newIndex == newDeletes.size()) {
+          // Done with new, add the rest of old to merged and return
+          mergeDown(mergeDeletes, deletes, oldIndex);
+          finalize(mergeDeletes);
+          return;
+        }
+        newDelete = this.newDeletes.get(newIndex);
+        break;
+      }
+
+      case INCLUDE_NEW_NEXT_BOTH: {
+        mergeDeletes.add(newDelete);
+        ++oldIndex;
+        ++newIndex;
+        if(oldIndex == deletes.size()) {
+          if(newIndex == newDeletes.size()) {
+            finalize(mergeDeletes);
+            return;
+          }
+          mergeDown(mergeDeletes, newDeletes, newIndex);
+          finalize(mergeDeletes);
+          return;
+        } else if(newIndex == newDeletes.size()) {
+          mergeDown(mergeDeletes, deletes, oldIndex);
+          finalize(mergeDeletes);
+          return;
+        }
+        oldDelete = this.deletes.get(oldIndex);
+        newDelete = this.newDeletes.get(newIndex);
+        break;
+      }
+
+      case INCLUDE_OLD_NEXT_BOTH: {
+        mergeDeletes.add(oldDelete);
+        ++oldIndex;
+        ++newIndex;
+        if(oldIndex == deletes.size()) {
+          if(newIndex == newDeletes.size()) {
+            finalize(mergeDeletes);
+            return;
+          }
+          mergeDown(mergeDeletes, newDeletes, newIndex);
+          finalize(mergeDeletes);
+          return;
+        } else if(newIndex == newDeletes.size()) {
+          mergeDown(mergeDeletes, deletes, oldIndex);
+          finalize(mergeDeletes);
+          return;
+        }
+        oldDelete = this.deletes.get(oldIndex);
+        newDelete = this.newDeletes.get(newIndex);
+        break;
+      }
+
+      case INCLUDE_OLD_NEXT_OLD: {
+        mergeDeletes.add(oldDelete);
+        if(++oldIndex == deletes.size()) {
+          mergeDown(mergeDeletes, newDeletes, newIndex);
+          finalize(mergeDeletes);
+          return;
+        }
+        oldDelete = this.deletes.get(oldIndex);
+        break;
+      }
+
+      case NEXT_OLD: {
+        if(++oldIndex == deletes.size()) {
+          // Done with old, add the rest of new to merged and return
+          mergeDown(mergeDeletes, newDeletes, newIndex);
+          finalize(mergeDeletes);
+          return;
+        }
+        oldDelete = this.deletes.get(oldIndex);
+      }
+      }
+    }
+  }
+
+  private void finalize(List<Delete> mergeDeletes) {
+    this.deletes = mergeDeletes;
+    this.newDeletes = new ArrayList<Delete>();
+    if(this.deletes.size() > 0){
+      this.iterator = deletes.iterator();
+      this.delete = iterator.next();
+    }
+  }
+
+  private void mergeDown(List<Delete> mergeDeletes, List<Delete> srcDeletes, 
+      int srcIndex) {
+    int index = srcIndex;
+    while(index < srcDeletes.size()) {
+      mergeDeletes.add(srcDeletes.get(index++));
+    }
+  }
+
+
+  protected DeleteCompare compareDeletes(Delete oldDelete, Delete newDelete) {
+
+    // Compare columns
+    // Just compairing qualifier portion, can keep on using Bytes.compareTo().
+    int ret = Bytes.compareTo(oldDelete.buffer, oldDelete.qualifierOffset,
+        oldDelete.qualifierLength, newDelete.buffer, newDelete.qualifierOffset,
+        newDelete.qualifierLength);
+
+    if(ret <= -1) {
+      return DeleteCompare.INCLUDE_OLD_NEXT_OLD;
+    } else if(ret >= 1) {
+      return DeleteCompare.INCLUDE_NEW_NEXT_NEW;
+    }
+
+    // Same column
+
+    // Branches below can be optimized.  Keeping like this until testing
+    // is complete.
+    if(oldDelete.type == newDelete.type) {
+      // the one case where we can merge 2 deletes -> 1 delete.
+      if(oldDelete.type == KeyValue.Type.Delete.getCode()){
+        if(oldDelete.timestamp > newDelete.timestamp) {
+          return DeleteCompare.INCLUDE_OLD_NEXT_OLD;
+        } else if(oldDelete.timestamp < newDelete.timestamp) {
+          return DeleteCompare.INCLUDE_NEW_NEXT_NEW;
+        } else {
+          return DeleteCompare.INCLUDE_OLD_NEXT_BOTH;
+        }
+      }
+      if(oldDelete.timestamp < newDelete.timestamp) {
+        return DeleteCompare.INCLUDE_NEW_NEXT_BOTH;
+      } 
+      return DeleteCompare.INCLUDE_OLD_NEXT_BOTH;
+    }
+
+    // old delete is more specific than the new delete.
+    // if the olddelete is newer then the newdelete, we have to
+    //  keep it
+    if(oldDelete.type < newDelete.type) {
+      if(oldDelete.timestamp > newDelete.timestamp) {
+        return DeleteCompare.INCLUDE_OLD_NEXT_OLD;
+      } else if(oldDelete.timestamp < newDelete.timestamp) {
+        return DeleteCompare.NEXT_OLD;
+      } else {
+        return DeleteCompare.NEXT_OLD;
+      }
+    }
+
+    // new delete is more specific than the old delete.
+    if(oldDelete.type > newDelete.type) {
+      if(oldDelete.timestamp > newDelete.timestamp) {
+        return DeleteCompare.NEXT_NEW;
+      } else if(oldDelete.timestamp < newDelete.timestamp) {
+        return DeleteCompare.INCLUDE_NEW_NEXT_NEW;
+      } else {
+        return DeleteCompare.NEXT_NEW;
+      }
+    }
+
+    // Should never reach,
+    // throw exception for assertion?
+    throw new RuntimeException("GetDeleteTracker:compareDelete reached terminal state");
+  }
+
+  /**
+   * Internal class used to store the necessary information for a Delete.
+   * <p>
+   * Rather than reparsing the KeyValue, or copying fields, this class points
+   * to the underlying KeyValue buffer with pointers to the qualifier and fields
+   * for type and timestamp.  No parsing work is done in DeleteTracker now.
+   * <p>
+   * Fields are public because they are accessed often, directly, and only
+   * within this class.
+   */
+  protected class Delete {
+    byte [] buffer;
+    int qualifierOffset;
+    int qualifierLength;
+    byte type;
+    long timestamp;
+    /**
+     * Constructor
+     * @param buffer
+     * @param qualifierOffset
+     * @param qualifierLength
+     * @param type
+     * @param timestamp
+     */
+    public Delete(byte [] buffer, int qualifierOffset, int qualifierLength,
+        byte type, long timestamp) {
+      this.buffer = buffer;
+      this.qualifierOffset = qualifierOffset;
+      this.qualifierLength = qualifierLength;
+      this.type = type;
+      this.timestamp = timestamp;
+    }
+  }
+}

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java?rev=782445&r1=782444&r2=782445&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLog.java Sun Jun  7 19:57:37 2009
@@ -100,7 +100,7 @@
 public class HLog implements HConstants, Syncable {
   static final Log LOG = LogFactory.getLog(HLog.class);
   private static final String HLOG_DATFILE = "hlog.dat.";
-  static final byte [] METACOLUMN = Bytes.toBytes("METACOLUMN:");
+  static final byte [] METAFAMILY = Bytes.toBytes("METAFAMILY");
   static final byte [] METAROW = Bytes.toBytes("METAROW");
   private final FileSystem fs;
   private final Path dir;
@@ -701,8 +701,8 @@
   }
 
   private KeyValue completeCacheFlushLogEdit() {
-    return new KeyValue(METAROW, METACOLUMN, System.currentTimeMillis(),
-      COMPLETE_CACHE_FLUSH);
+    return new KeyValue(METAROW, METAFAMILY, null,
+      System.currentTimeMillis(), COMPLETE_CACHE_FLUSH);
   }
 
   /**
@@ -716,11 +716,11 @@
   }
 
   /**
-   * @param column
+   * @param family
    * @return true if the column is a meta column
    */
-  public static boolean isMetaColumn(byte [] column) {
-    return Bytes.equals(METACOLUMN, column);
+  public static boolean isMetaFamily(byte [] family) {
+    return Bytes.equals(METAFAMILY, family);
   }
   
   /**
@@ -870,6 +870,7 @@
           Executors.newFixedThreadPool(DEFAULT_NUMBER_LOG_WRITER_THREAD);
         for (final byte[] key : logEntries.keySet()) {
           Thread thread = new Thread(Bytes.toString(key)) {
+            @Override
             public void run() {
               LinkedList<HLogEntry> entries = logEntries.get(key);
               LOG.debug("Thread got " + entries.size() + " to process");

Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLogKey.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLogKey.java?rev=782445&r1=782444&r2=782445&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLogKey.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/HLogKey.java Sun Jun  7 19:57:37 2009
@@ -87,6 +87,9 @@
     return logSeqNum;
   }
 
+  /**
+   * @return the write time
+   */
   public long getWriteTime() {
     return this.writeTime;
   }