Mailing-List: contact hbase-commits-help@hadoop.apache.org; run by ezmlm
Precedence: bulk
Reply-To: hbase-dev@hadoop.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r813209 - in /hadoop/hbase/branches/0.20: CHANGES.txt
 src/java/org/apache/hadoop/hbase/client/Result.java
 src/test/org/apache/hadoop/hbase/TestSerialization.java
Date: Thu, 10 Sep 2009 03:39:10 -0000
To: hbase-commits@hadoop.apache.org
From: jgray@apache.org
Message-Id: <20090910033910.6DB7B2388868@eris.apache.org>

Author: jgray
Date: Thu Sep 10 03:39:09 2009
New Revision: 813209

URL: http://svn.apache.org/viewvc?rev=813209&view=rev
Log:
HBASE-1765 Delay Result deserialization until asked for and permit access to the raw binary to prevent forced deserialization

Modified:
    hadoop/hbase/branches/0.20/CHANGES.txt
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/Result.java
    hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestSerialization.java

Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=813209&r1=813208&r2=813209&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Thu Sep 10 03:39:09 2009
@@ -3,6 +3,8 @@
 Release 0.20.1 - Unreleased
   IMPROVEMENTS
    HBASE-1819  Update to 0.20.1 hadoop and zk 3.2.1
+   HBASE-1765  Delay Result deserialization until asked for and permit
+               access to the raw binary to prevent forced deserialization
 
 Release 0.20.0 - Tue Sep  8 12:48:41 PDT 2009
 

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/Result.java?rev=813209&r1=813208&r2=813209&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/client/Result.java Thu Sep 10 03:39:09 2009
@@ -23,6 +23,7 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
@@ -32,6 +33,7 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.SplitKeyValue;
 import org.apache.hadoop.hbase.io.Cell;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.io.RowResult;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -69,6 +71,7 @@
   // We're not using java serialization.  Transient here is just a marker to say
   // that this is where we cache row if we're ever asked for it.
   private transient byte [] row = null;
+  private ImmutableBytesWritable bytes = null;
 
   /**
    * Constructor used for Writable.
@@ -92,6 +95,15 @@
   public Result(List<KeyValue> kvs) {
     this(kvs.toArray(new KeyValue[0]));
   }
+  
+  /**
+   * Instantiate a Result from the specified raw binary format.
+   * @param bytes raw binary format of Result
+   * @param numKeys number of KeyValues in Result
+   */
+  public Result(ImmutableBytesWritable bytes) {
+    this.bytes = bytes;
+  }
 
   /**
    * Method for retrieving the row that this result is for
@@ -99,8 +111,10 @@
    */
   public synchronized byte [] getRow() {
     if (this.row == null) {
-      this.row =
-        this.kvs == null || this.kvs.length == 0? null: this.kvs[0].getRow();
+      if(this.kvs == null) {
+        readFields();
+      }
+      this.row = this.kvs.length == 0? null: this.kvs[0].getRow();
     }
     return this.row;
   }
@@ -110,6 +124,9 @@
    * @return unsorted array of KeyValues
    */
   public KeyValue[] raw() {
+    if(this.kvs == null) {
+      readFields();
+    }
     return kvs;
   }
 
@@ -119,6 +136,9 @@
    * @return The sorted list of KeyValue's.
    */
   public List<KeyValue> list() {
+    if(this.kvs == null) {
+      readFields();
+    }
     return Arrays.asList(sorted());
   }
 
@@ -352,6 +372,9 @@
    * @return a RowResult
    */
   public RowResult getRowResult() {
+    if(this.kvs == null) {
+      readFields();
+    }
     return RowResult.createRowResult(Arrays.asList(kvs));
   }
   
@@ -367,10 +390,25 @@
   }
   
   /**
+   * Returns the raw binary encoding of this Result.<p>
+   * 
+   * Please note, there may be an offset into the underlying byte array of the
+   * returned ImmutableBytesWritable.  Be sure to use both 
+   * {@link ImmutableBytesWritable#get()} and {@link ImmutableBytesWritable#getOffset()}
+   * @return pointer to raw binary of Result
+   */
+  public ImmutableBytesWritable getBytes() {
+    return this.bytes;
+  }
+  
+  /**
    * Check if the underlying KeyValue [] is empty or not
    * @return true if empty
    */
   public boolean isEmpty() {
+    if(this.kvs == null) {
+      readFields();
+    }
     return this.kvs == null || this.kvs.length == 0;
   }
   
@@ -378,6 +416,9 @@
    * @return the size of the underlying KeyValue []
    */
   public int size() {
+    if(this.kvs == null) {
+      readFields();
+    }
     return this.kvs == null? 0: this.kvs.length;
   }
   
@@ -411,20 +452,33 @@
   throws IOException {
     familyMap = null;
     row = null;
-    int numKeys = in.readInt();
-    this.kvs = new KeyValue[numKeys];
-    if(numKeys == 0) {
+    int totalBuffer = in.readInt();
+    if(totalBuffer == 0) {
+      bytes = null;
       return;
     }
-    int totalBuffer = in.readInt();
-    byte [] buf = new byte[totalBuffer];
-    int offset = 0;
-    for(int i=0; i<numKeys; i++) {
-      int keyLength = in.readInt();
-      in.readFully(buf, offset, keyLength);
-      kvs[i] = new KeyValue(buf, offset, keyLength);
+    byte [] raw = new byte[totalBuffer];
+    in.readFully(raw, 0, totalBuffer);
+    bytes = new ImmutableBytesWritable(raw, 0, totalBuffer);
+  }
+  
+  //Create KeyValue[] when needed
+  private void readFields() {
+    if(bytes == null) {
+      this.kvs = new KeyValue[0];
+      return;
+    }
+    byte [] buf = bytes.get();
+    int offset = bytes.getOffset();
+    int finalOffset = bytes.getSize() + offset;
+    List<KeyValue> kvs = new ArrayList<KeyValue>();
+    while(offset < finalOffset) {
+      int keyLength = Bytes.toInt(buf, offset);
+      offset += Bytes.SIZEOF_INT;
+      kvs.add(new KeyValue(buf, offset, keyLength));
       offset += keyLength;
     }
+    this.kvs = kvs.toArray(new KeyValue[kvs.size()]);
   }
   
   public void write(final DataOutput out)
@@ -432,11 +486,9 @@
     if(isEmpty()) {
       out.writeInt(0);
     } else {
-      int len = this.kvs.length;
-      out.writeInt(len);
       int totalLen = 0;
       for(KeyValue kv : kvs) {
-        totalLen += kv.getLength();
+        totalLen += kv.getLength() + Bytes.SIZEOF_INT;
       }
       out.writeInt(totalLen);
       for(KeyValue kv : kvs) {
@@ -455,11 +507,12 @@
     out.writeInt(results.length);
     int bufLen = 0;
     for(Result result : results) {
+      bufLen += Bytes.SIZEOF_INT;
       if(result == null || result.isEmpty()) {
         continue;
       }
       for(KeyValue key : result.raw()) {
-        bufLen += key.getLength();
+        bufLen += key.getLength() + Bytes.SIZEOF_INT;
       }
     }
     out.writeInt(bufLen);
@@ -488,14 +541,22 @@
     int offset = 0;
     for(int i=0;i<numResults;i++) {
       int numKeys = in.readInt();
-      KeyValue [] keys = new KeyValue[numKeys];
+      offset += Bytes.SIZEOF_INT;
+      if(numKeys == 0) {
+        results[i] = new Result((ImmutableBytesWritable)null);
+        continue;
+      }
+      int initialOffset = offset;
       for(int j=0;j<numKeys;j++) {
         int keyLen = in.readInt();
+        Bytes.putInt(buf, offset, keyLen);
+        offset += Bytes.SIZEOF_INT;
         in.readFully(buf, offset, keyLen);
-        keys[j] = new KeyValue(buf, offset, keyLen);
         offset += keyLen;
       }
-      results[i] = new Result(keys);
+      int totalLength = offset - initialOffset;
+      results[i] = new Result(new ImmutableBytesWritable(buf, initialOffset, 
+          totalLength));
     }
     return results;
   }

Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestSerialization.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestSerialization.java?rev=813209&r1=813208&r2=813209&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestSerialization.java (original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/TestSerialization.java Thu Sep 10 03:39:09 2009
@@ -429,6 +429,47 @@
     Result deserialized = (Result)Writables.getWritable(b, new Result());
     assertEquals(r.size(), deserialized.size());
   }
+
+  public void testResultDynamicBuild() throws Exception {
+    byte [] rowA = Bytes.toBytes("rowA");
+    byte [] famA = Bytes.toBytes("famA");
+    byte [] qfA = Bytes.toBytes("qfA");
+    byte [] valueA = Bytes.toBytes("valueA");
+    
+    byte [] rowB = Bytes.toBytes("rowB");
+    byte [] famB = Bytes.toBytes("famB");
+    byte [] qfB = Bytes.toBytes("qfB");
+    byte [] valueB = Bytes.toBytes("valueB");
+    
+    KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
+    KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
+    
+    Result result = new Result(new KeyValue[]{kvA, kvB});
+    
+    byte [] rb = Writables.getBytes(result);
+    
+    
+    // Call getRow() first
+    Result deResult = (Result)Writables.getWritable(rb, new Result());
+    byte [] row = deResult.getRow();
+    assertTrue(Bytes.equals(row, rowA));
+    
+    // Call sorted() first
+    deResult = (Result)Writables.getWritable(rb, new Result());
+    assertTrue("results are not equivalent, first key mismatch",
+        result.sorted()[0].equals(deResult.sorted()[0]));
+    assertTrue("results are not equivalent, second key mismatch",
+        result.sorted()[1].equals(deResult.sorted()[1]));
+
+    // Call raw() first
+    deResult = (Result)Writables.getWritable(rb, new Result());
+    assertTrue("results are not equivalent, first key mismatch",
+        result.raw()[0].equals(deResult.raw()[0]));
+    assertTrue("results are not equivalent, second key mismatch",
+        result.raw()[1].equals(deResult.raw()[1]));
+    
+    
+  }
   
   public void testResultArray() throws Exception {
     byte [] rowA = Bytes.toBytes("rowA");