Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@hbase.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1557780 - in /hbase/branches/0.98/hbase-common/src:
 main/java/org/apache/hadoop/hbase/codec/
 test/java/org/apache/hadoop/hbase/codec/
Date: Mon, 13 Jan 2014 17:29:37 -0000
To: commits@hbase.apache.org
From: anoopsamjohn@apache.org
Message-Id: <20140113172937.7E181238890D@eris.apache.org>

Author: anoopsamjohn
Date: Mon Jan 13 17:29:36 2014
New Revision: 1557780

URL: http://svn.apache.org/r1557780
Log:
HBASE-10321 CellCodec has broken the 96 client to 98 server compatibility

Added:
    hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecV2.java
    hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecV2.java
Modified:
    hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java
    hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java

Modified: hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java?rev=1557780&r1=1557779&r2=1557780&view=diff
==============================================================================
--- hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java (original)
+++ hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodec.java Mon Jan 13 17:29:36 2014
@@ -29,7 +29,8 @@ import org.apache.hadoop.hbase.util.Byte
 
 /**
  * Basic Cell codec that just writes out all the individual elements of a Cell.  Uses ints
- * delimiting all lengths. Profligate. Needs tune up. 
+ * delimiting all lengths. Profligate. Needs tune up.
+ * Note: This will not write tags of a Cell.
  */
 @InterfaceAudience.Private
 public class CellCodec implements Codec {
@@ -53,8 +54,6 @@ public class CellCodec implements Codec 
       this.out.write(cell.getTypeByte());
       // Value
       write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
-      // Write tags
-      write(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength());
       // MvccVersion
       this.out.write(Bytes.toBytes(cell.getMvccVersion()));
     }
@@ -86,13 +85,12 @@ public class CellCodec implements Codec 
       IOUtils.readFully(this.in, longArray);
       long timestamp = Bytes.toLong(longArray);
       byte type = (byte) this.in.read();
-      byte [] value = readByteArray(in);
-      byte[] tags = readByteArray(in);
+      byte[] value = readByteArray(in);
       // Read memstore version
       byte[] memstoreTSArray = new byte[Bytes.SIZEOF_LONG];
       IOUtils.readFully(this.in, memstoreTSArray);
       long memstoreTS = Bytes.toLong(memstoreTSArray);
-      return CellUtil.createCell(row, family, qualifier, timestamp, type, value, tags, memstoreTS);
+      return CellUtil.createCell(row, family, qualifier, timestamp, type, value, memstoreTS);
     }
 
     /**

Added: hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecV2.java?rev=1557780&view=auto
==============================================================================
--- hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecV2.java (added)
+++ hbase/branches/0.98/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/CellCodecV2.java Mon Jan 13 17:29:36 2014
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.codec;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Basic Cell codec that just writes out all the individual elements of a Cell including the tags.
+ * Uses ints delimiting all lengths. Profligate. Needs tune up.
+ */
+@InterfaceAudience.Private
+public class CellCodecV2 implements Codec {
+  static class CellEncoder extends BaseEncoder {
+    CellEncoder(final OutputStream out) {
+      super(out);
+    }
+
+    @Override
+    public void write(Cell cell) throws IOException {
+      checkFlushed();
+      // Row
+      write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
+      // Column family
+      write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
+      // Qualifier
+      write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
+      // Version
+      this.out.write(Bytes.toBytes(cell.getTimestamp()));
+      // Type
+      this.out.write(cell.getTypeByte());
+      // Value
+      write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
+      // Tags
+      write(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength());
+      // MvccVersion
+      this.out.write(Bytes.toBytes(cell.getMvccVersion()));
+    }
+
+    /**
+     * Write int length followed by array bytes.
+     * 
+     * @param bytes
+     * @param offset
+     * @param length
+     * @throws IOException
+     */
+    private void write(final byte[] bytes, final int offset, final int length) throws IOException {
+      this.out.write(Bytes.toBytes(length));
+      this.out.write(bytes, offset, length);
+    }
+  }
+
+  static class CellDecoder extends BaseDecoder {
+    public CellDecoder(final InputStream in) {
+      super(in);
+    }
+
+    protected Cell parseCell() throws IOException {
+      byte[] row = readByteArray(this.in);
+      byte[] family = readByteArray(in);
+      byte[] qualifier = readByteArray(in);
+      byte[] longArray = new byte[Bytes.SIZEOF_LONG];
+      IOUtils.readFully(this.in, longArray);
+      long timestamp = Bytes.toLong(longArray);
+      byte type = (byte) this.in.read();
+      byte[] value = readByteArray(in);
+      byte[] tags = readByteArray(in);
+      // Read memstore version
+      byte[] memstoreTSArray = new byte[Bytes.SIZEOF_LONG];
+      IOUtils.readFully(this.in, memstoreTSArray);
+      long memstoreTS = Bytes.toLong(memstoreTSArray);
+      return CellUtil.createCell(row, family, qualifier, timestamp, type, value, tags, memstoreTS);
+    }
+
+    /**
+     * @return Byte array read from the stream.
+     * @throws IOException
+     */
+    private byte[] readByteArray(final InputStream in) throws IOException {
+      byte[] intArray = new byte[Bytes.SIZEOF_INT];
+      IOUtils.readFully(in, intArray);
+      int length = Bytes.toInt(intArray);
+      byte[] bytes = new byte[length];
+      IOUtils.readFully(in, bytes);
+      return bytes;
+    }
+  }
+
+  @Override
+  public Decoder getDecoder(InputStream is) {
+    return new CellDecoder(is);
+  }
+
+  @Override
+  public Encoder getEncoder(OutputStream os) {
+    return new CellEncoder(os);
+  }
+}

Modified: hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java?rev=1557780&r1=1557779&r2=1557780&view=diff
==============================================================================
--- hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java (original)
+++ hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodec.java Mon Jan 13 17:29:36 2014
@@ -27,10 +27,8 @@ import java.io.IOException;
 
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.Tag;
 import org.apache.hadoop.hbase.codec.CellCodec;
 import org.apache.hadoop.hbase.codec.Codec;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -124,47 +122,4 @@ public class TestCellCodec {
     dis.close();
     assertEquals(offset, cis.getCount());
   }
-
-  @Test
-  public void testThreeWithTag() throws IOException {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    CountingOutputStream cos = new CountingOutputStream(baos);
-    DataOutputStream dos = new DataOutputStream(cos);
-    Codec codec = new CellCodec();
-    Codec.Encoder encoder = codec.getEncoder(dos);
-    final KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"),
-        HConstants.LATEST_TIMESTAMP, Bytes.toBytes("1"), new Tag[] {
-            new Tag((byte) 1, Bytes.toBytes("teststring1")),
-            new Tag((byte) 2, Bytes.toBytes("testString2")) });
-    final KeyValue kv2 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("2"),
-        HConstants.LATEST_TIMESTAMP, Bytes.toBytes("2"), new Tag[] { new Tag((byte) 1,
-            Bytes.toBytes("teststring3")), });
-    final KeyValue kv3 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("3"),
-        HConstants.LATEST_TIMESTAMP, Bytes.toBytes("3"), new Tag[] {
-            new Tag((byte) 2, Bytes.toBytes("teststring4")),
-            new Tag((byte) 2, Bytes.toBytes("teststring5")),
-            new Tag((byte) 1, Bytes.toBytes("teststring6")) });
-
-    encoder.write(kv1);
-    encoder.write(kv2);
-    encoder.write(kv3);
-    encoder.flush();
-    dos.close();
-    long offset = cos.getCount();
-    CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray()));
-    DataInputStream dis = new DataInputStream(cis);
-    Codec.Decoder decoder = codec.getDecoder(dis);
-    assertTrue(decoder.advance());
-    Cell c = decoder.current();
-    assertTrue(CellComparator.equals(c, kv1));
-    assertTrue(decoder.advance());
-    c = decoder.current();
-    assertTrue(CellComparator.equals(c, kv2));
-    assertTrue(decoder.advance());
-    c = decoder.current();
-    assertTrue(CellComparator.equals(c, kv3));
-    assertFalse(decoder.advance());
-    dis.close();
-    assertEquals(offset, cis.getCount());
-  }
 }
\ No newline at end of file

Added: hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecV2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecV2.java?rev=1557780&view=auto
==============================================================================
--- hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecV2.java (added)
+++ hbase/branches/0.98/hbase-common/src/test/java/org/apache/hadoop/hbase/codec/TestCellCodecV2.java Mon Jan 13 17:29:36 2014
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.codec;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.io.CountingInputStream;
+import com.google.common.io.CountingOutputStream;
+
+@Category(SmallTests.class)
+public class TestCellCodecV2 {
+
+  @Test
+  public void testCellWithTag() throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    CountingOutputStream cos = new CountingOutputStream(baos);
+    DataOutputStream dos = new DataOutputStream(cos);
+    Codec codec = new CellCodecV2();
+    Codec.Encoder encoder = codec.getEncoder(dos);
+    final Cell cell1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"),
+        HConstants.LATEST_TIMESTAMP, Bytes.toBytes("1"), new Tag[] {
+            new Tag((byte) 1, Bytes.toBytes("teststring1")),
+            new Tag((byte) 2, Bytes.toBytes("teststring2")) });
+    final Cell cell2 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("2"),
+        HConstants.LATEST_TIMESTAMP, Bytes.toBytes("2"), new Tag[] { new Tag((byte) 1,
+            Bytes.toBytes("teststring3")), });
+    final Cell cell3 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("3"),
+        HConstants.LATEST_TIMESTAMP, Bytes.toBytes("3"), new Tag[] {
+            new Tag((byte) 2, Bytes.toBytes("teststring4")),
+            new Tag((byte) 2, Bytes.toBytes("teststring5")),
+            new Tag((byte) 1, Bytes.toBytes("teststring6")) });
+
+    encoder.write(cell1);
+    encoder.write(cell2);
+    encoder.write(cell3);
+    encoder.flush();
+    dos.close();
+    long offset = cos.getCount();
+    CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray()));
+    DataInputStream dis = new DataInputStream(cis);
+    Codec.Decoder decoder = codec.getDecoder(dis);
+    assertTrue(decoder.advance());
+    Cell c = decoder.current();
+    assertTrue(CellComparator.equals(c, cell1));
+    List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
+    assertEquals(2, tags.size());
+    Tag tag = tags.get(0);
+    assertEquals(1, tag.getType());
+    assertTrue(Bytes.equals(Bytes.toBytes("teststring1"), tag.getValue()));
+    tag = tags.get(1);
+    assertEquals(2, tag.getType());
+    assertTrue(Bytes.equals(Bytes.toBytes("teststring2"), tag.getValue()));
+    assertTrue(decoder.advance());
+    c = decoder.current();
+    assertTrue(CellComparator.equals(c, cell2));
+    tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
+    assertEquals(1, tags.size());
+    tag = tags.get(0);
+    assertEquals(1, tag.getType());
+    assertTrue(Bytes.equals(Bytes.toBytes("teststring3"), tag.getValue()));
+    assertTrue(decoder.advance());
+    c = decoder.current();
+    assertTrue(CellComparator.equals(c, cell3));
+    tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
+    assertEquals(3, tags.size());
+    tag = tags.get(0);
+    assertEquals(2, tag.getType());
+    assertTrue(Bytes.equals(Bytes.toBytes("teststring4"), tag.getValue()));
+    tag = tags.get(1);
+    assertEquals(2, tag.getType());
+    assertTrue(Bytes.equals(Bytes.toBytes("teststring5"), tag.getValue()));
+    tag = tags.get(2);
+    assertEquals(1, tag.getType());
+    assertTrue(Bytes.equals(Bytes.toBytes("teststring6"), tag.getValue()));
+    assertFalse(decoder.advance());
+    dis.close();
+    assertEquals(offset, cis.getCount());
+  }
+}