avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1435314 - in /avro/trunk: ./ lang/java/tools/src/main/java/org/apache/avro/tool/ lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/ lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/ lang/java/trevni/core/src/main/java/or...
Date Fri, 18 Jan 2013 19:34:34 GMT
Author: cutting
Date: Fri Jan 18 19:34:34 2013
New Revision: 1435314

URL: http://svn.apache.org/viewvc?rev=1435314&view=rev
Log:
AVRO-1229. Add support for booleans to Trevni.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java
    avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
    avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
    avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java
    avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java
    avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java
    avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java
    avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java
    avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java
    avro/trunk/lang/java/trevni/doc/apt/spec.apt

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Jan 18 19:34:34 2013
@@ -4,6 +4,8 @@ Trunk (not yet released)
 
   NEW FEATURES
 
+    AVRO-1229. Add support for booleans to Trevni. (cutting)
+
   IMPROVEMENTS
 
     AVRO-1211. Add MR guide to documentation. (Skye Wanderman-Milne via

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java Fri
Jan 18 19:34:34 2013
@@ -143,6 +143,8 @@ public class TrevniToJsonTool implements
     switch (column.getType()) {
     case NULL:
       generator.writeNull();                        break;
+    case BOOLEAN:
+      generator.writeBoolean((Boolean)value);       break;
     case INT:
       generator.writeNumber((Integer)value);        break;
     case LONG:

Modified: avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
(original)
+++ avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
Fri Jan 18 19:34:34 2013
@@ -137,7 +137,7 @@ class AvroColumnator {
 
   static boolean isSimple(Schema s) {
     switch (s.getType()) {
-    case NULL:
+    case NULL: case BOOLEAN:
     case INT: case LONG:
     case FLOAT: case DOUBLE: 
     case BYTES: case STRING: 
@@ -151,6 +151,7 @@ class AvroColumnator {
   private ValueType simpleValueType(Schema s) {
     switch (s.getType()) {
     case NULL:   return ValueType.NULL;
+    case BOOLEAN:return ValueType.BOOLEAN;
     case INT:    return ValueType.INT;
     case LONG:   return ValueType.LONG;
     case FLOAT:  return ValueType.FLOAT;

Modified: avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
(original)
+++ avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
Fri Jan 18 19:34:34 2013
@@ -38,6 +38,8 @@ public class TestShredder {
   @Test public void testPrimitives() throws Exception {
     check(Schema.create(Schema.Type.NULL),
           new ColumnMetaData("null", ValueType.NULL));
+    check(Schema.create(Schema.Type.BOOLEAN),
+          new ColumnMetaData("boolean", ValueType.BOOLEAN));
 
     check(Schema.create(Schema.Type.INT),
           new ColumnMetaData("int", ValueType.INT));

Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java
(original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java
Fri Jan 18 19:34:34 2013
@@ -33,7 +33,7 @@ class ArrayColumnOutputBuffer extends Co
   @Override public void writeLength(int length) throws IOException {
     assert this.length == 0;
     this.length = length;
-    getBuffer().writeInt(length);
+    getBuffer().writeLength(length);
   }
 
   @Override public void writeValue(Object value) throws IOException {

Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java Fri
Jan 18 19:34:34 2013
@@ -136,7 +136,7 @@ public class ColumnValues<T extends Comp
       throw new TrevniRuntimeException
         ("Column is not array: " +column.metaData.getName());
     assert arrayLength == 0;
-    return arrayLength = values.readInt();
+    return arrayLength = values.readLength();
   }
 
   /** Expert: Returns the next value in a column. */

Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java Fri
Jan 18 19:34:34 2013
@@ -33,6 +33,8 @@ class InputBuffer {
   private int pos;                                // position within buffer
   private int limit;                              // end of valid buffer data
 
+  private int bitCount;                           // position in booleans
+
   public InputBuffer(Input in) throws IOException { this(in, 0); }
 
   public InputBuffer(Input in, long position) throws IOException {
@@ -68,6 +70,8 @@ class InputBuffer {
     switch (type) {
     case NULL:
       return (T)null;
+    case BOOLEAN:
+      return (T)Boolean.valueOf(readBoolean());
     case INT:
       return (T)Integer.valueOf(readInt());
     case LONG:
@@ -93,6 +97,8 @@ class InputBuffer {
     switch (type) {
     case NULL:
                     break;
+    case BOOLEAN:
+      readBoolean(); break;
     case INT:
       readInt();    break;
     case LONG:
@@ -111,6 +117,22 @@ class InputBuffer {
     }
   }
 
+  public boolean readBoolean() throws IOException {
+    if (bitCount == 0)
+      read();
+    int bits = buf[pos-1] & 0xff;
+    int bit = (bits >> bitCount) & 1;
+    bitCount++;
+    if (bitCount == 8)
+      bitCount = 0;
+    return bit == 0 ? false : true;
+  }
+
+  public int readLength() throws IOException {
+    bitCount = 0;
+    return readInt();
+  }
+
   public int readInt() throws IOException {
     if ((limit - pos) < 5) {                      // maybe not in buffer
       int b = read();

Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java Fri
Jan 18 19:34:34 2013
@@ -27,6 +27,8 @@ import java.util.Arrays;
 class OutputBuffer extends ByteArrayOutputStream {
   static final int BLOCK_SIZE = 64 * 1024;
 
+  private int bitCount;                           // position in booleans
+
   public OutputBuffer() { super(BLOCK_SIZE + BLOCK_SIZE >> 2); }
 
   public boolean isFull() { return size() >= BLOCK_SIZE; }
@@ -38,6 +40,8 @@ class OutputBuffer extends ByteArrayOutp
     switch (type) {
     case NULL:
                                               break;
+    case BOOLEAN:
+      writeBoolean((Boolean)value);           break;
     case INT:
       writeInt((Integer)value);               break;
     case LONG:
@@ -63,6 +67,23 @@ class OutputBuffer extends ByteArrayOutp
     }
   }
 
+  public void writeBoolean(boolean value) {
+    if (bitCount == 0) {                           // first bool in byte
+      ensure(1);
+      count++;
+    }
+    if (value)
+      buf[count-1] |= (byte)(1 << bitCount);
+    bitCount++;
+    if (bitCount == 8)
+      bitCount = 0;
+  }
+
+  public void writeLength(int length) throws IOException {
+    bitCount = 0;
+    writeInt(length);
+  }
+
   private static final Charset UTF8 = Charset.forName("UTF-8");
 
   public void writeString(String string) throws IOException {

Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java Fri Jan
18 19:34:34 2013
@@ -19,7 +19,7 @@ package org.apache.trevni;
 
 /** The datatypes that may be stored in a column. */
 public enum ValueType {
-  NULL, INT, LONG, FIXED32, FIXED64, FLOAT, DOUBLE, STRING, BYTES;
+  NULL, BOOLEAN, INT, LONG, FIXED32, FIXED64, FLOAT, DOUBLE, STRING, BYTES;
   private String name;
   private ValueType() { this.name = this.name().toLowerCase(); }
 

Modified: avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java (original)
+++ avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java Fri
Jan 18 19:34:34 2013
@@ -26,7 +26,7 @@ import org.junit.Test;
 
 public class TestIOBuffers {
 
-  private static final int COUNT = 1000;
+  private static final int COUNT = 1001;
 
   @Test public void testEmpty() throws Exception {
     OutputBuffer out = new OutputBuffer();
@@ -47,6 +47,19 @@ public class TestIOBuffers {
     Assert.assertEquals(0, in.readInt());
   }
 
+  @Test public void testBoolean() throws Exception {
+    Random random = TestUtil.createRandom();
+    OutputBuffer out = new OutputBuffer();
+    for (int i = 0; i < COUNT; i++)
+      out.writeValue(random.nextBoolean(), ValueType.BOOLEAN);
+    
+    InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
+    random = TestUtil.createRandom();
+    for (int i = 0; i < COUNT; i++)
+      Assert.assertEquals(random.nextBoolean(),
+                          in.readValue(ValueType.BOOLEAN));
+  }
+
   @Test public void testInt() throws Exception {
     Random random = TestUtil.createRandom();
     OutputBuffer out = new OutputBuffer();
@@ -150,6 +163,16 @@ public class TestIOBuffers {
     in.skipValue(ValueType.NULL);
     Assert.assertEquals(sentinel, in.readLong());
   }
+  @Test public void testSkipBoolean() throws Exception {
+    long sentinel = Long.MAX_VALUE;
+    OutputBuffer out = new OutputBuffer();
+    out.writeValue(false, ValueType.BOOLEAN);
+    out.writeLong(sentinel);
+    
+    InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
+    in.skipValue(ValueType.BOOLEAN);
+    Assert.assertEquals(sentinel, in.readLong());
+  }
   @Test public void testSkipInt() throws Exception {
     long sentinel = Long.MAX_VALUE;
     OutputBuffer out = new OutputBuffer();

Modified: avro/trunk/lang/java/trevni/doc/apt/spec.apt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/doc/apt/spec.apt?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/doc/apt/spec.apt (original)
+++ avro/trunk/lang/java/trevni/doc/apt/spec.apt Fri Jan 18 19:34:34 2013
@@ -190,6 +190,8 @@ whether subsequent bytes are present.  F
 
   * <<null>>, requires zero bytes.  Sometimes used in array columns.
 
+  * <<boolean>>, one bit, packed into bytes, little-endian;
+
   * <<int>>, like <long>, but restricted to 32-bit signed values
 
   * <<long>> 64-bit signed values, represented as above



Mime
View raw message