avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r911995 [2/2] - in /hadoop/avro/trunk/lang/java/src: java/org/apache/avro/file/ java/org/apache/avro/io/ java/org/apache/avro/ipc/ java/org/apache/avro/tool/ test/java/org/apache/avro/ test/java/org/apache/avro/io/
Date Fri, 19 Feb 2010 21:45:11 GMT
Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java Fri Feb 19 21:45:10
2010
@@ -17,7 +17,6 @@
  */
 package org.apache.avro.io;
 
-import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
 import java.io.IOException;
@@ -35,20 +34,26 @@
  * Avro encoding and decoding.
  */
 public class Perf {
-  private static final int COUNT = 200000; // needs to be a multiple of 4
-  private static final int CYCLES = 150;
+  private static final int COUNT = 100000; // needs to be a multiple of 4
+  private static final int CYCLES = 500;
   
   public static void main(String[] args) throws IOException {
     List<Test> tests = new ArrayList<Test>();
+    ReadInt readIntTest = null;
     for (String a : args) {
       if (a.equals("-i")) {
-        tests.add(new ReadInt());
+        readIntTest = new ReadInt();
+        tests.add(readIntTest);
       } else if (a.equals("-f")) {
         tests.add(new ReadFloat());
       } else if (a.equals("-d")) {
         tests.add(new ReadDouble());
       } else if (a.equals("-l")) {
         tests.add(new ReadLong());
+      } else if (a.equals("-ls")) {
+        tests.add(new ReadLongSmall(readIntTest));
+      } else if (a.equals("-b")) {
+        tests.add(new ReadBoolean());
       } else if (a.equals("-R")) {
         tests.add(new RepeaterTest());
       } else if (a.equals("-N")) {
@@ -71,32 +76,47 @@
       }
     }
     if (tests.isEmpty()) {
+      readIntTest = new ReadInt();
       tests.addAll(Arrays.asList(new Test[] {
-          new ReadInt(), new ReadLong(),
-          new ReadFloat(), new ReadDouble(),
+          readIntTest, 
+          new ReadLongSmall(readIntTest), 
+          new ReadLong(),
+          new ReadFloat(), 
+          new ReadDouble(),
+          new ReadBoolean(),
           new RepeaterTest(), new NestedRecordTest(),
           new ResolverTest(), new MigrationTest(),
           new GenericReaderTest(), new GenericReaderWithDefaultTest(),
           new GenericReaderWithOutOfOrderTest(),
-          new GenericReaderWithPromotionTest(),
+          new GenericReaderWithPromotionTest()
       }));
     }
     
-    for (Test t : tests) {
+    for (int k = 0; k < tests.size(); k++) {
+      Test t = tests.get(k);
+      // get everything to compile once 
+      t.read();
+    }
+    for (int k = 0; k < tests.size(); k++) {
+      Test t = tests.get(k);
       // warmup JVM 
-      for (int i = 0; i < CYCLES; i++) {
+      for (int i = 0; i < t.cycles; i++) {
         t.read();
     }
     // test
     long s = 0;
-    for (int i = 0; i < CYCLES; i++) {
+    for (int i = 0; i < t.cycles; i++) {
       long l = t.read();
       // System.out.println("** " + l);
       s += l;
     }
     s /= 1000;
+    double entries = (t.cycles * (double) t.count);
+    double bytes = t.cycles * (double) t.data.length;
     System.out.println(t.name + ": " + (s / 1000) + " ms, "
-        +  ((CYCLES * (double) COUNT) / s) + " million entries/sec");
+        +  (entries / s) + " million entries/sec.  "
+        +  (bytes / s) + " million bytes/sec" );
+    tests.set(k, null);
     }
   }
   
@@ -106,6 +126,10 @@
      * Name of the test.
      */
     public final String name;
+    public final int count;
+    public final int cycles;
+    protected byte[] data;
+    protected static DecoderFactory factory = new DecoderFactory();
     
     /**
      * Reads the contents and returns the time taken in nanoseconds.
@@ -114,20 +138,40 @@
      */
     abstract long read() throws IOException;
     
-    public Test(String name) {
+    public Test(String name, int cycles, int count) {
       this.name = name;
+      this.cycles = cycles;
+      this.count = count;
+    }
+    
+    protected void generateRepeaterData(Encoder e) throws IOException {
+      e.writeArrayStart();
+      e.setItemCount(count);
+      Random r = newRandom();
+      for (int i = 0; i < count; i++) {
+        e.writeDouble(r.nextDouble());
+        e.writeDouble(r.nextDouble());
+        e.writeDouble(r.nextDouble());
+        e.writeInt(r.nextInt());
+        e.writeInt(r.nextInt());
+        e.writeInt(r.nextInt());
+  }
+      e.writeArrayEnd();
     }
   }
   
   private static abstract class DecoderTest extends Test {
     public final Schema schema;
-    protected byte[] data;
     public DecoderTest(String name, String json) throws IOException {
-      super(name);
+      this(name, json, 1);
+    }
+    public DecoderTest(String name, String json, int factor) throws IOException {
+      super(name, CYCLES, COUNT/factor);
       this.schema = Schema.parse(json);
       ByteArrayOutputStream bao = new ByteArrayOutputStream();
       Encoder e = new BinaryEncoder(bao);
       genData(e);
+      e.flush();
       data = bao.toByteArray();
     }
 
@@ -148,7 +192,7 @@
     }
 
     protected static Decoder newDecoder(byte[] data) {
-      return new BinaryDecoder(new ByteArrayInputStream(data));
+      return factory.createBinaryDecoder(data, null);
     }
 
     abstract void genData(Encoder e) throws IOException;
@@ -176,9 +220,9 @@
 
     @Override void genData(Encoder e) throws IOException {
       e.writeArrayStart();
-      e.setItemCount((COUNT/4) * 4); //next lowest multiple of 4  
+      e.setItemCount((count/4) * 4); //next lowest multiple of 4  
       Random r = newRandom();
-      for (int i = 0; i < COUNT/4; i++) {
+      for (int i = 0; i < count/4; i++) {
         e.writeInt(r.nextInt(50)); // fits in 1 byte
         e.writeInt(r.nextInt(5000)); // fits in 2 bytes
         e.writeInt(r.nextInt(500000)); // fits in 3 bytes
@@ -193,6 +237,21 @@
     }
   }
 
+  // This is the same data as ReadInt, but using readLong.
+  private static class ReadLongSmall extends DecoderTest {
+    public ReadLongSmall(ReadInt dataFrom) throws IOException {
+      super("ReadLongSmall", "{ \"type\": \"array\", \"items\": \"long\"} ");
+      data = dataFrom.data;
+    }
+    @Override void genData(Encoder e) throws IOException {
+    }
+    @Override
+    void readInternal(Decoder d) throws IOException {
+       d.readLong();
+    }
+  }
+ 
+  // this tests reading Longs that are sometimes very large
   private static class ReadLong extends DecoderTest {
     public ReadLong() throws IOException {
       super("ReadLong", "{ \"type\": \"array\", \"items\": \"long\"} ");
@@ -201,9 +260,9 @@
     @Override
     void genData(Encoder e) throws IOException {
       e.writeArrayStart();
-      e.setItemCount((COUNT / 4) *4);
+      e.setItemCount((count / 4) *4);
       Random r = newRandom();
-      for (int i = 0; i < COUNT /4; i++) {
+      for (int i = 0; i < count /4; i++) {
         e.writeLong(r.nextLong() % 0x7FL); // half fit in 1, half in 2 
         e.writeLong(r.nextLong() % 0x1FFFFFL); // half fit in <=3, half in 4
         e.writeLong(r.nextLong() % 0x3FFFFFFFFL); // half in <=5, half in 6
@@ -226,9 +285,9 @@
     @Override
     void genData(Encoder e) throws IOException {
       e.writeArrayStart();
-      e.setItemCount(COUNT);
+      e.setItemCount(count);
       Random r = newRandom();
-      for (int i = 0; i < COUNT; i++) {
+      for (int i = 0; i < count; i++) {
         e.writeFloat(r.nextFloat());
       }
       e.writeArrayEnd();
@@ -240,6 +299,25 @@
     }
   }
 
+  private static class ReadBoolean extends DecoderTest {
+    public ReadBoolean() throws IOException {
+      super("ReadBoolean", "{ \"type\": \"array\", \"items\": \"boolean\"} ");
+    }
+    @Override void genData(Encoder e) throws IOException {
+       e.writeArrayStart();
+      e.setItemCount(count);
+      Random r = newRandom();
+      for (int i = 0; i < count; i++) {
+        e.writeBoolean(r.nextBoolean());
+      }
+      e.writeArrayEnd();
+    }
+    @Override
+    void readInternal(Decoder d) throws IOException {
+      d.readBoolean();
+    }
+  }
+
   private static class ReadDouble extends DecoderTest {
     public ReadDouble() throws IOException {
       super("ReadDouble", "{ \"type\": \"array\", \"items\": \"double\"} ");
@@ -248,9 +326,9 @@
     @Override
     void genData(Encoder e) throws IOException {
       e.writeArrayStart();
-      e.setItemCount(COUNT);
+      e.setItemCount(count);
       Random r = newRandom();
-      for (int i = 0; i < COUNT; i++) {
+      for (int i = 0; i < count; i++) {
         e.writeDouble(r.nextFloat());
       }
       e.writeArrayEnd();
@@ -272,27 +350,13 @@
     + "{ \"name\": \"f6\", \"type\": \"int\" }\n"
     + "] } }";
 
-  private static void generateRepeaterData(Encoder e) throws IOException {
-    e.writeArrayStart();
-    e.setItemCount(COUNT);
-    Random r = newRandom();
-    for (int i = 0; i < COUNT; i++) {
-      e.writeDouble(r.nextDouble());
-      e.writeDouble(r.nextDouble());
-      e.writeDouble(r.nextDouble());
-      e.writeInt(r.nextInt());
-      e.writeInt(r.nextInt());
-      e.writeInt(r.nextInt());
-    }
-    e.writeArrayEnd();
-  }
   private static class RepeaterTest extends DecoderTest {
     public RepeaterTest() throws IOException {
       this("RepeaterTest");
     }
     
     public RepeaterTest(String name) throws IOException {
-      super(name, REPEATER_SCHEMA);
+      super(name, REPEATER_SCHEMA, 6);
     }
     
     @Override
@@ -401,14 +465,13 @@
   
   private static class GenericReaderTest extends Test {
     public final Schema writerSchema;
-    protected byte[] data;
 
     public GenericReaderTest() throws IOException {
       this("GenericReaderTest");
     }
 
     public GenericReaderTest(String name) throws IOException {
-      super(name);
+      super(name, CYCLES, COUNT/12);
       this.writerSchema = Schema.parse(REPEATER_SCHEMA);
       ByteArrayOutputStream bao = new ByteArrayOutputStream();
       Encoder e = new BinaryEncoder(bao);
@@ -421,10 +484,11 @@
       GenericDatumReader<Object> r = getReader();
       long t = System.nanoTime();
       Decoder d =
-        new BinaryDecoder(new ByteArrayInputStream(data));
+        DecoderFactory.defaultFactory().createBinaryDecoder(data, null);
+      Object reuse = null;
       for (; ;) {
         try {
-          r.read(null, d);
+          reuse = r.read(reuse, d);
         } catch (EOFException e) {
           break;
         }
@@ -490,11 +554,14 @@
   }
 
   private static void usage() {
-    System.out.println("Usage: Perf { -i | -l | -f | -d }");
+    System.out.println("Usage: Perf { -i | -ls | -l | -f | -d | -b | -R | -N " +
+      "| -S | -M | -G | -Gd | -Go | Gp }");
     System.out.println("  -i readInt()");
+    System.out.println("  -ls readLongSmall()");
     System.out.println("  -l readLong()");
     System.out.println("  -f readFloat()");
     System.out.println("  -d readDouble()");
+    System.out.println("  -b readBoolean()");
     System.out.println("  -R repeater in validating decoder");
     System.out.println("  -N nested record in validating decoder");
     System.out.println("  -S resolving decoder");

Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBinaryDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBinaryDecoder.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBinaryDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBinaryDecoder.java Fri
Feb 19 21:45:10 2010
@@ -18,61 +18,385 @@
 package org.apache.avro.io;
 
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.EOFException;
 import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 
+import org.apache.avro.RandomData;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
 import org.apache.avro.util.Utf8;
+import org.junit.Assert;
+import org.junit.BeforeClass;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
 
-
+@RunWith(Parameterized.class)
 public class TestBinaryDecoder {
+  // prime number buffer size so that looping tests hit the buffer edge
+  // at different points in the loop.
+  DecoderFactory factory;
+  public TestBinaryDecoder(boolean useDirect) {
+    factory = new DecoderFactory().configureDecoderBufferSize(521);
+    factory.configureDirectDecoder(useDirect);
+  }
+  
+  @Parameters
+  public static Collection<Object[]> data() {
+    return Arrays.asList(new Object[][] {
+        { true },
+        { false },
+    });
+  }
+  
+  private Decoder newDecoderWithNoData() throws IOException {
+    return newDecoder(new byte[0]);
+  }
+
+  private Decoder newDecoder(byte[] bytes, int start, int len)
+    throws IOException {
+    return factory.createBinaryDecoder(bytes, start, len, null);
+    
+  }
+
+  private Decoder newDecoder(InputStream in) {
+    return factory.createBinaryDecoder(in, null);
+  }
+
+  private Decoder newDecoder(byte[] bytes) throws IOException {
+    return factory.createBinaryDecoder(bytes, null);
+  }
+
   /** Verify EOFException throw at EOF */
 
   @Test(expected=EOFException.class)
   public void testEOFBoolean() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readBoolean();
+    newDecoderWithNoData().readBoolean();
   }
   
   @Test(expected=EOFException.class)
   public void testEOFInt() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readInt();
+    newDecoderWithNoData().readInt();
   }
   
   @Test(expected=EOFException.class)
   public void testEOFLong() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readLong();
+    newDecoderWithNoData().readLong();
   }
   
   @Test(expected=EOFException.class)
   public void testEOFFloat() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readFloat();
+    newDecoderWithNoData().readFloat();
   }
   
   @Test(expected=EOFException.class)
   public void testEOFDouble() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readDouble();
+    newDecoderWithNoData().readDouble();
   }
   
   @Test(expected=EOFException.class)
   public void testEOFBytes() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readBytes(null);
+    newDecoderWithNoData().readBytes(null);
   }
   
   @Test(expected=EOFException.class)
   public void testEOFString() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).
-      readString(new Utf8("a"));
+    newDecoderWithNoData().readString(new Utf8("a"));
   }
   
   @Test(expected=EOFException.class)
   public void testEOFFixed() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).
-      readFixed(new byte[1]);
+    newDecoderWithNoData().readFixed(new byte[1]);
   }
 
   @Test(expected=EOFException.class)
   public void testEOFEnum() throws IOException {
-    new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readEnum();
+    newDecoderWithNoData().readEnum();
   }
   
+  private static byte[] data = null;
+  private static int seed = -1;
+  private static Schema schema = null;
+  private static int count = 200;
+  private static ArrayList<Object> records = new ArrayList<Object>(count);
+  @BeforeClass
+  public static void generateData() throws IOException {
+    seed = (int)System.currentTimeMillis();
+    // note some tests (testSkipping) rely on this explicitly
+    String jsonSchema =
+      "{\"type\": \"record\", \"name\": \"Test\", \"fields\": ["
+      +"{\"name\":\"intField\", \"type\":\"int\"},"
+      +"{\"name\":\"bytesField\", \"type\":\"bytes\"},"
+      +"{\"name\":\"booleanField\", \"type\":\"boolean\"},"
+      +"{\"name\":\"stringField\", \"type\":\"string\"},"
+      +"{\"name\":\"floatField\", \"type\":\"float\"},"
+      +"{\"name\":\"doubleField\", \"type\":\"double\"},"
+      +"{\"name\":\"arrayField\", \"type\": " +
+          "{\"type\":\"array\", \"items\":\"boolean\"}},"
+      +"{\"name\":\"longField\", \"type\":\"long\"}]}";
+    schema = Schema.parse(jsonSchema);
+    GenericDatumWriter<Object> writer = new GenericDatumWriter<Object>();
+    writer.setSchema(schema);
+    ByteArrayOutputStream baos = new ByteArrayOutputStream(8192);
+    BinaryEncoder encoder = new BinaryEncoder(baos);
+    
+    for (Object datum : new RandomData(schema, count, seed)) {
+      writer.write(datum, encoder);
+      records.add(datum);
+    }
+    data = baos.toByteArray();
+  }
+
+  @Test
+  public void testDecodeFromSources() throws IOException {
+    GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
+    reader.setSchema(schema);
+    
+    ByteArrayInputStream is = new ByteArrayInputStream(data);
+    ByteArrayInputStream is2 = new ByteArrayInputStream(data);
+    ByteArrayInputStream is3 = new ByteArrayInputStream(data);
+
+    Decoder fromInputStream = newDecoder(is);
+    Decoder fromArray = newDecoder(data);
+    
+    byte[] data2 = new byte[data.length + 30];
+    Arrays.fill(data2, (byte)0xff);
+    System.arraycopy(data, 0, data2, 15, data.length);
+
+    Decoder fromOffsetArray = newDecoder(data2, 15, data.length);
+
+    BinaryDecoder initOnInputStream = factory.createBinaryDecoder(
+        new byte[50], 0, 30, null);
+    initOnInputStream = factory.createBinaryDecoder(is2, initOnInputStream);
+    BinaryDecoder initOnArray = factory.createBinaryDecoder(is3, null);
+    initOnArray = factory.createBinaryDecoder(
+        data, 0, data.length, initOnArray);
+    
+    for (Object datum : records) {
+      Assert.assertEquals(
+          "InputStream based BinaryDecoder result does not match",
+          datum, reader.read(null, fromInputStream));
+      Assert.assertEquals(
+          "Array based BinaryDecoder result does not match",
+          datum, reader.read(null, fromArray));
+      Assert.assertEquals(
+          "offset Array based BinaryDecoder result does not match",
+          datum, reader.read(null, fromOffsetArray));
+      Assert.assertEquals(
+          "InputStream initialized BinaryDecoder result does not match",
+          datum, reader.read(null, initOnInputStream));
+      Assert.assertEquals(
+          "Array initialized BinaryDecoder result does not match",
+          datum, reader.read(null, initOnArray));
+    }
+  }
+
+  @Test
+  public void testInputStreamProxy() throws IOException {
+    Decoder d = newDecoder(data);
+    if (d instanceof BinaryDecoder) {
+      BinaryDecoder bd = (BinaryDecoder) d;
+      InputStream test = bd.inputStream();
+      InputStream check = new ByteArrayInputStream(data);
+      validateInputStreamReads(test, check);
+      bd = factory.createBinaryDecoder(data, bd);
+      test = bd.inputStream();
+      check = new ByteArrayInputStream(data);
+      validateInputStreamSkips(test, check);
+      // with input stream sources
+      bd = factory.createBinaryDecoder(new ByteArrayInputStream(data), bd);
+      test = bd.inputStream();
+      check = new ByteArrayInputStream(data);
+      validateInputStreamReads(test, check);
+      bd = factory.createBinaryDecoder(new ByteArrayInputStream(data), bd);
+      test = bd.inputStream();
+      check = new ByteArrayInputStream(data);
+      validateInputStreamSkips(test, check);
+    }
+  }
+
+  @Test
+  public void testInputStreamProxyDetached() throws IOException {
+    Decoder d = newDecoder(data);
+    if (d instanceof BinaryDecoder) {
+      BinaryDecoder bd = (BinaryDecoder) d;
+      InputStream test = bd.inputStream();
+      InputStream check = new ByteArrayInputStream(data);
+      // detach input stream and decoder from old source
+      factory.createBinaryDecoder(new byte[56], null);
+      InputStream bad = bd.inputStream();
+      InputStream check2 = new ByteArrayInputStream(data);
+      validateInputStreamReads(test, check);
+      Assert.assertFalse(bad.read() == check2.read());
+    }
+  }
+  
+  @Test
+  public void testInputStreamPartiallyUsed() throws IOException {
+    BinaryDecoder bd = factory.createBinaryDecoder(
+        new ByteArrayInputStream(data), null);
+    InputStream test = bd.inputStream();
+    InputStream check = new ByteArrayInputStream(data);
+    // triggers buffer fill if unused and tests isEnd()
+    try {
+      Assert.assertFalse(bd.isEnd()); 
+    } catch (UnsupportedOperationException e) {
+      // this is ok if its a DirectBinaryDecoder.
+      if (bd.getClass() != DirectBinaryDecoder.class) {
+        throw e;
+      }
+    }
+    bd.readFloat(); // use data, and otherwise trigger buffer fill
+    check.skip(4); // skip the same # of bytes here
+    validateInputStreamReads(test, check);
+  }
+
+  private void validateInputStreamReads(InputStream test, InputStream check)
+      throws IOException {
+    byte[] bt = new byte[7];
+    byte[] bc = new byte[7]; 
+    while (true) {
+      int t = test.read();
+      int c = check.read();
+      Assert.assertEquals(c, t);
+      if (-1 == t) break;
+      t = test.read(bt);
+      c = check.read(bc);
+      Assert.assertEquals(c, t);
+      Assert.assertArrayEquals(bt, bc);
+      if (-1 == t) break;
+      t = test.read(bt, 1, 4);
+      c = check.read(bc, 1, 4);
+      Assert.assertEquals(c, t);
+      Assert.assertArrayEquals(bt, bc);
+      if (-1 == t) break;
+    }
+    Assert.assertEquals(0, test.skip(5));
+    Assert.assertEquals(0, test.available());
+    Assert.assertFalse(test.getClass() != ByteArrayInputStream.class && test.markSupported());
+    test.close();
+  }
+  
+  private void validateInputStreamSkips(InputStream test, InputStream check) throws IOException
{
+    while(true) {
+      long t2 = test.skip(19);
+      long c2 = check.skip(19);
+      Assert.assertEquals(c2, t2);
+      if (0 == t2) break;
+    }
+    Assert.assertEquals(-1, test.read());
+  }
+
+  @Test
+  public void testBadIntEncoding() throws IOException {
+    byte[] badint = new byte[5];
+    Arrays.fill(badint, (byte)0xff);
+    Decoder bd = factory.createBinaryDecoder(badint, null);
+    String message = "";
+    try {
+      bd.readInt();
+    } catch (IOException ioe) {
+      message = ioe.getMessage();
+    }
+    Assert.assertEquals("Invalid int encoding", message);
+  }
+
+  @Test
+  public void testBadLongEncoding() throws IOException {
+    byte[] badint = new byte[10];
+    Arrays.fill(badint, (byte)0xff);
+    Decoder bd = factory.createBinaryDecoder(badint, null);
+    String message = "";
+    try {
+      bd.readLong();
+    } catch (IOException ioe) {
+      message = ioe.getMessage();
+    }
+    Assert.assertEquals("Invalid long encoding", message);
+  }
+
+  @Test(expected=EOFException.class)
+  public void testIntTooShort() throws IOException {
+    byte[] badint = new byte[4];
+    Arrays.fill(badint, (byte)0xff);
+    newDecoder(badint).readInt();
+  }
+
+  @Test(expected=EOFException.class)
+  public void testLongTooShort() throws IOException {
+    byte[] badint = new byte[9];
+    Arrays.fill(badint, (byte)0xff);
+    newDecoder(badint).readLong();
+  }
+  
+  @Test(expected=EOFException.class)
+  public void testFloatTooShort() throws IOException {
+    byte[] badint = new byte[3];
+    Arrays.fill(badint, (byte)0xff);
+    newDecoder(badint).readInt();
+  }
+
+  @Test(expected=EOFException.class)
+  public void testDoubleTooShort() throws IOException {
+    byte[] badint = new byte[7];
+    Arrays.fill(badint, (byte)0xff);
+    newDecoder(badint).readLong();
+  }
+
+  @Test
+  public void testSkipping() throws IOException {
+    Decoder d = newDecoder(data);
+    skipGenerated(d);
+    if (d instanceof BinaryDecoder) {
+      BinaryDecoder bd = (BinaryDecoder) d;
+      try {
+        Assert.assertTrue(bd.isEnd());
+      } catch (UnsupportedOperationException e) {
+        // this is ok if its a DirectBinaryDecoder.
+        if (bd.getClass() != DirectBinaryDecoder.class) {
+          throw e;
+        }
+      }
+      bd = factory.createBinaryDecoder(new ByteArrayInputStream(data), bd);
+      skipGenerated(bd);
+      try {
+        Assert.assertTrue(bd.isEnd());
+      } catch (UnsupportedOperationException e) {
+        // this is ok if its a DirectBinaryDecoder.
+        if (bd.getClass() != DirectBinaryDecoder.class) {
+          throw e;
+        }
+      }
+    }
+  }
+
+  private void skipGenerated(Decoder bd) throws IOException {
+    for (int i = 0; i < records.size(); i++) {
+      bd.readInt();
+      bd.skipBytes();
+      bd.skipFixed(1);
+      bd.skipString();
+      bd.skipFixed(4);
+      bd.skipFixed(8);
+      long leftover = bd.skipArray();
+      // booleans are one byte, array trailer is one byte
+      bd.skipFixed((int)leftover + 1); 
+      bd.skipFixed(0);
+      bd.readLong();
+    }
+    EOFException eof = null;
+    try {
+      bd.skipFixed(4);
+    } catch (EOFException e) {
+      eof = e;
+    }
+    Assert.assertTrue(null != eof);
+  }
 }

Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO.java Fri Feb
19 21:45:10 2010
@@ -68,7 +68,7 @@
       
       byte[] bb = os.toByteArray();
       // dump(bb);
-      this.input = new BinaryDecoder(new ByteArrayInputStream(bb));
+      this.input = DecoderFactory.defaultFactory().createBinaryDecoder(bb, null);
       this.parser =  f.createJsonParser(new ByteArrayInputStream(in));
     }
     

Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO2.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO2.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO2.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO2.java Fri
Feb 19 21:45:10 2010
@@ -17,10 +17,8 @@
  */
 package org.apache.avro.io;
 
-import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.InputStream;
 import java.util.Collection;
 import java.util.Arrays;
 
@@ -53,8 +51,7 @@
     
     byte[] bb = os.toByteArray();
     
-    InputStream in = new ByteArrayInputStream(bb);
-    decoder = new BinaryDecoder(in);
+    decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bb, null);
     this.calls = calls;
   }
     

Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestResolvingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestResolvingIO.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestResolvingIO.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestResolvingIO.java Fri
Feb 19 21:45:10 2010
@@ -97,14 +97,14 @@
       throws IOException {
     // TestValidatingIO.dump(bytes);
     // System.out.println(new String(bytes, "UTF-8"));
-    InputStream in = new ByteArrayInputStream(bytes);
     Decoder bvi = null;
     switch (encoding) {
     case BINARY:
     case BLOCKING_BINARY:
-      bvi = new BinaryDecoder(in);
+      bvi = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
       break;
     case JSON:
+      InputStream in = new ByteArrayInputStream(bytes);
       bvi = new JsonDecoder(wsc, in);
       break;
     }

Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestValidatingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestValidatingIO.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestValidatingIO.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestValidatingIO.java Fri
Feb 19 21:45:10 2010
@@ -298,14 +298,14 @@
     throws IOException {
     // dump(bytes);
     // System.out.println(new String(bytes, "UTF-8"));
-    InputStream in = new ByteArrayInputStream(bytes);
     Decoder bvi = null;
     switch (encoding) {
     case BINARY:
     case BLOCKING_BINARY:
-      bvi = new BinaryDecoder(in);
+      bvi = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
       break;
     case JSON:
+      InputStream in = new ByteArrayInputStream(bytes);
       bvi = new JsonDecoder(sc, in);
     }
     Decoder vi = new ValidatingDecoder(sc, bvi);



Mime
View raw message