avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r782092 [2/2] - in /hadoop/avro/trunk: ./ src/java/org/apache/avro/file/ src/java/org/apache/avro/generic/ src/java/org/apache/avro/io/ src/java/org/apache/avro/ipc/ src/test/java/org/apache/avro/ src/test/java/org/apache/avro/io/
Date Fri, 05 Jun 2009 18:31:45 GMT
Added: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java?rev=782092&view=auto
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java (added)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java Fri Jun  5 18:31:45
2009
@@ -0,0 +1,450 @@
+package org.apache.avro.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.Stack;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.JsonParser;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class TestBlockingIO {
+  private static final String UTF_8 = "UTF-8";
+  
+  private static class Tests {
+    private final JsonParser parser;
+    private final ValueReader input;
+    private final int depth;
+    public Tests(int bufferSize, int depth, String input)
+      throws JsonParseException, IOException {
+  
+      this.depth = depth;
+      byte[] in = input.getBytes("UTF-8");
+      JsonFactory f = new JsonFactory();
+      JsonParser p = f.createJsonParser(
+          new ByteArrayInputStream(input.getBytes("UTF-8")));
+      
+      ByteArrayOutputStream os = new ByteArrayOutputStream();
+      ValueWriter cos = new BlockingValueWriter(os, bufferSize);
+      serialize(cos, p, os);
+      cos.flush();
+      
+      byte[] bb = os.toByteArray();
+      // dump(bb);
+      this.input = new ValueReader(new ByteArrayInputStream(bb));
+      this.parser =  f.createJsonParser(new ByteArrayInputStream(in));
+    }
+    
+    public void scan()
+      throws JsonParseException, UnsupportedEncodingException, IOException {
+      Stack<S> countStack = new Stack<S>();
+      long count = 0;
+      while (parser.nextToken() != null) {
+        switch (parser.getCurrentToken()) {
+        case END_ARRAY:
+          Assert.assertEquals(0, count);
+          Assert.assertTrue(countStack.peek().isArray);
+          count = countStack.pop().count;
+          break;
+        case END_OBJECT:
+          Assert.assertEquals(0, count);
+          Assert.assertFalse(countStack.peek().isArray);
+          count = countStack.pop().count;
+          break;
+        case START_ARRAY:
+          countStack.push(new S(count, true));
+          count = input.readArrayStart();
+          continue;
+        case VALUE_STRING:
+        {
+          String s = parser.getText();
+          int n = s.getBytes(UTF_8).length;
+          checkString(s, input, n);
+          break;
+        }
+        case FIELD_NAME:
+        {
+          String s = parser.getCurrentName();
+          int n = s.getBytes(UTF_8).length;
+          checkString(s, input, n);
+          continue;
+        }
+        case START_OBJECT:
+          countStack.push(new S(count, false));
+          count = input.readMapStart();
+          if (count < 0) {
+            count = -count;
+            input.readLong();  // byte count
+          }
+          continue;
+        default:
+          throw new RuntimeException("Unsupported: " + parser.getCurrentToken());
+        }
+        count--;
+        if (count == 0) {
+          count = countStack.peek().isArray ? input.arrayNext() :
+            input.mapNext();
+        }
+      }
+    }
+
+    public void skip(int skipLevel) throws
+      JsonParseException, UnsupportedEncodingException, IOException {
+      Stack<S> countStack = new Stack<S>();
+      long count = 0;
+      while (parser.nextToken() != null) {
+        switch (parser.getCurrentToken()) {
+        case END_ARRAY:
+          // assertEquals(0, count);
+          Assert.assertTrue(countStack.peek().isArray);
+          count = countStack.pop().count;
+          break;
+        case END_OBJECT:
+          // assertEquals(0, count);
+          Assert.assertFalse(countStack.peek().isArray);
+          count = countStack.pop().count;
+          break;
+        case START_ARRAY:
+          if (countStack.size() == skipLevel) {
+            skipArray(parser, input, depth - skipLevel);
+            break;
+          } else {
+            countStack.push(new S(count, true));
+            count = input.readArrayStart();
+            continue;
+          }
+        case VALUE_STRING:
+        {
+          if (countStack.size() == skipLevel) {
+            input.skipBytes();
+          } else {
+            String s = parser.getText();
+            int n = s.getBytes(UTF_8).length;
+            checkString(s, input, n);
+          }
+          break;
+        }
+        case FIELD_NAME:
+        {
+          String s = parser.getCurrentName();
+          int n = s.getBytes(UTF_8).length;
+          checkString(s, input, n);
+          continue;
+        }
+        case START_OBJECT:
+          if (countStack.size() == skipLevel) {
+            skipMap(parser, input, depth - skipLevel);
+            break;
+          } else {
+            countStack.push(new S(count, false));
+            count = input.readMapStart();
+            if (count < 0) {
+              count = -count;
+              input.readLong();  // byte count
+            }
+            continue;
+          }
+        default:
+          throw new RuntimeException("Unsupported: " + parser.getCurrentToken());
+        }
+        count--;
+        if (count == 0) {
+          count = countStack.peek().isArray ? input.arrayNext() :
+            input.mapNext();
+        }
+      }
+    }
+  }
+
+  protected static void dump(byte[] bb) {
+    int col = 0;
+    for (byte b : bb) {
+      if (col % 16 == 0) {
+        System.out.println();
+      }
+      col++;
+      System.out.print(Integer.toHexString(b & 0xff) + " ");
+    }
+    System.out.println();
+  }
+
+  private static class S {
+    public final long count;
+    public final boolean isArray;
+    
+    public S(long count, boolean isArray) {
+      this.count = count;
+      this.isArray = isArray;
+    }
+  }
+
+  @Test(dataProvider="data")
+  public void testScan(int bufferSize, int depth, String input)
+    throws JsonParseException, IOException {
+    Tests t = new Tests(bufferSize, depth, input);
+    t.scan();
+  }
+
+  @Test(dataProvider="data")
+  public void testSkip_1(int bufferSize, int depth, String input)
+    throws JsonParseException, IOException {
+    testSkip(bufferSize, depth, input, 0);
+  }
+
+  @Test(dataProvider="data")
+  public void testSkip_2(int bufferSize, int depth, String input)
+    throws JsonParseException, IOException {
+    testSkip(bufferSize, depth, input, 1);
+  }
+
+  @Test(dataProvider="data")
+  public void testSkip_3(int bufferSize, int depth, String input)
+    throws JsonParseException, IOException {
+    testSkip(bufferSize, depth, input, 2);
+  }
+
+  private void testSkip(int bufferSize, int depth, String input,
+      int skipLevel)
+    throws JsonParseException, IOException {
+    Tests t = new Tests(bufferSize, depth, input);
+    t.skip(skipLevel);
+  }
+
+  private static void skipMap(JsonParser parser, ValueReader input, int depth)
+    throws IOException, JsonParseException {
+    for (long l = input.skipMap(); l != 0; l = input.skipMap()) {
+      for (long i = 0; i < l; i++) {
+        if (depth == 0) {
+          input.skipBytes();
+        } else {
+          skipArray(parser, input, depth - 1);
+        }
+      }
+    }
+    parser.skipChildren();
+  }
+
+  private static void skipArray(JsonParser parser, ValueReader input, int depth)
+    throws IOException, JsonParseException {
+    for (long l = input.skipArray(); l != 0; l = input.skipArray()) {
+      for (long i = 0; i < l; i++) {
+        if (depth == 1) {
+          input.skipBytes();
+        } else {
+          skipArray(parser, input, depth - 1);
+        }
+      }
+    }
+    parser.skipChildren();
+  }
+ 
+  private static void checkString(String s, ValueReader input, int n)
+    throws IOException, UnsupportedEncodingException {
+    ByteBuffer buf = input.readBytes(null);
+    Assert.assertEquals(n, buf.remaining());
+    String s2 = new String(buf.array(), buf.position(),
+        buf.remaining(), UTF_8);
+    Assert.assertEquals(s, s2);
+  }
+  
+  private static void serialize(ValueWriter cos, JsonParser p,
+      ByteArrayOutputStream os)
+    throws JsonParseException, IOException {
+    boolean[] isArray = new boolean[100];
+    int[] counts = new int[100];
+    int stackTop = -1;
+    
+    while (p.nextToken() != null) {
+      switch (p.getCurrentToken()) {
+      case END_ARRAY:
+        Assert.assertTrue(isArray[stackTop]);
+        cos.writeArrayEnd();
+        stackTop--;
+        break;
+      case END_OBJECT:
+        Assert.assertFalse(isArray[stackTop]);
+        cos.writeMapEnd();
+        stackTop--;
+        break;
+      case START_ARRAY:
+        if (stackTop >= 0 && isArray[stackTop]) {
+          cos.setItemCount(1);
+          cos.startItem();
+          counts[stackTop]++;
+        }
+        cos.writeArrayStart();
+        isArray[++stackTop] = true;
+        counts[stackTop] = 0;
+        continue;
+      case VALUE_STRING:
+        if (stackTop >= 0 && isArray[stackTop]) {
+          cos.setItemCount(1);
+          cos.startItem();
+          counts[stackTop]++;
+        }
+        byte[] bb = p.getText().getBytes(UTF_8);
+        cos.writeBytes(bb);
+        break;
+      case START_OBJECT:
+        if (stackTop >= 0 && isArray[stackTop]) {
+          cos.setItemCount(1);
+          cos.startItem();
+          counts[stackTop]++;
+        }
+        cos.writeMapStart();
+        isArray[++stackTop] = false;
+        counts[stackTop] = 0;
+        continue;
+      case FIELD_NAME:
+        cos.setItemCount(1);
+        cos.startItem();
+        counts[stackTop]++;
+        cos.writeBytes(p.getCurrentName().getBytes(UTF_8));
+        break;
+     default:
+       throw new RuntimeException("Unsupported: " + p.getCurrentToken());
+      }
+    }
+  }
+
+  @DataProvider
+  public static Object[][] data() {
+    return new Object[][] {
+        { 64, 0, "" },
+        { 64, 0, jss(0, 'a') },
+        { 64, 0, jss(3, 'a') },
+        { 64, 0, jss(64, 'a') },
+        { 64, 0, jss(65, 'a') },
+        { 64, 0, jss(100, 'a') },
+        { 64, 1, "[]" },
+        { 64, 1, "[" + jss(0, 'a') + "]" },
+        { 64, 1, "[" + jss(3, 'a') + "]" },
+        { 64, 1, "[" + jss(61, 'a') + "]" },
+        { 64, 1, "[" + jss(62, 'a') + "]" },
+        { 64, 1, "[" + jss(64, 'a') + "]" },
+        { 64, 1, "[" + jss(65, 'a') + "]" },
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(0, '0') + "]" },
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(10, '0') + "]" },
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(63, '0') + "]" },
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(64, '0') + "]" },
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(65, '0') + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(0, '0') + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(10, '0') + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(51, '0') + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(52, '0') + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(54, '0') + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(55, '0') + "]" },
+
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(0, '0')
+               + "]" },
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(63, '0')
+               + "]" },
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(64, '0')
+               + "]" },
+        { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(65, '0')
+                 + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(10, '0')
+                 + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(23, '0')
+                 + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(24, '0')
+                 + "]" },
+        { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(25, '0')
+                 + "]" },
+        { 64, 2, "[[]]"},
+        { 64, 2, "[[" + jss(0, 'a') + "], []]" },
+        { 64, 2, "[[" + jss(10, 'a') + "], []]" },
+        { 64, 2, "[[" + jss(59, 'a') + "], []]" },
+        { 64, 2, "[[" + jss(60, 'a') + "], []]" },
+        { 64, 2, "[[" + jss(100, 'a') + "], []]" },
+        { 64, 2, "[[" + jss(10, '0') + ", " + jss(53, 'a') + "], []]" },
+        { 64, 2, "[[" + jss(10, '0') + ", "  + jss(54, 'a') + "], []]" },
+        { 64, 2, "[[" + jss(10, '0') + ", "  + jss(55, 'a') + "], []]" },
+
+        { 64, 2, "[[], [" + jss(0, 'a') + "]]" },
+        { 64, 2, "[[], [" + jss(10, 'a') + "]]" },
+        { 64, 2, "[[], [" + jss(63, 'a') + "]]" },
+        { 64, 2, "[[], [" + jss(64, 'a') + "]]" },
+        { 64, 2, "[[], [" + jss(65, 'a') + "]]" },
+        { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(53, 'a') + "]]" },
+        { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(54, 'a') + "]]" },
+        { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(55, 'a') + "]]" },
+
+        { 64, 2, "[[" + jss(10, '0') + "]]"},
+        { 64, 2, "[[" + jss(62, '0') + "]]"},
+        { 64, 2, "[[" + jss(63, '0') + "]]"},
+        { 64, 2, "[[" + jss(64, '0') + "]]"},
+        { 64, 2, "[[" + jss(10, 'a') + ", " + jss(10, '0') + "]]"},
+        { 64, 2, "[[" + jss(10, 'a') + ", " + jss(52, '0') + "]]"},
+        { 64, 2, "[[" + jss(10, 'a') + ", " + jss(53, '0') + "]]"},
+        { 64, 2, "[[" + jss(10, 'a') + ", " + jss(54, '0') + "]]"},
+        { 64, 3, "[[[" + jss(10, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(62, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(63, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(64, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(10, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(52, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(53, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(54, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(10, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(52, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(53, '0') + "]]]"},
+        { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(54, '0') + "]]]"},
+
+        { 64, 2, "[[\"p\"], [\"mn\"]]"},
+        { 64, 2, "[[\"pqr\"], [\"mn\"]]"},
+        { 64, 2, "[[\"pqrstuvwxyz\"], [\"mn\"]]"},
+        { 64, 2, "[[\"abc\", \"pqrstuvwxyz\"], [\"mn\"]]"},
+        { 64, 2, "[[\"mn\"], [\"\"]]"},
+        { 64, 2, "[[\"mn\"], \"abc\"]"},
+        { 64, 2, "[[\"mn\"], \"abcdefghijk\"]"},
+        { 64, 2, "[[\"mn\"], \"pqr\", \"abc\"]"},
+        { 64, 2, "[[\"mn\"]]"},
+        { 64, 2, "[[\"p\"], [\"mnopqrstuvwx\"]]"},
+        { 64, 2, "[[\"pqr\"], [\"mnopqrstuvwx\"]]"},
+        { 64, 2, "[[\"pqrstuvwxyz\"], [\"mnopqrstuvwx\"]]"},
+        { 64, 2, "[[\"abc\"], \"pqrstuvwxyz\", [\"mnopqrstuvwx\"]]"},
+        { 64, 2, "[[\"mnopqrstuvwx\"], [\"\"]]"},
+        { 64, 2, "[[\"mnopqrstuvwx\"], [\"abc\"]]"},
+        { 64, 2, "[[\"mnopqrstuvwx\"], [\"abcdefghijk\"]]"},
+        { 64, 2, "[[\"mnopqrstuvwx\"], [\"pqr\", \"abc\"]]"},
+        { 100, 2, "[[\"pqr\", \"mnopqrstuvwx\"]]"},
+        { 100, 2, "[[\"pqr\", \"ab\", \"mnopqrstuvwx\"]]"},
+        { 64, 2, "[[[\"pqr\"]], [[\"ab\"], [\"mnopqrstuvwx\"]]]"},
+
+        { 64, 1, "{}" },
+        { 64, 1, "{\"n\": \"v\"}" },
+        { 64, 1, "{\"n1\": \"v\", \"n2\": []}" },
+        { 100, 1, "{\"n1\": \"v\", \"n2\": []}" },
+        { 100, 1, "{\"n1\": \"v\", \"n2\": [\"abc\"]}" },
+    };
+  }
+
+  /**
+   * Returns a new JSON String {@code n} bytes long with
+   * consecutive characters starting with {@code c}.
+   */
+  private static String jss(final int n, char c) {
+    char[] cc = new char[n + 2];
+    cc[0] = cc[n + 1] = '"';
+    for (int i = 1; i < n + 1; i++) {
+      if (c == 'Z') {
+        c = 'a';
+      } else if (c == 'z') {
+        c = '0';
+      } else if (c == '9') {
+        c = 'A';
+      } else {
+        c++;
+      }
+      cc[i] = c;
+    }
+    return new String(cc);
+  }
+}

Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java?rev=782092&r1=782091&r2=782092&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java (original)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java Fri Jun  5 18:31:45
2009
@@ -30,46 +30,51 @@
 public class TestValueReader {
   /** Verify EOFException throw at EOF */
 
-  @Test
-  public void testEOFHandling() throws IOException {
-    InputStream is = new ByteArrayInputStream(new byte[0]);
-    ValueReader vr = new ValueReader(is);
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_boolean() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).readBoolean();
+  }
+  
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_int() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).readInt();
+  }
+  
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_long() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).readLong();
+  }
+  
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_float() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).readFloat();
+  }
+  
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_double() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).readDouble();
+  }
+  
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_bytes() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).readBytes(null);
+  }
+  
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_string() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).
+      readString(new Utf8("a"));
+  }
+  
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_fixed() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).
+      readFixed(new byte[1]);
+  }
 
-    try {
-      vr.readBoolean();
-      fail();
-    } catch (EOFException e) { /* this is good */ }
-    try {
-      vr.readBuffer(null);
-      fail();
-    } catch (EOFException e) { /* this is good */ }
-    try {
-      vr.readBytes(new byte[1]);
-      fail();
-    } catch (EOFException e) { /* this is good */ }
-    try {
-      vr.readBytes(new byte[1], 0, 1);
-      fail();
-    } catch (EOFException e) { /* this is good */ }
-    try {
-      vr.readDouble();
-      fail();
-    } catch (EOFException e) { /* this is good */ }
-    try {
-      vr.readFloat();
-      fail();
-    } catch (EOFException e) { /* this is good */ }
-    try {
-      vr.readInt();
-      fail();
-    } catch (EOFException e) { /* this is good */ }
-    try {
-      vr.readLong();
-      fail();
-    } catch (EOFException e) { /* this is good */ }
-    try {
-      vr.readUtf8(new Utf8("a"));
-      fail();
-    } catch (EOFException e) { /* this is good */ }
+  @Test(expectedExceptions=EOFException.class)
+  public void testEOF_enum() throws IOException {
+    new ValueReader(new ByteArrayInputStream(new byte[0])).readEnum();
   }
+  
 }



Mime
View raw message