avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From th...@apache.org
Subject svn commit: r899195 - in /hadoop/avro/trunk: ./ lang/java/src/java/org/apache/avro/io/ lang/java/src/java/org/apache/avro/io/parsing/ lang/java/src/test/java/org/apache/avro/io/
Date Thu, 14 Jan 2010 13:40:26 GMT
Author: thiru
Date: Thu Jan 14 13:40:25 2010
New Revision: 899195

URL: http://svn.apache.org/viewvc?rev=899195&view=rev
Log:
AVRO-316. Optiminzing inner loop functions of Avro io

Modified:
    hadoop/avro/trunk/CHANGES.txt
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java
    hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
    hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java

Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Thu Jan 14 13:40:25 2010
@@ -224,6 +224,8 @@
 
     AVRO-315. Performance improvements to BinaryDecoder (thiru)
 
+    AVRO-316. Optiminzing inner loop functions of Avro io (thiru)
+
   BUG FIXES
  
     AVRO-176. Safeguard against bad istreams before reading. (sbanacho)

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java Thu Jan 14 13:40:25
2010
@@ -389,7 +389,7 @@
         String fn = in.getCurrentName();
         if (fa.fname.equals(fn)) {
           in.nextToken();
-          return Symbol.CONTINUE;
+          return null;
         } else {
           throw new AvroTypeException("Expected field name " + fa.fname +
               " got " + in.getCurrentName());
@@ -410,7 +410,7 @@
     } else {
       throw new AvroTypeException("Unknown action symbol " + top);
     }
-    return Symbol.CONTINUE;
+    return null;
   }
 
   private AvroTypeException error(String type) {

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java Thu Jan 14 13:40:25
2010
@@ -233,7 +233,7 @@
     } else {
       throw new AvroTypeException("Unknown action symbol " + top);
     }
-    return Symbol.CONTINUE;
+    return null;
   }
 }
 

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java Thu Jan
14 13:40:25 2010
@@ -157,7 +157,7 @@
   @Override
   public Symbol doAction(Symbol input, Symbol top) throws IOException {
     if (top instanceof Symbol.FieldAdjustAction) {
-      return input == Symbol.FIELD_ACTION ? top : Symbol.CONTINUE;
+      return input == Symbol.FIELD_ACTION ? top : null;
     } if (top instanceof Symbol.ResolvingAction) {
       Symbol.ResolvingAction t = (Symbol.ResolvingAction) top;
       if (t.reader != input) {
@@ -183,7 +183,7 @@
     } else {
       throw new AvroTypeException("Unknown action: " + top);
     }
-    return Symbol.CONTINUE;
+    return null;
   }
 
   @Override

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java Thu Jan
14 13:40:25 2010
@@ -226,7 +226,7 @@
   }
   
   public Symbol doAction(Symbol input, Symbol top) throws IOException {
-    return Symbol.CONTINUE;
+    return null;
   }
 }
 

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java Thu Jan
14 13:40:25 2010
@@ -192,7 +192,7 @@
 
   @Override
   public Symbol doAction(Symbol input, Symbol top) throws IOException {
-    return Symbol.CONTINUE;
+    return null;
   }
 
   /** Have we written at least one item into the current collection? */

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
(original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
Thu Jan 14 13:40:25 2010
@@ -64,13 +64,11 @@
       return Symbol.seq(new Symbol.EnumLabelsAction(sc.getEnumSymbols()),
           Symbol.ENUM);
     case ARRAY:
-      return Symbol.seq(Symbol.ARRAY_END,
-          Symbol.repeat(Symbol.ARRAY_END,
+      return Symbol.seq(Symbol.repeat(Symbol.ARRAY_END,
               Symbol.ITEM_END, generate(sc.getElementType(), seen)),
           Symbol.ARRAY_START);
     case MAP:
-      return Symbol.seq(Symbol.MAP_END,
-          Symbol.repeat(Symbol.MAP_END,
+      return Symbol.seq(Symbol.repeat(Symbol.MAP_END,
               Symbol.ITEM_END, generate(sc.getValueType(), seen),
               Symbol.MAP_KEY_MARKER, Symbol.STRING),
           Symbol.MAP_START);

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java Thu Jan 14
13:40:25 2010
@@ -35,6 +35,16 @@
    * provide this help.
    */
   public interface ActionHandler {
+    /**
+     * Handle the action symbol <tt>top</tt> when the <tt>input</tt>
is
+     * sought to be taken off the stack.
+     * @param input The input symbol from the caller of advance
+     * @param top The symbol at the top the stack.
+     * @return  <tt>null</tt> if advance() is to continue processing the
+     * stack. If not <tt>null</tt> the return value will be returned
+     * by advance().
+     * @throws IOException
+     */
     Symbol doAction(Symbol input, Symbol top) throws IOException;
   }
 
@@ -70,21 +80,25 @@
   public final Symbol advance(Symbol input) throws IOException {
     for (; ;) {
       Symbol top = stack[--pos];
-      if (top.kind == Symbol.Kind.TERMINAL) {
-        if (top == input) {
-          return top; // A common case
-        } else {
-          throw new AvroTypeException("Attempt to process a "
-              + input + " when a "
-              + top + " was expected.");
+      if (top == input) {
+        return top; // A common case
+      }
+
+      Symbol.Kind k = top.kind;
+      if (k == Symbol.Kind.IMPLICIT_ACTION) {
+        Symbol result = symbolHandler.doAction(input, top);
+        if (result != null) {
+          return result;
         }
-      } else if (top.kind == Symbol.Kind.IMPLICIT_ACTION) {
-          Symbol result = symbolHandler.doAction(input, top);
-          if (result != Symbol.CONTINUE) {
-            return result;
-          }
+      } else if (k == Symbol.Kind.TERMINAL) {
+        throw new AvroTypeException("Attempt to process a "
+                + input + " when a "
+                + top + " was expected.");
+      } else if (k == Symbol.Kind.REPEATER
+          && input == ((Symbol.Repeater) top).end) {
+        return input;
       } else {
-        pushProduction(input, top);
+        pushProduction(top);
       }
     }
   }
@@ -112,16 +126,13 @@
    * @param input
    * @param sym
    */
-  public final void pushProduction(Symbol input, Symbol sym) {
-    if (sym.kind != Symbol.Kind.REPEATER ||
-        input != ((Symbol.Repeater) sym).end) {
-      Symbol[] p = sym.production;
-      while (pos + p.length > stack.length) {
-        expandStack();
-      }
-      System.arraycopy(p, 0, stack, pos, p.length);
-      pos += p.length;
+  public final void pushProduction(Symbol sym) {
+    Symbol[] p = sym.production;
+    while (pos + p.length > stack.length) {
+      expandStack();
     }
+    System.arraycopy(p, 0, stack, pos, p.length);
+    pos += p.length;
   }
 
   /**

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
(original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
Thu Jan 14 13:40:25 2010
@@ -98,15 +98,13 @@
         break;
 
       case ARRAY:
-        return Symbol.seq(Symbol.ARRAY_END,
-            Symbol.repeat(Symbol.ARRAY_END,
+        return Symbol.seq(Symbol.repeat(Symbol.ARRAY_END,
                 generate(writer.getElementType(),
                 reader.getElementType(), seen)),
             Symbol.ARRAY_START);
       
       case MAP:
-        return Symbol.seq(Symbol.MAP_END,
-            Symbol.repeat(Symbol.MAP_END,
+        return Symbol.seq(Symbol.repeat(Symbol.MAP_END,
                 generate(writer.getValueType(),
                 reader.getValueType(), seen), Symbol.STRING),
             Symbol.MAP_START);

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java Thu Jan
14 13:40:25 2010
@@ -64,7 +64,7 @@
           skipHandler.skipAction();
         } else {
           --pos;
-          pushProduction(null, top);
+          pushProduction(top);
         }
         continue outer;
       }
@@ -79,7 +79,7 @@
     int target = pos;
     Symbol repeater = stack[--pos];
     assert repeater.kind == Symbol.Kind.REPEATER;
-    pushProduction(null, repeater);
+    pushProduction(repeater);
     skipTo(target);
   }
 

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java Thu Jan 14
13:40:25 2010
@@ -379,7 +379,6 @@
   public static final Symbol ITEM_END = new Symbol.Terminal("item-end");
 
   /* a pseudo terminal used by parsers */
-  public static final Symbol CONTINUE = new Symbol.Terminal("continue");
   public static final Symbol FIELD_ACTION =
     new Symbol.Terminal("field-action");
 

Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
(original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
Thu Jan 14 13:40:25 2010
@@ -71,12 +71,10 @@
       return Symbol.seq(new Symbol.IntCheckAction(sc.getEnumSymbols().size()),
           Symbol.ENUM);
     case ARRAY:
-      return Symbol.seq(Symbol.ARRAY_END,
-          Symbol.repeat(Symbol.ARRAY_END, generate(sc.getElementType(), seen)),
+      return Symbol.seq(Symbol.repeat(Symbol.ARRAY_END, generate(sc.getElementType(), seen)),
           Symbol.ARRAY_START);
     case MAP:
-      return Symbol.seq(Symbol.MAP_END,
-          Symbol.repeat(Symbol.MAP_END,
+      return Symbol.seq(Symbol.repeat(Symbol.MAP_END,
               generate(sc.getValueType(), seen), Symbol.STRING),
           Symbol.MAP_START);
     case RECORD: {

Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java Thu Jan 14 13:40:25
2010
@@ -20,8 +20,13 @@
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Random;
 
+import junit.extensions.RepeatedTest;
+
 import org.apache.avro.Schema;
 
 /**
@@ -34,39 +39,45 @@
   
   
   public static void main(String[] args) throws IOException {
-    Test[] tests = null;
-    if (args.length == 0) {
-      tests = new Test[] { new ReadInt(),
-          new ReadLong(), new ReadFloat(), new ReadDouble() };
-    } else if (args.length == 1) {
-      if (args[0].equals("-i")) {
-        tests = new Test[] { new ReadInt() };
-      } else if (args[0].equals("-f")) {
-        tests = new Test[] { new ReadFloat() };
-      } else if (args[0].equals("-d")) {
-        tests = new Test[] { new ReadDouble() };
-      } else if (args[0].equals("-l")) {
-        tests = new Test[] { new ReadLong() };
-      }
-    } else {
-      usage();
-      System.exit(1);
+    List<Test> tests = new ArrayList<Test>();
+    for (String arg : args) {
+      if (arg.equals("-i")) {
+        tests.add(new ReadInt());
+      } else if (arg.equals("-f")) {
+        tests.add(new ReadFloat());
+      } else if (arg.equals("-d")) {
+        tests.add(new ReadDouble());
+      } else if (arg.equals("-l")) {
+        tests.add(new ReadLong());
+      } else if (arg.equals("-R")) {
+        tests.add(new RepeaterTest());
+      } else {
+        usage();
+      }
+    }
+    if (tests.isEmpty()) {
+      tests.addAll(Arrays.asList(new Test[] {
+         new ReadInt(), new ReadLong(), new ReadFloat(), new ReadDouble(),
+         new RepeaterTest(),
+      }));
     }
     
     for (Test t : tests) {
-      // warmup JVM 
+      // warmup JVM
+      // System.out.println("Warming up...");
       for (int i = 0; i < CYCLES; i++) {
-        t.read();
+        t.test();
       }
       // test
       long s = 0;
       for (int i = 0; i < CYCLES; i++) {
-        long l = t.read();
+        long l = t.test();
         // System.out.println("** " + l);
         s += l;
       }
-       s /= 1000;
-      System.out.println(t.name + "(" + t.schema + "): " + s/1000 + " ms, " +  (CYCLES *
(double)COUNT)/s + " million numbers decoded /sec" );
+      s /= 1000;
+      System.out.println(t.name + "(" + t.schema + "): " + s/1000 + " ms, "
+          +  ((CYCLES * (double)COUNT) / s) + " million numbers decoded/sec" );
     }
   }
   
@@ -74,6 +85,7 @@
     public final String name;
     public final Schema schema;
     protected byte[] data;
+    
     public Test(String name, String json) throws IOException {
       this.name = name;
       this.schema = Schema.parse(json);
@@ -82,8 +94,9 @@
       genData(e);
       data = bao.toByteArray();
     }
-    public final long read() throws IOException {
-      Decoder d = new BinaryDecoder(new ByteArrayInputStream(data));
+    
+    public final long test() throws IOException {
+      Decoder d = getDecoder();
       long t = System.nanoTime();
       for (long l = d.readArrayStart(); l > 0; l = d.arrayNext()) {
         for (int j = 0; j < l; j++) {
@@ -92,13 +105,22 @@
       }
       return (System.nanoTime() - t);
     }
+
+    protected Decoder getDecoder() throws IOException {
+      return new BinaryDecoder(new ByteArrayInputStream(data));
+    }
+
     abstract void genData(Encoder e) throws IOException;
     abstract void readInternal(Decoder d) throws IOException;
   }
   
   private static class ReadInt extends Test {
     public ReadInt() throws IOException {
-      super("ReadInt", "{ \"type\": \"array\", \"items\": \"int\"} ");
+      this("ReadInt", "{ \"type\": \"array\", \"items\": \"int\"} ");
+    }
+
+    public ReadInt(String name, String json) throws IOException {
+      super(name, json);
     }
     @Override void genData(Encoder e) throws IOException {
       e.writeArrayStart();
@@ -179,11 +201,26 @@
       d.readDouble();
     }
   }
+  
+  private static class RepeaterTest extends ReadInt {
+      public RepeaterTest() throws IOException {
+        super("RepeaterTest",
+            "{ \"type\": \"array\", \"items\": \"int\"} ");
+      }
+
+      @Override
+      public Decoder getDecoder() throws IOException {
+        return new ValidatingDecoder(schema, super.getDecoder());
+      }
+    }
+
   private static void usage() {
     System.out.println("Usage: Perf { -i | -l | -f | -d }");
     System.out.println("    -i      measures readInt() performance");
     System.out.println("    -l      measures readLong() performance");
     System.out.println("    -f      measures readFloat() performance");
     System.out.println("    -d      measures readDouble() performance");
+    System.out.println("    -R      measures repeater performance in "
+        + "validating encoder");
   }
 }



Mime
View raw message