Return-Path: 
 <user-return-1071-apmail-avro-user-archive=avro.apache.org@avro.apache.org>
X-Original-To: apmail-avro-user-archive@www.apache.org
Delivered-To: apmail-avro-user-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id CBA227C44
	for <apmail-avro-user-archive@www.apache.org>;
 Thu,  1 Sep 2011 01:50:56 +0000 (UTC)
Received: (qmail 73180 invoked by uid 500); 1 Sep 2011 01:50:56 -0000
Delivered-To: apmail-avro-user-archive@avro.apache.org
Received: (qmail 72879 invoked by uid 500); 1 Sep 2011 01:50:55 -0000
Mailing-List: contact user-help@avro.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:user-help@avro.apache.org>
List-Unsubscribe: <mailto:user-unsubscribe@avro.apache.org>
List-Post: <mailto:user@avro.apache.org>
List-Id: <user.avro.apache.org>
Reply-To: user@avro.apache.org
Delivered-To: mailing list user@avro.apache.org
Received: (qmail 72871 invoked by uid 99); 1 Sep 2011 01:50:54 -0000
Received: from minotaur.apache.org (HELO minotaur.apache.org) (140.211.11.9)
    by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 01 Sep 2011 01:50:54 +0000
Received: from localhost (HELO [10.0.0.231]) (127.0.0.1)
  (smtp-auth username scottcarey, mechanism login)
  by minotaur.apache.org (qpsmtpd/0.29) with ESMTP;
 Thu, 01 Sep 2011 01:50:54 +0000
User-Agent: Microsoft-MacOutlook/14.12.0.110505
Date: Wed, 31 Aug 2011 18:52:25 -0700
Subject: Re: avro BinaryDecoder bug ?
From: Scott Carey <scottcarey@apache.org>
Sender: Scott Carey <scott@richrelevance.com>
To: "user@avro.apache.org" <user@avro.apache.org>
Message-ID: <CA843352.4F413%scott@richrelevance.com>
Thread-Topic: avro BinaryDecoder bug ?
In-Reply-To: 
 <CAAnh3_9_CQqbmt4VQYZg8-iKfO4nnLPcUZbBwd4kQoAvPEKGKQ@mail.gmail.com>
Mime-version: 1.0
Content-type: text/plain;
	charset="US-ASCII"
Content-transfer-encoding: 7bit

Looks like a bug to me.

Can you file a JIRA ticket?

Thanks!

On 8/29/11 1:24 PM, "Yang" <teddyyyy123@gmail.com> wrote:

>if I read on a empty file with BinaryDecoder, I get EOF, good,
>
>but with the current code, if I read it again with the same decoder, I
>get a IndexOutofBoundException, not EOF.
>
>it seems that always giving EOF should be a more desirable behavior.
>
>you can see from this test code:
>
>import static org.junit.Assert.assertEquals;
>
>import java.io.IOException;
>
>import org.apache.avro.specific.SpecificRecord;
>import org.junit.Test;
>
>import myavro.Apple;
>
>import java.io.File;
>import java.io.FileInputStream;
>import java.io.FileNotFoundException;
>import java.io.FileOutputStream;
>import java.io.InputStream;
>import java.io.OutputStream;
>
>import org.apache.avro.io.Decoder;
>import org.apache.avro.io.DecoderFactory;
>import org.apache.avro.io.Encoder;
>import org.apache.avro.io.EncoderFactory;
>import org.apache.avro.specific.SpecificDatumReader;
>import org.apache.avro.specific.SpecificDatumWriter;
>
>class MyWriter {
>
>    SpecificDatumWriter<SpecificRecord> wr;
>    Encoder enc;
>    OutputStream ostream;
>
>    public MyWriter() throws FileNotFoundException {
>        wr = new SpecificDatumWriter<SpecificRecord>(new
>Apple().getSchema());
>        ostream = new FileOutputStream(new File("/tmp/testavro"));
>        enc = EncoderFactory.get().binaryEncoder(ostream, null);
>    }
>
>    public synchronized void dump(SpecificRecord event) throws
>IOException {
>        wr.write(event, enc);
>        enc.flush();
>    }
>
>}
>
>class MyReader {
>
>    SpecificDatumReader<SpecificRecord> rd;
>    Decoder dec;
>    InputStream istream;
>
>    public MyReader() throws FileNotFoundException {
>        rd = new SpecificDatumReader<SpecificRecord>(new
>Apple().getSchema());
>        istream = new FileInputStream(new File("/tmp/testavro"));
>        dec = DecoderFactory.get().binaryDecoder(istream, null);
>    }
>
>    public synchronized SpecificRecord read() throws IOException {
>        Object r = rd.read(null, dec);
>        return (SpecificRecord) r;
>    }
>
>}
>
>public class AvroWriteAndReadSameTime {
>    @Test
>    public void testWritingAndReadingAtSameTime() throws Exception {
>
>        MyWriter dumper = new MyWriter();
>        final Apple apple = new Apple();
>        apple.taste = "sweet";
>        dumper.dump(apple);
>
>        final MyReader rd = new MyReader();
>        rd.read();
>
>
>        try {
>        rd.read();
>        } catch (Exception e) {
>            e.printStackTrace();
>        }
>
>        // the second one somehow generates a NPE, we hope to get EOF...
>        try {
>        rd.read();
>        } catch (Exception e) {
>            e.printStackTrace();
>        }
>
>    }
>}
>
>
>
>
>
>the issue is in BinaryDecoder.readInt(), right now even when it hits
>EOF, it still advances the pos pointer.
>all the other APIs (readLong readFloat ...) do not do this. changing
>to the following  makes it work:
>
>
>  @Override
>  public int readInt() throws IOException {
>    ensureBounds(5); // won't throw index out of bounds
>    int len = 1;
>    int b = buf[pos] & 0xff;
>    int n = b & 0x7f;
>    if (b > 0x7f) {
>      b = buf[pos + len++] & 0xff;
>      n ^= (b & 0x7f) << 7;
>      if (b > 0x7f) {
>        b = buf[pos + len++] & 0xff;
>        n ^= (b & 0x7f) << 14;
>        if (b > 0x7f) {
>          b = buf[pos + len++] & 0xff;
>          n ^= (b & 0x7f) << 21;
>          if (b > 0x7f) {
>            b = buf[pos + len++] & 0xff;
>            n ^= (b & 0x7f) << 28;
>            if (b > 0x7f) {
>              throw new IOException("Invalid int encoding");
>            }
>          }
>        }
>      }
>    }
>    if (pos+len > limit) {
>      throw new EOFException();
>    }
>    pos += len;             //<================== CHANGE, used to be
>above the EOF throw
>
>    return (n >>> 1) ^ -(n & 1); // back to two's-complement
>  }