avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1124971 - in /avro/trunk: CHANGES.txt lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
Date Thu, 19 May 2011 15:50:41 GMT
Author: cutting
Date: Thu May 19 15:50:40 2011
New Revision: 1124971

URL: http://svn.apache.org/viewvc?rev=1124971&view=rev
Log:
AVRO-820. Java: Permit applications to catch exceptions thrown while writing data to a file
and then continue writing to that file.  Contributed by scottcarey.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
    avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1124971&r1=1124970&r2=1124971&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu May 19 15:50:40 2011
@@ -20,6 +20,10 @@ Avro 1.5.2 (unreleased)
 
   IMPROVEMENTS
 
+    AVRO-820. Java: Permit applications to catch exceptions thrown
+    while writing data to a file and then continue writing to that
+    file.  (scottcarey & cutting)
+
   BUG FIXES
 
     AVRO-818. C: Fix data file corruption bug in C library (dcreager)

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java?rev=1124971&r1=1124970&r2=1124971&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java Thu May
19 15:50:40 2011
@@ -238,13 +238,41 @@ public class DataFileWriter<D> implement
     return setMeta(key, Long.toString(value));
   }
 
-  /** Append a datum to the file. */
+  /** Thrown by {@link #append(Object)} when an exception occurs while writing a
+   * datum to the buffer.  When this is thrown, the file is unaltered and may
+   * continue to be appended to. */
+  public static class AppendWriteException extends RuntimeException {
+    public AppendWriteException(Exception e) { super(e); }
+  }
+
+  /** Append a datum to the file.
+   * @see AppendWriteException
+   */
   public void append(D datum) throws IOException {
     assertOpen();
-    dout.write(datum, bufOut);
+    int usedBuffer = bufferInUse();
+    try {
+      dout.write(datum, bufOut);
+    } catch (IOException e) {
+      resetBufferTo(usedBuffer);
+      throw new AppendWriteException(e);
+    } catch (RuntimeException re) {
+      resetBufferTo(usedBuffer);
+      throw new AppendWriteException(re);
+    }
     blockCount++;
     writeIfBlockFull();
   }
+  
+  // if there is an error encoding, flush the encoder and then
+  // reset the buffer position to contain size bytes, discarding the rest.
+  // Otherwise the file will be corrupt with a partial record.
+  private void resetBufferTo(int size) throws IOException {
+    bufOut.flush();
+    byte[] data = buffer.toByteArray();
+    buffer.reset();
+    buffer.write(data, 0, size);
+  }
 
   /** Expert: Append a pre-encoded datum to the file.  No validation is
    * performed to check that the encoding conforms to the file's schema.
@@ -256,9 +284,13 @@ public class DataFileWriter<D> implement
     blockCount++;
     writeIfBlockFull();
   }
+  
+  private int bufferInUse() {
+    return (buffer.size() + bufOut.bytesBuffered());
+  }
 
   private void writeIfBlockFull() throws IOException {
-    if ((buffer.size() + bufOut.bytesBuffered()) >= syncInterval)
+    if (bufferInUse() >= syncInterval)
       writeBlock();
   }
 

Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java?rev=1124971&r1=1124970&r2=1124971&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java Thu May 19 15:50:40
2011
@@ -26,12 +26,15 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
 
+import junit.framework.Assert;
+
 import org.apache.avro.file.CodecFactory;
 import org.apache.avro.file.FileReader;
 import org.apache.avro.file.DataFileReader;
 import org.apache.avro.file.DataFileStream;
 import org.apache.avro.file.DataFileWriter;
 import org.apache.avro.file.SeekableFileInput;
+import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericDatumWriter;
 import org.apache.avro.io.DatumReader;
@@ -97,6 +100,20 @@ public class TestDataFile {
         writer.append(datum);
         if (++count%(COUNT/3) == 0)
           writer.sync();                          // force some syncs mid-file
+        if (count == 5) {
+          // force a write of an invalid record
+          boolean threwProperly = false;
+          try {
+            GenericData.Record record = (GenericData.Record) datum;
+            record.put(1, null);
+            threwProperly = true;
+            writer.append(record);
+            threwProperly = false;
+          } catch (DataFileWriter.AppendWriteException e) {
+            System.out.println("Ignoring: "+e);
+          }
+          Assert.assertTrue("failed to throw when expected", threwProperly);
+        }
       }
     } finally {
       writer.close();



Mime
View raw message