avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r1687559 - in /avro/trunk: CHANGES.txt lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java
Date Thu, 25 Jun 2015 15:38:03 GMT
Author: tomwhite
Date: Thu Jun 25 15:38:03 2015
New Revision: 1687559

URL: http://svn.apache.org/r1687559
Log:
AVRO-1685. Allow specifying sync in DataFileWriter.create. Contributed by Sehrope Sarkuni.

Added:
    avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1687559&r1=1687558&r2=1687559&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Jun 25 15:38:03 2015
@@ -44,6 +44,9 @@ Trunk (not yet released)
 
     AVRO-1497. Java: Add support for logical types. (blue)
 
+    AVRO-1685. Java: Allow specifying sync in DataFileWriter.create
+    (Sehrope Sarkuni via tomwhite)
+
   OPTIMIZATIONS
 
   IMPROVEMENTS

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java?rev=1687559&r1=1687558&r2=1687559&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java Thu Jun
25 15:38:03 2015
@@ -124,19 +124,31 @@ public class DataFileWriter<D> implement
     return this;
   }
 
-  /** Open a new file for data matching a schema. */
+  /** Open a new file for data matching a schema with a random sync. */
   public DataFileWriter<D> create(Schema schema, File file) throws IOException {
-    return create(schema, new SyncableFileOutputStream(file));
+    return create(schema, new SyncableFileOutputStream(file), null);
   }
 
-  /** Open a new file for data matching a schema. */
+  /** Open a new file for data matching a schema with a random sync. */
   public DataFileWriter<D> create(Schema schema, OutputStream outs)
     throws IOException {
+    return create(schema, outs, null);
+  }
+
+  /** Open a new file for data matching a schema with an explicit sync. */
+  public DataFileWriter<D> create(Schema schema, OutputStream outs, byte[] sync)
+    throws IOException {
     assertNotOpen();
 
     this.schema = schema;
     setMetaInternal(DataFileConstants.SCHEMA, schema.toString());
-    this.sync = generateSync();
+    if (sync == null ) {
+      this.sync = generateSync();
+    } else if (sync.length == 16) {
+      this.sync = sync;
+    } else {
+      throw new IOException("sync must be exactly 16 bytes");
+    }
 
     init(outs);
 
@@ -150,7 +162,7 @@ public class DataFileWriter<D> implement
       vout.writeBytes(entry.getValue());
     }
     vout.writeMapEnd();
-    vout.writeFixed(sync);                       // write initial sync
+    vout.writeFixed(this.sync); // write initial sync
     vout.flush(); //vout may be buffered, flush before writing to out
     return this;
   }

Added: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java?rev=1687559&view=auto
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java (added)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestDataFileCustomSync.java Thu
Jun 25 15:38:03 2015
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.UUID;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileConstants;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.util.Utf8;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestDataFileCustomSync {
+  private byte[] createDataFile(byte[] sync) throws IOException {
+    Schema schema = Schema.create(Type.STRING);
+    DataFileWriter<Utf8> w = new DataFileWriter<Utf8>(new GenericDatumWriter<Utf8>(schema));
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    w.create(schema, baos, sync);
+    w.append(new Utf8("apple"));
+    w.append(new Utf8("banana"));
+    w.sync();
+    w.append(new Utf8("celery"));
+    w.append(new Utf8("date"));
+    w.sync();
+    w.append(new Utf8("endive"));
+    w.append(new Utf8("fig"));
+    w.close();
+    return baos.toByteArray();
+  }
+
+  private static byte[] generateSync() {
+    try {
+      MessageDigest digester = MessageDigest.getInstance("MD5");
+      long time = System.currentTimeMillis();
+      digester.update((UUID.randomUUID()+"@"+time).getBytes());
+      return digester.digest();
+    } catch (NoSuchAlgorithmException e) {
+      throw new RuntimeException(e);
+    }
+  }  
+
+  @Test(expected = IOException.class)
+  public void testInvalidSync() throws IOException {
+    // Invalid size (must be 16):
+    byte[] sync = new byte[8];
+    createDataFile(sync);
+  }
+
+  @Test
+  public void testRandomSync() throws IOException {
+    byte[] sync = generateSync();
+    byte[] randSyncFile = createDataFile(null);
+    byte[] customSyncFile = createDataFile(sync);
+    assertFalse(Arrays.equals(randSyncFile, customSyncFile));
+  }
+
+  @Test
+  public void testCustomSync() throws IOException {
+    byte[] sync = generateSync();
+    byte[] customSyncFile = createDataFile(sync);
+    byte[] sameCustomSyncFile = createDataFile(sync);
+    assertTrue(Arrays.equals(customSyncFile, sameCustomSyncFile));
+  }
+}



Mime
View raw message