avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject avro git commit: AVRO-1985: Unreleased resources
Date Fri, 16 Jun 2017 07:19:12 GMT
Repository: avro
Updated Branches:
  refs/heads/master edd8edda0 -> 528a1c8a9


AVRO-1985: Unreleased resources

This closes #177


Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/528a1c8a
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/528a1c8a
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/528a1c8a

Branch: refs/heads/master
Commit: 528a1c8a997a0831e83b525173c6a70b08ee681a
Parents: edd8edd
Author: Gabor Szadovszky <gabor@apache.org>
Authored: Thu Dec 29 12:27:38 2016 +0100
Committer: Gabor Szadovszky <gabor@apache.org>
Committed: Fri Jun 16 09:18:40 2017 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |  2 ++
 .../main/java/org/apache/avro/data/Json.java    |  9 ++++--
 .../org/apache/avro/file/DataFileWriter.java    | 14 +++++++--
 .../test/java/org/apache/avro/TestDataFile.java | 32 +++++++++++---------
 .../compiler/specific/SpecificCompiler.java     | 19 +++++++-----
 .../avro/mapreduce/AvroKeyOutputFormat.java     | 12 ++++++--
 .../mapreduce/AvroKeyValueOutputFormat.java     | 13 ++++++--
 .../main/java/org/apache/avro/tool/IdlTool.java |  9 ++++--
 .../main/java/org/apache/avro/tool/Main.java    | 14 +++++++--
 9 files changed, 88 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index c387de6..d7b0ec1 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -160,6 +160,8 @@ Trunk (not yet released)
 
     AVRO-1401: @Nullable does not work with byte[] (Nandor Kollar via gabor)
 
+    AVRO-1985: Unreleased resources (gabor)
+
 Avro 1.8.1 (14 May 2016)
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java b/lang/java/avro/src/main/java/org/apache/avro/data/Json.java
index daa8482..66b92ef 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/data/Json.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/data/Json.java
@@ -18,6 +18,7 @@
 package org.apache.avro.data;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.StringReader;
 import java.util.Iterator;
 
@@ -55,8 +56,12 @@ public class Json {
   public static final Schema SCHEMA;
   static {
     try {
-      SCHEMA = Schema.parse
-        (Json.class.getResourceAsStream("/org/apache/avro/data/Json.avsc"));
+      InputStream in = Json.class.getResourceAsStream("/org/apache/avro/data/Json.avsc");
+      try {
+        SCHEMA = Schema.parse(in);
+      } finally {
+        in.close();
+      }
     } catch (IOException e) {
       throw new AvroRuntimeException(e);
     }

http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
index fe916dc..d70d4a4 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/file/DataFileWriter.java
@@ -190,11 +190,20 @@ public class DataFileWriter<D> implements Closeable, Flushable
{
 
   /** Open a writer appending to an existing file. */
   public DataFileWriter<D> appendTo(File file) throws IOException {
-    return appendTo(new SeekableFileInput(file),
-                    new SyncableFileOutputStream(file, true));
+    SeekableInput input = null;
+    try {
+      input = new SeekableFileInput(file);
+      OutputStream output = new SyncableFileOutputStream(file, true);
+      return appendTo(input, output);
+    } finally {
+      if (input != null)
+        input.close();
+      // output does not need to be closed here. It will be closed by invoking close() of
this writer.
+    }
   }
 
   /** Open a writer appending to an existing file.
+   * <strong>Since 1.9.0 this method does not close in.</strong>
    * @param in reading the existing file.
    * @param out positioned at the end of the existing file.
    */
@@ -213,7 +222,6 @@ public class DataFileWriter<D> implements Closeable, Flushable {
     } else {
       this.codec = CodecFactory.nullCodec().createInstance();
     }
-    reader.close();
 
     init(out);
 

http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
----------------------------------------------------------------------
diff --git a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java b/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
index e067496..608d960 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/TestDataFile.java
@@ -324,19 +324,23 @@ public class TestDataFile {
   private void testFSync(boolean useFile) throws IOException {
     DataFileWriter<Object> writer =
       new DataFileWriter<Object>(new GenericDatumWriter<Object>());
-    writer.setFlushOnEveryBlock(false);
-    TestingByteArrayOutputStream out = new TestingByteArrayOutputStream();
-    if (useFile) {
-      File f = makeFile();
-      SeekableFileInput in = new SeekableFileInput(f);
-      writer.appendTo(in, out);
-    } else {
-      writer.create(SCHEMA, out);
-    }
-    int currentCount = 0;
-    int syncCounter = 0;
     try {
-      for (Object datum : new RandomData(SCHEMA, COUNT, SEED+1)) {
+      writer.setFlushOnEveryBlock(false);
+      TestingByteArrayOutputStream out = new TestingByteArrayOutputStream();
+      if (useFile) {
+        File f = makeFile();
+        SeekableFileInput in = new SeekableFileInput(f);
+        try {
+          writer.appendTo(in, out);
+        } finally {
+          in.close();
+        }
+      } else {
+        writer.create(SCHEMA, out);
+      }
+      int currentCount = 0;
+      int syncCounter = 0;
+      for (Object datum : new RandomData(SCHEMA, COUNT, SEED + 1)) {
         currentCount++;
         writer.append(datum);
         if (currentCount % 10 == 0) {
@@ -344,11 +348,11 @@ public class TestDataFile {
           syncCounter++;
         }
       }
+      System.out.println("Total number of syncs: " + out.syncCount);
+      Assert.assertEquals(syncCounter, out.syncCount);
     } finally {
       writer.close();
     }
-    System.out.println("Total number of syncs: " + out.syncCount);
-    Assert.assertEquals(syncCounter, out.syncCount);
   }
 
 

http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
----------------------------------------------------------------------
diff --git a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
index 2959493..7a6f5f1 100644
--- a/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
+++ b/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
@@ -270,17 +270,22 @@ public class SpecificCompiler {
       if (src != null && f.exists() && f.lastModified() >= src.lastModified())
         return f;                                 // already up to date: ignore
       f.getParentFile().mkdirs();
-      Writer fw;
-      if (outputCharacterEncoding != null) {
-        fw = new OutputStreamWriter(new FileOutputStream(f), outputCharacterEncoding);
-      } else {
-        fw = new FileWriter(f);
-      }
+      Writer fw = null;
+      FileOutputStream fos = null;
       try {
+        if (outputCharacterEncoding != null) {
+          fos = new FileOutputStream(f);
+          fw = new OutputStreamWriter(fos, outputCharacterEncoding);
+        } else {
+          fw = new FileWriter(f);
+        }
         fw.write(FILE_HEADER);
         fw.write(contents);
       } finally {
-        fw.close();
+        if (fw != null)
+          fw.close();
+        if (fos != null)
+          fos.close();
       }
       return f;
     }

http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
index 4c86112..125833c 100644
--- a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
+++ b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyOutputFormat.java
@@ -102,8 +102,14 @@ public class AvroKeyOutputFormat<T> extends AvroOutputFormatBase<AvroKey<T>,
Nul
 
     GenericData dataModel = AvroSerialization.createDataModel(conf);
 
-    return mRecordWriterFactory.create
-      (writerSchema, dataModel, getCompressionCodec(context),
-       getAvroFileOutputStream(context), getSyncInterval(context));
+    OutputStream out = getAvroFileOutputStream(context);
+    try {
+      return mRecordWriterFactory.create
+              (writerSchema, dataModel, getCompressionCodec(context),
+                      out, getSyncInterval(context));
+    } catch (IOException e) {
+      out.close();
+      throw e;
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
----------------------------------------------------------------------
diff --git a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
index b844abd..7fd5d38 100644
--- a/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
+++ b/lang/java/mapred/src/main/java/org/apache/avro/mapreduce/AvroKeyValueOutputFormat.java
@@ -19,6 +19,7 @@
 package org.apache.avro.mapreduce;
 
 import java.io.IOException;
+import java.io.OutputStream;
 
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.hadoop.io.AvroDatumConverter;
@@ -59,8 +60,14 @@ public class AvroKeyValueOutputFormat<K, V> extends AvroOutputFormatBase<K,
V> {
 
     GenericData dataModel = AvroSerialization.createDataModel(conf);
 
-    return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter,
-        dataModel, getCompressionCodec(context), getAvroFileOutputStream(context),
-        getSyncInterval(context));
+    OutputStream out = getAvroFileOutputStream(context);
+    try {
+      return new AvroKeyValueRecordWriter<K, V>(keyConverter, valueConverter,
+              dataModel, getCompressionCodec(context), out,
+              getSyncInterval(context));
+    } catch (IOException e) {
+      out.close();
+      throw e;
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java
index f3cfe24..4095740 100644
--- a/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java
+++ b/lang/java/tools/src/main/java/org/apache/avro/tool/IdlTool.java
@@ -58,12 +58,17 @@ public class IdlTool implements Tool {
       parser = new Idl(in);
     }
 
-    if (args.size() == 2 && ! "-".equals(args.get(1))) {
+    if (args.size() == 2 && !"-".equals(args.get(1))) {
       parseOut = new PrintStream(new FileOutputStream(args.get(1)));
     }
 
     Protocol p = parser.CompilationUnit();
-    parseOut.print(p.toString(true));
+    try {
+      parseOut.print(p.toString(true));
+    } finally {
+      if (parseOut != out) // Close only the newly created FileOutputStream
+        parseOut.close();
+    }
     return 0;
   }
 

http://git-wip-us.apache.org/repos/asf/avro/blob/528a1c8a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
----------------------------------------------------------------------
diff --git a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java b/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
index 93c827c..aa689ec 100644
--- a/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
+++ b/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
@@ -89,9 +89,19 @@ public class Main {
       }
     }
     System.err.print("Version ");
-    printStream(Main.class.getClassLoader().getResourceAsStream("VERSION.txt"));
+    InputStream versionInput = Main.class.getClassLoader().getResourceAsStream("VERSION.txt");
+    try {
+      printStream(versionInput);
+    } finally {
+      versionInput.close();
+    }
     System.err.print(" of ");
-    printHead(Main.class.getClassLoader().getResourceAsStream("META-INF/NOTICE"), 5);
+    InputStream noticeInput = Main.class.getClassLoader().getResourceAsStream("META-INF/NOTICE");
+    try {
+      printHead(noticeInput, 5);
+    } finally {
+      noticeInput.close();
+    }
     System.err.println("----------------");
 
     System.err.println("Available tools:");


Mime
View raw message