parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [33/50] [abbrv] parquet-mr git commit: PARQUET-544: Add closed flag to allow for closeable contract adherence
Date Thu, 19 Jan 2017 01:27:44 GMT
PARQUET-544: Add closed flag to allow for closeable contract adherence

The closeable interface states:
> Closes this stream and releases any system resources associated with it. If the stream
is already closed then invoking this method has no effect.

As InternalParquetRecordWriter implements this interface we should adhere to this contract.

Author: Mark Reddy <mark.l.reddy@gmail.com>

Closes #345 from markreddy/PARQUET-544-adhere-to-closeable-contract and squashes the following
commits:

135db9b [Mark Reddy] PARQUET-544: add closed flag to allow for adherence to closeable contract


Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/4240cb97
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/4240cb97
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/4240cb97

Branch: refs/heads/parquet-1.8.x
Commit: 4240cb975a3a8a76b27fc2f25faa1d791220a20d
Parents: e3ee35a
Author: Mark Reddy <mark.l.reddy@gmail.com>
Authored: Thu Jun 30 09:41:51 2016 -0700
Committer: Ryan Blue <blue@apache.org>
Committed: Mon Jan 9 16:54:54 2017 -0800

----------------------------------------------------------------------
 .../hadoop/InternalParquetRecordWriter.java     | 21 ++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/4240cb97/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
index 26d9c50..d5c57ab 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordWriter.java
@@ -56,6 +56,8 @@ class InternalParquetRecordWriter<T> {
   private final boolean validating;
   private final ParquetProperties props;
 
+  private boolean closed;
+
   private long recordCount = 0;
   private long recordCountForNextMemCheck = MINIMUM_RECORD_COUNT_FOR_CHECK;
   private long lastRowGroupEndPos = 0;
@@ -103,15 +105,18 @@ class InternalParquetRecordWriter<T> {
   }
 
   public void close() throws IOException, InterruptedException {
-    flushRowGroupToStore();
-    FinalizedWriteContext finalWriteContext = writeSupport.finalizeWrite();
-    Map<String, String> finalMetadata = new HashMap<String, String>(extraMetaData);
-    String modelName = writeSupport.getName();
-    if (modelName != null) {
-      finalMetadata.put(ParquetWriter.OBJECT_MODEL_NAME_PROP, modelName);
+    if (!closed) {
+      flushRowGroupToStore();
+      FinalizedWriteContext finalWriteContext = writeSupport.finalizeWrite();
+      Map<String, String> finalMetadata = new HashMap<String, String>(extraMetaData);
+      String modelName = writeSupport.getName();
+      if (modelName != null) {
+        finalMetadata.put(ParquetWriter.OBJECT_MODEL_NAME_PROP, modelName);
+      }
+      finalMetadata.putAll(finalWriteContext.getExtraMetaData());
+      parquetFileWriter.end(finalMetadata);
+      closed = true;
     }
-    finalMetadata.putAll(finalWriteContext.getExtraMetaData());
-    parquetFileWriter.end(finalMetadata);
   }
 
   public void write(T value) throws IOException, InterruptedException {


Mime
View raw message