hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r688986 - in /hadoop/core/branches/branch-0.18: ./ src/core/org/apache/hadoop/io/ src/core/org/apache/hadoop/io/compress/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/io/
Date Tue, 26 Aug 2008 07:43:07 GMT
Author: cdouglas
Date: Tue Aug 26 00:43:07 2008
New Revision: 688986

URL: http://svn.apache.org/viewvc?rev=688986&view=rev
Log:
HADOOP-3821. Prevent SequenceFile and IFile from duplicating codecs in
CodecPool when closed more than once. Contributed by Arun Murthy.

Modified:
    hadoop/core/branches/branch-0.18/CHANGES.txt
    hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/SequenceFile.java
    hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/compress/CodecPool.java
    hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/IFile.java
    hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/io/TestSequenceFile.java

Modified: hadoop/core/branches/branch-0.18/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/CHANGES.txt?rev=688986&r1=688985&r2=688986&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.18/CHANGES.txt Tue Aug 26 00:43:07 2008
@@ -1,5 +1,11 @@
 Hadoop Change Log
 
+Release 0.18.1 - Unreleased
+
+  BUG FIXES
+
+    HADOOP-3821. Prevent SequenceFile and IFile from duplicating codecs in
+    CodecPool when closed more than once. (Arun Murthy via cdouglas)
 
 Release 0.18.0 - 2008-08-19
 

Modified: hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/SequenceFile.java?rev=688986&r1=688985&r2=688986&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/SequenceFile.java (original)
+++ hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/SequenceFile.java Tue Aug
26 00:43:07 2008
@@ -947,6 +947,7 @@
     /** Close the file. */
     public synchronized void close() throws IOException {
       CodecPool.returnCompressor(compressor);
+      compressor = null;
       
       keySerializer.close();
       uncompressedValSerializer.close();
@@ -1572,6 +1573,8 @@
       CodecPool.returnDecompressor(keyDecompressor);
       CodecPool.returnDecompressor(valLenDecompressor);
       CodecPool.returnDecompressor(valDecompressor);
+      keyLenDecompressor = keyDecompressor = null;
+      valLenDecompressor = valDecompressor = null;
       
       if (keyDeserializer != null) {
     	keyDeserializer.close();

Modified: hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/compress/CodecPool.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/compress/CodecPool.java?rev=688986&r1=688985&r2=688986&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/compress/CodecPool.java
(original)
+++ hadoop/core/branches/branch-0.18/src/core/org/apache/hadoop/io/compress/CodecPool.java
Tue Aug 26 00:43:07 2008
@@ -59,7 +59,7 @@
         if (codecList != null) {
           synchronized (codecList) {
             if (!codecList.isEmpty()) {
-              codec = codecList.remove(0);
+              codec = codecList.remove(codecList.size()-1);
             }
           }
         }

Modified: hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/IFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/IFile.java?rev=688986&r1=688985&r2=688986&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/IFile.java (original)
+++ hadoop/core/branches/branch-0.18/src/mapred/org/apache/hadoop/mapred/IFile.java Tue Aug
26 00:43:07 2008
@@ -122,6 +122,7 @@
         compressedOut.finish();
         compressedOut.resetState();
         CodecPool.returnCompressor(compressor);
+        compressor = null;
       }
 
       // Close the stream
@@ -371,6 +372,7 @@
       if (decompressor != null) {
         decompressor.reset();
         CodecPool.returnDecompressor(decompressor);
+        decompressor = null;
       }
       
       // Close the underlying stream

Modified: hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/io/TestSequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/io/TestSequenceFile.java?rev=688986&r1=688985&r2=688986&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/io/TestSequenceFile.java (original)
+++ hadoop/core/branches/branch-0.18/src/test/org/apache/hadoop/io/TestSequenceFile.java Tue
Aug 26 00:43:07 2008
@@ -404,7 +404,61 @@
     }
     writer.close();
   }
+
+  public void testClose() throws IOException {
+    Configuration conf = new Configuration();
+    LocalFileSystem fs = new LocalFileSystem();
+    fs.setConf(conf);
+    fs.getRawFileSystem().setConf(conf);
+  
+    // create a sequence file 1
+    Path path1 = new Path(System.getProperty("test.build.data",".")+"/test1.seq");
+    SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, path1,
+        Text.class, NullWritable.class, CompressionType.BLOCK);
+    writer.append(new Text("file1-1"), NullWritable.get());
+    writer.append(new Text("file1-2"), NullWritable.get());
+    writer.close();
+  
+    Path path2 = new Path(System.getProperty("test.build.data",".")+"/test2.seq");
+    writer = SequenceFile.createWriter(fs, conf, path2, Text.class,
+        NullWritable.class, CompressionType.BLOCK);
+    writer.append(new Text("file2-1"), NullWritable.get());
+    writer.append(new Text("file2-2"), NullWritable.get());
+    writer.close();
+  
+    // Create a reader which uses 4 BuiltInZLibInflater instances
+    SequenceFile.Reader reader = new SequenceFile.Reader(fs, path1, conf);
+    // Returns the 4 BuiltInZLibInflater instances to the CodecPool
+    reader.close();
+    // The second close _could_ erroneously returns the same 
+    // 4 BuiltInZLibInflater instances to the CodecPool again
+    reader.close();
   
+    // The first reader gets 4 BuiltInZLibInflater instances from the CodecPool
+    SequenceFile.Reader reader1 = new SequenceFile.Reader(fs, path1, conf);
+    // read first value from reader1
+    Text text = new Text();
+    reader1.next(text);
+    assertEquals("file1-1", text.toString());
+    
+    // The second reader _could_ get the same 4 BuiltInZLibInflater 
+    // instances from the CodePool as reader1
+    SequenceFile.Reader reader2 = new SequenceFile.Reader(fs, path2, conf);
+    
+    // read first value from reader2
+    reader2.next(text);
+    assertEquals("file2-1", text.toString());
+    // read second value from reader1
+    reader1.next(text);
+    assertEquals("file1-2", text.toString());
+    // read second value from reader2 (this throws an exception)
+    reader2.next(text);
+    assertEquals("file2-2", text.toString());
+  
+    assertFalse(reader1.next(text));
+    assertFalse(reader2.next(text));
+  }
+
   /** For debugging and testing. */
   public static void main(String[] args) throws Exception {
     int count = 1024 * 1024;



Mime
View raw message