Author: omalley Date: Tue May 3 19:03:15 2011 New Revision: 1099191 URL: http://svn.apache.org/viewvc?rev=1099191&view=rev Log: HADOOP-7258. The Gzip codec should not return null decompressors. (omalley) Added: hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/DoNotPool.java Modified: hadoop/common/branches/branch-0.20-security-203/CHANGES.txt hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/CodecPool.java hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/GzipCodec.java hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java hadoop/common/branches/branch-0.20-security-203/src/test/org/apache/hadoop/io/compress/TestCodec.java Modified: hadoop/common/branches/branch-0.20-security-203/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/CHANGES.txt?rev=1099191&r1=1099190&r2=1099191&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-203/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.20-security-203/CHANGES.txt Tue May 3 19:03:15 2011 @@ -74,6 +74,8 @@ Release 0.20.203.0 - unreleased HADOOP-7232. Fix Javadoc warnings. (omalley) + HADOOP-7258. The Gzip codec should not return null decompressors. (omalley) + Release 0.20.202.0 - unreleased MAPREDUCE-2355. Add a configuration knob Modified: hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/CodecPool.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/CodecPool.java?rev=1099191&r1=1099190&r2=1099191&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/CodecPool.java (original) +++ hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/CodecPool.java Tue May 3 19:03:15 2011 @@ -141,6 +141,10 @@ public class CodecPool { if (compressor == null) { return; } + // if the compressor can't be reused, don't pool it. + if (compressor.getClass().isAnnotationPresent(DoNotPool.class)) { + return; + } compressor.reset(); payback(compressorPool, compressor); } @@ -155,6 +159,10 @@ public class CodecPool { if (decompressor == null) { return; } + // if the decompressor can't be reused, don't pool it. + if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) { + return; + } decompressor.reset(); payback(decompressorPool, decompressor); } Added: hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/DoNotPool.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/DoNotPool.java?rev=1099191&view=auto ============================================================================== --- hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/DoNotPool.java (added) +++ hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/DoNotPool.java Tue May 3 19:03:15 2011 @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.io.compress; + +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * This is a marker annotation that marks a compressor or decompressor + * type as not to be pooled. + */ +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface DoNotPool { + +} Modified: hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/GzipCodec.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/GzipCodec.java?rev=1099191&r1=1099190&r2=1099191&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/GzipCodec.java (original) +++ hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/GzipCodec.java Tue May 3 19:03:15 2011 @@ -141,7 +141,7 @@ public class GzipCodec extends DefaultCo public Class getDecompressorType() { return ZlibFactory.isNativeZlibLoaded(conf) ? GzipZlibDecompressor.class - : null; + : BuiltInGzipDecompressor.class; } public String getDefaultExtension() { Modified: hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java?rev=1099191&r1=1099190&r2=1099191&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java (original) +++ hadoop/common/branches/branch-0.20-security-203/src/core/org/apache/hadoop/io/compress/zlib/BuiltInGzipDecompressor.java Tue May 3 19:03:15 2011 @@ -24,6 +24,7 @@ import java.util.zip.Inflater; import java.util.zip.CRC32; import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.io.compress.DoNotPool; /** * A {@link Decompressor} based on the popular gzip compressed file format. @@ -108,6 +109,7 @@ public class BuiltInGzipDecompressor imp /** * Creates a new (pure Java) gzip decompressor. */ + @DoNotPool public BuiltInGzipDecompressor() { state = GzipStateLabel.HEADER_BASIC; crc.reset(); Modified: hadoop/common/branches/branch-0.20-security-203/src/test/org/apache/hadoop/io/compress/TestCodec.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-203/src/test/org/apache/hadoop/io/compress/TestCodec.java?rev=1099191&r1=1099190&r2=1099191&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-203/src/test/org/apache/hadoop/io/compress/TestCodec.java (original) +++ hadoop/common/branches/branch-0.20-security-203/src/test/org/apache/hadoop/io/compress/TestCodec.java Tue May 3 19:03:15 2011 @@ -480,27 +480,26 @@ public class TestCodec extends TestCase // java.util.zip.GZIPOutputStream. CompressionCodecFactory ccf = new CompressionCodecFactory(conf); CompressionCodec codec = ccf.getCodec(new Path("foo.gz")); - assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec); + assertTrue("Codec for .gz file is not GzipCodec", + codec instanceof GzipCodec); Decompressor codecDecompressor = codec.createDecompressor(); - if (null != codecDecompressor) { - fail("Got non-null codecDecompressor: " + codecDecompressor); + if (null == codecDecompressor) { + fail("Got null codecDecompressor"); } // Asking the CodecPool for a decompressor for GzipCodec // should return null as well. Decompressor poolDecompressor = CodecPool.getDecompressor(codec); - if (null != poolDecompressor) { - fail("Got non-null poolDecompressor: " + poolDecompressor); + if (null == poolDecompressor) { + fail("Got null poolDecompressor: " + poolDecompressor); } - // If we then ensure that the pool is populated... CodecPool.returnDecompressor(zlibDecompressor); - - // Asking the pool another time should still not bind this to GzipCodec. - poolDecompressor = CodecPool.getDecompressor(codec); - if (null != poolDecompressor) { - fail("Second time, got non-null poolDecompressor: " - + poolDecompressor); + // return the decompressor + CodecPool.returnDecompressor(poolDecompressor); + Decompressor poolDecompressor2 = CodecPool.getDecompressor(codec); + if (poolDecompressor == poolDecompressor2) { + fail("Reused gzip decompressor in pool"); } }