hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject svn commit: r634431 - in /hadoop/core/trunk: ./ conf/ src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/ src/java/org/apache/hadoop/io/ src/java/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/io/
Date Thu, 06 Mar 2008 21:52:43 GMT
Author: cdouglas
Date: Thu Mar  6 13:52:35 2008
New Revision: 634431

URL: http://svn.apache.org/viewvc?rev=634431&view=rev
Log:
HADOOP-2869. Deprecate SequenceFile.setCompressionType in favor of
SequenceFile.createWriter, SequenceFileOutputFormat.setCompressionType,
and JobConf.setMapOutputCompressionType. Contributed by Arun C Murthy
Configuration changes to hadoop-default.xml:
  deprecated io.seqfile.compression.type



Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/conf/hadoop-default.xml
    hadoop/core/trunk/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java
    hadoop/core/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
    hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
    hadoop/core/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
    hadoop/core/trunk/src/test/org/apache/hadoop/io/FileBench.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=634431&r1=634430&r2=634431&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Mar  6 13:52:35 2008
@@ -156,6 +156,12 @@
     HADOOP-2861. Improve the user interface for the HOD commands. 
     Command line structure has changed. (Hemanth Yamijala via nigel)
 
+    HADOOP-2869. Deprecate SequenceFile.setCompressionType in favor of
+    SequenceFile.createWriter, SequenceFileOutputFormat.setCompressionType,
+    and JobConf.setMapOutputCompressionType. (Arun C Murthy via cdouglas)
+    Configuration changes to hadoop-default.xml:
+      deprecated io.seqfile.compression.type
+
   IMPROVEMENTS
 
     HADOOP-2371. User guide for file permissions in HDFS.

Modified: hadoop/core/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/conf/hadoop-default.xml?rev=634431&r1=634430&r2=634431&view=diff
==============================================================================
--- hadoop/core/trunk/conf/hadoop-default.xml (original)
+++ hadoop/core/trunk/conf/hadoop-default.xml Thu Mar  6 13:52:35 2008
@@ -884,13 +884,6 @@
 </property>
 
 <property>
-  <name>io.seqfile.compression.type</name>
-  <value>RECORD</value>
-  <description>The default compression type for SequenceFile.Writer.
-  </description>
-</property>
-
-<property>
   <name>map.sort.class</name>
   <value>org.apache.hadoop.mapred.MergeSorter</value>
   <description>The default sort class for sorting keys.

Modified: hadoop/core/trunk/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java?rev=634431&r1=634430&r2=634431&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java
(original)
+++ hadoop/core/trunk/src/contrib/data_join/src/java/org/apache/hadoop/contrib/utils/join/DataJoinJob.java
Thu Mar  6 13:52:35 2008
@@ -102,8 +102,8 @@
     job.setMapperClass(mapper);
     job.setOutputPath(new Path(outputDir));
     job.setOutputFormat(outputFormat);
-    SequenceFile.setCompressionType(job,
-                                    SequenceFile.CompressionType.BLOCK);
+    SequenceFileOutputFormat.setOutputCompressionType(job,
+            SequenceFile.CompressionType.BLOCK);
     job.setMapOutputKeyClass(Text.class);
     job.setMapOutputValueClass(mapoutputValueClass);
     job.setOutputKeyClass(Text.class);

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/io/SequenceFile.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/io/SequenceFile.java?rev=634431&r1=634430&r2=634431&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/io/SequenceFile.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/io/SequenceFile.java Thu Mar  6 13:52:35
2008
@@ -219,7 +219,12 @@
    * Get the compression type for the reduce outputs
    * @param job the job config to look in
    * @return the kind of compression to use
+   * @deprecated Use {@link org.apache.hadoop.mapred.JobConf#getMapOutputCompressionType()}
+   *             to get {@link CompressionType} for intermediate map-outputs or
+   *             {@link org.apache.hadoop.mapred.SequenceFileOutputFormat#getOutputCompressionType(org.apache.hadoop.mapred.JobConf)}

+   *             to get {@link CompressionType} for job-outputs.
    */
+  @Deprecated
   static public CompressionType getCompressionType(Configuration job) {
     String name = job.get("io.seqfile.compression.type");
     return name == null ? CompressionType.RECORD : 
@@ -230,7 +235,15 @@
    * Set the compression type for sequence files.
    * @param job the configuration to modify
    * @param val the new compression type (none, block, record)
+   * @deprecated Use the one of the many SequenceFile.createWriter methods to specify
+   *             the {@link CompressionType} while creating the {@link SequenceFile} or
+   *             {@link org.apache.hadoop.mapred.JobConf#setMapOutputCompressionType(org.apache.hadoop.io.SequenceFile.CompressionType)}
+   *             to specify the {@link CompressionType} for intermediate map-outputs or 
+   *             {@link org.apache.hadoop.mapred.SequenceFileOutputFormat#setOutputCompressionType(org.apache.hadoop.mapred.JobConf,
org.apache.hadoop.io.SequenceFile.CompressionType)}
+   *             to specify the {@link CompressionType} for job-outputs. 
+   * or 
    */
+  @Deprecated
   static public void setCompressionType(Configuration job, 
                                         CompressionType val) {
     job.set("io.seqfile.compression.type", val.toString());

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobConf.java?rev=634431&r1=634430&r2=634431&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/JobConf.java Thu Mar  6 13:52:35 2008
@@ -484,6 +484,7 @@
    *              are compressed.
    */
   public void setMapOutputCompressionType(CompressionType style) {
+    setCompressMapOutput(true);
     set("mapred.map.output.compression.type", style.toString());
   }
   
@@ -507,6 +508,7 @@
    */
   public void 
   setMapOutputCompressorClass(Class<? extends CompressionCodec> codecClass) {
+    setCompressMapOutput(true);
     setClass("mapred.map.output.compression.codec", codecClass, 
              CompressionCodec.class);
   }

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java?rev=634431&r1=634430&r2=634431&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/SequenceFileOutputFormat.java Thu
Mar  6 13:52:35 2008
@@ -113,6 +113,7 @@
    */
   public static void setOutputCompressionType(JobConf conf, 
 		                                          CompressionType style) {
+    setCompressOutput(conf, true);
     conf.set("mapred.output.compression.type", style.toString());
   }
 

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/io/FileBench.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/io/FileBench.java?rev=634431&r1=634430&r2=634431&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/io/FileBench.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/io/FileBench.java Thu Mar  6 13:52:35 2008
@@ -268,7 +268,6 @@
     public void configure(JobConf job) {
       job.set("mapred.map.output.compression.type", typ);
       job.set("mapred.output.compression.type", typ);
-      job.set("io.seqfile.compression.type", typ);
     }
   }
   enum Format {



Mime
View raw message