hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r700291 - in /hadoop/core/trunk: ./ src/core/org/apache/hadoop/conf/ src/mapred/org/apache/hadoop/mapred/ src/mapred/org/apache/hadoop/mapred/lib/ src/test/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/pipes/
Date Mon, 29 Sep 2008 22:36:08 GMT
Author: omalley
Date: Mon Sep 29 15:36:07 2008
New Revision: 700291

URL: http://svn.apache.org/viewvc?rev=700291&view=rev
Log:
HADOOP-4293. Make Configuration Writable and remove unreleased 
WritableJobConf. Configuration.write is renamed to writeXml. (omalley)

Removed:
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/WritableJobConf.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java
    hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Sep 29 15:36:07 2008
@@ -108,6 +108,9 @@
     HADOOP-3938. Disk space quotas for HDFS. This is similar to namespace
     quotas in 0.18. (rangadi)
 
+    HADOOP-4293. Make Configuration Writable and remove unreleased 
+    WritableJobConf. Configuration.write is renamed to writeXml. (omalley)
+
   NEW FEATURES
 
     HADOOP-3341. Allow streaming jobs to specify the field separator for map

Modified: hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java (original)
+++ hadoop/core/trunk/src/core/org/apache/hadoop/conf/Configuration.java Mon Sep 29 15:36:07
2008
@@ -19,6 +19,8 @@
 package org.apache.hadoop.conf;
 
 import java.io.BufferedInputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -54,6 +56,8 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.Document;
@@ -128,7 +132,8 @@
  * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved
to the value
  * of the System property with that name.
  */
-public class Configuration implements Iterable<Map.Entry<String,String>> {
+public class Configuration implements Iterable<Map.Entry<String,String>>,
+                                      Writable {
   private static final Log LOG =
     LogFactory.getLog(Configuration.class);
 
@@ -1063,7 +1068,7 @@
    * 
    * @param out the output stream to write to.
    */
-  public void write(OutputStream out) throws IOException {
+  public void writeXml(OutputStream out) throws IOException {
     Properties properties = getProps();
     try {
       Document doc =
@@ -1154,7 +1159,27 @@
 
   /** For debugging.  List non-default properties to the terminal and exit. */
   public static void main(String[] args) throws Exception {
-    new Configuration().write(System.out);
+    new Configuration().writeXml(System.out);
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+    clear();
+    int size = WritableUtils.readVInt(in);
+    for(int i=0; i < size; ++i) {
+      set(org.apache.hadoop.io.Text.readString(in), 
+          org.apache.hadoop.io.Text.readString(in));
+    }
+  }
+
+  //@Override
+  public void write(DataOutput out) throws IOException {
+    Properties props = getProps();
+    WritableUtils.writeVInt(out, props.size());
+    for(Map.Entry<Object, Object> item: props.entrySet()) {
+      org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
+      org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
+    }
   }
 
 }

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobClient.java Mon Sep 29 15:36:07
2008
@@ -808,7 +808,7 @@
         new FsPermission(JOB_FILE_PERMISSION));
 
     try {
-      job.write(out);
+      job.writeXml(out);
     } finally {
       out.close();
     }

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/JobHistory.java Mon Sep 29 15:36:07
2008
@@ -860,7 +860,7 @@
       FileOutputStream jobOut = null;
       try {
         jobOut = new FileOutputStream(localJobFile);
-        jobConf.write(jobOut);
+        jobConf.writeXml(jobOut);
         if (LOG.isDebugEnabled()) {
           LOG.debug("Job conf for " + jobId + " stored at " 
                     + localJobFile.getAbsolutePath());
@@ -895,14 +895,14 @@
           fs = new Path(LOG_DIR).getFileSystem(jobConf);
           if (!fs.exists(jobFilePath)) {
             jobFileOut = fs.create(jobFilePath);
-            jobConf.write(jobFileOut);
+            jobConf.writeXml(jobFileOut);
             jobFileOut.close();
           }
         } 
         if (userLogDir != null) {
           fs = new Path(userLogDir).getFileSystem(jobConf);
           jobFileOut = fs.create(userJobFilePath);
-          jobConf.write(jobFileOut);
+          jobConf.writeXml(jobFileOut);
         }
         if (LOG.isDebugEnabled()) {
           LOG.debug("Job conf for " + jobId + " stored at " 

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Mon Sep 29 15:36:07
2008
@@ -187,7 +187,7 @@
         localFs.delete(localTaskFile, true);
         OutputStream out = localFs.create(localTaskFile);
         try {
-          conf.write(out);
+          conf.writeXml(out);
         } finally {
           out.close();
         }

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Mon Sep 29 15:36:07
2008
@@ -765,7 +765,7 @@
           localJobConf.setJar(localJarFile.toString());
           OutputStream out = localFs.create(localJobFile);
           try {
-            localJobConf.write(out);
+            localJobConf.writeXml(out);
           } finally {
             out.close();
           }
@@ -1842,7 +1842,7 @@
       }
       OutputStream out = localFs.create(localTaskFile);
       try {
-        localJobConf.write(out);
+        localJobConf.writeXml(out);
       } finally {
         out.close();
       }

Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/Chain.java Mon Sep 29 15:36:07
2008
@@ -122,8 +122,8 @@
   private static JobConf getChainElementConf(JobConf jobConf, String confKey) {
     JobConf conf;
     try {
-      Stringifier<WritableJobConf> stringifier =
-        new DefaultStringifier<WritableJobConf>(jobConf, WritableJobConf.class);
+      Stringifier<JobConf> stringifier =
+        new DefaultStringifier<JobConf>(jobConf, JobConf.class);
       conf = stringifier.fromString(jobConf.get(confKey, null));
     } catch (IOException ioex) {
       throw new RuntimeException(ioex);
@@ -233,11 +233,11 @@
                         Object.class);
 
     // serialize the private mapper jobconf in the chain jobconf.
-    Stringifier<WritableJobConf> stringifier =
-      new DefaultStringifier<WritableJobConf>(jobConf, WritableJobConf.class);
+    Stringifier<JobConf> stringifier =
+      new DefaultStringifier<JobConf>(jobConf, JobConf.class);
     try {
       jobConf.set(prefix + CHAIN_MAPPER_CONFIG + index,
-                  stringifier.toString(new WritableJobConf(mapperConf)));
+                  stringifier.toString(new JobConf(mapperConf)));
     }
     catch (IOException ioEx) {
       throw new RuntimeException(ioEx);
@@ -300,11 +300,11 @@
                          Object.class);
 
     // serialize the private mapper jobconf in the chain jobconf.
-    Stringifier<WritableJobConf> stringifier =
-      new DefaultStringifier<WritableJobConf>(jobConf, WritableJobConf.class);
+    Stringifier<JobConf> stringifier =
+      new DefaultStringifier<JobConf>(jobConf, JobConf.class);
     try {
       jobConf.set(prefix + CHAIN_REDUCER_CONFIG,
-                  stringifier.toString(new WritableJobConf(reducerConf)));
+                  stringifier.toString(new JobConf(reducerConf)));
     }
     catch (IOException ioEx) {
       throw new RuntimeException(ioEx);

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/TestWritableJobConf.java Mon Sep 29
15:36:07 2008
@@ -77,13 +77,13 @@
   }
 
   public void testEmptyConfiguration() throws Exception {
-    WritableJobConf conf = new WritableJobConf();
+    JobConf conf = new JobConf();
     Configuration deser = serDeser(conf);
     assertEquals(conf, deser);
   }
 
   public void testNonEmptyConfiguration() throws Exception {
-    WritableJobConf conf = new WritableJobConf();
+    JobConf conf = new JobConf();
     conf.set("a", "A");
     conf.set("b", "B");
     Configuration deser = serDeser(conf);
@@ -91,7 +91,7 @@
   }
 
   public void testConfigurationWithDefaults() throws Exception {
-    WritableJobConf conf = new WritableJobConf(false);
+    JobConf conf = new JobConf(false);
     conf.set("a", "A");
     conf.set("b", "B");
     Configuration deser = serDeser(conf);

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java?rev=700291&r1=700290&r2=700291&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/pipes/TestPipes.java Mon Sep 29 15:36:07
2008
@@ -231,7 +231,7 @@
     local.delete(outDir, true);
     local.mkdirs(outDir);
     out = local.create(jobXml);
-    job.write(out);
+    job.writeXml(out);
     out.close();
     System.err.println("About to run: Submitter -conf " + jobXml + 
                        " -input " + inDir + " -output " + outDir + 



Mime
View raw message