hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r1176694 - in /hadoop/common/branches/branch-0.22/mapreduce: ./ ivy/ src/contrib/capacity-scheduler/ src/contrib/fairscheduler/ src/contrib/gridmix/ src/contrib/mrunit/ src/contrib/mumak/ src/contrib/raid/ src/contrib/streaming/ src/java/or...
Date Wed, 28 Sep 2011 03:11:41 GMT
Author: shv
Date: Wed Sep 28 03:11:40 2011
New Revision: 1176694

URL: http://svn.apache.org/viewvc?rev=1176694&view=rev
Log:
MAPREDUCE-3039. Upgrade to Avro 1.5.3. Contributed by Joep Rottinghuis.

Modified:
    hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt
    hadoop/common/branches/branch-0.22/mapreduce/build.xml
    hadoop/common/branches/branch-0.22/mapreduce/ivy.xml
    hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties
    hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml
    hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml
    hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml
    hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml
    hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml
    hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml
    hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml
    hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
    hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
    hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java

Modified: hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt Wed Sep 28 03:11:40 2011
@@ -223,6 +223,8 @@ Release 0.22.0 - Unreleased
     HADOOP-7106. Reorganize project SVN layout to "unsplit" the projects.
     (todd, nigel)
 
+    MAPREDUCE-3039. Upgrade to Avro 1.5.3. (Joep Rottinghuis via shv)
+
   OPTIMIZATIONS
     
     MAPREDUCE-1354. Enhancements to JobTracker for better performance and

Modified: hadoop/common/branches/branch-0.22/mapreduce/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/build.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/build.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/build.xml Wed Sep 28 03:11:40 2011
@@ -339,7 +339,7 @@
 
   <target name="avro-generate" depends="init">
     <mkdir dir="${build.src}/org/apache/hadoop/mapreduce/jobhistory"/>
-    <taskdef name="protocol" classname="org.apache.avro.specific.ProtocolTask">
+    <taskdef name="protocol" classname="org.apache.avro.compiler.specific.ProtocolTask">
       <classpath refid="classpath" />
     </taskdef>
     <protocol destdir="${build.src}">

Modified: hadoop/common/branches/branch-0.22/mapreduce/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/ivy.xml Wed Sep 28 03:11:40 2011
@@ -97,12 +97,18 @@
 
    <dependency org="org.apache.lucene" name="lucene-core" 
                rev="${lucene-core.version}" conf="javadoc->default"/>
-   <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}" 
-               conf="common->default">
+   <dependency org="org.apache.avro" name="avro"
+               rev="${avro.version}" conf="common->default">
       <exclude module="ant"/>
       <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
+      <exclude module="slf4j-api"/>
+   </dependency>
+   <dependency org="org.apache.avro" name="avro-compiler"
+               rev="${avro.version}" conf="common->default">
+      <exclude module="ant"/>
+      <exclude module="jetty"/>
+      <exclude module="slf4j-api"/>
+   </dependency>
    <dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}" 
                conf="test->default"/>
    <dependency org="org.mortbay.jetty" name="jetty-servlet-tester" rev="${jetty.version}"

Modified: hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties Wed Sep 28 03:11:40
2011
@@ -20,7 +20,7 @@ ant-task.version=2.0.10
 #This property has to be updated synchronously with aop.xml
 aspectj.version=1.6.5
 
-avro.version=1.3.2
+avro.version=1.5.3
 
 checkstyle.version=4.2
 

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml Wed
Sep 28 03:11:40 2011
@@ -69,11 +69,5 @@
                 rev="${servlet-api-2.5.version}" conf="common->master"/>
     <dependency org="commons-httpclient" name="commons-httpclient" 
                 rev="${commons-httpclient.version}" conf="common->master"/>
-    <dependency org="org.apache.hadoop" name="avro" 
-                rev="${avro.version}" conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
   </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml Wed Sep
28 03:11:40 2011
@@ -64,26 +64,10 @@
       name="junit"
       rev="${junit.version}"
       conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="avro"
-      rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
     <dependency org="org.codehaus.jackson"
       name="jackson-mapper-asl"
       rev="${jackson.version}"
       conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer"
-      rev="${paranamer.version}"
-      conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer-ant"
-      rev="${paranamer.version}"
-      conf="common->default"/>
     <dependency org="org.mortbay.jetty"
       name="jetty-util"
       rev="${jetty-util.version}"

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml Wed Sep 28 03:11:40
2011
@@ -102,14 +102,6 @@
       name="commons-cli"
       rev="${commons-cli.version}"
       conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="avro"
-      rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
     <dependency org="org.codehaus.jackson"
       name="jackson-mapper-asl"
       rev="${jackson.version}"
@@ -118,9 +110,5 @@
       name="jackson-core-asl"
       rev="${jackson.version}"
       conf="common->default"/>
-    <dependency org="com.thoughtworks.paranamer"
-      name="paranamer"
-      rev="${paranamer.version}"
-      conf="common->default"/>
   </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml Wed Sep 28 03:11:40
2011
@@ -45,11 +45,6 @@
     <dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}"
conf="common->default"/>
     <dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
     <dependency org="junit" name="junit" rev="${junit.version}" conf="common->master"/>
-    <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}" conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
   </dependencies>
 
 </ivy-module>

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml Wed Sep 28 03:11:40
2011
@@ -114,13 +114,5 @@
       name="commons-cli"
       rev="${commons-cli.version}"
       conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="avro"
-	  rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
   </dependencies>
 </ivy-module>

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml Wed Sep 28 03:11:40
2011
@@ -102,14 +102,6 @@
       name="commons-cli"
       rev="${commons-cli.version}"
       conf="common->default"/>
-    <dependency org="org.apache.hadoop"
-      name="avro"
-      rev="${avro.version}"
-      conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
     <dependency org="org.codehaus.jackson"
       name="jackson-mapper-asl"
       rev="${jackson.version}"

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml Wed Sep 28
03:11:40 2011
@@ -68,12 +68,6 @@
                 rev="${commons-httpclient.version}" conf="common->default"/>
     <dependency org="log4j" name="log4j" 
                 rev="${log4j.version}" conf="common->master"/>
-    <dependency org="org.apache.hadoop" name="avro" 
-                rev="${avro.version}" conf="common->default">
-      <exclude module="ant"/>
-      <exclude module="jetty"/>
-      <exclude module="slf4j-simple"/>
-    </dependency>
     <dependency org="org.slf4j" name="slf4j-api" 
                 rev="${slf4j-api.version}" conf="common->master"/>
   </dependencies>

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
(original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
Wed Sep 28 03:11:40 2011
@@ -20,10 +20,14 @@ package org.apache.hadoop.mapreduce.jobh
 
 import java.io.Closeable;
 import java.io.DataInputStream;
-import java.io.IOException;
 import java.io.EOFException;
-import java.io.StringBufferInputStream;
+import java.io.IOException;
 
+import org.apache.avro.Schema;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.specific.SpecificDatumReader;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.FileSystem;
@@ -32,13 +36,6 @@ import org.apache.hadoop.mapreduce.Count
 import org.apache.hadoop.mapreduce.CounterGroup;
 import org.apache.hadoop.mapreduce.Counters;
 
-import org.apache.avro.Schema;
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.io.Decoder;
-import org.apache.avro.io.JsonDecoder;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.specific.SpecificDatumReader;
-
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class EventReader implements Closeable {
@@ -46,7 +43,7 @@ public class EventReader implements Clos
   private Schema schema;
   private DataInputStream in;
   private Decoder decoder;
-  private DatumReader reader;
+  private DatumReader<Event> reader;
 
   /**
    * Create a new Event Reader
@@ -73,8 +70,8 @@ public class EventReader implements Clos
     }
     
     this.schema = Schema.parse(in.readLine());
-    this.reader = new SpecificDatumReader(schema);
-    this.decoder = new JsonDecoder(schema, in);
+    this.reader = new SpecificDatumReader<Event>(schema);
+    this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
   }
   
   /**
@@ -82,11 +79,10 @@ public class EventReader implements Clos
    * @return the next event
    * @throws IOException
    */
-  @SuppressWarnings("unchecked")
   public HistoryEvent getNextEvent() throws IOException {
     Event wrapper;
     try {
-      wrapper = (Event)reader.read(null, decoder);
+      wrapper = reader.read(null, decoder);
     } catch (EOFException e) {            // at EOF
       return null;
     }

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
(original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
Wed Sep 28 03:11:40 2011
@@ -19,20 +19,18 @@
 package org.apache.hadoop.mapreduce.jobhistory;
 
 import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
 
 import org.apache.avro.Schema;
-import org.apache.avro.io.Encoder;
-import org.apache.avro.io.JsonEncoder;
+import org.apache.avro.generic.GenericData;
 import org.apache.avro.io.DatumWriter;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.io.EncoderFactory;
 import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.generic.GenericData;
 import org.apache.avro.util.Utf8;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
 
 /**
  * Event Writer is an utility class used to write events to the underlying
@@ -54,7 +52,7 @@ class EventWriter {
     out.writeBytes("\n");
     out.writeBytes(Event.SCHEMA$.toString());
     out.writeBytes("\n");
-    this.encoder = new JsonEncoder(Event.SCHEMA$, out);
+    this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
   }
   
   synchronized void write(HistoryEvent event) throws IOException { 

Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
(original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
Wed Sep 28 03:11:40 2011
@@ -57,7 +57,7 @@ public class JobSubmittedEvent implement
     datum.userName = new Utf8(userName);
     datum.submitTime = submitTime;
     datum.jobConfPath = new Utf8(jobConfPath);
-    Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>();
+    Map<CharSequence, CharSequence> jobAcls = new HashMap<CharSequence, CharSequence>();
     for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
       jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8(
           entry.getValue().getAclString()));



Mime
View raw message