avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1368249 - in /avro/trunk: ./ lang/java/avro/src/main/java/org/apache/avro/reflect/ lang/java/avro/src/main/java/org/apache/avro/specific/ lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/ lang/java/mapred/src/test/java/org/apache/a...
Date Wed, 01 Aug 2012 21:14:31 GMT
Author: cutting
Date: Wed Aug  1 21:14:30 2012
New Revision: 1368249

URL: http://svn.apache.org/viewvc?rev=1368249&view=rev
Log:
AVRO-1103. Java: Fix SpecificData and mapreduce to use correct classloader.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java
    avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java
    avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java
    avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java
    avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Aug  1 21:14:30 2012
@@ -14,6 +14,9 @@ Avro 1.7.2 (unreleased)
     AVRO-1131. Generated build makefiles for MSYS/MinGW use Visual
     Studio compiler flags (Laurent Moss via thiru)
 
+    AVRO-1103. Java: Fix SpecificData and mapreduce to use correct
+    classloader. (cutting)
+
 Avro 1.7.1 (16 July 2012)
 
   NEW FEATURES

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectData.java Wed Aug
 1 21:14:30 2012
@@ -76,6 +76,11 @@ public class ReflectData extends Specifi
 
   protected ReflectData() {}
   
+  /** Construct with a particular classloader. */
+  public ReflectData(ClassLoader classLoader) {
+    super(classLoader);
+  }
+  
   /** Return the singleton instance. */
   public static ReflectData get() { return INSTANCE; }
 

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
(original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/reflect/ReflectDatumReader.java
Wed Aug  1 21:14:30 2012
@@ -52,7 +52,8 @@ public class ReflectDatumReader<T> exten
     this(writer, reader, ReflectData.get());
   }
 
-  protected ReflectDatumReader(Schema writer, Schema reader, ReflectData data){
+  /** Construct given writer's and reader's schema and the data model. */
+  public ReflectDatumReader(Schema writer, Schema reader, ReflectData data) {
     super(writer, reader, data);
   }
 

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/specific/SpecificData.java Wed
Aug  1 21:14:30 2012
@@ -55,6 +55,9 @@ public class SpecificData extends Generi
     this.classLoader = classLoader;
   }
   
+  /** Return the class loader that's used. */
+  public ClassLoader getClassLoader() { return classLoader; }
+
   @Override
   public DatumReader createDatumReader(Schema schema) {
     return new SpecificDatumReader(schema, schema, this);
@@ -267,14 +270,14 @@ public class SpecificData extends Generi
   
   @Override
   public Object createFixed(Object old, Schema schema) {
-    Class c = SpecificData.get().getClass(schema);
+    Class c = getClass(schema);
     if (c == null) return super.createFixed(old, schema); // punt to generic
     return c.isInstance(old) ? old : newInstance(c, schema);
   }
   
   @Override
   public Object newRecord(Object old, Schema schema) {
-    Class c = SpecificData.get().getClass(schema);
+    Class c = getClass(schema);
     if (c == null) return super.newRecord(old, schema); // punt to generic
     return (c.isInstance(old) ? old : newInstance(c, schema));
   }

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java
(original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroDeserializer.java
Wed Aug  1 21:14:30 2012
@@ -26,6 +26,7 @@ import org.apache.avro.io.DatumReader;
 import org.apache.avro.io.BinaryDecoder;
 import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.mapred.AvroWrapper;
+import org.apache.avro.reflect.ReflectData;
 import org.apache.avro.reflect.ReflectDatumReader;
 import org.apache.hadoop.io.serializer.Deserializer;
 
@@ -52,7 +53,7 @@ public abstract class AvroDeserializer<T
   private final Schema mReaderSchema;
 
   /** The Avro datum reader for deserializing. */
-  private final DatumReader<D> mAvroDatumReader;
+  final DatumReader<D> mAvroDatumReader;
 
   /** An Avro binary decoder for deserializing. */
   private BinaryDecoder mAvroDecoder;
@@ -63,10 +64,12 @@ public abstract class AvroDeserializer<T
    * @param writerSchema The Avro writer schema for the data to deserialize.
    * @param readerSchema The Avro reader schema for the data to deserialize (may be null).
    */
-  protected AvroDeserializer(Schema writerSchema, Schema readerSchema) {
+  protected AvroDeserializer(Schema writerSchema, Schema readerSchema,
+                             ClassLoader classLoader) {
     mWriterSchema = writerSchema;
     mReaderSchema = null != readerSchema ? readerSchema : writerSchema;
-    mAvroDatumReader = new ReflectDatumReader<D>(mWriterSchema, mReaderSchema);
+    mAvroDatumReader = new ReflectDatumReader<D>(mWriterSchema, mReaderSchema,
+                                                 new ReflectData(classLoader));
   }
 
   /**

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java
(original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroKeyDeserializer.java
Wed Aug  1 21:14:30 2012
@@ -36,8 +36,9 @@ public class AvroKeyDeserializer<D> exte
    * @param writerSchema The Avro writer schema for the data to deserialize.
    * @param readerSchema The Avro reader schema for the data to deserialize.
    */
-  public AvroKeyDeserializer(Schema writerSchema, Schema readerSchema) {
-    super(writerSchema, readerSchema);
+  public AvroKeyDeserializer(Schema writerSchema, Schema readerSchema,
+                             ClassLoader classLoader) {
+    super(writerSchema, readerSchema, classLoader);
   }
 
   /**

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java
(original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroSerialization.java
Wed Aug  1 21:14:30 2012
@@ -65,9 +65,13 @@ public class AvroSerialization<T> extend
   public Deserializer<AvroWrapper<T>> getDeserializer(Class<AvroWrapper<T>>
c) {
     Configuration conf = getConf();
     if (AvroKey.class.isAssignableFrom(c)) {
-      return new AvroKeyDeserializer<T>(getKeyWriterSchema(conf), getKeyReaderSchema(conf));
+      return new AvroKeyDeserializer<T>(getKeyWriterSchema(conf),
+                                        getKeyReaderSchema(conf),
+                                        conf.getClassLoader());
     } else if (AvroValue.class.isAssignableFrom(c)) {
-      return new AvroValueDeserializer<T>(getValueWriterSchema(conf), getValueReaderSchema(conf));
+      return new AvroValueDeserializer<T>(getValueWriterSchema(conf),
+                                          getValueReaderSchema(conf),
+                                          conf.getClassLoader());
     } else {
       throw new IllegalStateException("Only AvroKey and AvroValue are supported.");
     }

Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java
(original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/hadoop/io/AvroValueDeserializer.java
Wed Aug  1 21:14:30 2012
@@ -36,8 +36,9 @@ public class AvroValueDeserializer<D> ex
    * @param writerSchema The Avro writer schema for the data to deserialize.
    * @param readerSchema The Avro reader schema for the data to deserialize.
    */
-  public AvroValueDeserializer(Schema writerSchema, Schema readerSchema) {
-    super(writerSchema, readerSchema);
+  public AvroValueDeserializer(Schema writerSchema, Schema readerSchema,
+                               ClassLoader classLoader) {
+    super(writerSchema, readerSchema, classLoader);
   }
 
   /**

Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java
(original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroKeyDeserializer.java
Wed Aug  1 21:14:30 2012
@@ -38,8 +38,10 @@ public class TestAvroKeyDeserializer {
     // Create a deserializer.
     Schema writerSchema = Schema.create(Schema.Type.STRING);
     Schema readerSchema = Schema.create(Schema.Type.STRING);
+    ClassLoader classLoader = this.getClass().getClassLoader();
     AvroKeyDeserializer<CharSequence> deserializer
-        = new AvroKeyDeserializer<CharSequence>(writerSchema, readerSchema);
+      = new AvroKeyDeserializer<CharSequence>(writerSchema, readerSchema,
+                                              classLoader);
 
     // Check the schemas.
     assertEquals(writerSchema, deserializer.getWriterSchema());

Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java
(original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroSerialization.java
Wed Aug  1 21:14:30 2012
@@ -22,6 +22,8 @@ import static org.junit.Assert.*;
 import java.io.IOException;
 
 import org.apache.avro.Schema;
+import org.apache.avro.reflect.ReflectData;
+import org.apache.avro.reflect.ReflectDatumReader;
 import org.apache.avro.mapred.AvroKey;
 import org.apache.avro.mapred.AvroValue;
 import org.apache.avro.mapred.AvroWrapper;
@@ -29,8 +31,10 @@ import org.apache.avro.mapreduce.AvroJob
 import org.apache.hadoop.io.serializer.Deserializer;
 import org.apache.hadoop.io.serializer.Serializer;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.junit.Test;
+import org.junit.Assert;
 
 public class TestAvroSerialization {
   @Test
@@ -118,4 +122,17 @@ public class TestAvroSerialization {
     // Check that the reader schema is set correctly on the deserializer.
     assertEquals(readerSchema, avroDeserializer.getReaderSchema());
   }
+
+  @Test public void testClassPath() throws Exception {
+    Configuration conf = new Configuration();
+    ClassLoader loader = new ClassLoader() {};
+    conf.setClassLoader(loader);
+    AvroSerialization serialization = new AvroSerialization();
+    serialization.setConf(conf);
+    AvroDeserializer des =
+      (AvroDeserializer)serialization.getDeserializer(AvroKey.class);
+    ReflectData data =
+      (ReflectData)((ReflectDatumReader)des.mAvroDatumReader).getData();
+    Assert.assertEquals(loader, data.getClassLoader());
+  }
 }

Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java?rev=1368249&r1=1368248&r2=1368249&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java
(original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/hadoop/io/TestAvroValueDeserializer.java
Wed Aug  1 21:14:30 2012
@@ -38,8 +38,10 @@ public class TestAvroValueDeserializer {
     // Create a deserializer.
     Schema writerSchema = Schema.create(Schema.Type.STRING);
     Schema readerSchema = Schema.create(Schema.Type.STRING);
+    ClassLoader classLoader = this.getClass().getClassLoader();
     AvroValueDeserializer<CharSequence> deserializer
-        = new AvroValueDeserializer<CharSequence>(writerSchema, readerSchema);
+      = new AvroValueDeserializer<CharSequence>(writerSchema, readerSchema,
+                                                classLoader);
 
     // Check the schemas.
     assertEquals(writerSchema, deserializer.getWriterSchema());



Mime
View raw message