crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gr...@apache.org
Subject git commit: CRUNCH-289 Allow materializing Avro SpecificRecords
Date Fri, 01 Nov 2013 04:11:26 GMT
Updated Branches:
  refs/heads/master a3b56322d -> cb92723af


CRUNCH-289 Allow materializing Avro SpecificRecords

Due to issues outlined in AVRO-1240, the correct classloader must
be supplied to the SpecificData class to allow reading
SpecificRecords. This patch passes a custom classloader to the
SpecificData constructor to allow locally materializing
SpecificRecords when running with "hadoop jar".


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/cb92723a
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/cb92723a
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/cb92723a

Branch: refs/heads/master
Commit: cb92723af09cee63735cbf6b5dac5e86dd480e9c
Parents: a3b5632
Author: Gabriel Reid <greid@apache.org>
Authored: Wed Oct 30 21:42:14 2013 +0100
Committer: Gabriel Reid <greid@apache.org>
Committed: Fri Nov 1 05:10:27 2013 +0100

----------------------------------------------------------------------
 .../apache/crunch/io/avro/AvroFileReaderFactory.java  | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/cb92723a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
b/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
index becde73..efde4ec 100644
--- a/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
+++ b/crunch-core/src/main/java/org/apache/crunch/io/avro/AvroFileReaderFactory.java
@@ -26,6 +26,7 @@ import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.io.DatumReader;
 import org.apache.avro.mapred.FsInput;
 import org.apache.avro.reflect.ReflectDatumReader;
+import org.apache.avro.specific.SpecificData;
 import org.apache.avro.specific.SpecificDatumReader;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -61,7 +62,7 @@ public class AvroFileReaderFactory<T> implements FileReaderFactory<T>
{
     this.recordReader = new GenericDatumReader<T>(schema);
     this.mapFn = IdentityFn.<T>getInstance();
   }
-  
+
   static <T> DatumReader<T> createDatumReader(AvroType<T> avroType) {
     if (avroType.hasReflect()) {
       if (avroType.hasSpecific()) {
@@ -69,7 +70,16 @@ public class AvroFileReaderFactory<T> implements FileReaderFactory<T>
{
       }
       return new ReflectDatumReader<T>(avroType.getSchema());
     } else if (avroType.hasSpecific()) {
-      return new SpecificDatumReader<T>(avroType.getSchema());
+      // Use the classloader of the avro type as the classloader
+      // for loading SpecificData classes. This is a best-effort
+      // approach to avoid running into AVRO-1240. We can't just
+      // use the SpecificDatumReader(Class) constructor because the
+      // type class here isn't necessarily a SpecificData class, it
+      // might just contain one as a subtype.
+      return new SpecificDatumReader<T>(
+                        avroType.getSchema(),
+                        avroType.getSchema(),
+                        new SpecificData(avroType.getClass().getClassLoader()));
     } else {
       return new GenericDatumReader<T>(avroType.getSchema());
     }


Mime
View raw message