crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gr...@apache.org
Subject [2/2] crunch git commit: CRUNCH-539 Fix reading WritableComparables bimap
Date Thu, 09 Jul 2015 18:32:20 GMT
CRUNCH-539 Fix reading WritableComparables bimap

Correct the reading of the registered WritableComparables (as
serialized in Writables) to get around the fact that the context
class loader is not used by ObjectInputStream when resolving
classes when this code is called.


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/a727606e
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/a727606e
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/a727606e

Branch: refs/heads/master
Commit: a727606eaeaea2d78a161685d9d29217864e6846
Parents: 8d9c540
Author: Gabriel Reid <greid@apache.org>
Authored: Wed Jul 8 07:27:32 2015 +0200
Committer: Gabriel Reid <greid@apache.org>
Committed: Thu Jul 9 20:12:51 2015 +0200

----------------------------------------------------------------------
 .../apache/crunch/types/writable/Writables.java | 34 +++++++++++---------
 .../crunch/types/writable/WritablesTest.java    |  8 +++++
 2 files changed, 26 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/a727606e/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java b/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java
index 23775ed..89d08ee 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/writable/Writables.java
@@ -18,21 +18,20 @@
 package org.apache.crunch.types.writable;
 
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
 import com.google.common.collect.BiMap;
 import com.google.common.collect.HashBiMap;
 import com.google.common.collect.ImmutableBiMap;
-import org.apache.commons.codec.binary.Base64;
 import org.apache.crunch.CrunchRuntimeException;
 import org.apache.crunch.MapFn;
 import org.apache.crunch.Pair;
@@ -130,22 +129,25 @@ public class Writables {
 
   private static final String WRITABLE_COMPARABLE_CODES = "crunch.writable.comparable.codes";
 
-  private static void serializeWritableComparableCodes(Configuration conf) throws IOException
{
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    ObjectOutputStream oos = new ObjectOutputStream(baos);
-    oos.writeObject(WRITABLE_CODES);
-    oos.close();
-    conf.set(WRITABLE_COMPARABLE_CODES, Base64.encodeBase64String(baos.toByteArray()));
+  static void serializeWritableComparableCodes(Configuration conf) throws IOException {
+    Map<Integer, String> codeToClassNameMap = Maps.transformValues(WRITABLE_CODES,
+        new Function<Class<? extends Writable>, String>() {
+          @Override
+          public String apply(Class<? extends Writable> input) {
+            return input.getName();
+          }
+        });
+    conf.set(WRITABLE_COMPARABLE_CODES, Joiner.on(';').withKeyValueSeparator(":").join(codeToClassNameMap));
   }
 
   static void reloadWritableComparableCodes(Configuration conf) throws Exception {
     if (conf.get(WRITABLE_COMPARABLE_CODES) != null) {
-      ByteArrayInputStream bais = new ByteArrayInputStream(Base64.decodeBase64(conf.get(WRITABLE_COMPARABLE_CODES)));
-      ObjectInputStream ois = new ObjectInputStream(bais);
-      BiMap<Integer, Class<? extends Writable>> codes = (BiMap<Integer, Class<?
extends Writable>>) ois.readObject();
-      ois.close();
-      for (Map.Entry<Integer, Class<? extends Writable>> e : codes.entrySet())
{
-        WRITABLE_CODES.put(e.getKey(), e.getValue());
+      Map<String, String> codeToClassName = Splitter.on(';')
+          .withKeyValueSeparator(":").split(conf.get(WRITABLE_COMPARABLE_CODES));
+      for (Map.Entry<String, String> codeToClassNameEntry : codeToClassName.entrySet())
{
+        WRITABLE_CODES.put(
+            Integer.parseInt(codeToClassNameEntry.getKey()),
+            (Class<? extends Writable>) Class.forName(codeToClassNameEntry.getValue()));
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/crunch/blob/a727606e/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
b/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
index 9af3dea..f5c40d7 100644
--- a/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
+++ b/crunch-core/src/test/java/org/apache/crunch/types/writable/WritablesTest.java
@@ -34,6 +34,7 @@ import org.apache.crunch.Tuple4;
 import org.apache.crunch.TupleN;
 import org.apache.crunch.types.PTableType;
 import org.apache.crunch.types.PType;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.DoubleWritable;
@@ -252,6 +253,13 @@ public class WritablesTest {
   public void testRegisterComparable() throws Exception {
     Writables.registerComparable(TestWritable.class);
     assertNotNull(Writables.WRITABLE_CODES.inverse().get(TestWritable.class));
+
+    // Also try serializing/deserializing from a configuration, just to make sure this works
+    Configuration conf = new Configuration();
+    Writables.serializeWritableComparableCodes(conf);
+    Writables.WRITABLE_CODES.clear();
+    Writables.reloadWritableComparableCodes(conf);
+    assertNotNull(Writables.WRITABLE_CODES.inverse().get(TestWritable.class));
   }
 
   @SuppressWarnings({ "unchecked", "rawtypes" })


Mime
View raw message