crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject crunch git commit: CRUNCH-583: Scrunch classloader failure in distcache
Date Thu, 10 Dec 2015 13:27:36 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 46b33437a -> 775de6cd8


CRUNCH-583: Scrunch classloader failure in distcache


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/775de6cd
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/775de6cd
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/775de6cd

Branch: refs/heads/master
Commit: 775de6cd8c35f128b5426e0383550218140674d1
Parents: 46b3343
Author: Tom White <tomwhite@apache.org>
Authored: Thu Dec 10 13:19:24 2015 +0000
Committer: Tom White <tomwhite@apache.org>
Committed: Thu Dec 10 13:19:24 2015 +0000

----------------------------------------------------------------------
 .../crunch/impl/mem/collect/MemCollection.java  | 14 +-----
 .../ClassloaderFallbackObjectInputStream.java   | 46 ++++++++++++++++++++
 .../java/org/apache/crunch/util/DistCache.java  |  3 +-
 3 files changed, 50 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/775de6cd/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java
b/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java
index e5f04d5..89671a3 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java
@@ -60,6 +60,7 @@ import org.apache.crunch.materialize.pobject.FirstElementPObject;
 import org.apache.crunch.types.PTableType;
 import org.apache.crunch.types.PType;
 import org.apache.crunch.types.PTypeFamily;
+import org.apache.crunch.util.ClassloaderFallbackObjectInputStream;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.RecordWriter;
@@ -126,18 +127,7 @@ public class MemCollection<S> implements PCollection<S> {
     ObjectInputStream in = null;
     try {
       // stream closed in the finally
-      in = new ObjectInputStream(inputStream) {
-        @Override
-        protected Class<?> resolveClass(ObjectStreamClass desc) throws IOException,
-            ClassNotFoundException {
-          try {
-            return super.resolveClass(desc);
-          } catch (ClassNotFoundException e) {
-            ClassLoader cl = Thread.currentThread().getContextClassLoader();
-            return Class.forName(desc.getName(), false, cl);
-          }
-        }
-      };
+      in = new ClassloaderFallbackObjectInputStream(inputStream);
       return in.readObject();
 
     } catch (ClassNotFoundException ex) {

http://git-wip-us.apache.org/repos/asf/crunch/blob/775de6cd/crunch-core/src/main/java/org/apache/crunch/util/ClassloaderFallbackObjectInputStream.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/util/ClassloaderFallbackObjectInputStream.java
b/crunch-core/src/main/java/org/apache/crunch/util/ClassloaderFallbackObjectInputStream.java
new file mode 100644
index 0000000..572ebf4
--- /dev/null
+++ b/crunch-core/src/main/java/org/apache/crunch/util/ClassloaderFallbackObjectInputStream.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectStreamClass;
+
+/**
+ * A custom {@link ObjectInputStream} that falls back to the thread context classloader
+ * if the class can't be found with the usual classloader that {@link
+ * ObjectInputStream} uses. This is needed when running in the Scala REPL.
+ * See https://issues.scala-lang.org/browse/SI-2403.
+ */
+public class ClassloaderFallbackObjectInputStream extends ObjectInputStream {
+  public ClassloaderFallbackObjectInputStream(InputStream in) throws IOException {
+    super(in);
+  }
+
+  @Override
+  protected Class<?> resolveClass(ObjectStreamClass desc) throws IOException,
+      ClassNotFoundException {
+    try {
+      return super.resolveClass(desc);
+    } catch (ClassNotFoundException e) {
+      ClassLoader cl = Thread.currentThread().getContextClassLoader();
+      return Class.forName(desc.getName(), false, cl);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/775de6cd/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java b/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java
index 3e49930..0325e12 100644
--- a/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java
+++ b/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java
@@ -69,7 +69,8 @@ public class DistCache {
     Object value = null;
     if (target != null) {
       Path targetPath = new Path(target.toString());
-      ObjectInputStream ois = new ObjectInputStream(targetPath.getFileSystem(conf).open(targetPath));
+      ObjectInputStream ois = new ClassloaderFallbackObjectInputStream(
+          targetPath.getFileSystem(conf).open(targetPath));
       try {
         value = ois.readObject();
       } catch (ClassNotFoundException e) {


Mime
View raw message