Return-Path: X-Original-To: apmail-crunch-commits-archive@www.apache.org Delivered-To: apmail-crunch-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7DCED182F9 for ; Thu, 10 Dec 2015 13:27:41 +0000 (UTC) Received: (qmail 72034 invoked by uid 500); 10 Dec 2015 13:27:40 -0000 Delivered-To: apmail-crunch-commits-archive@crunch.apache.org Received: (qmail 71991 invoked by uid 500); 10 Dec 2015 13:27:36 -0000 Mailing-List: contact commits-help@crunch.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@crunch.apache.org Delivered-To: mailing list commits@crunch.apache.org Received: (qmail 71976 invoked by uid 99); 10 Dec 2015 13:27:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Dec 2015 13:27:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2ECCCE1799; Thu, 10 Dec 2015 13:27:36 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tomwhite@apache.org To: commits@crunch.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: crunch git commit: CRUNCH-583: Scrunch classloader failure in distcache Date: Thu, 10 Dec 2015 13:27:36 +0000 (UTC) Repository: crunch Updated Branches: refs/heads/master 46b33437a -> 775de6cd8 CRUNCH-583: Scrunch classloader failure in distcache Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/775de6cd Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/775de6cd Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/775de6cd Branch: refs/heads/master Commit: 775de6cd8c35f128b5426e0383550218140674d1 Parents: 46b3343 Author: Tom White Authored: Thu Dec 10 13:19:24 2015 +0000 Committer: Tom White Committed: Thu Dec 10 13:19:24 2015 +0000 ---------------------------------------------------------------------- .../crunch/impl/mem/collect/MemCollection.java | 14 +----- .../ClassloaderFallbackObjectInputStream.java | 46 ++++++++++++++++++++ .../java/org/apache/crunch/util/DistCache.java | 3 +- 3 files changed, 50 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/775de6cd/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java b/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java index e5f04d5..89671a3 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mem/collect/MemCollection.java @@ -60,6 +60,7 @@ import org.apache.crunch.materialize.pobject.FirstElementPObject; import org.apache.crunch.types.PTableType; import org.apache.crunch.types.PType; import org.apache.crunch.types.PTypeFamily; +import org.apache.crunch.util.ClassloaderFallbackObjectInputStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.RecordWriter; @@ -126,18 +127,7 @@ public class MemCollection implements PCollection { ObjectInputStream in = null; try { // stream closed in the finally - in = new ObjectInputStream(inputStream) { - @Override - protected Class resolveClass(ObjectStreamClass desc) throws IOException, - ClassNotFoundException { - try { - return super.resolveClass(desc); - } catch (ClassNotFoundException e) { - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - return Class.forName(desc.getName(), false, cl); - } - } - }; + in = new ClassloaderFallbackObjectInputStream(inputStream); return in.readObject(); } catch (ClassNotFoundException ex) { http://git-wip-us.apache.org/repos/asf/crunch/blob/775de6cd/crunch-core/src/main/java/org/apache/crunch/util/ClassloaderFallbackObjectInputStream.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/util/ClassloaderFallbackObjectInputStream.java b/crunch-core/src/main/java/org/apache/crunch/util/ClassloaderFallbackObjectInputStream.java new file mode 100644 index 0000000..572ebf4 --- /dev/null +++ b/crunch-core/src/main/java/org/apache/crunch/util/ClassloaderFallbackObjectInputStream.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.crunch.util; + +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectStreamClass; + +/** + * A custom {@link ObjectInputStream} that falls back to the thread context classloader + * if the class can't be found with the usual classloader that {@link + * ObjectInputStream} uses. This is needed when running in the Scala REPL. + * See https://issues.scala-lang.org/browse/SI-2403. + */ +public class ClassloaderFallbackObjectInputStream extends ObjectInputStream { + public ClassloaderFallbackObjectInputStream(InputStream in) throws IOException { + super(in); + } + + @Override + protected Class resolveClass(ObjectStreamClass desc) throws IOException, + ClassNotFoundException { + try { + return super.resolveClass(desc); + } catch (ClassNotFoundException e) { + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + return Class.forName(desc.getName(), false, cl); + } + } +} http://git-wip-us.apache.org/repos/asf/crunch/blob/775de6cd/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java b/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java index 3e49930..0325e12 100644 --- a/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java +++ b/crunch-core/src/main/java/org/apache/crunch/util/DistCache.java @@ -69,7 +69,8 @@ public class DistCache { Object value = null; if (target != null) { Path targetPath = new Path(target.toString()); - ObjectInputStream ois = new ObjectInputStream(targetPath.getFileSystem(conf).open(targetPath)); + ObjectInputStream ois = new ClassloaderFallbackObjectInputStream( + targetPath.getFileSystem(conf).open(targetPath)); try { value = ois.readObject(); } catch (ClassNotFoundException e) {