hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From heyongqi...@apache.org
Subject svn commit: r1229885 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: exec/Utilities.java processors/AddResourceProcessor.java processors/DeleteResourceProcessor.java session/SessionState.java
Date Wed, 11 Jan 2012 03:44:17 GMT
Author: heyongqiang
Date: Wed Jan 11 03:44:17 2012
New Revision: 1229885

URL: http://svn.apache.org/viewvc?rev=1229885&view=rev
Log:
HIVE-2706 [jira] StackOverflowError when using custom UDF after adding archive
after adding jars
(Kevin Wilfong via Yongqiang He)

Summary:
https://issues.apache.org/jira/browse/HIVE-2706

The issue was that the current thread's classloader and the classloader in the
conf differed due to the prehook updating only the current thread's classloader
with new jars.  Now, it updates both classloaders, fixing the issue.

When a custom UDF is used in a query after add an archive, such as a zip file,
after adding jars, the XMLEncoder enters an infinite loop when serializing the
map reduce task, as part of sending it to be executed. This results in a stack
overflow error.

Test Plan: Verified it fixed the stack overflow error.

Reviewers: JIRA, heyongqiang, njain

Reviewed By: heyongqiang

CC: heyongqiang

Differential Revision: https://reviews.facebook.net/D1167

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1229885&r1=1229884&r2=1229885&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Jan 11 03:44:17
2012
@@ -120,6 +120,7 @@ import org.apache.hadoop.hive.ql.plan.Pa
 import org.apache.hadoop.hive.ql.plan.PlanUtils;
 import org.apache.hadoop.hive.ql.plan.PlanUtils.ExpressionTypes;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.stats.StatsFactory;
 import org.apache.hadoop.hive.ql.stats.StatsPublisher;
 import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
@@ -144,7 +145,6 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.hive.ql.session.SessionState;
 
 /**
  * Utilities.
@@ -1501,18 +1501,6 @@ public final class Utilities {
     return new URLClassLoader(curPath.toArray(new URL[0]), loader);
   }
 
-  public static ClassLoader addResourceFilesToClassPath(Configuration conf) {
-    try {
-      String addedJars = getResourceFiles(conf, SessionState.ResourceType.JAR);
-      if (!StringUtils.isNotBlank(addedJars)) {
-        return conf.getClassLoader();
-      }
-      return addToClassPath(conf.getClassLoader(), StringUtils.split(addedJars, ','));
-    } catch (Exception e) {
-      throw new RuntimeException("Error in adding jars to conf ", e);
-    }
-  }
-
   /**
    * remove elements from the classpath.
    *
@@ -1536,6 +1524,7 @@ public final class Utilities {
 
     loader = new URLClassLoader(newPath.toArray(new URL[0]));
     curThread.setContextClassLoader(loader);
+    SessionState.get().getConf().setClassLoader(loader);
   }
 
   public static String formatBinaryString(byte[] array, int start, int length) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java?rev=1229885&r1=1229884&r2=1229885&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/AddResourceProcessor.java
Wed Jan 11 03:44:17 2012
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.proces
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -59,9 +58,6 @@ public class AddResourceProcessor implem
       }
     }
 
-    Thread.currentThread().setContextClassLoader(
-        Utilities.addResourceFilesToClassPath(ss.getConf()));
-
     return new CommandProcessorResponse(0);
   }
 

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java?rev=1229885&r1=1229884&r2=1229885&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/DeleteResourceProcessor.java
Wed Jan 11 03:44:17 2012
@@ -21,7 +21,6 @@ package org.apache.hadoop.hive.ql.proces
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -60,9 +59,6 @@ public class DeleteResourceProcessor imp
       ss.delete_resource(t);
     }
 
-    Thread.currentThread().setContextClassLoader(
-        Utilities.addResourceFilesToClassPath(ss.getConf()));
-
     return new CommandProcessorResponse(0);
   }
 }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1229885&r1=1229884&r2=1229885&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Wed Jan 11
03:44:17 2012
@@ -418,8 +418,9 @@ public class SessionState {
     LogHelper console = getConsole();
     try {
       ClassLoader loader = Thread.currentThread().getContextClassLoader();
-      Thread.currentThread().setContextClassLoader(
-          Utilities.addToClassPath(loader, StringUtils.split(newJar, ",")));
+      ClassLoader newLoader = Utilities.addToClassPath(loader, StringUtils.split(newJar,
","));
+      Thread.currentThread().setContextClassLoader(newLoader);
+      SessionState.get().getConf().setClassLoader(newLoader);
       console.printInfo("Added " + newJar + " to class path");
       return true;
     } catch (Exception e) {



Mime
View raw message