hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1580532 - in /hive/branches/branch-0.13: common/src/java/org/apache/hadoop/hive/common/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ ql/src/java/org/apache/hadoop/hive/ql/session/ ql/...
Date Sun, 23 Mar 2014 16:54:47 GMT
Author: hashutosh
Date: Sun Mar 23 16:54:47 2014
New Revision: 1580532

URL: http://svn.apache.org/r1580532
Log:
HIVE-3969 : Session state for hive server should be cleaned-up (Navis via Ashutosh Chauhan)

Modified:
    hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
    hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
    hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
    hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java
    hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
    hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java

Modified: hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
(original)
+++ hive/branches/branch-0.13/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java
Sun Mar 23 16:54:47 2014
@@ -18,12 +18,40 @@
 
 package org.apache.hadoop.hive.common;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URLClassLoader;
+import java.util.Arrays;
+
 /**
  * Collection of Java class loading/reflection related utilities common across
  * Hive.
  */
 public final class JavaUtils {
 
+  private static final Log LOG = LogFactory.getLog(JavaUtils.class);
+  private static final Method SUN_MISC_UTIL_RELEASE;
+
+  static {
+    if (Closeable.class.isAssignableFrom(URLClassLoader.class)) {
+      SUN_MISC_UTIL_RELEASE = null;
+    } else {
+      Method release = null;
+      try {
+        Class<?> clazz = Class.forName("sun.misc.ClassLoaderUtil");
+        release = clazz.getMethod("releaseLoader", URLClassLoader.class);
+      } catch (Exception e) {
+        // ignore
+      }
+      SUN_MISC_UTIL_RELEASE = release;
+    }
+  }
+
   /**
    * Standard way of getting classloader in Hive code (outside of Hadoop).
    * 
@@ -40,6 +68,49 @@ public final class JavaUtils {
     return classLoader;
   }
 
+  public static void closeClassLoadersTo(ClassLoader current, ClassLoader stop) {
+    if (!isValidHierarchy(current, stop)) {
+      return;
+    }
+    for (; current != null && current != stop; current = current.getParent()) {
+      try {
+        closeClassLoader(current);
+      } catch (IOException e) {
+        LOG.info("Failed to close class loader " + current +
+            Arrays.toString(((URLClassLoader) current).getURLs()), e);
+      }
+    }
+  }
+
+  // check before closing loaders, not to close app-classloader, etc. by mistake
+  private static boolean isValidHierarchy(ClassLoader current, ClassLoader stop) {
+    if (current == null || stop == null || current == stop) {
+      return false;
+    }
+    for (; current != null && current != stop; current = current.getParent()) {
+    }
+    return current == stop;
+  }
+
+  // best effort to close
+  // see https://issues.apache.org/jira/browse/HIVE-3969 for detail
+  public static void closeClassLoader(ClassLoader loader) throws IOException {
+    if (loader instanceof Closeable) {
+      ((Closeable)loader).close();
+    } else if (SUN_MISC_UTIL_RELEASE != null && loader instanceof URLClassLoader)
{
+      try {
+        SUN_MISC_UTIL_RELEASE.invoke(null, loader);
+      } catch (InvocationTargetException e) {
+        if (e.getTargetException() instanceof IOException) {
+          throw (IOException)e.getTargetException();
+        }
+        throw new IOException(e.getTargetException());
+      } catch (Exception e) {
+        throw new IOException(e);
+      }
+    }
+  }
+
   private JavaUtils() {
     // prevent instantiation
   }

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/exec/PTFOperator.java
Sun Mar 23 16:54:47 2014
@@ -96,7 +96,14 @@ public class PTFOperator extends Operato
       }
     }
     inputPart.close();
-	}
+    inputPart = null;
+
+    for (PTFInputDef iDef = conf.getFuncDef(); iDef != null; iDef = iDef.getInput()) {
+      if (iDef instanceof PartitionedTableFunctionDef) {
+        ((PartitionedTableFunctionDef)iDef).getTFunction().close();
+      }
+    }
+  }
 
 	@Override
 	public void processOp(Object row, int tag) throws HiveException

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
Sun Mar 23 16:54:47 2014
@@ -123,6 +123,10 @@ public class LineageCtx implements NodeP
         old_dep.setExpr(null);
       }
     }
+
+    public void clear() {
+      depMap.clear();
+    }
   }
 
   /**

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
Sun Mar 23 16:54:47 2014
@@ -40,6 +40,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.MapRedStats;
@@ -74,6 +75,8 @@ import org.apache.hadoop.util.Reflection
 public class SessionState {
   private static final Log LOG = LogFactory.getLog(SessionState.class);
 
+  protected ClassLoader parentLoader;
+
   /**
    * current configuration.
    */
@@ -237,6 +240,7 @@ public class SessionState {
     if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) {
       conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
     }
+    parentLoader = JavaUtils.getClassLoader();
   }
 
   private static final SimpleDateFormat DATE_FORMAT =
@@ -286,8 +290,13 @@ public class SessionState {
   /**
    * Sets the given session state in the thread local var for sessions.
    */
-  public static void setCurrentSessionState(SessionState session) {
-    tss.set(session);
+  public static void setCurrentSessionState(SessionState startSs) {
+    tss.set(startSs);
+    Thread.currentThread().setContextClassLoader(startSs.getConf().getClassLoader());
+  }
+
+  public static void detachSession() {
+    tss.remove();
   }
 
   /**
@@ -930,6 +939,7 @@ public class SessionState {
   }
 
   public void close() throws IOException {
+    JavaUtils.closeClassLoadersTo(conf.getClassLoader(), parentLoader);
     File resourceDir =
         new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
     LOG.debug("Removing resource dir " + resourceDir);
@@ -939,6 +949,8 @@ public class SessionState {
       }
     } catch (IOException e) {
       LOG.info("Error removing session resource dir " + resourceDir, e);
+    } finally {
+      detachSession();
     }
 
     try {

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
(original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/udf/ptf/TableFunctionEvaluator.java
Sun Mar 23 16:54:47 2014
@@ -137,5 +137,12 @@ public abstract class TableFunctionEvalu
   protected PTFPartition _transformRawInput(PTFPartition iPart) throws HiveException {
     return null;
   }
+
+  public void close() {
+    if (outputPartition != null) {
+      outputPartition.close();
+    }
+    outputPartition = null;
+  }
 }
 

Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
(original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/session/TestSessionState.java
Sun Mar 23 16:54:47 2014
@@ -88,4 +88,45 @@ public class TestSessionState {
     ss.close();
     assertNull(ss.getTezSession());
   }
+
+  class RegisterJarRunnable implements Runnable {
+    String jar;
+    ClassLoader loader;
+    SessionState ss;
+
+    public RegisterJarRunnable(String jar, SessionState ss) {
+      this.jar = jar;
+      this.ss = ss;
+    }
+
+    public void run() {
+      SessionState.start(ss);
+      SessionState.registerJar(jar);
+      loader = Thread.currentThread().getContextClassLoader();
+    }
+  }
+
+  @Test
+  public void testClassLoaderEquality() throws Exception {
+    HiveConf conf = new HiveConf();
+    final SessionState ss1 = new SessionState(conf);
+    RegisterJarRunnable otherThread = new RegisterJarRunnable("./build/contrib/test/test-udfs.jar",
ss1);
+    Thread th1 = new Thread(otherThread);
+    th1.start();
+    th1.join();
+
+    // set state in current thread
+    SessionState.start(ss1);
+    SessionState ss2 = SessionState.get();
+    ClassLoader loader2 = ss2.conf.getClassLoader();
+
+    System.out.println("Loader1:(Set in other thread) " + otherThread.loader);
+    System.out.println("Loader2:(Set in SessionState.conf) " + loader2);
+    System.out.println("Loader3:(CurrentThread.getContextClassLoader()) " +
+        Thread.currentThread().getContextClassLoader());
+    assertEquals("Other thread loader and session state loader",
+        otherThread.loader, loader2);
+    assertEquals("Other thread loader and current thread loader",
+        otherThread.loader, Thread.currentThread().getContextClassLoader());
+  }
 }

Modified: hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java
(original)
+++ hive/branches/branch-0.13/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFOPNumeric.java
Sun Mar 23 16:54:47 2014
@@ -8,7 +8,6 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hive.common.HiveCompat;
 import org.junit.Assert;
 
 public abstract class TestGenericUDFOPNumeric {

Modified: hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
(original)
+++ hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java
Sun Mar 23 16:54:47 2014
@@ -204,6 +204,7 @@ public class SQLOperation extends Execut
       driver.close();
       driver.destroy();
     }
+    driver = null;
 
     SessionState ss = SessionState.get();
     if (ss.getTmpOutputFile() != null) {

Modified: hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1580532&r1=1580531&r2=1580532&view=diff
==============================================================================
--- hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
(original)
+++ hive/branches/branch-0.13/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
Sun Mar 23 16:54:47 2014
@@ -35,7 +35,6 @@ import org.apache.hadoop.hive.ql.exec.Fe
 import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
 import org.apache.hadoop.hive.ql.history.HiveHistory;
 import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hive.common.util.HiveVersionInfo;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.FetchOrientation;
@@ -144,7 +143,7 @@ public class HiveSessionImpl implements 
 
   protected synchronized void release() {
     assert sessionState != null;
-    // no need to release sessionState...
+    SessionState.detachSession();
   }
 
   @Override
@@ -411,10 +410,10 @@ public class HiveSessionImpl implements 
         hiveHist.closeStream();
       }
       sessionState.close();
-      release();
     } catch (IOException ioe) {
-      release();
       throw new HiveSQLException("Failure to close", ioe);
+    } finally {
+      release();
     }
   }
 



Mime
View raw message