hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject hive git commit: HIVE-11878: ClassNotFoundException can possibly occur if multiple jars are registered one at a time in Hive (Ratandeep Ratti via Jason Dere)
Date Fri, 11 Dec 2015 01:18:32 GMT
Repository: hive
Updated Branches:
  refs/heads/master 688a3e17f -> db5a52548


HIVE-11878: ClassNotFoundException can possibly occur if multiple jars are registered one
at a time in Hive (Ratandeep Ratti via Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db5a5254
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db5a5254
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db5a5254

Branch: refs/heads/master
Commit: db5a52548356312aa1c18c6f5f2ea9a84ec905bd
Parents: 688a3e1
Author: Jason Dere <jdere@hortonworks.com>
Authored: Thu Dec 10 17:17:29 2015 -0800
Committer: Jason Dere <jdere@hortonworks.com>
Committed: Thu Dec 10 17:17:29 2015 -0800

----------------------------------------------------------------------
 conf/ivysettings.xml                            | 12 +++-
 itests/custom-udfs/pom.xml                      | 62 +++++++++++++++++
 itests/custom-udfs/udf-classloader-udf1/pom.xml | 43 ++++++++++++
 .../src/main/java/hive/it/custom/udfs/UDF1.java | 58 ++++++++++++++++
 itests/custom-udfs/udf-classloader-udf2/pom.xml | 43 ++++++++++++
 .../src/main/java/hive/it/custom/udfs/UDF2.java | 60 +++++++++++++++++
 itests/custom-udfs/udf-classloader-util/pom.xml | 35 ++++++++++
 .../src/main/java/hive/it/custom/udfs/Util.java | 25 +++++++
 .../plugin/TestHiveAuthorizerShowFilters.java   | 36 +++++-----
 itests/pom.xml                                  |  1 +
 itests/qtest/pom.xml                            |  1 -
 .../hadoop/hive/ql/exec/UDFClassLoader.java     | 70 ++++++++++++++++++++
 .../apache/hadoop/hive/ql/exec/Utilities.java   | 50 +++++++++-----
 .../hadoop/hive/ql/session/SessionState.java    |  9 ++-
 .../queries/clientpositive/udf_classloader.q    | 16 +++++
 ..._classloader_dynamic_dependency_resolution.q | 16 +++++
 .../clientpositive/udf_classloader.q.out        | 43 ++++++++++++
 ...ssloader_dynamic_dependency_resolution.q.out | 43 ++++++++++++
 18 files changed, 585 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/conf/ivysettings.xml
----------------------------------------------------------------------
diff --git a/conf/ivysettings.xml b/conf/ivysettings.xml
index bda842a..aa10f43 100644
--- a/conf/ivysettings.xml
+++ b/conf/ivysettings.xml
@@ -1,4 +1,3 @@
-
 <!--
    Licensed to the Apache Software Foundation (ASF) under one or more
    contributor license agreements.  See the NOTICE file distributed with
@@ -23,15 +22,24 @@
 <ivysettings>
   <!--name of the defaultResolver should always be 'downloadGrapes'. -->
   <settings defaultResolver="downloadGrapes"/>
+  <!-- Only set maven.local.repository if not already set -->
+  <property name="maven.local.repository" value="${user.home}/.m2/repository" override="false"
/>
+  <property name="m2-pattern"
+            value="file:${maven.local.repository}/[organisation]/[module]/[revision]/[module]-[revision](-[classifier]).[ext]"
+            override="false"/>
   <resolvers>
     <!-- more resolvers can be added here -->
     <chain name="downloadGrapes">
       <!-- This resolver uses ibiblio to find artifacts, compatible with maven2 repository
-->
       <ibiblio name="central" m2compatible="true"/>
+      <url name="local-maven2" m2compatible="true">
+        <artifact pattern="${m2-pattern}"/>
+      </url>
       <!-- File resolver to add jars from the local system. -->
       <filesystem name="test" checkmodified="true">
-        <artifact pattern="/tmp/[module]-[revision](-[classifier]).jar" />
+        <artifact pattern="/tmp/[module]-[revision](-[classifier]).jar"/>
       </filesystem>
+
     </chain>
   </resolvers>
 </ivysettings>

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/pom.xml
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/pom.xml b/itests/custom-udfs/pom.xml
new file mode 100644
index 0000000..69e1f70
--- /dev/null
+++ b/itests/custom-udfs/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hive</groupId>
+    <artifactId>hive-it</artifactId>
+    <version>2.1.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>hive-it-custom-udfs</artifactId>
+  <packaging>pom</packaging>
+  <name>Hive Integration - Custom udfs</name>
+  <description>Custom udfs used in hive itest can be defined under this module</description>
+
+  <properties>
+    <hive.path.to.root>../..</hive.path.to.root>
+  </properties>
+
+  <modules>
+    <!--
+    The following three modules are defined for itests udf_classloader.q
+    and udf_classloader_dynamic_dependency_resolution.q . They need to be
+    defined in separate modules instead of a single module to correctly
+     test the said itests
+    -->
+    <module>udf-classloader-util</module>
+    <module>udf-classloader-udf1</module>
+    <module>udf-classloader-udf2</module>
+  </modules>
+
+  <dependencies>
+    <!-- intra-project -->
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <version>${project.version}</version>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop.version}</version>
+      <optional>true</optional>
+    </dependency>
+  </dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-udf1/pom.xml
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-udf1/pom.xml b/itests/custom-udfs/udf-classloader-udf1/pom.xml
new file mode 100644
index 0000000..5f35c63
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-udf1/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hive</groupId>
+    <artifactId>hive-it-custom-udfs</artifactId>
+    <version>2.1.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+  <artifactId>udf-classloader-udf1</artifactId>
+  <packaging>jar</packaging>
+  <name>Hive Integration - Custom UDFs - udf-classloader-udf1</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+      <artifactId>udf-classloader-util</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+
+  <properties>
+    <hive.path.to.root>../../..</hive.path.to.root>
+  </properties>
+
+</project>

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-udf1/src/main/java/hive/it/custom/udfs/UDF1.java
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-udf1/src/main/java/hive/it/custom/udfs/UDF1.java
b/itests/custom-udfs/udf-classloader-udf1/src/main/java/hive/it/custom/udfs/UDF1.java
new file mode 100644
index 0000000..6816190
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-udf1/src/main/java/hive/it/custom/udfs/UDF1.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hive.it.custom.udfs;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+
+public class UDF1 extends GenericUDF {
+  public UDF1() {
+    Util util = new Util();
+    System.out.println(
+        "constructor: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader()
+ ", " + util + " classloader: "
+            + util.getClass().getClassLoader());
+  }
+
+  @Override
+  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException
{
+    Util util = new Util();
+    System.out.println(
+        "initialize: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader()
+ ", " + util + " classloader: "
+            + util.getClass().getClassLoader());
+    return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+  }
+
+  @Override
+  public Object evaluate(DeferredObject[] arguments) throws HiveException {
+    Util util = new Util();
+    System.out.println(
+        "evaluate: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader()
+ ", " + util + " classloader: "
+            + util.getClass().getClassLoader());
+    return getClass().getSimpleName();
+  }
+
+  @Override
+  public String getDisplayString(String[] children) {
+    return getClass().getName();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-udf2/pom.xml
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-udf2/pom.xml b/itests/custom-udfs/udf-classloader-udf2/pom.xml
new file mode 100644
index 0000000..8edca63
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-udf2/pom.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hive</groupId>
+    <artifactId>hive-it-custom-udfs</artifactId>
+    <version>2.1.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+  <artifactId>udf-classloader-udf2</artifactId>
+  <packaging>jar</packaging>
+  <name>Hive Integration - Custom UDFs - udf-classloader-udf2</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+      <artifactId>udf-classloader-util</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+
+  <properties>
+    <hive.path.to.root>../../..</hive.path.to.root>
+  </properties>
+
+</project>

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-udf2/src/main/java/hive/it/custom/udfs/UDF2.java
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-udf2/src/main/java/hive/it/custom/udfs/UDF2.java
b/itests/custom-udfs/udf-classloader-udf2/src/main/java/hive/it/custom/udfs/UDF2.java
new file mode 100644
index 0000000..4bb8772
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-udf2/src/main/java/hive/it/custom/udfs/UDF2.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hive.it.custom.udfs;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+
+
+public class UDF2 extends GenericUDF {
+  public UDF2() {
+    Util util = new Util();
+    System.out.println(
+        "constructor: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader()
+ ", " + util
+            + " classloader: "
+            + util.getClass().getClassLoader());
+  }
+
+  @Override
+  public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException
{
+    Util util = new Util();
+    System.out.println(
+        "initialize: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader()
+ ", " + util
+            + " classloader: "
+            + util.getClass().getClassLoader());
+    return PrimitiveObjectInspectorFactory.javaStringObjectInspector;
+  }
+
+  @Override
+  public Object evaluate(DeferredObject[] arguments) throws HiveException {
+    Util util = new Util();
+    System.out.println(
+        "evaluate: " + getClass().getSimpleName() + " classloader: " + getClass().getClassLoader()
+ ", " + util
+            + " classloader: "
+            + util.getClass().getClassLoader());
+    return getClass().getSimpleName();
+  }
+
+  @Override
+  public String getDisplayString(String[] children) {
+    return getClass().getName();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-util/pom.xml
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-util/pom.xml b/itests/custom-udfs/udf-classloader-util/pom.xml
new file mode 100644
index 0000000..b6475f5
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-util/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hive</groupId>
+    <artifactId>hive-it-custom-udfs</artifactId>
+    <version>2.1.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <groupId>org.apache.hive.hive-it-custom-udfs</groupId>
+  <artifactId>udf-classloader-util</artifactId>
+  <packaging>jar</packaging>
+  <name>Hive Integration - Custom UDFs - udf-classloader-util</name>
+
+  <properties>
+    <hive.path.to.root>../../..</hive.path.to.root>
+  </properties>
+
+</project>

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/custom-udfs/udf-classloader-util/src/main/java/hive/it/custom/udfs/Util.java
----------------------------------------------------------------------
diff --git a/itests/custom-udfs/udf-classloader-util/src/main/java/hive/it/custom/udfs/Util.java
b/itests/custom-udfs/udf-classloader-util/src/main/java/hive/it/custom/udfs/Util.java
new file mode 100644
index 0000000..8881ec7
--- /dev/null
+++ b/itests/custom-udfs/udf-classloader-util/src/main/java/hive/it/custom/udfs/Util.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hive.it.custom.udfs;
+
+public class Util {
+  @Override
+  public String toString() {
+    return getClass().getSimpleName();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
index 0c03a00..5922a8c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerShowFilters.java
@@ -58,7 +58,7 @@ public class TestHiveAuthorizerShowFilters {
   private static final String dbName2 = (TestHiveAuthorizerShowFilters.class.getSimpleName()
+ "db2")
       .toLowerCase();
 
-  static HiveAuthorizer mockedAuthorizer;
+  protected static HiveAuthorizer mockedAuthorizer;
 
   static final List<String> AllTables = getSortedList(tableName1, tableName2);
   static final List<String> AllDbs = getSortedList("default", dbName1, dbName2);
@@ -73,28 +73,28 @@ public class TestHiveAuthorizerShowFilters {
    * HiveAuthorizer.filterListCmdObjects, and stores the list argument in
    * filterArguments
    */
-  static class MockedHiveAuthorizerFactory implements HiveAuthorizerFactory {
+  protected static class MockedHiveAuthorizerFactory implements HiveAuthorizerFactory {
+    protected abstract class AuthorizerWithFilterCmdImpl implements HiveAuthorizer {
+      @Override
+      public List<HivePrivilegeObject> filterListCmdObjects(List<HivePrivilegeObject>
listObjs,
+          HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException
{
+        // capture arguments in static
+        filterArguments = listObjs;
+        // return static variable with results, if it is set to some set of
+        // values
+        // otherwise return the arguments
+        if (filteredResults.size() == 0) {
+          return filterArguments;
+        }
+        return filteredResults;
+      }
+    }
+    
     @Override
     public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreClientFactory,
         HiveConf conf, HiveAuthenticationProvider authenticator, HiveAuthzSessionContext
ctx) {
       Mockito.validateMockitoUsage();
 
-      abstract class AuthorizerWithFilterCmdImpl implements HiveAuthorizer {
-        @Override
-        public List<HivePrivilegeObject> filterListCmdObjects(List<HivePrivilegeObject>
listObjs,
-            HiveAuthzContext context) throws HiveAuthzPluginException, HiveAccessControlException
{
-          // capture arguments in static
-          filterArguments = listObjs;
-          // return static variable with results, if it is set to some set of
-          // values
-          // otherwise return the arguments
-          if (filteredResults.size() == 0) {
-            return filterArguments;
-          }
-          return filteredResults;
-        }
-      }
-
       mockedAuthorizer = Mockito.mock(AuthorizerWithFilterCmdImpl.class, Mockito.withSettings()
           .verboseLogging());
 

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index 5d8249f..fb0a613 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -33,6 +33,7 @@
 
   <modules>
    <module>custom-serde</module>
+   <module>custom-udfs</module>
    <module>hcatalog-unit</module>
    <module>hive-unit</module>
    <module>util</module>

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 8f6807a..ec2fc85 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -656,7 +656,6 @@
                   initScript="${initScript}"
                   cleanupScript="q_test_cleanup.sql"/>
 
-
               </target>
             </configuration>
             <goals>

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFClassLoader.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFClassLoader.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFClassLoader.java
new file mode 100644
index 0000000..c363803
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UDFClassLoader.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import org.apache.hadoop.hive.ql.session.SessionState;
+
+
+/**
+ * {@link UDFClassLoader} is used to dynamically register
+ * udf (and related) jars
+ *
+ * This was introducted to fix HIVE-11878
+ *
+ * Each session will have its own instance of {@link UDFClassLoader}
+ * This is to support HiveServer2 where there can be multiple
+ * active sessions. Addition/removal of jars/resources in one
+ * session should not affect other sessions.
+ */
+public class UDFClassLoader extends URLClassLoader {
+  private boolean isClosed;
+
+  public UDFClassLoader(URL[] urls) {
+    super(urls);
+    isClosed = false;
+  }
+
+  public UDFClassLoader(URL[] urls, ClassLoader parent) {
+    super(urls, parent);
+    isClosed = false;
+  }
+
+  @Override
+  public void addURL(URL url) {
+    Preconditions.checkState(!isClosed, getClass().getSimpleName() + " is already closed");
+    super.addURL(url);
+  }
+
+  /**
+   * See {@link URLClassLoader#close}
+   */
+  public boolean isClosed() {
+    return isClosed;
+  }
+
+  @Override
+  public void close() throws IOException {
+    isClosed = true;
+    super.close();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
index c01994f..54abdef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.hive.ql.exec;
 
+import java.util.ArrayList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 import java.beans.DefaultPersistenceDelegate;
 import java.beans.Encoder;
 import java.beans.Expression;
@@ -1748,6 +1751,19 @@ public final class Utilities {
         return result;
     }
 
+  private static boolean useExistingClassLoader(ClassLoader cl) {
+    if (!(cl instanceof UDFClassLoader)) {
+      // Cannot use the same classloader if it is not an instance of {@code UDFClassLoader}
+      return false;
+    }
+    final UDFClassLoader udfClassLoader = (UDFClassLoader) cl;
+    if (udfClassLoader.isClosed()) {
+      // The classloader may have been closed, Cannot add to the same instance
+      return false;
+    }
+    return true;
+  }
+
   /**
    * Add new elements to the classpath.
    *
@@ -1755,24 +1771,28 @@ public final class Utilities {
    *          Array of classpath elements
    */
   public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws
Exception {
-    URLClassLoader loader = (URLClassLoader) cloader;
-    List<URL> curPath = Arrays.asList(loader.getURLs());
-    ArrayList<URL> newPath = new ArrayList<URL>();
-
-    // get a list with the current classpath components
-    for (URL onePath : curPath) {
-      newPath.add(onePath);
+    final URLClassLoader loader = (URLClassLoader) cloader;
+    if (useExistingClassLoader(cloader)) {
+      final UDFClassLoader udfClassLoader = (UDFClassLoader) loader;
+      for (String path : newPaths) {
+        udfClassLoader.addURL(urlFromPathString(path));
+      }
+      return udfClassLoader;
+    } else {
+      return createUDFClassLoader(loader, newPaths);
     }
-    curPath = newPath;
+  }
 
+  public static ClassLoader createUDFClassLoader(URLClassLoader loader, String[] newPaths)
{
+    final Set<URL> curPathsSet = Sets.newHashSet(loader.getURLs());
+    final List<URL> curPaths = Lists.newArrayList(curPathsSet);
     for (String onestr : newPaths) {
-      URL oneurl = urlFromPathString(onestr);
-      if (oneurl != null && !curPath.contains(oneurl)) {
-        curPath.add(oneurl);
+      final URL oneurl = urlFromPathString(onestr);
+      if (oneurl != null && !curPathsSet.contains(oneurl)) {
+        curPaths.add(oneurl);
       }
     }
-
-    return new URLClassLoader(curPath.toArray(new URL[0]), loader);
+    return new UDFClassLoader(curPaths.toArray(new URL[0]), loader);
   }
 
   /**
@@ -1793,13 +1813,13 @@ public final class Utilities {
       }
     }
     JavaUtils.closeClassLoader(loader);
-//this loader is closed, remove it from cached registry loaders to avoid remove it again.
+   // This loader is closed, remove it from cached registry loaders to avoid removing it
again.
     Registry reg = SessionState.getRegistry();
     if(reg != null) {
       reg.removeFromUDFLoaders(loader);
     }
 
-    loader = new URLClassLoader(newPath.toArray(new URL[0]));
+    loader = new UDFClassLoader(newPath.toArray(new URL[0]));
     curThread.setContextClassLoader(loader);
     SessionState.get().getConf().setClassLoader(loader);
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 5c69fb6..c066c7a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -362,7 +362,12 @@ public class SessionState {
     if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) {
       conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
     }
-    parentLoader = JavaUtils.getClassLoader();
+    // Using system classloader as the parent. Using thread context
+    // classloader as parent can pollute the session. See HIVE-11878
+    parentLoader = SessionState.class.getClassLoader();
+    // Make sure that each session has its own UDFClassloader. For details see {@link UDFClassLoader}
+    final ClassLoader currentLoader = Utilities.createUDFClassLoader((URLClassLoader) parentLoader,
new String[]{});
+    this.conf.setClassLoader(currentLoader);
   }
 
   public void setCmd(String cmdString) {
@@ -1273,7 +1278,7 @@ public class SessionState {
     return scheme;
   }
 
-  List<URI> resolveAndDownload(ResourceType t, String value, boolean convertToUnix)
throws URISyntaxException,
+  protected List<URI> resolveAndDownload(ResourceType t, String value, boolean convertToUnix)
throws URISyntaxException,
       IOException {
     URI uri = createURI(value);
     if (getURLType(value).equals("file")) {

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/test/queries/clientpositive/udf_classloader.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_classloader.q b/ql/src/test/queries/clientpositive/udf_classloader.q
new file mode 100644
index 0000000..82678c2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_classloader.q
@@ -0,0 +1,16 @@
+ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf1/${system:hive.version}/udf-classloader-udf1-${system:hive.version}.jar;
+ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-util/${system:hive.version}/udf-classloader-util-${system:hive.version}.jar;
+ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf2/${system:hive.version}/udf-classloader-udf2-${system:hive.version}.jar;
+
+CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1';
+CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2';
+
+-- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1;
+
+DELETE JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf2/${system:hive.version}/udf-classloader-udf2-${system:hive.version}.jar;
+SELECT f1(*) FROM SRC limit 1;
+
+ADD JAR ${system:maven.local.repository}/org/apache/hive/hive-it-custom-udfs/udf-classloader-udf2/${system:hive.version}/udf-classloader-udf2-${system:hive.version}.jar;
+SELECT f2(*) FROM SRC limit 1;

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
b/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
new file mode 100644
index 0000000..2ceeaa3
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_classloader_dynamic_dependency_resolution.q
@@ -0,0 +1,16 @@
+ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf1:+;
+ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-util:+;
+ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf2:+;
+
+CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1';
+CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2';
+
+-- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1;
+
+DELETE JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf2:+;
+SELECT f1(*) FROM SRC limit 1;
+
+ADD JAR ivy://org.apache.hive.hive-it-custom-udfs:udf-classloader-udf2:+;
+SELECT f2(*) FROM SRC limit 1;

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/test/results/clientpositive/udf_classloader.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_classloader.q.out b/ql/src/test/results/clientpositive/udf_classloader.q.out
new file mode 100644
index 0000000..031d88a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_classloader.q.out
@@ -0,0 +1,43 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: f1
+POSTHOOK: query: CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: f1
+PREHOOK: query: CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: f2
+POSTHOOK: query: CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: f2
+PREHOOK: query: -- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF1	UDF2
+PREHOOK: query: SELECT f1(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT f1(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF1
+PREHOOK: query: SELECT f2(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT f2(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF2

http://git-wip-us.apache.org/repos/asf/hive/blob/db5a5254/ql/src/test/results/clientpositive/udf_classloader_dynamic_dependency_resolution.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_classloader_dynamic_dependency_resolution.q.out
b/ql/src/test/results/clientpositive/udf_classloader_dynamic_dependency_resolution.q.out
new file mode 100644
index 0000000..031d88a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_classloader_dynamic_dependency_resolution.q.out
@@ -0,0 +1,43 @@
+PREHOOK: query: CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: f1
+POSTHOOK: query: CREATE TEMPORARY FUNCTION f1 AS 'hive.it.custom.udfs.UDF1'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: f1
+PREHOOK: query: CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: f2
+POSTHOOK: query: CREATE TEMPORARY FUNCTION f2 AS 'hive.it.custom.udfs.UDF2'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: f2
+PREHOOK: query: -- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: -- udf-classloader-udf1.jar contains f1 which relies on udf-classloader-util.jar,
+-- similiary udf-classloader-udf2.jar contains f2 which also relies on udf-classloader-util.jar.
+SELECT f1(*), f2(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF1	UDF2
+PREHOOK: query: SELECT f1(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT f1(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF1
+PREHOOK: query: SELECT f2(*) FROM SRC limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT f2(*) FROM SRC limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+UDF2


Mime
View raw message