drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject [5/5] drill git commit: DRILL-4995: Allow lazy init when dynamic UDF support is disabled
Date Wed, 23 Nov 2016 23:42:13 GMT
DRILL-4995: Allow lazy init when dynamic UDF support is disabled

This closes #645


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/04fb0be1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/04fb0be1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/04fb0be1

Branch: refs/heads/master
Commit: 04fb0be191ef09409c00ca7173cb903dfbe2abb0
Parents: 3f38118
Author: Arina Ielchiieva <arina.yelchiyeva@gmail.com>
Authored: Thu Nov 3 16:20:04 2016 +0000
Committer: Parth Chandra <parthc@apache.org>
Committed: Wed Nov 23 09:33:30 2016 -0800

----------------------------------------------------------------------
 .../expr/fn/FunctionImplementationRegistry.java | 79 ++++++++++----------
 .../drill/exec/planner/sql/DrillSqlWorker.java  | 15 ++--
 .../org/apache/drill/TestDynamicUDFSupport.java | 25 +++++++
 .../record/ExpressionTreeMaterializerTest.java  | 10 +++
 4 files changed, 79 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/04fb0be1/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index 988a9f6..d0aa33f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -140,9 +140,9 @@ public class FunctionImplementationRegistry implements FunctionLookupContext,
Au
   }
 
   /**
-   * Using the given <code>functionResolver</code> find Drill function implementation
for given
-   * <code>functionCall</code>
-   * If function implementation was not found and in case if Dynamic UDF Support is enabled
+   * Using the given <code>functionResolver</code>
+   * finds Drill function implementation for given <code>functionCall</code>.
+   * If function implementation was not found,
    * loads all missing remote functions and tries to find Drill implementation one more time.
    */
   @Override
@@ -154,12 +154,8 @@ public class FunctionImplementationRegistry implements FunctionLookupContext,
Au
     AtomicLong version = new AtomicLong();
     DrillFuncHolder holder = functionResolver.getBestMatch(
         localFunctionRegistry.getMethods(functionReplacement(functionCall), version), functionCall);
-    if (holder == null && retry) {
-      if (optionManager != null && optionManager.getOption(ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED).bool_val)
{
-        if (loadRemoteFunctions(version.get())) {
-          return findDrillFunction(functionResolver, functionCall, false);
-        }
-      }
+    if (holder == null && retry && loadRemoteFunctions(version.get())) {
+      return findDrillFunction(functionResolver, functionCall, false);
     }
     return holder;
   }
@@ -183,7 +179,7 @@ public class FunctionImplementationRegistry implements FunctionLookupContext,
Au
 
   /**
    * Find the Drill function implementation that matches the name, arg types and return type.
-   * If exact function implementation was not found and in case if Dynamic UDF Support is
enabled
+   * If exact function implementation was not found,
    * loads all missing remote functions and tries to find Drill implementation one more time.
    */
   public DrillFuncHolder findExactMatchingDrillFunction(String name, List<MajorType>
argTypes, MajorType returnType) {
@@ -197,11 +193,8 @@ public class FunctionImplementationRegistry implements FunctionLookupContext,
Au
         return h;
       }
     }
-
-    if (retry && optionManager != null && optionManager.getOption(ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED).bool_val)
{
-      if (loadRemoteFunctions(version.get())) {
-        return findExactMatchingDrillFunction(name, argTypes, returnType, false);
-      }
+    if (retry && loadRemoteFunctions(version.get())) {
+      return findExactMatchingDrillFunction(name, argTypes, returnType, false);
     }
     return null;
   }
@@ -287,35 +280,39 @@ public class FunctionImplementationRegistry implements FunctionLookupContext,
Au
     if (!missingJars.isEmpty()) {
       synchronized (this) {
         missingJars = getMissingJars(remoteFunctionRegistry, localFunctionRegistry);
-        List<JarScan> jars = Lists.newArrayList();
-        for (String jarName : missingJars) {
-          Path binary = null;
-          Path source = null;
-          URLClassLoader classLoader = null;
-          try {
-            binary = copyJarToLocal(jarName, remoteFunctionRegistry);
-            source = copyJarToLocal(JarUtil.getSourceName(jarName), remoteFunctionRegistry);
-            URL[] urls = {binary.toUri().toURL(), source.toUri().toURL()};
-            classLoader = new URLClassLoader(urls);
-            ScanResult scanResult = scan(classLoader, binary, urls);
-            localFunctionRegistry.validate(jarName, scanResult);
-            jars.add(new JarScan(jarName, scanResult, classLoader));
-          } catch (Exception e) {
-            deleteQuietlyLocalJar(binary);
-            deleteQuietlyLocalJar(source);
-            if (classLoader != null) {
-              try {
-                classLoader.close();
-              } catch (Exception ex) {
-                logger.warn("Problem during closing class loader for {}", jarName, e);
+        if (!missingJars.isEmpty()) {
+          logger.info("Starting dynamic UDFs lazy-init process.\n" +
+              "The following jars are going to be downloaded and registered locally: " +
missingJars);
+          List<JarScan> jars = Lists.newArrayList();
+          for (String jarName : missingJars) {
+            Path binary = null;
+            Path source = null;
+            URLClassLoader classLoader = null;
+            try {
+              binary = copyJarToLocal(jarName, remoteFunctionRegistry);
+              source = copyJarToLocal(JarUtil.getSourceName(jarName), remoteFunctionRegistry);
+              URL[] urls = {binary.toUri().toURL(), source.toUri().toURL()};
+              classLoader = new URLClassLoader(urls);
+              ScanResult scanResult = scan(classLoader, binary, urls);
+              localFunctionRegistry.validate(jarName, scanResult);
+              jars.add(new JarScan(jarName, scanResult, classLoader));
+            } catch (Exception e) {
+              deleteQuietlyLocalJar(binary);
+              deleteQuietlyLocalJar(source);
+              if (classLoader != null) {
+                try {
+                  classLoader.close();
+                } catch (Exception ex) {
+                  logger.warn("Problem during closing class loader for {}", jarName, e);
+                }
               }
+              logger.error("Problem during remote functions load from {}", jarName, e);
             }
-            logger.error("Problem during remote functions load from {}", jarName, e);
           }
-        }
-        if (!jars.isEmpty()) {
-          localFunctionRegistry.register(jars);
-          return true;
+          if (!jars.isEmpty()) {
+            localFunctionRegistry.register(jars);
+            return true;
+          }
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/04fb0be1/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index 19123d0..76529d4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -24,7 +24,6 @@ import org.apache.calcite.sql.parser.SqlParseException;
 import org.apache.calcite.tools.RelConversionException;
 import org.apache.calcite.tools.ValidationException;
 import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.exception.FunctionNotFoundException;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.ops.QueryContext;
@@ -113,7 +112,7 @@ public class DrillSqlWorker {
 
   /**
    * Returns query physical plan.
-   * In case of {@link FunctionNotFoundException} and dynamic udf support is enabled, attempts
to load remote functions.
+   * In case of {@link FunctionNotFoundException} attempts to load remote functions.
    * If at least one function was loaded or local function function registry version has
changed,
    * makes one more attempt to get query physical plan.
    */
@@ -122,13 +121,11 @@ public class DrillSqlWorker {
     try {
       return handler.getPlan(sqlNode);
     } catch (FunctionNotFoundException e) {
-      if (context.getOption(ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED).bool_val) {
-        DrillOperatorTable drillOperatorTable = context.getDrillOperatorTable();
-        FunctionImplementationRegistry functionRegistry = context.getFunctionRegistry();
-        if (functionRegistry.loadRemoteFunctions(drillOperatorTable.getFunctionRegistryVersion()))
{
-          drillOperatorTable.reloadOperators(functionRegistry);
-          return handler.getPlan(sqlNode);
-        }
+      DrillOperatorTable drillOperatorTable = context.getDrillOperatorTable();
+      FunctionImplementationRegistry functionRegistry = context.getFunctionRegistry();
+      if (functionRegistry.loadRemoteFunctions(drillOperatorTable.getFunctionRegistryVersion()))
{
+        drillOperatorTable.reloadOperators(functionRegistry);
+        return handler.getPlan(sqlNode);
       }
       throw e;
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/04fb0be1/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
index ae73a3d..4676e39 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestDynamicUDFSupport.java
@@ -339,6 +339,31 @@ public class TestDynamicUDFSupport extends BaseTestQuery {
   }
 
   @Test
+  public void testLazyInitWhenDynamicUdfSupportIsDisabled() throws Exception {
+    try {
+      test("select custom_lower('A') from (values(1))");
+    } catch (UserRemoteException e){
+      assertThat(e.getMessage(), containsString("No match found for function signature custom_lower(<CHARACTER>)"));
+    }
+
+    copyDefaultJarsToStagingArea();
+    test("create function using jar '%s'", default_binary_name);
+
+    try {
+      testBuilder()
+          .sqlQuery("select custom_lower('A') as res from (values(1))")
+          .optionSettingQueriesForTestQuery("alter system set `exec.udf.enable_dynamic_support`
= false")
+          .unOrdered()
+          .baselineColumns("res")
+          .baselineValues("a")
+          .go();
+    } finally {
+      test("alter system reset `exec.udf.enable_dynamic_support`");
+    }
+  }
+
+
+  @Test
   public void testDropFunction() throws Exception {
     copyDefaultJarsToStagingArea();
     test("create function using jar '%s'", default_binary_name);

http://git-wip-us.apache.org/repos/asf/drill/blob/04fb0be1/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
index 7d28c9b..8b338af 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
@@ -21,6 +21,8 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import mockit.Injectable;
+import mockit.Mock;
+import mockit.MockUp;
 import mockit.NonStrictExpectations;
 
 import org.apache.drill.common.config.DrillConfig;
@@ -42,6 +44,8 @@ import org.apache.drill.exec.ExecTest;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry;
+import org.apache.drill.exec.proto.UserBitShared.Registry;
 import org.junit.Test;
 
 import com.google.common.collect.ImmutableList;
@@ -196,6 +200,12 @@ public class ExpressionTreeMaterializerTest extends ExecTest {
       }
     };
 
+    new MockUp<RemoteFunctionRegistry>() {
+      @Mock
+      Registry getRegistry() {
+        return Registry.getDefaultInstance();
+      }
+    };
 
     LogicalExpression functionCallExpr = new FunctionCall("testFunc",
       ImmutableList.of((LogicalExpression) new FieldReference("test", ExpressionPosition.UNKNOWN)
),


Mime
View raw message