drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [26/27] drill git commit: DRILL-5290: Provide an option to build operator table once for built-in static functions and reuse it across queries.
Date Thu, 02 Mar 2017 20:59:53 GMT
DRILL-5290: Provide an option to build operator table once for built-in static functions and
reuse it across queries.

close #757


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2b5a6f0b
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2b5a6f0b
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2b5a6f0b

Branch: refs/heads/master
Commit: 2b5a6f0b523859297f86298ef384cd146e425f72
Parents: 4a0fd56
Author: Padma Penumarthy <ppenumar97@yahoo.com>
Authored: Wed Feb 22 10:31:01 2017 -0800
Committer: Jinfeng Ni <jni@apache.org>
Committed: Thu Mar 2 10:46:48 2017 -0800

----------------------------------------------------------------------
 .../org/apache/drill/exec/ExecConstants.java    |  4 ++++
 .../org/apache/drill/exec/ops/QueryContext.java |  9 +++++++-
 .../drill/exec/server/DrillbitContext.java      | 23 ++++++++++++++++++++
 .../server/options/SystemOptionManager.java     |  3 ++-
 4 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/2b5a6f0b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 60d6265..4f0f4d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -437,4 +437,8 @@ public interface ExecConstants {
   String QUERY_PROFILE_DEBUG_OPTION = "exec.query_profile.debug_mode";
   BooleanValidator QUERY_PROFILE_DEBUG_VALIDATOR = new BooleanValidator(
       QUERY_PROFILE_DEBUG_OPTION, false, false);
+
+  String USE_DYNAMIC_UDFS_KEY = "exec.udf.use_dynamic";
+  BooleanValidator USE_DYNAMIC_UDFS = new BooleanValidator(USE_DYNAMIC_UDFS_KEY, true);
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/2b5a6f0b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index 707815a..df3f4f4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -91,7 +91,14 @@ public class QueryContext implements AutoCloseable, OptimizerRulesContext,
Schem
     executionControls = new ExecutionControls(queryOptions, drillbitContext.getEndpoint());
     plannerSettings = new PlannerSettings(queryOptions, getFunctionRegistry());
     plannerSettings.setNumEndPoints(drillbitContext.getBits().size());
-    table = new DrillOperatorTable(getFunctionRegistry(), drillbitContext.getOptionManager());
+
+    // If we do not need to support dynamic UDFs for this query, just use static operator
table
+    // built at the startup. Else, build new operator table from latest version of function
registry.
+    if (queryOptions.getOption(ExecConstants.USE_DYNAMIC_UDFS)) {
+      this.table = new DrillOperatorTable(drillbitContext.getFunctionImplementationRegistry(),
drillbitContext.getOptionManager());
+    } else {
+      this.table = drillbitContext.getOperatorTable();
+    }
 
     queryContextInfo = Utilities.createQueryContextInfo(session.getDefaultSchemaPath(), session.getSessionId());
     contextInformation = new ContextInformation(session.getCredentials(), queryContextInfo);

http://git-wip-us.apache.org/repos/asf/drill/blob/2b5a6f0b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 23dc30c..6c68ab2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -33,6 +33,7 @@ import org.apache.drill.exec.expr.fn.registry.RemoteFunctionRegistry;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
 import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.planner.sql.DrillOperatorTable;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 import org.apache.drill.exec.rpc.control.Controller;
 import org.apache.drill.exec.rpc.control.WorkEventBus;
@@ -62,6 +63,8 @@ public class DrillbitContext implements AutoCloseable {
   private final CodeCompiler compiler;
   private final ScanResult classpathScan;
   private final LogicalPlanPersistence lpPersistence;
+  // operator table for standard SQL operators and functions, Drill built-in UDFs
+  private final DrillOperatorTable table;
 
 
   public DrillbitContext(
@@ -91,6 +94,9 @@ public class DrillbitContext implements AutoCloseable {
     this.systemOptions = new SystemOptionManager(lpPersistence, provider);
     this.functionRegistry = new FunctionImplementationRegistry(context.getConfig(), classpathScan,
systemOptions);
     this.compiler = new CodeCompiler(context.getConfig(), systemOptions);
+
+    // This operator table is built once and used for all queries which do not need dynamic
UDF support.
+    this.table = new DrillOperatorTable(functionRegistry, systemOptions);
   }
 
   public FunctionImplementationRegistry getFunctionImplementationRegistry() {
@@ -189,6 +195,23 @@ public class DrillbitContext implements AutoCloseable {
 
   public RemoteFunctionRegistry getRemoteFunctionRegistry() { return functionRegistry.getRemoteFunctionRegistry();
}
 
+  /**
+   * Use the operator table built during startup when "exec.udf.use_dynamic" option
+   * is set to false.
+   * This operator table has standard SQL functions, operators and drill
+   * built-in user defined functions (UDFs).
+   * It does not include dynamic user defined functions (UDFs) that get added/removed
+   * at run time.
+   * This operator table is meant to be used for high throughput,
+   * low latency operational queries, for which cost of building operator table is
+   * high, both in terms of CPU and heap memory usage.
+   *
+   * @return - Operator table
+   */
+  public DrillOperatorTable getOperatorTable() {
+    return table;
+  }
+
   @Override
   public void close() throws Exception {
     getOptionManager().close();

http://git-wip-us.apache.org/repos/asf/drill/blob/2b5a6f0b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 425131c..4a846c0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -165,7 +165,8 @@ public class SystemOptionManager extends BaseOptionManager implements
AutoClosea
       ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR,
       ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED_OPTION,
       ExecConstants.ENABLE_QUERY_PROFILE_VALIDATOR,
-      ExecConstants.QUERY_PROFILE_DEBUG_VALIDATOR
+      ExecConstants.QUERY_PROFILE_DEBUG_VALIDATOR,
+      ExecConstants.USE_DYNAMIC_UDFS
     };
     final Map<String, OptionValidator> tmp = new HashMap<>();
     for (final OptionValidator validator : validators) {


Mime
View raw message