drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prog...@apache.org
Subject [1/2] drill git commit: DRILL-5547: Linking config options with system option manager
Date Sat, 26 Aug 2017 03:52:11 GMT
Repository: drill
Updated Branches:
  refs/heads/master a3442021b -> a51c98b8b


http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
index 3604eb7..ecaa7b7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
@@ -20,10 +20,11 @@ package org.apache.drill.exec.server.options;
 import java.util.Set;
 
 import com.google.common.collect.Sets;
+import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.server.options.OptionValue.Kind;
 import org.apache.drill.exec.server.options.OptionValue.OptionType;
-
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
 import static com.google.common.base.Preconditions.checkArgument;
 
 public class TypeValidators {
@@ -31,8 +32,8 @@ public class TypeValidators {
   public static class PositiveLongValidator extends LongValidator {
     private final long max;
 
-    public PositiveLongValidator(String name, long max, long def) {
-      super(name, def);
+    public PositiveLongValidator(String name, long max) {
+      super(name);
       this.max = max;
     }
 
@@ -49,8 +50,8 @@ public class TypeValidators {
 
   public static class PowerOfTwoLongValidator extends PositiveLongValidator {
 
-    public PowerOfTwoLongValidator(String name, long max, long def) {
-      super(name, max, def);
+    public PowerOfTwoLongValidator(String name, long max) {
+      super(name, max);
     }
 
     @Override
@@ -72,8 +73,8 @@ public class TypeValidators {
     private final double min;
     private final double max;
 
-    public RangeDoubleValidator(String name, double min, double max, double def) {
-      super(name, def);
+    public RangeDoubleValidator(String name, double min, double max) {
+      super(name);
       this.min = min;
       this.max = max;
     }
@@ -93,7 +94,7 @@ public class TypeValidators {
     private final String maxValidatorName;
 
     public MinRangeDoubleValidator(String name, double min, double max, double def, String maxValidatorName) {
-      super(name, min, max, def);
+      super(name, min, max);
       this.maxValidatorName = maxValidatorName;
     }
 
@@ -114,7 +115,7 @@ public class TypeValidators {
     private final String minValidatorName;
 
     public MaxRangeDoubleValidator(String name, double min, double max, double def, String minValidatorName) {
-      super(name, min, max, def);
+      super(name, min, max);
       this.minValidatorName = minValidatorName;
     }
 
@@ -132,42 +133,61 @@ public class TypeValidators {
   }
 
   public static class BooleanValidator extends TypeValidator {
-    public BooleanValidator(String name, boolean def) {
-      this(name, def, false);
+    public BooleanValidator(String name) {
+      this(name, false);
+    }
+
+    public BooleanValidator(String name, boolean isAdminOption) {
+      super(name, Kind.BOOLEAN, isAdminOption);
     }
 
-    public BooleanValidator(String name, boolean def, boolean isAdminOption) {
-      super(name, Kind.BOOLEAN, OptionValue.createBoolean(OptionType.SYSTEM, name, def), isAdminOption);
+    public void loadDefault(DrillConfig bootConfig){
+      OptionValue value = OptionValue.createBoolean(OptionType.SYSTEM, getOptionName(), bootConfig.getBoolean(getConfigProperty()), OptionScope.BOOT);
+      setDefaultValue(value);
     }
   }
 
   public static class StringValidator extends TypeValidator {
-    public StringValidator(String name, String def) {
-      this(name, def, false);
+    public StringValidator(String name) {
+      this(name, false);
+    }
+    public StringValidator(String name, boolean isAdminOption) {
+      super(name, Kind.STRING, isAdminOption);
     }
 
-    public StringValidator(String name, String def, boolean isAdminOption) {
-      super(name, Kind.STRING, OptionValue.createString(OptionType.SYSTEM, name, def), isAdminOption);
+    public void loadDefault(DrillConfig bootConfig){
+      OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionScope.BOOT);
+      setDefaultValue(value);
     }
   }
 
   public static class LongValidator extends TypeValidator {
-    public LongValidator(String name, long def) {
-      this(name, def, false);
+    public LongValidator(String name) {
+      this(name, false);
     }
 
-    public LongValidator(String name, long def, boolean isAdminOption) {
-      super(name, Kind.LONG, OptionValue.createLong(OptionType.SYSTEM, name, def), isAdminOption);
+    public LongValidator(String name, boolean isAdminOption) {
+      super(name, Kind.LONG, isAdminOption);
+    }
+
+    public void loadDefault(DrillConfig bootConfig){
+      OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT);
+      setDefaultValue(value);
     }
   }
 
   public static class DoubleValidator extends TypeValidator {
-    public DoubleValidator(String name, double def) {
-      this(name, def, false);
+    public DoubleValidator(String name) {
+      this(name,false);
     }
 
-    public DoubleValidator(String name, double def, boolean isAdminOption) {
-      super(name, Kind.DOUBLE, OptionValue.createDouble(OptionType.SYSTEM, name, def), isAdminOption);
+    public DoubleValidator(String name, boolean isAdminOption) {
+      super(name, Kind.DOUBLE, isAdminOption);
+    }
+
+    public void loadDefault(DrillConfig bootConfig){
+      OptionValue value = OptionValue.createDouble(OptionType.SYSTEM, getOptionName(),bootConfig.getDouble(getConfigProperty()), OptionScope.BOOT);
+      setDefaultValue(value);
     }
   }
 
@@ -175,8 +195,8 @@ public class TypeValidators {
     private final long min;
     private final long max;
 
-    public RangeLongValidator(String name, long min, long max, long def) {
-      super(name, def);
+    public RangeLongValidator(String name, long min, long max) {
+      super(name);
       this.min = min;
       this.max = max;
     }
@@ -198,9 +218,8 @@ public class TypeValidators {
   public static class EnumeratedStringValidator extends StringValidator {
     private final Set<String> valuesSet = Sets.newLinkedHashSet();
 
-    public EnumeratedStringValidator(String name, String def, String... values) {
-      super(name, def);
-      valuesSet.add(def.toLowerCase());
+    public EnumeratedStringValidator(String name, String... values) {
+      super(name);
       for (String value : values) {
         valuesSet.add(value.toLowerCase());
       }
@@ -217,19 +236,51 @@ public class TypeValidators {
     }
   }
 
+  /** Max width is a special validator which computes and validates
+   *  the maxwidth. If the maxwidth is already set in system/session
+   * the value is returned or else it is computed dynamically based on
+   * the available number of processors and cpu load average
+   */
+  public static class MaxWidthValidator extends LongValidator{
+
+    public MaxWidthValidator(String name) {
+      this(name, false);
+    }
+
+    public MaxWidthValidator(String name, boolean isAdminOption) {
+      super(name, isAdminOption);
+    }
+
+    public void loadDefault(DrillConfig bootConfig) {
+      OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT);
+      setDefaultValue(value);
+    }
+
+    public int computeMaxWidth(double cpuLoadAverage, long maxWidth) {
+      // if maxwidth is already set return it
+      if (maxWidth != 0) {
+        return (int) maxWidth;
+      }
+      // else compute the value and return
+      else {
+        int availProc = Runtime.getRuntime().availableProcessors();
+        long maxWidthPerNode = Math.max(1, Math.min(availProc, Math.round(availProc * cpuLoadAverage)));
+        return (int) maxWidthPerNode;
+      }
+    }
+  }
+
   public static abstract class TypeValidator extends OptionValidator {
     private final Kind kind;
-    private final OptionValue defaultValue;
+    private OptionValue defaultValue = null;
 
-    public TypeValidator(final String name, final Kind kind, final OptionValue defValue) {
-      this(name, kind, defValue, false);
+    public TypeValidator(final String name, final Kind kind) {
+      this(name, kind, false);
     }
 
-    public TypeValidator(final String name, final Kind kind, final OptionValue defValue, final boolean isAdminOption) {
+    public TypeValidator(final String name, final Kind kind, final boolean isAdminOption) {
       super(name, isAdminOption);
-      checkArgument(defValue.type == OptionType.SYSTEM, "Default value must be SYSTEM type.");
       this.kind = kind;
-      this.defaultValue = defValue;
     }
 
     @Override
@@ -256,5 +307,13 @@ public class TypeValidators {
     public Kind getKind() {
       return kind;
     }
+
+    protected void setDefaultValue(OptionValue defaultValue) {
+      this.defaultValue = defaultValue;
+    }
+
+    public String getConfigProperty() {
+      return OPTION_DEFAULTS_ROOT + getOptionName();
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
index d0007d3..1caf10b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
@@ -102,7 +102,7 @@ public class StatusResources {
           OptionValue.Kind.valueOf(kind),
           OptionValue.OptionType.SYSTEM,
           name,
-          value));
+          value, OptionValue.OptionScope.SYSTEM));
     } catch (Exception e) {
       logger.debug("Could not update.", e);
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
new file mode 100644
index 0000000..fdd5092
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+
+import com.google.common.collect.Lists;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionValue.Kind;
+import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
+
+
+/*
+ * Extends the original Option iterator. The idea is to hide the implementation details and present the
+ * user with the rows which have values set at the top level of hierarchy and exclude the values set
+ * at lower levels. This is done by examining the scope and the precedence order of scope is session - system - default.
+ * All the values are represented as String instead of having multiple
+ * columns and the data type is provided as kind to preserve type information about the option.
+ * The query output is as follows  -
+ *               name,kind,type,val,optionScope
+ *        planner.slice_target,BIGINT,SESSION,20,SESSION
+ *        planner.width.max_per_node,BIGINT,SYSTEM,0,BOOT
+ *        planner.affinity_factor,FLOAT,SYSTEM,1.7,SYSTEM
+ *  In the above example, the query output contains single row for each option
+ *  and we can infer that slice target is set at the session level and max width
+ *  per node is set at the BOOT level and affinity factor is set at the SYSTEM level.
+ *  The options set in the top level of hierarchy always takes precedence and they are returned
+ *  in the query output. For example if the option is set at both SESSION level and
+ *  SYSTEM level the value set at SESSION level takes precedence and query output has
+ *  only the value set at SESSION level.
+ */
+public class ExtendedOptionIterator implements Iterator<Object> {
+//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OptionIterator.class);
+
+  private final OptionManager fragmentOptions;
+  private final Iterator<OptionValue> mergedOptions;
+
+  public ExtendedOptionIterator(FragmentContext context) {
+    fragmentOptions = context.getOptions();
+    mergedOptions = sortOptions(fragmentOptions.iterator());
+  }
+   /* *
+    * Remove the redundant rows for the same option based on the scope and return
+    * the value that takes precedence over others. For example the option set in session
+    * scope takes precedence over system and boot and etc.,
+    */
+  public Iterator<OptionValue> sortOptions(Iterator<OptionValue> options) {
+    List<OptionValue> optionslist = Lists.newArrayList(options);
+    HashMap<String, OptionValue> optionsmap = new HashMap<>();
+    final Map<OptionScope, Integer> preference = new HashMap<OptionScope, Integer>() {{
+      put(OptionScope.SESSION, 0);
+      put(OptionScope.SYSTEM, 1);
+      put(OptionScope.BOOT, 2);
+    }};
+
+    for (OptionValue option : optionslist) {
+      if (optionsmap.containsKey(option.getName())) {
+
+        if (preference.get(option.scope) < preference.get(optionsmap.get(option.getName()).scope)) {
+          optionsmap.put(option.getName(), option);
+        }
+
+      } else {
+        optionsmap.put(option.getName(), option);
+      }
+    }
+    optionslist.clear();
+    for (String name : optionsmap.keySet()) {
+      optionslist.add(optionsmap.get(name));
+    }
+
+    Collections.sort(optionslist, new Comparator<OptionValue>() {
+      @Override
+      public int compare(OptionValue v1, OptionValue v2) {
+        return v1.name.compareTo(v2.name);
+      }
+    });
+
+    return optionslist.iterator();
+  }
+
+  @Override
+  public boolean hasNext() {
+    return mergedOptions.hasNext();
+  }
+
+  @Override
+  public ExtendedOptionValueWrapper next() {
+    final OptionValue value = mergedOptions.next();
+    final HashMap<OptionValue.Kind,String> typeMapping = new HashMap() {{
+      put(Kind.STRING,"VARCHAR");
+      put(Kind.DOUBLE,"FLOAT");
+      put(Kind.LONG,"BIGINT");
+      put(Kind.BOOLEAN,"BIT");
+
+    }};
+    return new ExtendedOptionValueWrapper(value.name, typeMapping.get(value.kind), value.type,value.getValue().toString(), value.scope);
+  }
+
+  public enum Status {
+    BOOT, DEFAULT, CHANGED
+  }
+
+  /**
+   * Wrapper class for Extended Option Value
+   */
+  public static class ExtendedOptionValueWrapper {
+
+    public final String name;
+    public final String kind;
+    public final OptionType type;
+    public final String val;
+    public final OptionScope optionScope;
+
+
+    public ExtendedOptionValueWrapper(final String name, final String kind, final OptionType type, final String value, final OptionScope scope) {
+      this.name = name;
+      this.kind = kind;
+      this.type = type;
+      this.val = value;
+      this.optionScope = scope;
+    }
+  }
+
+  @Override
+  public void remove() {
+    throw new UnsupportedOperationException();
+  }
+}
+
+

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
index 5c8a641..424ff95 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
@@ -25,6 +25,7 @@ import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.server.options.DrillConfigIterator;
+import org.apache.drill.exec.server.options.FragmentOptionManager;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.OptionValue.Kind;
@@ -40,10 +41,12 @@ public class OptionIterator implements Iterator<Object> {
 
   private final OptionManager fragmentOptions;
   private final Iterator<OptionValue> mergedOptions;
+  private FragmentContext context;
 
   public OptionIterator(FragmentContext context, Mode mode){
     final DrillConfigIterator configOptions = new DrillConfigIterator(context.getConfig());
     fragmentOptions = context.getOptions();
+    this.context = context;
     final Iterator<OptionValue> optionList;
     switch(mode){
     case BOOT:
@@ -71,10 +74,12 @@ public class OptionIterator implements Iterator<Object> {
   public OptionValueWrapper next() {
     final OptionValue value = mergedOptions.next();
     final Status status;
+    final FragmentOptionManager fragmentOptionManager = (FragmentOptionManager) fragmentOptions;
+    final SystemOptionManager systemOptionManager = (SystemOptionManager) fragmentOptionManager.getFallback();
     if (value.type == OptionType.BOOT) {
       status = Status.BOOT;
     } else {
-      final OptionValue def = SystemOptionManager.getValidator(value.name).getDefault();
+      final OptionValue def = systemOptionManager.getValidator(value.name).getDefault();
       if (value.equalsIgnoreType(def)) {
         status = Status.DEFAULT;
         } else {

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
index 7f99fb7..fa6981b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
@@ -21,6 +21,7 @@ import java.util.Iterator;
 
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.store.sys.OptionIterator.OptionValueWrapper;
+import org.apache.drill.exec.store.sys.ExtendedOptionIterator.ExtendedOptionValueWrapper;
 
 /**
  * An enumeration of all tables in Drill's system ("sys") schema.
@@ -39,6 +40,13 @@ public enum SystemTable {
     }
   },
 
+  OPTION2("options2", false,ExtendedOptionIterator.ExtendedOptionValueWrapper.class ) {
+    @Override
+    public Iterator<Object> getIterator(final FragmentContext context) {
+      return new ExtendedOptionIterator(context);
+    }
+  },
+
   BOOT("boot", false, OptionValueWrapper.class) {
     @Override
     public Iterator<Object> getIterator(final FragmentContext context) {

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
index 000d90f..a046c77 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.ExecConstants;
@@ -75,13 +76,11 @@ public final class ExecutionControls {
 
     /**
      * Constructor for controls option validator.
-     *
-     * @param name the name of the validator
-     * @param def  the default JSON, specified as string
+     *  @param name the name of the validator
      * @param ttl  the number of queries for which this option should be valid
      */
-    public ControlsOptionValidator(final String name, final String def, final int ttl) {
-      super(name, OptionValue.Kind.STRING, OptionValue.createString(OptionType.SYSTEM, name, def));
+    public ControlsOptionValidator(final String name, final int ttl) {
+      super(name, OptionValue.Kind.STRING);
       assert ttl > 0;
       this.ttl = ttl;
     }
@@ -112,6 +111,11 @@ public final class ExecutionControls {
             .build(logger);
       }
     }
+
+    public void loadDefault(DrillConfig bootConfig){
+      OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionValue.OptionScope.BOOT);
+      setDefaultValue(value);
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
index 4580222..48724a4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/MemoryAllocationUtilities.java
@@ -58,7 +58,9 @@ public class MemoryAllocationUtilities {
     // if there are any sorts, compute the maximum allocation, and set it on them
     if (bufferedOpList.size() > 0) {
       final OptionManager optionManager = queryContext.getOptions();
-      final long maxWidthPerNode = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY).num_val;
+      double cpu_load_average = optionManager.getOption(ExecConstants.CPU_LOAD_AVERAGE);
+      final long maxWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE);
+      final long maxWidthPerNode = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(cpu_load_average,maxWidth);
       long maxAllocPerNode = Math.min(DrillConfig.getMaxDirectMemory(),
           queryContext.getConfig().getLong(RootAllocatorFactory.TOP_LEVEL_MAX_ALLOC));
       maxAllocPerNode = Math.min(maxAllocPerNode,

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 41ecc95..f387cda 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -349,3 +349,150 @@ drill.exec: {
 drill.jdbc: {
   batch_queue_throttling_threshold: 100
 }
+
+# The following are defaults for system and session options.
+# Provided here for easy customization in Drill distributions
+# like the drill-distrib.conf file.
+# An addition of new system option should go through following procedure.
+# Before adding a system option a validator should be added for that
+# option and then the option should be configured below under the name
+# space drill.exec.options. When the system options are being loaded
+# options config options are read to find the default value for a give option
+# if the option is not set using ALTER SYSTEM or ALTER SESSION.But
+# if the option is not configured in the conf file under the name space
+# "drill.exec.options", option missing exception will be thrown.
+# Users are not supposed to set these options in the drill-override.conf file.
+# Users should use ALTER SYSTEM and ALTER SESSION to set the options.
+
+drill.exec.options:  {
+    bootstrap-storage-plugins.json: .sys.drill,
+    debug.validate_iterators : false,
+    debug.validate_vectors :false,
+    drill.exec.functions.cast_empty_string_to_null: false,
+    drill.exec.hashagg.min_batches_per_partition : 3,
+    drill.exec.storage.file.partition.column.label: "dir",
+    drill.exec.storage.implicit.filename.column.label: "filename",
+    drill.exec.storage.implicit.filepath.column.label: "filepath",
+    drill.exec.storage.implicit.fqn.column.label: "fqn",
+    drill.exec.storage.implicit.suffix.column.label: "suffix",
+    drill.exec.testing.controls: "{}",
+    exec.bulk_load_table_list.bulk_size: 1000,
+    exec.compile.scalar_replacement: false,
+    exec.enable_bulk_load_table_list: false,
+    exec.enable_union_type: false,
+    exec.errors.verbose: false,
+    exec.hashagg.mem_limit : 0,
+    exec.hashagg.num_partitions :32,
+    exec.impersonation.inbound_policies: "[]",
+    exec.java.compiler.exp_in_method_size: 50,
+    exec.java_compiler : "DEFAULT",
+    exec.java_compiler_debug :true,
+    exec.java_compiler_janino_maxsize : 262144,
+    exec.max_hash_table_size: 1073741824,
+    exec.min_hash_table_size: 65536,
+    exec.persistent_table.umask: "002",
+    exec.query.progress.update: true,
+    exec.query_profile.debug_mode: false,
+    exec.query_profile.save: true,
+    exec.queue.enable: false,
+    exec.queue.large: 10,
+    exec.queue.small: 100,
+    exec.queue.threshold: 30000000,
+    exec.queue.timeout_millis: 300000,
+    exec.sort.disable_managed : false,
+    exec.storage.enable_new_text_reader: true,
+    exec.udf.enable_dynamic_support: true,
+    exec.udf.use_dynamic: true,
+    new_view_default_permissions: 700,
+    org.apache.drill.exec.compile.ClassTransformer.scalar_replacement : "try",
+    planner.add_producer_consumer:false,
+    planner.affinity_factor: 1.2,
+    planner.broadcast_factor:1.0,
+    planner.broadcast_threshold:10000000,
+    planner.cpu_load_average : 0.70,
+    planner.disable_exchanges:false,
+    planner.enable_broadcast_join:true,
+    planner.enable_constant_folding:true,
+    planner.enable_decimal_data_type:false,
+    planner.enable_demux_exchange:false,
+    planner.enable_hash_single_key:true,
+    planner.enable_hashagg:true,
+    planner.enable_hashjoin:true,
+    planner.enable_hashjoin_swap:true,
+    planner.enable_hep_opt:true,
+    planner.enable_hep_partition_pruning:true,
+    planner.enable_join_optimization: true,
+    planner.enable_limit0_optimization: false,
+    planner.enable_mergejoin:true,
+    planner.enable_multiphase_agg:true,
+    planner.enable_mux_exchange:true,
+    planner.enable_nestedloopjoin:true,
+    planner.enable_nljoin_for_scalar_only:true,
+    planner.enable_streamagg:true,
+    planner.enable_type_inference: true,
+    planner.enable_unionall_distribute:false,
+    planner.filter.max_selectivity_estimate_factor:1.0,
+    planner.filter.min_selectivity_estimate_factor:0.0,
+    planner.force_2phase_aggr : false,
+    planner.identifier_max_length:1024,
+    planner.in_subquery_threshold: 20,
+    planner.join.hash_join_swap_margin_factor:10,
+    planner.join.row_count_estimate_factor:1.0,
+    planner.memory.average_field_width: 8,
+    planner.memory.enable_memory_estimation: false,
+    planner.memory.hash_agg_table_factor: 1.1d,
+    planner.memory.hash_join_table_factor: 1.1d,
+    planner.memory.max_query_memory_per_node: 2147483648,
+    planner.memory.min_memory_per_buffered_op: 41943040,
+    planner.memory.non_blocking_operators_memory: 64,
+    planner.memory_limit:268435456,
+    planner.nestedloopjoin_factor:100.0,
+    planner.parser.quoting_identifiers :"`",
+    planner.partitioner_sender_max_threads:8,
+    planner.partitioner_sender_set_threads:-1,
+    planner.partitioner_sender_threads_factor:2,
+    planner.producer_consumer_queue_size:10,
+    planner.slice_target: 100000,
+    planner.store.parquet.rowgroup.filter.pushdown.enabled : true,
+    planner.store.parquet.rowgroup.filter.pushdown.threshold : 10000,
+    # Max per node should always be configured as zero and
+    # it is dynamically computed based on cpu_load_average
+    planner.width.max_per_node: 0,
+    planner.width.max_per_query: 1000,
+    prepare.statement.create_timeout_ms: 10000,
+    security.admin.user_groups: true,
+    security.admin.users: true,
+    store.format: "parquet",
+    store.hive.optimize_scan_with_native_readers: false,
+    store.json.all_text_mode: false,
+    store.json.extended_types: false,
+    store.json.read_numbers_as_double: false,
+    store.json.reader.print_skipped_invalid_record_number: false,
+    store.json.reader.skip_invalid_records: false,
+    store.json.writer.skip_null_fields: true,
+    store.json.writer.uglify: false,
+    store.mongo.all_text_mode: false,
+    store.mongo.bson.record.reader: true,
+    store.mongo.read_numbers_as_double: false,
+    store.parquet.block-size:  536870912,
+    store.parquet.compression: "none",
+    store.parquet.dictionary.page-size: 1048576,
+    store.parquet.enable_dictionary_encoding: false,
+    store.parquet.page-size: 1048576,
+    store.parquet.reader.columnreader.async: false,
+    store.parquet.reader.int96_as_timestamp: false,
+    store.parquet.reader.pagereader.async: true,
+    store.parquet.reader.pagereader.bufferedread: true,
+    store.parquet.reader.pagereader.buffersize: 1048576,
+    store.parquet.reader.pagereader.enforceTotalSize: false,
+    store.parquet.reader.pagereader.queuesize: 2,
+    store.parquet.reader.pagereader.usefadvise: false,
+    store.parquet.use_new_reader: false,
+    store.parquet.vector_fill_check_threshold: 10,
+    store.parquet.vector_fill_threshold: 85,
+    store.parquet.writer.use_single_fs_block : false,
+    store.partition.hash_distribute: false,
+    store.text.estimated_row_size_bytes: 100.0,
+    web.logs.max_lines: 10000,
+    window.enable: true,
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
index df0c89f..b00d4d6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanningBase.java
@@ -80,7 +80,7 @@ public class PlanningBase extends ExecTest{
     provider.start();
     final ScanResult scanResult = ClassPathScanner.fromPrescan(config);
     final LogicalPlanPersistence logicalPlanPersistence = new LogicalPlanPersistence(config, scanResult);
-    final SystemOptionManager systemOptions = new SystemOptionManager(logicalPlanPersistence , provider);
+    final SystemOptionManager systemOptions = new SystemOptionManager(logicalPlanPersistence , provider, config);
     systemOptions.init();
     @SuppressWarnings("resource")
     final UserSession userSession = UserSession.Builder.newBuilder().withOptionManager(systemOptions).build();

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
index 54ae774..8e767e4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
@@ -41,6 +41,7 @@ import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.server.RemoteServiceSet;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
 import org.apache.drill.exec.util.VectorUtil;
 
 /**
@@ -190,7 +191,7 @@ public class QueryTestUtil {
     final OptionManager optionManager = drillbitContext.getOptionManager();
     final OptionValue originalOptionValue = optionManager.getOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION);
     final OptionValue newOptionValue = OptionValue.createString(OptionValue.OptionType.SYSTEM,
-        ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase());
+        ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase(), OptionScope.SYSTEM);
     optionManager.setOption(newOptionValue);
 
     // flush the code cache

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
index dead858..7dd7680 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/ExecTest.java
@@ -65,7 +65,7 @@ public class ExecTest extends DrillTest {
   public static void setupOptionManager() throws Exception{
     final LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(c);
     provider.start();
-    optionManager = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), provider);
+    optionManager = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), provider,c);
     optionManager.init();
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
index f4903d8..7b74a3a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/CodeCompilerTestFactory.java
@@ -30,7 +30,7 @@ public class CodeCompilerTestFactory {
     DrillConfig config = checkNotNull(c);
     LogicalPlanPersistence persistence = new LogicalPlanPersistence(config, ClassPathScanner.fromPrescan(config));
     LocalPersistentStoreProvider provider = new LocalPersistentStoreProvider(config);
-    SystemOptionManager systemOptionManager = new SystemOptionManager(persistence, provider);
+    SystemOptionManager systemOptionManager = new SystemOptionManager(persistence, provider, config);
     return new CodeCompiler(config, systemOptionManager.init());
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
index 7728aae..3c8ca01 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java
@@ -29,6 +29,7 @@ import org.apache.drill.exec.expr.CodeGenerator;
 import org.apache.drill.exec.rpc.user.UserSession;
 import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
 import org.apache.drill.exec.server.options.SessionOptionManager;
 import org.codehaus.commons.compiler.CompileException;
 import org.junit.Assert;
@@ -58,7 +59,7 @@ public class TestClassTransformation extends BaseTestQuery {
   @Test
   public void testJaninoClassCompiler() throws Exception {
     logger.debug("Testing JaninoClassCompiler");
-    sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JANINO.name()));
+    sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JANINO.name(), OptionScope.SESSION));
     for (int i = 0; i < ITERATION_COUNT; i++) {
       compilationInnerClass(false); // Traditional byte-code manipulation
       compilationInnerClass(true); // Plain-old Java
@@ -68,7 +69,8 @@ public class TestClassTransformation extends BaseTestQuery {
   @Test
   public void testJDKClassCompiler() throws Exception {
     logger.debug("Testing JDKClassCompiler");
-    sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name()));
+    OptionType type = OptionType.SESSION;
+    sessionOptions.setOption(OptionValue.createString(type, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name(), OptionScope.SESSION));
     for (int i = 0; i < ITERATION_COUNT; i++) {
       compilationInnerClass(false); // Traditional byte-code manipulation
       compilationInnerClass(true); // Plain-old Java
@@ -80,9 +82,10 @@ public class TestClassTransformation extends BaseTestQuery {
     CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class);
     ClassSet classSet = new ClassSet(null, cg.getDefinition().getTemplateClassName(), cg.getMaterializedClassName());
     String sourceCode = cg.generateAndGet();
-    sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name()));
+    OptionType type = OptionType.SESSION;
+    sessionOptions.setOption(OptionValue.createString(type, ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name(), OptionScope.SESSION));
 
-    sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false));
+    sessionOptions.setOption(OptionValue.createBoolean(type, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false, OptionScope.SESSION));
     @SuppressWarnings("resource")
     QueryClassLoader loader = new QueryClassLoader(config, sessionOptions);
     final byte[][] codeWithoutDebug = loader.getClassByteCode(classSet.generated, sourceCode);
@@ -92,7 +95,7 @@ public class TestClassTransformation extends BaseTestQuery {
       sizeWithoutDebug += bs.length;
     }
 
-    sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, true));
+    sessionOptions.setOption(OptionValue.createBoolean(type, ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, true, OptionScope.SESSION));
     loader = new QueryClassLoader(config, sessionOptions);
     final byte[][] codeWithDebug = loader.getClassByteCode(classSet.generated, sourceCode);
     loader.close();

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
index 9b507fe..1c36c46 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java
@@ -56,7 +56,7 @@ public class ReplaceMethodInvoke {
     check(output);
 
     final DrillConfig c = DrillConfig.forClient();
-    final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPersistentStoreProvider(c));
+    final SystemOptionManager m = new SystemOptionManager(PhysicalPlanReaderTestFactory.defaultLogicalPlanPersistence(c), new LocalPersistentStoreProvider(c), c);
     m.init();
     try (QueryClassLoader ql = new QueryClassLoader(DrillConfig.create(), m)) {
       ql.injectByteCode("org.apache.drill.Pickle$OutgoingBatch", output);

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
index 0d5393e..f9a6358 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/impersonation/TestInboundImpersonationPrivileges.java
@@ -49,7 +49,7 @@ public class TestInboundImpersonationPrivileges extends BaseTestImpersonation {
     ExecConstants.IMPERSONATION_POLICY_VALIDATOR.validate(
         OptionValue.createString(OptionValue.OptionType.SYSTEM,
             ExecConstants.IMPERSONATION_POLICIES_KEY,
-            IMPERSONATION_POLICIES), null);
+            IMPERSONATION_POLICIES,OptionValue.OptionScope.SYSTEM), null);
     try {
       return InboundImpersonationManager.hasImpersonationPrivileges(proxyName, targetName, IMPERSONATION_POLICIES);
     } catch (final Exception e) {

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
index 82241d7..598bdc2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/partitionsender/TestPartitionSender.java
@@ -63,6 +63,7 @@ import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.server.options.OptionList;
 import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
 import org.apache.drill.exec.util.Utilities;
 import org.apache.drill.exec.work.QueryWorkUnit;
 import org.junit.AfterClass;
@@ -180,20 +181,20 @@ public class TestPartitionSender extends PlanTestBase {
 
     final OptionList options = new OptionList();
     // try multiple scenarios with different set of options
-    options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1));
+    options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1, OptionScope.SESSION));
     testThreadsHelper(hashToRandomExchange, drillbitContext, options,
         incoming, registry, planReader, planningSet, rootFragment, 1);
 
     options.clear();
-    options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1));
-    options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_max_threads", 10));
+    options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1, OptionScope.SESSION));
+    options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_max_threads", 10, OptionScope.SESSION));
     hashToRandomExchange.setCost(1000);
     testThreadsHelper(hashToRandomExchange, drillbitContext, options,
         incoming, registry, planReader, planningSet, rootFragment, 10);
 
     options.clear();
-    options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1000));
-    options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_threads_factor",2));
+    options.add(OptionValue.createLong(OptionType.SESSION, "planner.slice_target", 1000, OptionScope.SESSION));
+    options.add(OptionValue.createLong(OptionType.SESSION, "planner.partitioner_sender_threads_factor",2, OptionScope.SESSION));
     hashToRandomExchange.setCost(14000);
     testThreadsHelper(hashToRandomExchange, drillbitContext, options,
         incoming, registry, planReader, planningSet, rootFragment, 2);

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
index 5365773..39c9ea3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/testing/ControlsInjectionUtil.java
@@ -32,6 +32,7 @@ import org.apache.drill.exec.rpc.user.UserSession;
 import org.apache.drill.exec.rpc.user.UserSession.QueryCountIncrementer;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
 
 /**
  * Static methods for constructing exception and pause injections for testing purposes.
@@ -71,7 +72,7 @@ public class ControlsInjectionUtil {
   public static void setControls(final UserSession session, final String controls) {
     validateControlsString(controls);
     final OptionValue opValue = OptionValue.createString(OptionValue.OptionType.SESSION,
-      DRILLBIT_CONTROL_INJECTIONS, controls);
+      DRILLBIT_CONTROL_INJECTIONS, controls, OptionScope.SESSION);
 
     final OptionManager options = session.getOptions();
     try {

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
index f3bd63a..9ed3527 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/work/metadata/TestMetadataProvider.java
@@ -152,7 +152,7 @@ public class TestMetadataProvider extends BaseTestQuery {
 
     assertEquals(RequestStatus.OK, resp.getStatus());
     List<TableMetadata> tables = resp.getTablesList();
-    assertEquals(11, tables.size());
+    assertEquals(12, tables.size());
 
     verifyTable("INFORMATION_SCHEMA", "CATALOGS", tables);
     verifyTable("INFORMATION_SCHEMA", "COLUMNS", tables);
@@ -186,7 +186,7 @@ public class TestMetadataProvider extends BaseTestQuery {
 
     assertEquals(RequestStatus.OK, resp.getStatus());
     List<TableMetadata> tables = resp.getTablesList();
-    assertEquals(11, tables.size());
+    assertEquals(12, tables.size());
 
     verifyTable("INFORMATION_SCHEMA", "CATALOGS", tables);
     verifyTable("INFORMATION_SCHEMA", "COLUMNS", tables);
@@ -211,7 +211,7 @@ public class TestMetadataProvider extends BaseTestQuery {
 
     assertEquals(RequestStatus.OK, resp.getStatus());
     List<TableMetadata> tables = resp.getTablesList();
-    assertEquals(4, tables.size());
+    assertEquals(5, tables.size());
 
     verifyTable("sys", "boot", tables);
     verifyTable("sys", "memory", tables);
@@ -242,7 +242,7 @@ public class TestMetadataProvider extends BaseTestQuery {
 
     assertEquals(RequestStatus.OK, resp.getStatus());
     List<ColumnMetadata> columns = resp.getColumnsList();
-    assertEquals(71, columns.size());
+    assertEquals(76, columns.size());
     // too many records to verify the output.
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
index b305609..d6bb378 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/FixtureBuilder.java
@@ -49,6 +49,7 @@ public class FixtureBuilder {
   public static final int DEFAULT_ZK_REFRESH = 500; // ms
   public static final int DEFAULT_SERVER_RPC_THREADS = 10;
   public static final int DEFAULT_SCAN_THREADS = 8;
+  public static final String OPTION_DEFAULTS_ROOT = "drill.exec.options.";
 
   protected ConfigBuilder configBuilder = new ConfigBuilder();
   protected List<RuntimeOption> sessionOptions;
@@ -97,6 +98,14 @@ public class FixtureBuilder {
   }
 
   /**
+   *
+   */
+   public FixtureBuilder setOptionDefault(String key, Object value) {
+     String option_name = OPTION_DEFAULTS_ROOT + key;
+     configBuilder().put(option_name, value.toString());
+     return this;
+   }
+  /**
    * Add an additional boot-time property for the embedded Drillbit.
    * @param key config property name
    * @param value property value

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
index 976812c..4a63bb5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/OperatorFixture.java
@@ -46,6 +46,7 @@ import org.apache.drill.exec.server.options.BaseOptionManager;
 import org.apache.drill.exec.server.options.OptionSet;
 import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.OptionValue.OptionType;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
 import org.apache.drill.exec.testing.ExecutionControls;
 import org.apache.drill.test.rowSet.DirectRowSet;
 import org.apache.drill.test.rowSet.HyperRowSetImpl;
@@ -127,19 +128,19 @@ public class OperatorFixture extends BaseFixture implements AutoCloseable {
     }
 
     public void set(String key, long value) {
-      values.put(key, OptionValue.createLong(OptionType.SYSTEM, key, value));
+      values.put(key, OptionValue.createLong(OptionType.SYSTEM, key, value, OptionScope.SYSTEM));
     }
 
     public void set(String key, boolean value) {
-      values.put(key, OptionValue.createBoolean(OptionType.SYSTEM, key, value));
+      values.put(key, OptionValue.createBoolean(OptionType.SYSTEM, key, value, OptionScope.SYSTEM));
     }
 
     public void set(String key, double value) {
-      values.put(key, OptionValue.createDouble(OptionType.SYSTEM, key, value));
+      values.put(key, OptionValue.createDouble(OptionType.SYSTEM, key, value, OptionScope.SYSTEM));
     }
 
     public void set(String key, String value) {
-      values.put(key, OptionValue.createString(OptionType.SYSTEM, key, value));
+      values.put(key, OptionValue.createString(OptionType.SYSTEM, key, value, OptionScope.SYSTEM));
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
new file mode 100644
index 0000000..19f2f06
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/test/TestConfigLinkage.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.test;
+
+import org.apache.drill.exec.ExecConstants;
+import org.junit.Assert;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+/*
+ * Tests to test if the linkage between the two config option systems
+ * i.e., the linkage between boot-config system and system/session options.
+ * Tests to assert if the config options are read in the order of session , system, boot-config.
+ * Max width per node is slightly different from other options since it is set to zero by default
+ * in the config and the option value is computed dynamically everytime if the value is zero
+ * i.e., if the value is not set in system/session.
+ * */
+
+public class TestConfigLinkage {
+
+  /* Test if session option takes precedence */
+  @Test
+  public void testSessionOption() throws Exception {
+    FixtureBuilder builder = ClusterFixture.builder().sessionOption(ExecConstants.SLICE_TARGET, 10);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+      assertEquals(slice_target, "10");
+    }
+  }
+
+  /* Test if system option takes precedence over the boot option */
+  @Test
+  public void testSystemOption() throws Exception {
+    FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 10000);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+      assertEquals(slice_target, "10000");
+    }
+  }
+
+  /* Test if config option takes effect if system/session are not set */
+  @Test
+  public void testConfigOption() throws Exception {
+    FixtureBuilder builder = ClusterFixture.builder()
+            .setOptionDefault(ExecConstants.SLICE_TARGET, 1000);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+      assertEquals(slice_target, "1000");
+    }
+  }
+
+  /* Test if altering system option takes precedence over config option */
+  @Test
+  public void testAlterSystem() throws Exception {
+    try (ClusterFixture cluster = ClusterFixture.standardCluster();
+         ClientFixture client = cluster.clientFixture()) {
+      client.queryBuilder().sql("ALTER SYSTEM SET `planner.slice_target` = 10000").run();
+      String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+      assertEquals(slice_target, "10000");
+    }
+  }
+
+  /* Test if altering session option takes precedence over system option */
+  @Test
+  public void testSessionPrecedence() throws Exception {
+    FixtureBuilder builder = ClusterFixture.builder().systemOption(ExecConstants.SLICE_TARGET, 100000);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      client.queryBuilder().sql("ALTER SESSION SET `planner.slice_target` = 10000").run();
+      String slice_target = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.slice_target'").singletonString();
+      assertEquals(slice_target, "10000");
+    }
+  }
+
+  /* Test if setting maxwidth option through config takes effect */
+  @Test
+  public void testMaxWidthPerNodeConfig() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString();
+      assertEquals("2", maxWidth);
+    }
+  }
+
+  /* Test if setting maxwidth at system level takes precedence */
+  @Test
+  public void testMaxWidthPerNodeSystem() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 3);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString();
+      assertEquals("3", maxWidth);
+    }
+  }
+
+  /* Test if setting maxwidth at session level takes precedence */
+  @Test
+  public void testMaxWidthPerNodeSession() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.MAX_WIDTH_PER_NODE_KEY, 2);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String maxWidth = client.queryBuilder().sql("SELECT val FROM sys.options2 where name='planner.width.max_per_node'").singletonString();
+      assertEquals("2", maxWidth);
+    }
+  }
+
+  /* Test if max width is computed correctly using the cpu load average
+     when the option is not set at either system or session level
+  */
+  @Test
+  public void testMaxWidthPerNodeDefault() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.CPU_LOAD_AVERAGE_KEY, 0.70);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      long maxWidth = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(0.70, 0);
+      int availProc = Runtime.getRuntime().availableProcessors();
+      long maxWidthPerNode = Math.max(1, Math.min(availProc, Math.round(availProc * 0.70)));
+      assertEquals(maxWidthPerNode, maxWidth);
+    }
+  }
+
+  /* Test if the scope is set during BOOT time and scope is actually BOOT */
+  @Test
+  public void testScope() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder().setOptionDefault(ExecConstants.SLICE_TARGET, 100000);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String scope = client.queryBuilder()
+                          .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+                          .singletonString();
+      Assert.assertEquals("BOOT",scope);
+    }
+  }
+
+  /* Test if the option is set at SYSTEM scope and the scope is actually SYSTEM */
+  @Test
+  public void testScopeSystem() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder().systemOption(ExecConstants.SLICE_TARGET, 10000);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String scope = client.queryBuilder()
+              .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+              .singletonString();
+      Assert.assertEquals("SYSTEM",scope);
+    }
+  }
+
+  /* Test if the option is set at SESSION scope and the scope is actually SESSION */
+  @Test
+  public void testScopeSession() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder().sessionOption(ExecConstants.SLICE_TARGET, 100000);
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      String scope = client.queryBuilder()
+              .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+              .singletonString();
+      Assert.assertEquals("SESSION",scope);
+    }
+  }
+
+  /* Test if the option is altered at SYSTEM scope and the scope is actually SYSTEM */
+  @Test
+  public void testScopeAlterSystem() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder();
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      client.queryBuilder().sql("ALTER SYSTEM set `planner.slice_target`= 10000").run();
+      String scope = client.queryBuilder()
+              .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+              .singletonString();
+      Assert.assertEquals("SYSTEM",scope);
+    }
+  }
+
+  /* Test if the option is altered at SESSION scope and the scope is actually SESSION */
+  @Test
+  public void testScopeAlterSession() throws Exception {
+    FixtureBuilder builder = ClusterFixture.bareBuilder();
+    try (ClusterFixture cluster = builder.build();
+         ClientFixture client = cluster.clientFixture()) {
+      client.queryBuilder().sql("ALTER SESSION set `planner.slice_target`= 10000").run();
+      String scope = client.queryBuilder()
+              .sql("SELECT optionScope from sys.options2 where name='planner.slice_target'")
+              .singletonString();
+      Assert.assertEquals("SESSION",scope);
+    }
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/a51c98b8/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
index b859650..913834b 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestJdbcMetadata.java
@@ -81,7 +81,7 @@ public class TestJdbcMetadata extends JdbcTestActionBase {
       public ResultSet getResult(Connection c) throws SQLException {
         return c.getMetaData().getTables("DRILL", "sys", "opt%", new String[]{"SYSTEM_TABLE", "SYSTEM_VIEW"});
       }
-    }, 1);
+    }, 2);
   }
 
   @Test
@@ -101,6 +101,6 @@ public class TestJdbcMetadata extends JdbcTestActionBase {
       public ResultSet getResult(Connection c) throws SQLException {
         return c.getMetaData().getColumns("DRILL", "sys", "opt%", "%ame");
       }
-    }, 1);
+    }, 2);
   }
 }


Mime
View raw message