drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prog...@apache.org
Subject [3/4] drill git commit: DRILL-5723: Added System Internal Options That can be Modified at Runtime Changes include:
Date Sun, 17 Sep 2017 05:43:06 GMT
http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
index 5883480..0a49f5d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/OptionValue.java
@@ -25,15 +25,18 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.drill.exec.store.sys.PersistentStore;
 import com.google.common.base.Preconditions;
 
+import java.util.EnumSet;
+
 /**
  * <p>
- * An {@link OptionValue option value} is used by an {@link OptionManager} to store a run-time setting. This setting,
- * for example, could affect a query in execution stage. Instances of this class are JSON serializable and can be stored
+ * An {@link OptionValue option value} is used internally by an {@link OptionManager} to store a run-time setting. This setting,
+ * for example, could affect a query in an execution stage. Instances of this class are JSON serializable and can be stored
  * in a {@link PersistentStore persistent store} (see {@link SystemOptionManager#options}), or
  * in memory (see {@link InMemoryOptionManager#options}).
  * </p>
- *<p>  OptionType defines the permission level of the option.If it can be set at System level or Session level or so on.
- * Whereas OptionScope defines the scope at which the option is being set. If the option is being set at the BOOT time
+ * <p>
+ * {@link AccessibleScopes} defines the scopes at which the option can be set. If it can be set at System level or Session level or so on.
+ * Whereas {@link OptionScope} defines the scope at which the option is being set. If the option is being set at the BOOT time
  * the scope of the option is BOOT. If it is set at SYSTEM level the scope is SYSTEM. Although they look similar there
  * is a fine level which differentiates both of them which is at which level of hierarchy they can be set and
  * at what at level of hierarchy they were actually set.
@@ -42,61 +45,82 @@ import com.google.common.base.Preconditions;
 @JsonInclude(Include.NON_NULL)
 public class OptionValue implements Comparable<OptionValue> {
 
-  public enum OptionType {
-    BOOT, SYSTEM, SESSION, QUERY
+  /**
+   * Defines where an option can be configured.
+   */
+  public enum AccessibleScopes {
+    BOOT(EnumSet.of(OptionScope.BOOT)), // Can only be configured at boot time
+    SYSTEM(EnumSet.of(OptionScope.BOOT, OptionScope.SYSTEM)), // Can only be configured at the system level
+    SESSION(EnumSet.of(OptionScope.BOOT, OptionScope.SESSION)), // Can only be configured at the session level
+    QUERY(EnumSet.of(OptionScope.BOOT, OptionScope.QUERY)), // Can only be configured at the query level
+    SYSTEM_AND_SESSION(EnumSet.of(OptionScope.BOOT, OptionScope.SYSTEM, OptionScope.SESSION)), // Can only be configured at the system or session level
+    SESSION_AND_QUERY(EnumSet.of(OptionScope.BOOT, OptionScope.SESSION, OptionScope.QUERY)), // Can only be configured at the session or query level
+    ALL(EnumSet.of(OptionScope.BOOT, OptionScope.SYSTEM, OptionScope.SESSION, OptionScope.QUERY)); // Can be configured at the system, session, or query level
+
+    private EnumSet<OptionScope> scopes;
+
+    AccessibleScopes(EnumSet<OptionScope> scopes) {
+      this.scopes = Preconditions.checkNotNull(scopes);
+    }
+
+    public boolean inScopeOf(OptionScope scope) {
+      return scopes.contains(scope);
+    }
   }
 
   public enum Kind {
     BOOLEAN, LONG, STRING, DOUBLE
   }
 
+  /**
+   * This defines where an option was actually configured.
+   */
   public enum OptionScope {
-    BOOT, SYSTEM, SESSION, QUERY
+    BOOT, SYSTEM, SESSION, QUERY;
   }
 
   public final String name;
   public final Kind kind;
-  public final OptionType type;
+  public final AccessibleScopes accessibleScopes;
   public final Long num_val;
   public final String string_val;
   public final Boolean bool_val;
   public final Double float_val;
   public final OptionScope scope;
 
-  public static OptionValue createLong(OptionType type, String name, long val, OptionScope scope) {
+  public static OptionValue create(AccessibleScopes type, String name, long val, OptionScope scope) {
     return new OptionValue(Kind.LONG, type, name, val, null, null, null, scope);
   }
 
-  public static OptionValue createBoolean(OptionType type, String name, boolean bool, OptionScope scope) {
+  public static OptionValue create(AccessibleScopes type, String name, boolean bool, OptionScope scope) {
     return new OptionValue(Kind.BOOLEAN, type, name, null, null, bool, null, scope);
   }
 
-  public static OptionValue createString(OptionType type, String name, String val, OptionScope scope) {
+  public static OptionValue create(AccessibleScopes type, String name, String val, OptionScope scope) {
     return new OptionValue(Kind.STRING, type, name, null, val, null, null, scope);
   }
 
-  public static OptionValue createDouble(OptionType type, String name, double val, OptionScope scope) {
+  public static OptionValue create(AccessibleScopes type, String name, double val, OptionScope scope) {
     return new OptionValue(Kind.DOUBLE, type, name, null, null, null, val, scope);
   }
 
-  public static OptionValue createOption(Kind kind, OptionType type, String name, String val, OptionScope scope) {
-    switch (kind) {
-      case BOOLEAN:
-        return createBoolean(type, name, Boolean.valueOf(val), scope);
-      case LONG:
-        return createLong(type, name, Long.valueOf(val), scope);
-      case STRING:
-        return createString(type, name, val, scope);
-      case DOUBLE:
-        return createDouble(type, name, Double.valueOf(val), scope);
-      default:
-        return null;
+  public static OptionValue create(AccessibleScopes type, String name, Object val, OptionScope scope) {
+    if (val instanceof Boolean) {
+      return create(type, name, ((Boolean) val).booleanValue(), scope);
+    } else if (val instanceof Long) {
+      return create(type, name, ((Long) val).longValue(), scope);
+    } else if (val instanceof String) {
+      return create(type, name, (String) val, scope);
+    } else if (val instanceof Double) {
+      return create(type, name, ((Double) val).doubleValue(), scope);
     }
+
+    throw new IllegalArgumentException(String.format("Unsupported type %s", val.getClass()));
   }
 
   @JsonCreator
   private OptionValue(@JsonProperty("kind") Kind kind,
-                      @JsonProperty("type") OptionType type,
+                      @JsonProperty("accessibleScopes") AccessibleScopes accessibleScopes,
                       @JsonProperty("name") String name,
                       @JsonProperty("num_val") Long num_val,
                       @JsonProperty("string_val") String string_val,
@@ -105,7 +129,7 @@ public class OptionValue implements Comparable<OptionValue> {
                       @JsonProperty("scope") OptionScope scope) {
     Preconditions.checkArgument(num_val != null || string_val != null || bool_val != null || float_val != null);
     this.kind = kind;
-    this.type = type;
+    this.accessibleScopes = accessibleScopes;
     this.name = name;
     this.float_val = float_val;
     this.num_val = num_val;
@@ -144,7 +168,7 @@ public class OptionValue implements Comparable<OptionValue> {
     result = prime * result + ((name == null) ? 0 : name.hashCode());
     result = prime * result + ((num_val == null) ? 0 : num_val.hashCode());
     result = prime * result + ((string_val == null) ? 0 : string_val.hashCode());
-    result = prime * result + ((type == null) ? 0 : type.hashCode());
+    result = prime * result + ((accessibleScopes == null) ? 0 : accessibleScopes.hashCode());
     return result;
   }
 
@@ -206,7 +230,7 @@ public class OptionValue implements Comparable<OptionValue> {
       return false;
     }
     final OptionValue other = (OptionValue) obj;
-    return type == other.type;
+    return accessibleScopes == other.accessibleScopes;
   }
 
   @Override
@@ -216,6 +240,6 @@ public class OptionValue implements Comparable<OptionValue> {
 
   @Override
   public String toString() {
-    return "OptionValue [ type=" + type + ", name=" + name + ", value=" + getValue() + " ]";
+    return "OptionValue [ accessibleScopes=" + accessibleScopes + ", optionScope=" + scope + ", name=" + name + ", value=" + getValue() + " ]";
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java
index 77ca3d0..a111db4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/QueryOptionManager.java
@@ -18,7 +18,6 @@
 package org.apache.drill.exec.server.options;
 
 import org.apache.drill.common.map.CaseInsensitiveMap;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
 
 /**
  * {@link OptionManager} that holds options within {@link org.apache.drill.exec.ops.QueryContext}.
@@ -38,7 +37,31 @@ public class QueryOptionManager extends InMemoryOptionManager {
   }
 
   @Override
-  boolean supportsOptionType(OptionType type) {
-    return type == OptionType.QUERY;
+  public OptionValue getDefault(String optionName) {
+    return fallback.getDefault(optionName);
+  }
+
+  public SessionOptionManager getSessionOptionManager() {
+    return (SessionOptionManager) fallback;
+  }
+
+  public OptionManager getOptionManager(OptionValue.OptionScope scope) {
+    switch (scope) {
+      case SYSTEM:
+        return getSessionOptionManager().getSystemOptionManager();
+      case SESSION:
+        return getSessionOptionManager();
+      case QUERY:
+        return this;
+      case BOOT:
+        throw new UnsupportedOperationException("There is no option manager for " + OptionValue.OptionScope.BOOT);
+      default:
+        throw new UnsupportedOperationException("Invalid type: " + scope);
+    }
+  }
+
+  @Override
+  protected OptionValue.OptionScope getScope() {
+    return OptionValue.OptionScope.QUERY;
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java
index 512c8e9..52bf403 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SessionOptionManager.java
@@ -20,10 +20,8 @@ package org.apache.drill.exec.server.options;
 import com.google.common.base.Predicate;
 import com.google.common.collect.Collections2;
 import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.common.map.CaseInsensitiveMap;
 import org.apache.drill.exec.rpc.user.UserSession;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
 
 import java.util.Collection;
 import java.util.Map;
@@ -55,13 +53,11 @@ public class SessionOptionManager extends InMemoryOptionManager {
   }
 
   @Override
-  boolean setLocalOption(final OptionValue value) {
-    final boolean set = super.setLocalOption(value);
-    if (!set) {
-      return false;
-    }
+  public void setLocalOptionHelper(final OptionValue value) {
+    super.setLocalOptionHelper(value);
     final String name = value.name;
-    final OptionValidator validator = getFallbackOptionManager().getValidator(name); // if set, validator must exist.
+    final OptionDefinition definition = getOptionDefinition(name); // if set, validator must exist.
+    final OptionValidator validator = definition.getValidator();
     final boolean shortLived = validator.isShortLived();
     if (shortLived) {
       final int start = session.getQueryCount() + 1; // start from the next query
@@ -69,7 +65,6 @@ public class SessionOptionManager extends InMemoryOptionManager {
       final int end = start + ttl;
       shortLivedOptions.put(name, new ImmutablePair<>(start, end));
     }
-    return true;
   }
 
   @Override
@@ -83,7 +78,7 @@ public class SessionOptionManager extends InMemoryOptionManager {
       final int start = shortLivedOptions.get(name).getLeft();
       // option is not in effect if queryNumber < start
       if (queryNumber < start) {
-        return getFallbackOptionManager().getValidator(name).getDefault();
+        return fallback.getOption(name);
       // reset if queryNumber <= end
       } else {
         options.remove(name);
@@ -116,14 +111,22 @@ public class SessionOptionManager extends InMemoryOptionManager {
     return liveOptions;
   }
 
+  /**
+   * Gets the SystemOptionManager.
+   * @return The SystemOptionManager.
+   */
+  public SystemOptionManager getSystemOptionManager() {
+    final SystemOptionManager systemOptionManager = (SystemOptionManager) fallback;
+    return systemOptionManager;
+  }
+
   @Override
-  boolean supportsOptionType(OptionType type) {
-    return type == OptionType.SESSION;
+  public OptionValue getDefault(String optionName) {
+    return fallback.getDefault(optionName);
   }
 
-  /* Gets fallback manager and returns it as SystemOptionManager */
-  public SystemOptionManager getFallbackOptionManager() {
-    final SystemOptionManager systemOptionManager = (SystemOptionManager) getFallback();
-    return systemOptionManager;
+  @Override
+  protected OptionValue.OptionScope getScope() {
+    return OptionValue.OptionScope.SESSION;
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 4e362ff..75bcc1f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -26,7 +26,6 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
-import com.typesafe.config.ConfigException;
 import org.apache.commons.collections.IteratorUtils;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.config.LogicalPlanPersistence;
@@ -36,7 +35,6 @@ import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.compile.ClassCompilerSelector;
 import org.apache.drill.exec.compile.ClassTransformer;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
 import org.apache.drill.exec.store.sys.PersistentStore;
 import org.apache.drill.exec.store.sys.PersistentStoreConfig;
 import org.apache.drill.exec.store.sys.PersistentStoreProvider;
@@ -69,148 +67,154 @@ import com.google.common.collect.Sets;
  *  that is loaded into validator from the config will be returned.
  *  </p>
  */
-
-public class SystemOptionManager extends BaseOptionManager implements OptionManager, AutoCloseable {
+public class SystemOptionManager extends BaseOptionManager implements AutoCloseable {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SystemOptionManager.class);
 
-  private CaseInsensitiveMap<OptionValidator> VALIDATORS;
-  public void populateValidators() {
-    final OptionValidator[] validators = new OptionValidator[]{
-      PlannerSettings.CONSTANT_FOLDING,
-      PlannerSettings.EXCHANGE,
-      PlannerSettings.HASHAGG,
-      PlannerSettings.STREAMAGG,
-      PlannerSettings.HASHJOIN,
-      PlannerSettings.MERGEJOIN,
-      PlannerSettings.NESTEDLOOPJOIN,
-      PlannerSettings.MULTIPHASE,
-      PlannerSettings.BROADCAST,
-      PlannerSettings.BROADCAST_THRESHOLD,
-      PlannerSettings.BROADCAST_FACTOR,
-      PlannerSettings.NESTEDLOOPJOIN_FACTOR,
-      PlannerSettings.NLJOIN_FOR_SCALAR,
-      PlannerSettings.JOIN_ROW_COUNT_ESTIMATE_FACTOR,
-      PlannerSettings.MUX_EXCHANGE,
-      PlannerSettings.DEMUX_EXCHANGE,
-      PlannerSettings.PRODUCER_CONSUMER,
-      PlannerSettings.PRODUCER_CONSUMER_QUEUE_SIZE,
-      PlannerSettings.HASH_SINGLE_KEY,
-      PlannerSettings.IDENTIFIER_MAX_LENGTH,
-      PlannerSettings.HASH_JOIN_SWAP,
-      PlannerSettings.HASH_JOIN_SWAP_MARGIN_FACTOR,
-      PlannerSettings.PARTITION_SENDER_THREADS_FACTOR,
-      PlannerSettings.PARTITION_SENDER_MAX_THREADS,
-      PlannerSettings.PARTITION_SENDER_SET_THREADS,
-      PlannerSettings.ENABLE_DECIMAL_DATA_TYPE,
-      PlannerSettings.HEP_OPT,
-      PlannerSettings.PLANNER_MEMORY_LIMIT,
-      PlannerSettings.HEP_PARTITION_PRUNING,
-      PlannerSettings.FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR,
-      PlannerSettings.FILTER_MAX_SELECTIVITY_ESTIMATE_FACTOR,
-      PlannerSettings.TYPE_INFERENCE,
-      PlannerSettings.IN_SUBQUERY_THRESHOLD,
-      PlannerSettings.UNIONALL_DISTRIBUTE,
-      PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING,
-      PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD,
-      PlannerSettings.QUOTING_IDENTIFIERS,
-      PlannerSettings.JOIN_OPTIMIZATION,
-      PlannerSettings.FORCE_2PHASE_AGGR, // for testing
-      ExecConstants.HASHAGG_NUM_PARTITIONS_VALIDATOR,
-      ExecConstants.HASHAGG_MAX_MEMORY_VALIDATOR,
-      ExecConstants.HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR, // for tuning
-      ExecConstants.HASHAGG_FALLBACK_ENABLED_VALIDATOR, // for enable/disable unbounded HashAgg
-      ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
-      ExecConstants.OUTPUT_FORMAT_VALIDATOR,
-      ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
-      ExecConstants.PARQUET_WRITER_USE_SINGLE_FS_BLOCK_VALIDATOR,
-      ExecConstants.PARQUET_PAGE_SIZE_VALIDATOR,
-      ExecConstants.PARQUET_DICT_PAGE_SIZE_VALIDATOR,
-      ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE_VALIDATOR,
-      ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING_VALIDATOR,
-      ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR,
-      ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR,
-      ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR,
-      ExecConstants.PARQUET_PAGEREADER_ASYNC_VALIDATOR,
-      ExecConstants.PARQUET_PAGEREADER_QUEUE_SIZE_VALIDATOR,
-      ExecConstants.PARQUET_PAGEREADER_ENFORCETOTALSIZE_VALIDATOR,
-      ExecConstants.PARQUET_COLUMNREADER_ASYNC_VALIDATOR,
-      ExecConstants.PARQUET_PAGEREADER_USE_BUFFERED_READ_VALIDATOR,
-      ExecConstants.PARQUET_PAGEREADER_BUFFER_SIZE_VALIDATOR,
-      ExecConstants.PARQUET_PAGEREADER_USE_FADVISE_VALIDATOR,
-      ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR,
-      ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR,
-      ExecConstants.ENABLE_UNION_TYPE,
-      ExecConstants.TEXT_ESTIMATED_ROW_SIZE,
-      ExecConstants.JSON_EXTENDED_TYPES,
-      ExecConstants.JSON_WRITER_UGLIFY,
-      ExecConstants.JSON_WRITER_SKIPNULLFIELDS,
-      ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR,
-      ExecConstants.JSON_SKIP_MALFORMED_RECORDS_VALIDATOR,
-      ExecConstants.JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR,
-      ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR,
-      ExecConstants.MONGO_READER_ALL_TEXT_MODE_VALIDATOR,
-      ExecConstants.MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR,
-      ExecConstants.MONGO_BSON_RECORD_READER_VALIDATOR,
-      ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR,
-      ExecConstants.SLICE_TARGET_OPTION,
-      ExecConstants.AFFINITY_FACTOR,
-      ExecConstants.MAX_WIDTH_GLOBAL,
-      ExecConstants.MAX_WIDTH_PER_NODE,
-      ExecConstants.ENABLE_QUEUE,
-      ExecConstants.LARGE_QUEUE_SIZE,
-      ExecConstants.QUEUE_THRESHOLD_SIZE,
-      ExecConstants.QUEUE_TIMEOUT,
-      ExecConstants.SMALL_QUEUE_SIZE,
-      ExecConstants.MIN_HASH_TABLE_SIZE,
-      ExecConstants.MAX_HASH_TABLE_SIZE,
-      ExecConstants.EARLY_LIMIT0_OPT,
-      ExecConstants.ENABLE_MEMORY_ESTIMATION,
-      ExecConstants.MAX_QUERY_MEMORY_PER_NODE,
-      ExecConstants.MIN_MEMORY_PER_BUFFERED_OP,
-      ExecConstants.NON_BLOCKING_OPERATORS_MEMORY,
-      ExecConstants.HASH_JOIN_TABLE_FACTOR,
-      ExecConstants.HASH_AGG_TABLE_FACTOR,
-      ExecConstants.AVERAGE_FIELD_WIDTH,
-      ExecConstants.NEW_VIEW_DEFAULT_PERMS_VALIDATOR,
-      ExecConstants.CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR,
-      ExecConstants.ADMIN_USERS_VALIDATOR,
-      ExecConstants.ADMIN_USER_GROUPS_VALIDATOR,
-      ExecConstants.IMPERSONATION_POLICY_VALIDATOR,
-      ClassCompilerSelector.JAVA_COMPILER_VALIDATOR,
-      ClassCompilerSelector.JAVA_COMPILER_JANINO_MAXSIZE,
-      ClassCompilerSelector.JAVA_COMPILER_DEBUG,
-      ExecConstants.ENABLE_VERBOSE_ERRORS,
-      ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR,
-      ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR,
-      ExecConstants.ENABLE_NEW_TEXT_READER,
-      ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST,
-      ExecConstants.BULK_LOAD_TABLE_LIST_BULK_SIZE,
-      ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR,
-      ExecConstants.IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR,
-      ExecConstants.IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR,
-      ExecConstants.IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR,
-      ExecConstants.IMPLICIT_FILEPATH_COLUMN_LABEL_VALIDATOR,
-      ExecConstants.CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR,
-      ExecConstants.CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS_VALIDATOR,
-      ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR,
-      ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED_OPTION,
-      ExecConstants.ENABLE_QUERY_PROFILE_VALIDATOR,
-      ExecConstants.QUERY_PROFILE_DEBUG_VALIDATOR,
-      ExecConstants.USE_DYNAMIC_UDFS,
-      ExecConstants.QUERY_TRANSIENT_STATE_UPDATE,
-      ExecConstants.PERSISTENT_TABLE_UMASK_VALIDATOR,
-      ExecConstants.CPU_LOAD_AVERAGE,
-      ExecConstants.ENABLE_VECTOR_VALIDATOR,
-      ExecConstants.ENABLE_ITERATOR_VALIDATOR
+  /**
+   * Creates all the OptionDefinitions to be registered with the {@link SystemOptionManager}.
+   * @return A map
+   */
+  public static CaseInsensitiveMap<OptionDefinition> createDefaultOptionDefinitions() {
+    final OptionDefinition[] definitions = new OptionDefinition[]{
+      new OptionDefinition(PlannerSettings.CONSTANT_FOLDING),
+      new OptionDefinition(PlannerSettings.EXCHANGE),
+      new OptionDefinition(PlannerSettings.HASHAGG),
+      new OptionDefinition(PlannerSettings.STREAMAGG),
+      new OptionDefinition(PlannerSettings.HASHJOIN),
+      new OptionDefinition(PlannerSettings.MERGEJOIN),
+      new OptionDefinition(PlannerSettings.NESTEDLOOPJOIN),
+      new OptionDefinition(PlannerSettings.MULTIPHASE),
+      new OptionDefinition(PlannerSettings.BROADCAST),
+      new OptionDefinition(PlannerSettings.BROADCAST_THRESHOLD),
+      new OptionDefinition(PlannerSettings.BROADCAST_FACTOR),
+      new OptionDefinition(PlannerSettings.NESTEDLOOPJOIN_FACTOR),
+      new OptionDefinition(PlannerSettings.NLJOIN_FOR_SCALAR),
+      new OptionDefinition(PlannerSettings.JOIN_ROW_COUNT_ESTIMATE_FACTOR),
+      new OptionDefinition(PlannerSettings.MUX_EXCHANGE),
+      new OptionDefinition(PlannerSettings.DEMUX_EXCHANGE),
+      new OptionDefinition(PlannerSettings.PRODUCER_CONSUMER),
+      new OptionDefinition(PlannerSettings.PRODUCER_CONSUMER_QUEUE_SIZE),
+      new OptionDefinition(PlannerSettings.HASH_SINGLE_KEY),
+      new OptionDefinition(PlannerSettings.IDENTIFIER_MAX_LENGTH),
+      new OptionDefinition(PlannerSettings.HASH_JOIN_SWAP),
+      new OptionDefinition(PlannerSettings.HASH_JOIN_SWAP_MARGIN_FACTOR),
+      new OptionDefinition(PlannerSettings.PARTITION_SENDER_THREADS_FACTOR),
+      new OptionDefinition(PlannerSettings.PARTITION_SENDER_MAX_THREADS),
+      new OptionDefinition(PlannerSettings.PARTITION_SENDER_SET_THREADS),
+      new OptionDefinition(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE),
+      new OptionDefinition(PlannerSettings.HEP_OPT),
+      new OptionDefinition(PlannerSettings.PLANNER_MEMORY_LIMIT),
+      new OptionDefinition(PlannerSettings.HEP_PARTITION_PRUNING),
+      new OptionDefinition(PlannerSettings.FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR),
+      new OptionDefinition(PlannerSettings.FILTER_MAX_SELECTIVITY_ESTIMATE_FACTOR),
+      new OptionDefinition(PlannerSettings.TYPE_INFERENCE),
+      new OptionDefinition(PlannerSettings.IN_SUBQUERY_THRESHOLD),
+      new OptionDefinition(PlannerSettings.UNIONALL_DISTRIBUTE),
+      new OptionDefinition(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING),
+      new OptionDefinition(PlannerSettings.PARQUET_ROWGROUP_FILTER_PUSHDOWN_PLANNING_THRESHOLD),
+      new OptionDefinition(PlannerSettings.QUOTING_IDENTIFIERS),
+      new OptionDefinition(PlannerSettings.JOIN_OPTIMIZATION),
+      new OptionDefinition(PlannerSettings.FORCE_2PHASE_AGGR), // for testing
+      new OptionDefinition(ExecConstants.HASHAGG_NUM_PARTITIONS_VALIDATOR),
+      new OptionDefinition(ExecConstants.HASHAGG_MAX_MEMORY_VALIDATOR),
+      new OptionDefinition(ExecConstants.HASHAGG_MIN_BATCHES_PER_PARTITION_VALIDATOR), // for tuning
+      new OptionDefinition(ExecConstants.HASHAGG_FALLBACK_ENABLED_VALIDATOR), // for enable/disable unbounded HashAgg
+      new OptionDefinition(ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION),
+      new OptionDefinition(ExecConstants.OUTPUT_FORMAT_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_WRITER_USE_SINGLE_FS_BLOCK_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_PAGE_SIZE_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_DICT_PAGE_SIZE_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_WRITER_COMPRESSION_TYPE_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_WRITER_ENABLE_DICTIONARY_ENCODING_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_ASYNC_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_QUEUE_SIZE_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_ENFORCETOTALSIZE_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_COLUMNREADER_ASYNC_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_USE_BUFFERED_READ_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_BUFFER_SIZE_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_PAGEREADER_USE_FADVISE_VALIDATOR),
+      new OptionDefinition(ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP_VALIDATOR),
+      new OptionDefinition(ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR),
+      new OptionDefinition(ExecConstants.ENABLE_UNION_TYPE),
+      new OptionDefinition(ExecConstants.TEXT_ESTIMATED_ROW_SIZE),
+      new OptionDefinition(ExecConstants.JSON_EXTENDED_TYPES),
+      new OptionDefinition(ExecConstants.JSON_WRITER_UGLIFY),
+      new OptionDefinition(ExecConstants.JSON_WRITER_SKIPNULLFIELDS),
+      new OptionDefinition(ExecConstants.JSON_READ_NUMBERS_AS_DOUBLE_VALIDATOR),
+      new OptionDefinition(ExecConstants.JSON_SKIP_MALFORMED_RECORDS_VALIDATOR),
+      new OptionDefinition(ExecConstants.JSON_READER_PRINT_INVALID_RECORDS_LINE_NOS_FLAG_VALIDATOR),
+      new OptionDefinition(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR),
+      new OptionDefinition(ExecConstants.MONGO_READER_ALL_TEXT_MODE_VALIDATOR),
+      new OptionDefinition(ExecConstants.MONGO_READER_READ_NUMBERS_AS_DOUBLE_VALIDATOR),
+      new OptionDefinition(ExecConstants.MONGO_BSON_RECORD_READER_VALIDATOR),
+      new OptionDefinition(ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS_VALIDATOR),
+      new OptionDefinition(ExecConstants.SLICE_TARGET_OPTION),
+      new OptionDefinition(ExecConstants.AFFINITY_FACTOR),
+      new OptionDefinition(ExecConstants.MAX_WIDTH_GLOBAL),
+      new OptionDefinition(ExecConstants.MAX_WIDTH_PER_NODE),
+      new OptionDefinition(ExecConstants.ENABLE_QUEUE),
+      new OptionDefinition(ExecConstants.LARGE_QUEUE_SIZE),
+      new OptionDefinition(ExecConstants.QUEUE_THRESHOLD_SIZE),
+      new OptionDefinition(ExecConstants.QUEUE_TIMEOUT),
+      new OptionDefinition(ExecConstants.SMALL_QUEUE_SIZE),
+      new OptionDefinition(ExecConstants.MIN_HASH_TABLE_SIZE),
+      new OptionDefinition(ExecConstants.MAX_HASH_TABLE_SIZE),
+      new OptionDefinition(ExecConstants.EARLY_LIMIT0_OPT),
+      new OptionDefinition(ExecConstants.ENABLE_MEMORY_ESTIMATION),
+      new OptionDefinition(ExecConstants.MAX_QUERY_MEMORY_PER_NODE),
+      new OptionDefinition(ExecConstants.MIN_MEMORY_PER_BUFFERED_OP),
+      new OptionDefinition(ExecConstants.NON_BLOCKING_OPERATORS_MEMORY),
+      new OptionDefinition(ExecConstants.HASH_JOIN_TABLE_FACTOR),
+      new OptionDefinition(ExecConstants.HASH_AGG_TABLE_FACTOR),
+      new OptionDefinition(ExecConstants.AVERAGE_FIELD_WIDTH),
+      new OptionDefinition(ExecConstants.NEW_VIEW_DEFAULT_PERMS_VALIDATOR),
+      new OptionDefinition(ExecConstants.CTAS_PARTITIONING_HASH_DISTRIBUTE_VALIDATOR),
+      new OptionDefinition(ExecConstants.ADMIN_USERS_VALIDATOR, new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+      new OptionDefinition(ExecConstants.ADMIN_USER_GROUPS_VALIDATOR, new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+      new OptionDefinition(ExecConstants.IMPERSONATION_POLICY_VALIDATOR, new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+      new OptionDefinition(ClassCompilerSelector.JAVA_COMPILER_VALIDATOR),
+      new OptionDefinition(ClassCompilerSelector.JAVA_COMPILER_JANINO_MAXSIZE),
+      new OptionDefinition(ClassCompilerSelector.JAVA_COMPILER_DEBUG),
+      new OptionDefinition(ExecConstants.ENABLE_VERBOSE_ERRORS),
+      new OptionDefinition(ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR),
+      new OptionDefinition(ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR),
+      new OptionDefinition(ExecConstants.ENABLE_NEW_TEXT_READER),
+      new OptionDefinition(ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST),
+      new OptionDefinition(ExecConstants.BULK_LOAD_TABLE_LIST_BULK_SIZE),
+      new OptionDefinition(ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR),
+      new OptionDefinition(ExecConstants.IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR),
+      new OptionDefinition(ExecConstants.IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR),
+      new OptionDefinition(ExecConstants.IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR),
+      new OptionDefinition(ExecConstants.IMPLICIT_FILEPATH_COLUMN_LABEL_VALIDATOR),
+      new OptionDefinition(ExecConstants.CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR),
+      new OptionDefinition(ExecConstants.CREATE_PREPARE_STATEMENT_TIMEOUT_MILLIS_VALIDATOR),
+      new OptionDefinition(ExecConstants.DYNAMIC_UDF_SUPPORT_ENABLED_VALIDATOR,  new OptionMetaData(OptionValue.AccessibleScopes.SYSTEM, true, false)),
+      new OptionDefinition(ExecConstants.EXTERNAL_SORT_DISABLE_MANAGED_OPTION),
+      new OptionDefinition(ExecConstants.ENABLE_QUERY_PROFILE_VALIDATOR),
+      new OptionDefinition(ExecConstants.QUERY_PROFILE_DEBUG_VALIDATOR),
+      new OptionDefinition(ExecConstants.USE_DYNAMIC_UDFS),
+      new OptionDefinition(ExecConstants.QUERY_TRANSIENT_STATE_UPDATE),
+      new OptionDefinition(ExecConstants.PERSISTENT_TABLE_UMASK_VALIDATOR),
+      new OptionDefinition(ExecConstants.CPU_LOAD_AVERAGE),
+      new OptionDefinition(ExecConstants.ENABLE_VECTOR_VALIDATOR),
+      new OptionDefinition(ExecConstants.ENABLE_ITERATOR_VALIDATOR)
     };
-    final Map<String, OptionValidator> tmp = new HashMap<>();
-    for (final OptionValidator validator : validators) {
-      tmp.put(validator.getOptionName(), validator);
+
+    final CaseInsensitiveMap<OptionDefinition> map = CaseInsensitiveMap.newHashMap();
+
+    for (final OptionDefinition definition: definitions) {
+      map.put(definition.getValidator().getOptionName(), definition);
     }
+
     if (AssertionUtil.isAssertionsEnabled()) {
-      tmp.put(ExecConstants.DRILLBIT_CONTROL_INJECTIONS, ExecConstants.DRILLBIT_CONTROLS_VALIDATOR);
+      map.put(ExecConstants.DRILLBIT_CONTROL_INJECTIONS, new OptionDefinition(ExecConstants.DRILLBIT_CONTROLS_VALIDATOR));
     }
-    VALIDATORS = CaseInsensitiveMap.newImmutableMap(tmp);
+
+    return map;
   }
 
   private final PersistentStoreConfig<OptionValue> config;
@@ -223,37 +227,23 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
    * NOTE: CRUD operations must use lowercase keys.
    */
   private PersistentStore<OptionValue> options;
+  private CaseInsensitiveMap<OptionDefinition> definitions;
+  private CaseInsensitiveMap<OptionValue> defaults;
 
-  public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider) {
-    this(lpPersistence,provider,null);
+  public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider,
+                             final DrillConfig bootConfig) {
+    this(lpPersistence, provider, bootConfig, SystemOptionManager.createDefaultOptionDefinitions());
   }
 
-  public SystemOptionManager(LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider, final DrillConfig bootConfig) {
+  public SystemOptionManager(final LogicalPlanPersistence lpPersistence, final PersistentStoreProvider provider,
+                             final DrillConfig bootConfig, final CaseInsensitiveMap<OptionDefinition> definitions) {
     this.provider = provider;
     this.config = PersistentStoreConfig.newJacksonBuilder(lpPersistence.getMapper(), OptionValue.class)
           .name("sys.options")
           .build();
     this.bootConfig = bootConfig;
-    populateValidators();
-    populateDefaultValues();
-
-  }
-
-  /**
-   * Gets the {@link OptionValidator} associated with the name.
-   *
-   * @param name name of the option
-   * @return the associated validator
-   * @throws UserException - if the validator is not found
-   */
-  public OptionValidator getValidator(final String name) {
-    final OptionValidator validator = VALIDATORS.get(name);
-    if (validator == null) {
-      throw UserException.validationError()
-              .message(String.format("The option '%s' does not exist.", name.toLowerCase()))
-              .build(logger);
-    }
-    return validator;
+    this.definitions = definitions;
+    this.defaults = populateDefaultValues(definitions, bootConfig);
   }
 
   /**
@@ -267,12 +257,13 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
     // if necessary, deprecate and replace options from persistent store
     for (final Entry<String, OptionValue> option : Lists.newArrayList(options.getAll())) {
       final String name = option.getKey();
-      final OptionValidator validator = VALIDATORS.get(name);
-      if (validator == null) {
+      final OptionDefinition definition = definitions.get(name);
+      if (definition == null) {
         // deprecated option, delete.
         options.delete(name);
         logger.warn("Deleting deprecated option `{}`", name);
       } else {
+        OptionValidator validator = definition.getValidator();
         final String canonicalName = validator.getOptionName().toLowerCase();
         if (!name.equals(canonicalName)) {
           // for backwards compatibility <= 1.1, rename to lower case.
@@ -283,6 +274,7 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
         }
       }
     }
+
     return this;
   }
 
@@ -290,8 +282,8 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
   public Iterator<OptionValue> iterator() {
     final Map<String, OptionValue> buildList = CaseInsensitiveMap.newHashMap();
     // populate the default options
-    for (final Map.Entry<String, OptionValidator> entry : VALIDATORS.entrySet()) {
-      buildList.put(entry.getKey(), entry.getValue().getDefault());
+    for (final Map.Entry<String, OptionValue> entry : defaults.entrySet()) {
+      buildList.put(entry.getKey(), entry.getValue());
     }
     // override if changed
     for (final Map.Entry<String, OptionValue> entry : Lists.newArrayList(options.getAll())) {
@@ -304,39 +296,48 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
   public OptionValue getOption(final String name) {
     // check local space (persistent store)
     final OptionValue value = options.get(name.toLowerCase());
+
     if (value != null) {
       return value;
     }
 
-    // If option is not set return the default set in the validator.
-    final OptionValidator validator = getValidator(name);
-    return validator.getDefault();
+    // otherwise, return default set in the validator.
+    return defaults.get(name);
+  }
+
+  @Override
+  public OptionValue getDefault(String optionName) {
+    return defaults.get(optionName);
   }
 
   @Override
-  public void setOption(final OptionValue value) {
-    checkArgument(value.type == OptionType.SYSTEM, "OptionType must be SYSTEM.");
+  protected void setLocalOptionHelper(final OptionValue value) {
     final String name = value.name.toLowerCase();
-    final OptionValidator validator = getValidator(name);
+    final OptionDefinition definition = getOptionDefinition(name);
+    final OptionValidator validator = definition.getValidator();
+
+    validator.validate(value, definition.getMetaData(), this); // validate the option
 
-    validator.validate(value, this); // validate the option
-    if (options.get(name) == null && value.equals(validator.getDefault())) {
+    if (options.get(name) == null && value.equals(getDefault(name))) {
       return; // if the option is not overridden, ignore setting option to default
     }
+
     options.put(name, value);
   }
 
   @Override
-  public void deleteOption(final String name, OptionType type) {
-    checkArgument(type == OptionType.SYSTEM, "OptionType must be SYSTEM.");
+  protected OptionValue.OptionScope getScope() {
+    return OptionValue.OptionScope.SYSTEM;
+  }
 
-    getValidator(name); // ensure option exists
+  @Override
+  public void deleteLocalOption(final String name) {
+    getOptionDefinition(name); // ensure option exists
     options.delete(name.toLowerCase());
   }
 
   @Override
-  public void deleteAllOptions(OptionType type) {
-    checkArgument(type == OptionType.SYSTEM, "OptionType must be SYSTEM.");
+  public void deleteAllLocalOptions() {
     final Set<String> names = Sets.newHashSet();
     for (final Map.Entry<String, OptionValue> entry : Lists.newArrayList(options.getAll())) {
       names.add(entry.getKey());
@@ -346,17 +347,63 @@ public class SystemOptionManager extends BaseOptionManager implements OptionMana
     }
   }
 
-  public void populateDefaultValues() {
-
+  public static CaseInsensitiveMap<OptionValue> populateDefaultValues(Map<String, OptionDefinition> definitions, DrillConfig bootConfig) {
     // populate the options from the config
-    final Map<String, OptionValidator> tmp = new HashMap<>();
-    for (OptionValidator validator: VALIDATORS.values()) {
-      try {
-         validator.loadDefault(bootConfig);
-      } catch (ConfigException.Missing e) {
-        throw new IllegalStateException("Config Option is missing"+ validator.getOptionName());
+    final Map<String, OptionValue> defaults = new HashMap<>();
+
+    for (final Map.Entry<String, OptionDefinition> entry : definitions.entrySet()) {
+      final OptionDefinition definition = entry.getValue();
+      final OptionMetaData metaData = definition.getMetaData();
+      final OptionValue.AccessibleScopes type = metaData.getType();
+      final OptionValidator validator = definition.getValidator();
+      final String name = validator.getOptionName();
+      final String configName = validator.getConfigProperty();
+      final OptionValue.Kind kind = validator.getKind();
+      OptionValue optionValue;
+
+      switch (kind) {
+        case BOOLEAN:
+          optionValue = OptionValue.create(type, name,
+            bootConfig.getBoolean(configName), OptionValue.OptionScope.BOOT);
+          break;
+        case LONG:
+          optionValue = OptionValue.create(type, name,
+            bootConfig.getLong(configName), OptionValue.OptionScope.BOOT);
+          break;
+        case STRING:
+          optionValue = OptionValue.create(type, name,
+            bootConfig.getString(configName), OptionValue.OptionScope.BOOT);
+          break;
+        case DOUBLE:
+          optionValue = OptionValue.create(type, name,
+            bootConfig.getDouble(configName), OptionValue.OptionScope.BOOT);
+          break;
+        default:
+          throw new UnsupportedOperationException();
       }
+
+      defaults.put(name, optionValue);
+    }
+
+    return CaseInsensitiveMap.newImmutableMap(defaults);
+  }
+
+  /**
+   * Gets the {@link OptionDefinition} associated with the name.
+   *
+   * @param name name of the option
+   * @return the associated option definition
+   * @throws UserException - if the definition is not found
+   */
+  @Override
+  public OptionDefinition getOptionDefinition(String name) {
+    final OptionDefinition definition = definitions.get(name);
+    if (definition == null) {
+      throw UserException.validationError()
+        .message(String.format("The option '%s' does not exist.", name.toLowerCase()))
+        .build(logger);
     }
+    return definition;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
index ecaa7b7..b9861a6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java
@@ -20,17 +20,13 @@ package org.apache.drill.exec.server.options;
 import java.util.Set;
 
 import com.google.common.collect.Sets;
-import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.server.options.OptionValue.Kind;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
-import org.apache.drill.exec.server.options.OptionValue.OptionScope;
-import static com.google.common.base.Preconditions.checkArgument;
 
 public class TypeValidators {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeValidators.class);
   public static class PositiveLongValidator extends LongValidator {
-    private final long max;
+    protected final long max;
 
     public PositiveLongValidator(String name, long max) {
       super(name);
@@ -38,8 +34,8 @@ public class TypeValidators {
     }
 
     @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
-      super.validate(v, manager);
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+      super.validate(v, metaData, manager);
       if (v.num_val > max || v.num_val < 1) {
         throw UserException.validationError()
             .message(String.format("Option %s must be between %d and %d.", getOptionName(), 1, max))
@@ -55,8 +51,8 @@ public class TypeValidators {
     }
 
     @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
-      super.validate(v, manager);
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+      super.validate(v, metaData, manager);
       if (!isPowerOfTwo(v.num_val)) {
         throw UserException.validationError()
             .message(String.format("Option %s must be a power of two.", getOptionName()))
@@ -70,8 +66,8 @@ public class TypeValidators {
   }
 
   public static class RangeDoubleValidator extends DoubleValidator {
-    private final double min;
-    private final double max;
+    protected final double min;
+    protected final double max;
 
     public RangeDoubleValidator(String name, double min, double max) {
       super(name);
@@ -80,8 +76,8 @@ public class TypeValidators {
     }
 
     @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
-      super.validate(v, manager);
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+      super.validate(v, metaData, manager);
       if (v.float_val > max || v.float_val < min) {
         throw UserException.validationError()
             .message(String.format("Option %s must be between %f and %f.", getOptionName(), min, max))
@@ -93,14 +89,14 @@ public class TypeValidators {
   public static class MinRangeDoubleValidator extends RangeDoubleValidator {
     private final String maxValidatorName;
 
-    public MinRangeDoubleValidator(String name, double min, double max, double def, String maxValidatorName) {
+    public MinRangeDoubleValidator(String name, double min, double max, String maxValidatorName) {
       super(name, min, max);
       this.maxValidatorName = maxValidatorName;
     }
 
     @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
-      super.validate(v, manager);
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+      super.validate(v, metaData, manager);
       OptionValue maxValue = manager.getOption(maxValidatorName);
       if (v.float_val > maxValue.float_val) {
         throw UserException.validationError()
@@ -114,14 +110,14 @@ public class TypeValidators {
   public static class MaxRangeDoubleValidator extends RangeDoubleValidator {
     private final String minValidatorName;
 
-    public MaxRangeDoubleValidator(String name, double min, double max, double def, String minValidatorName) {
+    public MaxRangeDoubleValidator(String name, double min, double max, String minValidatorName) {
       super(name, min, max);
       this.minValidatorName = minValidatorName;
     }
 
     @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
-      super.validate(v, manager);
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+      super.validate(v, metaData, manager);
       OptionValue minValue = manager.getOption(minValidatorName);
       if (v.float_val < minValue.float_val) {
         throw UserException.validationError()
@@ -134,60 +130,25 @@ public class TypeValidators {
 
   public static class BooleanValidator extends TypeValidator {
     public BooleanValidator(String name) {
-      this(name, false);
-    }
-
-    public BooleanValidator(String name, boolean isAdminOption) {
-      super(name, Kind.BOOLEAN, isAdminOption);
-    }
-
-    public void loadDefault(DrillConfig bootConfig){
-      OptionValue value = OptionValue.createBoolean(OptionType.SYSTEM, getOptionName(), bootConfig.getBoolean(getConfigProperty()), OptionScope.BOOT);
-      setDefaultValue(value);
+      super(name, Kind.BOOLEAN);
     }
   }
 
   public static class StringValidator extends TypeValidator {
     public StringValidator(String name) {
-      this(name, false);
-    }
-    public StringValidator(String name, boolean isAdminOption) {
-      super(name, Kind.STRING, isAdminOption);
-    }
-
-    public void loadDefault(DrillConfig bootConfig){
-      OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionScope.BOOT);
-      setDefaultValue(value);
+      super(name, Kind.STRING);
     }
   }
 
   public static class LongValidator extends TypeValidator {
     public LongValidator(String name) {
-      this(name, false);
-    }
-
-    public LongValidator(String name, boolean isAdminOption) {
-      super(name, Kind.LONG, isAdminOption);
-    }
-
-    public void loadDefault(DrillConfig bootConfig){
-      OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT);
-      setDefaultValue(value);
+      super(name, Kind.LONG);
     }
   }
 
   public static class DoubleValidator extends TypeValidator {
     public DoubleValidator(String name) {
-      this(name,false);
-    }
-
-    public DoubleValidator(String name, boolean isAdminOption) {
-      super(name, Kind.DOUBLE, isAdminOption);
-    }
-
-    public void loadDefault(DrillConfig bootConfig){
-      OptionValue value = OptionValue.createDouble(OptionType.SYSTEM, getOptionName(),bootConfig.getDouble(getConfigProperty()), OptionScope.BOOT);
-      setDefaultValue(value);
+      super(name, Kind.DOUBLE);
     }
   }
 
@@ -202,8 +163,8 @@ public class TypeValidators {
     }
 
     @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
-      super.validate(v, manager);
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+      super.validate(v, metaData, manager);
       if (v.num_val > max || v.num_val < min) {
         throw UserException.validationError()
             .message(String.format("Option %s must be between %d and %d.", getOptionName(), min, max))
@@ -226,8 +187,8 @@ public class TypeValidators {
     }
 
     @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
-      super.validate(v, manager);
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
+      super.validate(v, metaData, manager);
       if (!valuesSet.contains(v.string_val.toLowerCase())) {
         throw UserException.validationError()
             .message(String.format("Option %s must be one of: %s.", getOptionName(), valuesSet))
@@ -242,18 +203,8 @@ public class TypeValidators {
    * the available number of processors and cpu load average
    */
   public static class MaxWidthValidator extends LongValidator{
-
     public MaxWidthValidator(String name) {
-      this(name, false);
-    }
-
-    public MaxWidthValidator(String name, boolean isAdminOption) {
-      super(name, isAdminOption);
-    }
-
-    public void loadDefault(DrillConfig bootConfig) {
-      OptionValue value = OptionValue.createLong(OptionType.SYSTEM, getOptionName(), bootConfig.getLong(getConfigProperty()), OptionScope.BOOT);
-      setDefaultValue(value);
+      super(name);
     }
 
     public int computeMaxWidth(double cpuLoadAverage, long maxWidth) {
@@ -272,48 +223,25 @@ public class TypeValidators {
 
   public static abstract class TypeValidator extends OptionValidator {
     private final Kind kind;
-    private OptionValue defaultValue = null;
 
     public TypeValidator(final String name, final Kind kind) {
-      this(name, kind, false);
-    }
-
-    public TypeValidator(final String name, final Kind kind, final boolean isAdminOption) {
-      super(name, isAdminOption);
+      super(name);
       this.kind = kind;
     }
 
     @Override
-    public OptionValue getDefault() {
-      return defaultValue;
-    }
-
-    @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
       if (v.kind != kind) {
         throw UserException.validationError()
             .message(String.format("Option %s must be of type %s but you tried to set to %s.", getOptionName(),
               kind.name(), v.kind.name()))
             .build(logger);
       }
-      if (isAdminOption() && v.type != OptionType.SYSTEM) {
-        throw UserException.validationError()
-            .message("Admin related settings can only be set at SYSTEM level scope. Given scope '%s'.", v.type)
-            .build(logger);
-      }
     }
 
     @Override
     public Kind getKind() {
       return kind;
     }
-
-    protected void setDefaultValue(OptionValue defaultValue) {
-      this.defaultValue = defaultValue;
-    }
-
-    public String getConfigProperty() {
-      return OPTION_DEFAULTS_ROOT + getOptionName();
-    }
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
index 1caf10b..4042a81 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/StatusResources.java
@@ -33,10 +33,14 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.SecurityContext;
 import javax.xml.bind.annotation.XmlRootElement;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.drill.exec.server.options.OptionList;
+import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.OptionValue.Kind;
+import org.apache.drill.exec.server.options.SystemOptionManager;
 import org.apache.drill.exec.server.rest.DrillRestServer.UserAuthEnabled;
 import org.apache.drill.exec.server.rest.auth.DrillUserPrincipal;
 import org.apache.drill.exec.work.WorkManager;
@@ -50,79 +54,128 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 public class StatusResources {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StatusResources.class);
 
+  public static final String REST_API_SUFFIX = ".json";
+  public static final String PATH_STATUS_JSON = "/status" + REST_API_SUFFIX;
+  public static final String PATH_STATUS = "/status";
+  public static final String PATH_OPTIONS_JSON = "/options" + REST_API_SUFFIX;
+  public static final String PATH_INTERNAL_OPTIONS_JSON = "/internal_options" + REST_API_SUFFIX;
+  public static final String PATH_OPTIONS = "/options";
+  public static final String PATH_INTERNAL_OPTIONS = "/internal_options";
+
   @Inject UserAuthEnabled authEnabled;
   @Inject WorkManager work;
   @Inject SecurityContext sc;
 
   @GET
-  @Path("/status.json")
+  @Path(StatusResources.PATH_STATUS_JSON)
   @Produces(MediaType.APPLICATION_JSON)
   public Pair<String, String> getStatusJSON() {
     return new ImmutablePair<>("status", "Running!");
   }
 
   @GET
-  @Path("/status")
+  @Path(StatusResources.PATH_STATUS)
   @Produces(MediaType.TEXT_HTML)
   public Viewable getStatus() {
     return ViewableWithPermissions.create(authEnabled.get(), "/rest/status.ftl", sc, getStatusJSON());
   }
 
-  @GET
-  @Path("/options.json")
-  @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
-  @Produces(MediaType.APPLICATION_JSON)
-  public List<OptionWrapper> getSystemOptionsJSON() {
+  private List<OptionWrapper> getSystemOptionsJSONHelper(boolean internal)
+  {
     List<OptionWrapper> options = new LinkedList<>();
-    for (OptionValue option : work.getContext().getOptionManager()) {
-      options.add(new OptionWrapper(option.name, option.getValue(), option.type, option.kind));
+    OptionManager optionManager = work.getContext().getOptionManager();
+    OptionList optionList = internal ? optionManager.getInternalOptionList(): optionManager.getPublicOptionList();
+
+    for (OptionValue option : optionList) {
+      options.add(new OptionWrapper(option.name, option.getValue(), option.accessibleScopes, option.kind, option.scope));
     }
+
     return options;
   }
 
   @GET
-  @Path("/options")
+  @Path(StatusResources.PATH_OPTIONS_JSON)
+  @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
+  @Produces(MediaType.APPLICATION_JSON)
+  public List<OptionWrapper> getSystemPublicOptionsJSON() {
+    return getSystemOptionsJSONHelper(false);
+  }
+
+  @GET
+  @Path(StatusResources.PATH_INTERNAL_OPTIONS_JSON)
+  @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
+  @Produces(MediaType.APPLICATION_JSON)
+  public List<OptionWrapper> getSystemInternalOptionsJSON() {
+    return getSystemOptionsJSONHelper(true);
+  }
+
+  private Viewable getSystemOptionsHelper(boolean internal) {
+    return ViewableWithPermissions.create(authEnabled.get(),
+      "/rest/options.ftl",
+      sc,
+      getSystemOptionsJSONHelper(internal));
+  }
+
+  @GET
+  @Path(StatusResources.PATH_OPTIONS)
+  @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
+  @Produces(MediaType.TEXT_HTML)
+  public Viewable getSystemPublicOptions() {
+    return getSystemOptionsHelper(false);
+  }
+
+  @GET
+  @Path(StatusResources.PATH_INTERNAL_OPTIONS)
   @RolesAllowed(DrillUserPrincipal.AUTHENTICATED_ROLE)
   @Produces(MediaType.TEXT_HTML)
-  public Viewable getSystemOptions() {
-    return ViewableWithPermissions.create(authEnabled.get(), "/rest/options.ftl", sc, getSystemOptionsJSON());
+  public Viewable getSystemInternalOptions() {
+    return getSystemOptionsHelper(true);
   }
 
   @POST
-  @Path("/option/{optionName}")
+  @Path("option/{optionName}")
   @RolesAllowed(DrillUserPrincipal.ADMIN_ROLE)
   @Consumes("application/x-www-form-urlencoded")
   @Produces(MediaType.TEXT_HTML)
-  public Viewable updateSystemOption(@FormParam("name") String name, @FormParam("value") String value,
+  public Viewable updateSystemOption(@FormParam("name") String name,
+                                   @FormParam("value") String value,
                                    @FormParam("kind") String kind) {
+    SystemOptionManager optionManager = work.getContext()
+      .getOptionManager();
+
     try {
-      work.getContext()
-        .getOptionManager()
-        .setOption(OptionValue.createOption(
-          OptionValue.Kind.valueOf(kind),
-          OptionValue.OptionType.SYSTEM,
-          name,
-          value, OptionValue.OptionScope.SYSTEM));
+      optionManager.setLocalOption(OptionValue.Kind.valueOf(kind), name, value);
     } catch (Exception e) {
       logger.debug("Could not update.", e);
     }
-    return getSystemOptions();
+
+    if (optionManager.getOptionDefinition(name).getMetaData().isInternal()) {
+      return getSystemInternalOptions();
+    } else {
+      return getSystemPublicOptions();
+    }
   }
 
   @XmlRootElement
-  public class OptionWrapper {
+  public static class OptionWrapper {
 
     private String name;
     private Object value;
-    private OptionValue.OptionType type;
+    private OptionValue.AccessibleScopes accessibleScopes;
     private String kind;
+    private String optionScope;
 
     @JsonCreator
-    public OptionWrapper(String name, Object value, OptionValue.OptionType type, Kind kind) {
+    public OptionWrapper(@JsonProperty("name") String name,
+                         @JsonProperty("value") Object value,
+                         @JsonProperty("accessibleScopes") OptionValue.AccessibleScopes type,
+                         @JsonProperty("kind") Kind kind,
+                         @JsonProperty("optionScope") OptionValue.OptionScope scope) {
       this.name = name;
       this.value = value;
-      this.type = type;
+      this.accessibleScopes = type;
       this.kind = kind.name();
+      this.optionScope = scope.name();
     }
 
     public String getName() {
@@ -138,13 +191,21 @@ public class StatusResources {
       return value;
     }
 
-    public OptionValue.OptionType getType() {
-      return type;
+    public OptionValue.AccessibleScopes getAccessibleScopes() {
+      return accessibleScopes;
     }
 
     public String getKind() {
       return kind;
     }
-  }
 
+    public String getOptionScope() {
+      return optionScope;
+    }
+
+    @Override
+    public String toString() {
+      return "OptionWrapper{" + "name='" + name + '\'' + ", value=" + value + ", accessibleScopes=" + accessibleScopes + ", kind='" + kind + '\'' + ", scope='" + optionScope + '\'' +'}';
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
index 8818349..0718fdd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebServer.java
@@ -71,6 +71,7 @@ import javax.servlet.DispatcherType;
 import javax.servlet.http.HttpSession;
 import javax.servlet.http.HttpSessionEvent;
 import javax.servlet.http.HttpSessionListener;
+import java.io.IOException;
 import java.math.BigInteger;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
@@ -91,16 +92,18 @@ import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.AUTHENTI
 public class WebServer implements AutoCloseable {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WebServer.class);
 
+  private static final int PORT_HUNT_TRIES = 10;
+
   private final DrillConfig config;
 
   private final MetricRegistry metrics;
 
   private final WorkManager workManager;
 
-  private final Server embeddedJetty;
-
   private final BootStrapContext context;
 
+  private Server embeddedJetty;
+
   /**
    * Create Jetty based web server.
    *
@@ -112,12 +115,6 @@ public class WebServer implements AutoCloseable {
     this.config = context.getConfig();
     this.metrics = context.getMetrics();
     this.workManager = workManager;
-
-    if (config.getBoolean(ExecConstants.HTTP_ENABLE)) {
-      embeddedJetty = new Server();
-    } else {
-      embeddedJetty = null;
-    }
   }
 
   private static final String BASE_STATIC_PATH = "/rest/static/";
@@ -139,9 +136,10 @@ public class WebServer implements AutoCloseable {
    * @throws Exception
    */
   public void start() throws Exception {
-    if (embeddedJetty == null) {
+    if (!config.getBoolean(ExecConstants.HTTP_ENABLE)) {
       return;
     }
+
     final boolean authEnabled = config.getBoolean(ExecConstants.USER_AUTHENTICATION_ENABLED);
     if (authEnabled && !context.getAuthProvider().containsFactory(PlainFactory.SIMPLE_NAME)) {
       logger.warn("Not starting web server. Currently Drill supports web authentication only through " +
@@ -149,19 +147,6 @@ public class WebServer implements AutoCloseable {
       return;
     }
 
-    final ServerConnector serverConnector;
-    if (config.getBoolean(ExecConstants.HTTP_ENABLE_SSL)) {
-      try {
-        serverConnector = createHttpsConnector();
-      }
-      catch(DrillException e){
-        throw new DrillbitStartupException(e.getMessage(), e);
-      }
-    } else {
-      serverConnector = createHttpConnector();
-    }
-    embeddedJetty.addConnector(serverConnector);
-
     // Add resources
     final ErrorHandler errorHandler = new ErrorHandler();
     errorHandler.setShowStacks(true);
@@ -216,8 +201,36 @@ public class WebServer implements AutoCloseable {
       }
     }
 
-    embeddedJetty.setHandler(servletContextHandler);
-    embeddedJetty.start();
+    int port = config.getInt(ExecConstants.HTTP_PORT);
+    boolean portHunt = config.getBoolean(ExecConstants.HTTP_PORT_HUNT);
+    int retry = 0;
+
+    System.out.println("Port hunting " + portHunt);
+
+    for (; retry < PORT_HUNT_TRIES; retry++) {
+      embeddedJetty = new Server();
+      embeddedJetty.setHandler(servletContextHandler);
+      embeddedJetty.addConnector(createConnector(port));
+
+      try {
+        embeddedJetty.start();
+      } catch (Exception e) {
+        if (portHunt) {
+          int nextPort = port++;
+          logger.info("Failed to start on port {}, trying port {}", port, nextPort);
+          logger.error("Attempted port failure:", e);
+          embeddedJetty.stop();
+        } else {
+          throw e;
+        }
+      }
+
+      break;
+    }
+
+    if (retry == PORT_HUNT_TRIES) {
+      throw new IOException("Failed to find a port");
+    }
   }
 
   /**
@@ -275,6 +288,22 @@ public class WebServer implements AutoCloseable {
     return security;
   }
 
+  private ServerConnector createConnector(int port) throws Exception {
+    final ServerConnector serverConnector;
+    if (config.getBoolean(ExecConstants.HTTP_ENABLE_SSL)) {
+      try {
+        serverConnector = createHttpsConnector(port);
+      }
+      catch(DrillException e){
+        throw new DrillbitStartupException(e.getMessage(), e);
+      }
+    } else {
+      serverConnector = createHttpConnector(port);
+    }
+
+    return serverConnector;
+  }
+
   /**
    * Create an HTTPS connector for given jetty server instance. If the admin has specified keystore/truststore settings
    * they will be used else a self-signed certificate is generated and used.
@@ -282,7 +311,7 @@ public class WebServer implements AutoCloseable {
    * @return Initialized {@link ServerConnector} for HTTPS connectios.
    * @throws Exception
    */
-  private ServerConnector createHttpsConnector() throws Exception {
+  private ServerConnector createHttpsConnector(int port) throws Exception {
     logger.info("Setting up HTTPS connector for web server");
 
     final SslContextFactory sslContextFactory = new SslContextFactory();
@@ -359,7 +388,7 @@ public class WebServer implements AutoCloseable {
     final ServerConnector sslConnector = new ServerConnector(embeddedJetty,
         new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString()),
         new HttpConnectionFactory(httpsConfig));
-    sslConnector.setPort(config.getInt(ExecConstants.HTTP_PORT));
+    sslConnector.setPort(port);
 
     return sslConnector;
   }
@@ -369,11 +398,11 @@ public class WebServer implements AutoCloseable {
    * @return Initialized {@link ServerConnector} instance for HTTP connections.
    * @throws Exception
    */
-  private ServerConnector createHttpConnector() throws Exception {
+  private ServerConnector createHttpConnector(int port) throws Exception {
     logger.info("Setting up HTTP connector for web server");
     final HttpConfiguration httpConfig = new HttpConfiguration();
     final ServerConnector httpConnector = new ServerConnector(embeddedJetty, new HttpConnectionFactory(httpConfig));
-    httpConnector.setPort(config.getInt(ExecConstants.HTTP_PORT));
+    httpConnector.setPort(port);
 
     return httpConnector;
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
index fdd5092..e329de3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java
@@ -30,10 +30,8 @@ import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.OptionValue.Kind;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
 import org.apache.drill.exec.server.options.OptionValue.OptionScope;
 
-
 /*
  * Extends the original Option iterator. The idea is to hide the implementation details and present the
  * user with the rows which have values set at the top level of hierarchy and exclude the values set
@@ -59,9 +57,15 @@ public class ExtendedOptionIterator implements Iterator<Object> {
   private final OptionManager fragmentOptions;
   private final Iterator<OptionValue> mergedOptions;
 
-  public ExtendedOptionIterator(FragmentContext context) {
+  public ExtendedOptionIterator(FragmentContext context, boolean internal) {
     fragmentOptions = context.getOptions();
-    mergedOptions = sortOptions(fragmentOptions.iterator());
+    final Iterator<OptionValue> optionList;
+
+    if (!internal) {
+      mergedOptions = sortOptions(fragmentOptions.getPublicOptionList().iterator());
+    } else {
+      mergedOptions = sortOptions(fragmentOptions.getInternalOptionList().iterator());
+    }
   }
    /* *
     * Remove the redundant rows for the same option based on the scope and return
@@ -118,7 +122,7 @@ public class ExtendedOptionIterator implements Iterator<Object> {
       put(Kind.BOOLEAN,"BIT");
 
     }};
-    return new ExtendedOptionValueWrapper(value.name, typeMapping.get(value.kind), value.type,value.getValue().toString(), value.scope);
+    return new ExtendedOptionValueWrapper(value.name, typeMapping.get(value.kind), value.accessibleScopes,value.getValue().toString(), value.scope);
   }
 
   public enum Status {
@@ -132,15 +136,15 @@ public class ExtendedOptionIterator implements Iterator<Object> {
 
     public final String name;
     public final String kind;
-    public final OptionType type;
+    public final OptionValue.AccessibleScopes accessibleScopes;
     public final String val;
     public final OptionScope optionScope;
 
 
-    public ExtendedOptionValueWrapper(final String name, final String kind, final OptionType type, final String value, final OptionScope scope) {
+    public ExtendedOptionValueWrapper(final String name, final String kind, final OptionValue.AccessibleScopes type, final String value, final OptionScope scope) {
       this.name = name;
       this.kind = kind;
-      this.type = type;
+      this.accessibleScopes = type;
       this.val = value;
       this.optionScope = scope;
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
index 424ff95..48d8c10 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java
@@ -25,35 +25,41 @@ import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.server.options.DrillConfigIterator;
-import org.apache.drill.exec.server.options.FragmentOptionManager;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.OptionValue.Kind;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
-import org.apache.drill.exec.server.options.SystemOptionManager;
+import org.apache.drill.exec.server.options.OptionValue.OptionScope;
+import org.apache.drill.exec.server.options.OptionValue.AccessibleScopes;
 
 public class OptionIterator implements Iterator<Object> {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OptionIterator.class);
 
   enum Mode {
-    BOOT, SYS_SESS, BOTH
+    BOOT,
+    // These represent System/Session options that are public. These are available for everyone to change and are considered safe to change.
+    SYS_SESS_PUBLIC,
+    // These represent System/Session options that are internal. These are segregated from the public options and are not visible in the same tables and rest endpoints as public
+    // options. These are technically available for anyone to change, but users are discouraged from doing so unless they absolutely know what they are doing. The general purpose
+    // for internal options is to enable support to easily debug.
+    SYS_SESS_INTERNAL
   };
 
   private final OptionManager fragmentOptions;
   private final Iterator<OptionValue> mergedOptions;
-  private FragmentContext context;
 
   public OptionIterator(FragmentContext context, Mode mode){
     final DrillConfigIterator configOptions = new DrillConfigIterator(context.getConfig());
     fragmentOptions = context.getOptions();
-    this.context = context;
     final Iterator<OptionValue> optionList;
     switch(mode){
     case BOOT:
       optionList = configOptions.iterator();
       break;
-    case SYS_SESS:
-      optionList = fragmentOptions.iterator();
+    case SYS_SESS_PUBLIC:
+      optionList = fragmentOptions.getPublicOptionList().iterator();
+      break;
+    case SYS_SESS_INTERNAL:
+      optionList = fragmentOptions.getInternalOptionList().iterator();
       break;
     default:
       optionList = Iterators.concat(configOptions.iterator(), fragmentOptions.iterator());
@@ -74,23 +80,22 @@ public class OptionIterator implements Iterator<Object> {
   public OptionValueWrapper next() {
     final OptionValue value = mergedOptions.next();
     final Status status;
-    final FragmentOptionManager fragmentOptionManager = (FragmentOptionManager) fragmentOptions;
-    final SystemOptionManager systemOptionManager = (SystemOptionManager) fragmentOptionManager.getFallback();
-    if (value.type == OptionType.BOOT) {
+
+    if (value.accessibleScopes == AccessibleScopes.BOOT) {
       status = Status.BOOT;
     } else {
-      final OptionValue def = systemOptionManager.getValidator(value.name).getDefault();
+      final OptionValue def = fragmentOptions.getDefault(value.name);
       if (value.equalsIgnoreType(def)) {
         status = Status.DEFAULT;
         } else {
         status = Status.CHANGED;
         }
       }
-    return new OptionValueWrapper(value.name, value.kind, value.type, value.num_val, value.string_val,
+    return new OptionValueWrapper(value.name, value.kind, value.accessibleScopes, value.scope, value.num_val, value.string_val,
         value.bool_val, value.float_val, status);
   }
 
-  public static enum Status {
+  public enum Status {
     BOOT, DEFAULT, CHANGED
   }
 
@@ -101,19 +106,21 @@ public class OptionIterator implements Iterator<Object> {
 
     public final String name;
     public final Kind kind;
-    public final OptionType type;
+    public final AccessibleScopes accessibleScopes;
+    public final OptionScope optionScope;
     public final Status status;
     public final Long num_val;
     public final String string_val;
     public final Boolean bool_val;
     public final Double float_val;
 
-    public OptionValueWrapper(final String name, final Kind kind, final OptionType type, final Long num_val,
-        final String string_val, final Boolean bool_val, final Double float_val,
-        final Status status) {
+    public OptionValueWrapper(final String name, final Kind kind, final OptionValue.AccessibleScopes type, final OptionScope scope, final Long num_val,
+                              final String string_val, final Boolean bool_val, final Double float_val,
+                              final Status status) {
       this.name = name;
       this.kind = kind;
-      this.type = type;
+      this.accessibleScopes = type;
+      this.optionScope = scope;
       this.num_val = num_val;
       this.string_val = string_val;
       this.bool_val = bool_val;

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
index fa6981b..ee798b9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/SystemTable.java
@@ -21,7 +21,6 @@ import java.util.Iterator;
 
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.store.sys.OptionIterator.OptionValueWrapper;
-import org.apache.drill.exec.store.sys.ExtendedOptionIterator.ExtendedOptionValueWrapper;
 
 /**
  * An enumeration of all tables in Drill's system ("sys") schema.
@@ -32,18 +31,31 @@ import org.apache.drill.exec.store.sys.ExtendedOptionIterator.ExtendedOptionValu
  * </p>
  */
 public enum SystemTable {
-
   OPTION("options", false, OptionValueWrapper.class) {
     @Override
     public Iterator<Object> getIterator(final FragmentContext context) {
-      return new OptionIterator(context, OptionIterator.Mode.SYS_SESS);
+      return new OptionIterator(context, OptionIterator.Mode.SYS_SESS_PUBLIC);
+    }
+  },
+
+  OPTION_VAL("options_val", false, ExtendedOptionIterator.ExtendedOptionValueWrapper.class) {
+    @Override
+    public Iterator<Object> getIterator(final FragmentContext context) {
+      return new ExtendedOptionIterator(context, false);
+    }
+  },
+
+  INTERNAL_OPTIONS("internal_options", false, OptionValueWrapper.class) {
+    @Override
+    public Iterator<Object> getIterator(final FragmentContext context) {
+      return new OptionIterator(context, OptionIterator.Mode.SYS_SESS_INTERNAL);
     }
   },
 
-  OPTION2("options2", false,ExtendedOptionIterator.ExtendedOptionValueWrapper.class ) {
+  INTERNAL_OPTIONS_VAL("internal_options_val", false, ExtendedOptionIterator.ExtendedOptionValueWrapper.class) {
     @Override
     public Iterator<Object> getIterator(final FragmentContext context) {
-      return new ExtendedOptionIterator(context);
+      return new ExtendedOptionIterator(context, true);
     }
   },
 

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
index a046c77..82646ea 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/testing/ExecutionControls.java
@@ -22,15 +22,14 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.UserException;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 import org.apache.drill.exec.server.options.OptionManager;
+import org.apache.drill.exec.server.options.OptionMetaData;
 import org.apache.drill.exec.server.options.OptionSet;
 import org.apache.drill.exec.server.options.OptionValue;
-import org.apache.drill.exec.server.options.OptionValue.OptionType;
 import org.apache.drill.exec.server.options.TypeValidators.TypeValidator;
 import org.apache.drill.exec.testing.InjectionSite.InjectionSiteKeyDeserializer;
 import org.apache.drill.exec.util.AssertionUtil;
@@ -96,12 +95,7 @@ public final class ExecutionControls {
     }
 
     @Override
-    public void validate(final OptionValue v, final OptionSet manager) {
-      if (v.type != OptionType.SESSION) {
-        throw UserException.validationError()
-            .message("Controls can be set only at SESSION level.")
-            .build(logger);
-      }
+    public void validate(final OptionValue v, final OptionMetaData metaData, final OptionSet manager) {
       final String jsonString = v.string_val;
       try {
         validateControlsString(jsonString);
@@ -111,11 +105,6 @@ public final class ExecutionControls {
             .build(logger);
       }
     }
-
-    public void loadDefault(DrillConfig bootConfig){
-      OptionValue value = OptionValue.createString(OptionType.SYSTEM, getOptionName(), bootConfig.getString(getConfigProperty()), OptionValue.OptionScope.BOOT);
-      setDefaultValue(value);
-    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index 26f0722..4e7bbdb 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -122,6 +122,7 @@ drill.exec: {
   http: {
     enabled: true,
     ssl_enabled: false,
+    porthunt: false,
     port: 8047,
     max_profiles: 100,
     session_max_idle_secs: 3600, # Default value 1hr

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index f7f41fc..e30c5e9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -460,6 +460,18 @@ public class BaseTestQuery extends ExecTest {
     return file.getPath();
   }
 
+  protected static void setSessionOption(final String option, final boolean value) {
+    setSessionOption(option, Boolean.toString(value));
+  }
+
+  protected static void setSessionOption(final String option, final long value) {
+    setSessionOption(option, Long.toString(value));
+  }
+
+  protected static void setSessionOption(final String option, final double value) {
+    setSessionOption(option, Double.toString(value));
+  }
+
   protected static void setSessionOption(final String option, final String value) {
     try {
       runSQL(String.format("alter session set `%s` = %s", option, value));

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
index 8e767e4..26a0537 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/QueryTestUtil.java
@@ -41,7 +41,7 @@ import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.server.RemoteServiceSet;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
-import org.apache.drill.exec.server.options.OptionValue.OptionScope;
+import org.apache.drill.exec.server.options.SystemOptionManager;
 import org.apache.drill.exec.util.VectorUtil;
 
 /**
@@ -188,11 +188,9 @@ public class QueryTestUtil {
       final Drillbit drillbit, final ClassTransformer.ScalarReplacementOption srOption) {
     // set the system option
     final DrillbitContext drillbitContext = drillbit.getContext();
-    final OptionManager optionManager = drillbitContext.getOptionManager();
+    final SystemOptionManager optionManager = drillbitContext.getOptionManager();
     final OptionValue originalOptionValue = optionManager.getOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION);
-    final OptionValue newOptionValue = OptionValue.createString(OptionValue.OptionType.SYSTEM,
-        ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase(), OptionScope.SYSTEM);
-    optionManager.setOption(newOptionValue);
+    optionManager.setLocalOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption.name().toLowerCase());
 
     // flush the code cache
     drillbitContext.getCompiler().flushCache();
@@ -209,12 +207,12 @@ public class QueryTestUtil {
    * @param drillbit the drillbit
    * @param srOption the scalar replacement option value to use
    */
-  public static void restoreScalarReplacementOption(final Drillbit drillbit, final OptionValue srOption) {
+  public static void restoreScalarReplacementOption(final Drillbit drillbit, final String srOption) {
     @SuppressWarnings("resource")
     final DrillbitContext drillbitContext = drillbit.getContext();
     @SuppressWarnings("resource")
     final OptionManager optionManager = drillbitContext.getOptionManager();
-    optionManager.setOption(srOption);
+    optionManager.setLocalOption(ClassTransformer.SCALAR_REPLACEMENT_OPTION, srOption);
 
     // flush the code cache
     drillbitContext.getCompiler().flushCache();

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
index b9dcacd..cf3e9b9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestStarQueries.java
@@ -244,7 +244,7 @@ public class TestStarQueries extends BaseTestQuery{
 
   @Test  // select star for a SchemaTable.
   public void testSelStarSubQSchemaTable() throws Exception {
-    test("select name, kind, type from (select * from sys.options);");
+    test("select name, kind, accessibleScopes from (select * from sys.options);");
   }
 
   @Test  // Join a select star of SchemaTable, with a select star of Schema-less table.

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java
index e63f085..2b3b475 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/DrillSystemTestBase.java
@@ -40,7 +40,7 @@ public class DrillSystemTestBase extends TestWithZookeeper {
     try {
       ImmutableList.Builder<Drillbit> servers = ImmutableList.builder();
       for (int i = 0; i < numServers; i++) {
-        servers.add(Drillbit.start(getConfig()));
+        servers.add(Drillbit.start(zkHelper.getConfig()));
       }
       this.servers = servers.build();
     } catch (DrillbitStartupException e) {

http://git-wip-us.apache.org/repos/asf/drill/blob/6adeb986/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java
index f65c638..842a4e1 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestWithZookeeper.java
@@ -18,26 +18,24 @@
 package org.apache.drill.exec;
 
 import org.apache.drill.common.config.DrillConfig;
+import org.junit.After;
 import org.junit.AfterClass;
+import org.junit.Before;
 import org.junit.BeforeClass;
 
 public class TestWithZookeeper extends ExecTest {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestWithZookeeper.class);
 
-  private static ZookeeperHelper zkHelper;
+  protected ZookeeperHelper zkHelper;
 
-  @BeforeClass
-  public static void setUp() throws Exception {
+  @Before
+  public void setUp() throws Exception {
     zkHelper = new ZookeeperHelper();
     zkHelper.startZookeeper(1);
   }
 
-  @AfterClass
-  public static void tearDown() throws Exception {
+  @After
+  public void tearDown() throws Exception {
     zkHelper.stopZookeeper();
   }
-
-  public static DrillConfig getConfig() {
-    return zkHelper.getConfig();
-  }
 }


Mime
View raw message