drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [12/13] drill git commit: DRILL-3742: Classpath scanning and build improvement
Date Mon, 26 Oct 2015 17:42:59 GMT
DRILL-3742: Classpath scanning and build improvement

Makes the classpath scanning a build time class discovery
Makes the fmpp generation incremental
Removes some slowness in DrillBit closing
Reduces the build time by 30%

This closes #148


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dbcab0fe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dbcab0fe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dbcab0fe

Branch: refs/heads/master
Commit: dbcab0fee8ff416724b770ca08018cf0f17d0035
Parents: 6576123
Author: Julien Le Dem <julien@ledem.net>
Authored: Thu Oct 15 10:59:18 2015 -0700
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Mon Oct 26 08:40:17 2015 -0700

----------------------------------------------------------------------
 common/pom.xml                                  |  19 +
 .../org/apache/drill/common/JSONOptions.java    |   7 +-
 .../drill/common/config/CommonConstants.java    |  16 -
 .../apache/drill/common/config/DrillConfig.java |  64 +--
 .../common/config/LogicalPlanPersistence.java   |  65 +++
 .../common/logical/FormatPluginConfigBase.java  |  25 +-
 .../drill/common/logical/LogicalPlan.java       |   9 +-
 .../common/logical/StoragePluginConfigBase.java |  24 +-
 .../logical/data/LogicalOperatorBase.java       |  18 +-
 .../drill/common/scanner/BuildTimeScan.java     | 143 ++++++
 .../drill/common/scanner/ClassPathScanner.java  | 446 +++++++++++++++++++
 .../drill/common/scanner/RunTimeScan.java       |  78 ++++
 .../persistence/AnnotatedClassDescriptor.java   |  93 ++++
 .../persistence/AnnotationDescriptor.java       | 156 +++++++
 .../persistence/AttributeDescriptor.java        |  55 +++
 .../persistence/ChildClassDescriptor.java       |  58 +++
 .../scanner/persistence/FieldDescriptor.java    | 112 +++++
 .../persistence/ParentClassDescriptor.java      |  61 +++
 .../common/scanner/persistence/ScanResult.java  | 213 +++++++++
 .../scanner/persistence/TypeDescriptor.java     | 115 +++++
 .../apache/drill/common/util/PathScanner.java   | 176 --------
 common/src/main/resources/drill-module.conf     |  15 +-
 .../drill/storage/CheckStorageConfig.java       |  14 +-
 common/src/test/resources/drill-module.conf     |   2 +-
 .../src/main/resources/drill-module.conf        |   9 +-
 .../drill/hbase/TestHBaseTableProvider.java     |   7 +-
 contrib/storage-hive/core/pom.xml               |  33 +-
 .../exec/expr/fn/HiveFunctionRegistry.java      |   9 +-
 .../core/src/main/resources/drill-module.conf   |  14 +-
 .../core/src/test/resources/drill-module.conf   |   2 +-
 .../src/main/resources/drill-module.conf        |   2 +-
 .../src/main/resources/drill-module.conf        |   3 +
 .../src/main/resources/drill-module.conf        |   6 +-
 .../exec/store/mongo/MongoTestConstants.java    |   1 +
 exec/java-exec/pom.xml                          | 154 ++++---
 .../org/apache/drill/exec/ExecConstants.java    |   3 -
 .../exec/client/PrintingResultsListener.java    |   2 +
 .../apache/drill/exec/compile/CodeCompiler.java |   5 +-
 .../drill/exec/dotdrill/DotDrillFile.java       |   5 +-
 .../drill/exec/expr/DrillFuncHolderExpr.java    |   2 +-
 .../exec/expr/annotations/FunctionTemplate.java |   2 +-
 .../drill/exec/expr/fn/DrillAggFuncHolder.java  |  60 ++-
 .../exec/expr/fn/DrillBooleanOPHolder.java      |  15 +-
 .../expr/fn/DrillComplexWriterFuncHolder.java   |  12 +-
 .../exec/expr/fn/DrillDecimalAddFuncHolder.java |   7 +-
 .../exec/expr/fn/DrillDecimalAggFuncHolder.java |   7 +-
 .../expr/fn/DrillDecimalCastFuncHolder.java     |  13 +-
 .../expr/fn/DrillDecimalDivScaleFuncHolder.java |  11 +-
 .../expr/fn/DrillDecimalMaxScaleFuncHolder.java |  12 +-
 .../expr/fn/DrillDecimalModScaleFuncHolder.java |   7 +-
 .../expr/fn/DrillDecimalSetScaleFuncHolder.java |  11 +-
 .../expr/fn/DrillDecimalSumAggFuncHolder.java   |   7 +-
 .../expr/fn/DrillDecimalSumScaleFuncHolder.java |  12 +-
 .../fn/DrillDecimalZeroScaleFuncHolder.java     |   7 +-
 .../drill/exec/expr/fn/DrillFuncHolder.java     |  78 ++--
 .../exec/expr/fn/DrillFunctionRegistry.java     |  33 +-
 .../exec/expr/fn/DrillSimpleFuncHolder.java     |  55 +--
 .../drill/exec/expr/fn/FunctionAttributes.java  |  98 ++++
 .../drill/exec/expr/fn/FunctionConverter.java   | 320 +++++--------
 .../expr/fn/FunctionImplementationRegistry.java |  22 +-
 .../drill/exec/expr/fn/FunctionInitializer.java | 152 +++++++
 .../apache/drill/exec/ops/FragmentContext.java  |   2 +-
 .../org/apache/drill/exec/ops/QueryContext.java |   5 +
 .../physical/base/PhysicalOperatorUtil.java     |  23 +-
 .../physical/impl/OperatorCreatorRegistry.java  |  16 +-
 .../drill/exec/planner/PhysicalPlanReader.java  |  24 +-
 .../exec/planner/logical/StoragePlugins.java    |  13 +-
 .../planner/sql/handlers/DefaultSqlHandler.java |   8 +-
 .../planner/sql/handlers/ExplainHandler.java    |   4 +-
 .../org/apache/drill/exec/rpc/BasicServer.java  |  16 +-
 .../exec/rpc/data/DataConnectionCreator.java    |   1 +
 .../apache/drill/exec/rpc/user/UserServer.java  |   6 +-
 .../user/security/UserAuthenticatorFactory.java |  17 +-
 .../drill/exec/server/BootStrapContext.java     |  17 +-
 .../org/apache/drill/exec/server/Drillbit.java  |  22 +-
 .../drill/exec/server/DrillbitContext.java      |  50 ++-
 .../server/options/SystemOptionManager.java     |   6 +-
 .../drill/exec/server/rest/DrillRestServer.java |   4 +-
 .../drill/exec/service/ServiceEngine.java       |  38 +-
 .../drill/exec/store/StoragePluginRegistry.java |  31 +-
 .../drill/exec/store/dfs/FileSystemPlugin.java  |  21 +-
 .../drill/exec/store/dfs/FormatCreator.java     |  19 +-
 .../exec/store/dfs/WorkspaceSchemaFactory.java  |  31 +-
 .../store/ischema/InfoSchemaStoragePlugin.java  |   2 +-
 .../drill/exec/store/sys/SystemTablePlugin.java |   2 +-
 .../org/apache/drill/exec/work/WorkManager.java |   9 +-
 .../apache/drill/exec/work/foreman/Foreman.java |   8 +-
 .../src/main/resources/drill-module.conf        |  31 +-
 .../java/org/apache/drill/BaseTestQuery.java    |   7 +-
 .../java/org/apache/drill/PlanningBase.java     |  18 +-
 .../common/scanner/TestClassPathScanner.java    | 175 ++++++++
 .../java/org/apache/drill/exec/RunRootExec.java |   3 +-
 .../apache/drill/exec/TestOpSerialization.java  |  10 +-
 .../drill/exec/cache/TestWriteToDisk.java       |   1 +
 .../apache/drill/exec/client/DumpCatTest.java   |  19 +-
 .../exec/compile/CodeCompilerTestFactory.java   |  38 ++
 .../compile/bytecode/ReplaceMethodInvoke.java   |   4 +-
 .../drill/exec/fn/impl/GeneratorFunctions.java  | 115 -----
 .../drill/exec/fn/impl/TestMathFunctions.java   |  16 +-
 .../exec/fn/impl/TestNewMathFunctions.java      |  17 +-
 .../exec/fn/impl/TestRepeatedFunction.java      |  16 +-
 .../fn/impl/testing/GeneratorFunctions.java     | 115 +++++
 .../drill/exec/memory/TestAllocators.java       |   8 +-
 .../drill/exec/opt/BasicOptimizerTest.java      |   8 +-
 .../physical/config/TestParsePhysicalPlan.java  |  16 +-
 .../exec/physical/impl/TestCastFunctions.java   |  59 +--
 .../physical/impl/TestComparisonFunctions.java  |  16 +-
 .../impl/TestImplicitCastFunctions.java         |  16 +-
 .../exec/physical/impl/TestOptiqPlans.java      |   4 +-
 .../exec/physical/impl/TestSimpleFunctions.java |  29 +-
 .../exec/physical/impl/TestStringFunctions.java |  16 +-
 .../drill/exec/physical/impl/agg/TestAgg.java   |  16 +-
 .../physical/impl/common/TestHashTable.java     |  23 +-
 .../physical/impl/filter/TestSimpleFilter.java  |  22 +-
 .../exec/physical/impl/join/TestHashJoin.java   |  22 +-
 .../exec/physical/impl/join/TestMergeJoin.java  |  41 +-
 .../physical/impl/limit/TestSimpleLimit.java    |  26 +-
 .../TestOrderedPartitionExchange.java           |   4 +-
 .../impl/project/TestSimpleProjection.java      |  16 +-
 .../exec/physical/impl/sort/TestSimpleSort.java |  22 +-
 .../impl/trace/TestTraceMultiRecordBatch.java   |  16 +-
 .../impl/trace/TestTraceOutputDump.java         |  16 +-
 .../physical/impl/union/TestSimpleUnion.java    |  16 +-
 .../planner/PhysicalPlanReaderTestFactory.java  |  63 +++
 .../drill/exec/pop/TestFragmentChecker.java     |   3 +-
 .../apache/drill/exec/pop/TestFragmenter.java   |   6 +-
 .../drill/exec/pop/TestInjectionValue.java      |   3 +-
 .../security/TestCustomUserAuthenticator.java   |   9 +-
 .../security/UserAuthenticatorTestImpl.java     |  77 ----
 .../testing/UserAuthenticatorTestImpl.java      |  80 ++++
 .../apache/drill/exec/server/TestBitRpc.java    |  20 +-
 .../exec/server/TestOptionsAuthEnabled.java     |  22 +-
 .../src/test/resources/drill-external-sort.conf |   9 +-
 .../src/test/resources/drill-module.conf        |  15 +-
 .../src/test/resources/drill-oom-xsort.conf     |  11 +-
 .../test/resources/drill-spool-test-module.conf |  17 +-
 exec/jdbc-all/example-conf/drill-module.conf    |  17 +-
 exec/jdbc-all/pom.xml                           |  11 -
 .../org/apache/drill/jdbc/test/JdbcAssert.java  |   5 +-
 pom.xml                                         |  12 +-
 src/main/resources/checkstyle-config.xml        |   1 +
 tools/fmpp/pom.xml                              |  82 ++++
 .../org/apache/drill/fmpp/mojo/FMPPMojo.java    | 227 ++++++++++
 tools/pom.xml                                   |  34 ++
 144 files changed, 3794 insertions(+), 1504 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index c3c0eb5..fa19e21 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -111,6 +111,25 @@
   <build>
     <plugins>
       <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <version>1.2.1</version>
+        <executions>
+          <execution>
+            <phase>process-classes</phase>
+            <goals>
+              <goal>java</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <mainClass>org.apache.drill.common.scanner.BuildTimeScan</mainClass>
+          <arguments>
+            <argument>${project.build.outputDirectory}</argument>
+          </arguments>
+        </configuration>
+      </plugin>
+      <plugin>
         <artifactId>maven-surefire-plugin</artifactId>
         <version>2.15</version>
         <configuration>

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/JSONOptions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/JSONOptions.java b/common/src/main/java/org/apache/drill/common/JSONOptions.java
index 9e0514c..e432135 100644
--- a/common/src/main/java/org/apache/drill/common/JSONOptions.java
+++ b/common/src/main/java/org/apache/drill/common/JSONOptions.java
@@ -24,6 +24,7 @@ import java.lang.reflect.Type;
 import org.apache.drill.common.JSONOptions.De;
 import org.apache.drill.common.JSONOptions.Se;
 import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.config.LogicalPlanPersistence;
 import org.apache.drill.common.exceptions.LogicalPlanParsingException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -66,7 +67,7 @@ public class JSONOptions {
   }
 
   @SuppressWarnings("unchecked")
-  public <T> T getWith(DrillConfig config, Class<T> c) {
+  public <T> T getWith(LogicalPlanPersistence lpPersistance, Class<T> c) {
     try {
       if (opaque != null) {
         final Class<?> opaqueClass = opaque.getClass();
@@ -88,7 +89,7 @@ public class JSONOptions {
       }
 
       //logger.debug("Read tree {}", root);
-      return config.getMapper().treeToValue(root, c);
+      return lpPersistance.getMapper().treeToValue(root, c);
     } catch (JsonProcessingException e) {
       throw new LogicalPlanParsingException(String.format("Failure while trying to convert late bound " +
         "json options to type of %s. Reference was originally located at line %d, column %d.",
@@ -96,7 +97,7 @@ public class JSONOptions {
     }
   }
 
-  public <T> T getListWith(DrillConfig config, TypeReference<T> t) throws IOException {
+  public <T> T getListWith(LogicalPlanPersistence config, TypeReference<T> t) throws IOException {
     return getListWith(config.getMapper(), t);
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java b/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
index 78c4356..d3d2b72 100644
--- a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
+++ b/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
@@ -28,20 +28,4 @@ public interface CommonConstants {
   /** Override configuration file name.  (Classpath resource pathname.) */
   String CONFIG_OVERRIDE_RESOURCE_PATHNAME = "drill-override.conf";
 
-  /** Configuration pathname to list of names of packages to scan for logical
-   *  operator subclasses. */
-  String LOGICAL_OPERATOR_SCAN_PACKAGES = "drill.logical.operator.packages";
-
-  /** Configuration pathname to list of names of packages to scan for physical
-   *  operator subclasses. */
-  String PHYSICAL_OPERATOR_SCAN_PACKAGES = "drill.physical.operator.packages";
-
-  /** Configuration pathname to list of names of packages to scan for function
-   *  subclasses. */
-  String LOGICAL_FUNCTION_SCAN_PACKAGES = "drill.logical.function.packages";
-
-  /** Configuration pathname to list of packages to scan for storage plugin
-   *  configuration subclasses. */
-  String STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES = "drill.logical.storage.packages";
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index d747b67..507c4f0 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -22,24 +22,16 @@ import java.lang.management.RuntimeMXBean;
 import java.net.URL;
 import java.util.Collection;
 import java.util.List;
+import java.util.Map.Entry;
 import java.util.Properties;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.drill.common.exceptions.DrillConfigurationException;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.FormatPluginConfigBase;
-import org.apache.drill.common.logical.StoragePluginConfigBase;
-import org.apache.drill.common.logical.data.LogicalOperatorBase;
-import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.common.scanner.ClassPathScanner;
 import org.reflections.util.ClasspathHelper;
 
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser.Feature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
+import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableList;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
@@ -48,7 +40,6 @@ import com.typesafe.config.ConfigRenderOptions;
 public final class DrillConfig extends NestedConfig{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConfig.class);
 
-  private final ObjectMapper mapper;
   private final ImmutableList<String> startupArguments;
 
   public static final boolean ON_OSX = System.getProperty("os.name").contains("OS X");
@@ -62,27 +53,10 @@ public final class DrillConfig extends NestedConfig{
     logger.debug("Setting up DrillConfig object.");
     logger.trace("Given Config object is:\n{}",
                  config.root().render(ConfigRenderOptions.defaults()));
-    mapper = new ObjectMapper();
-
-    if (enableServerConfigs) {
-      SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
-        .addDeserializer(LogicalExpression.class, new LogicalExpression.De(this))
-        .addDeserializer(SchemaPath.class, new SchemaPath.De());
-
-      mapper.registerModule(deserModule);
-      mapper.enable(SerializationFeature.INDENT_OUTPUT);
-      mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
-      mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
-      mapper.configure(Feature.ALLOW_COMMENTS, true);
-      mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this));
-      mapper.registerSubtypes(StoragePluginConfigBase.getSubTypes(this));
-      mapper.registerSubtypes(FormatPluginConfigBase.getSubTypes(this));
-    }
-
     RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
     this.startupArguments = ImmutableList.copyOf(bean.getInputArguments());
     logger.debug("DrillConfig object initialized.");
-  };
+  }
 
   public List<String> getStartupArguments() {
     return startupArguments;
@@ -168,6 +142,8 @@ public final class DrillConfig extends NestedConfig{
   private static DrillConfig create(String overrideFileResourcePathname,
                                     final Properties overriderProps,
                                     final boolean enableServerConfigs) {
+    final StringBuilder logString = new StringBuilder();
+    final Stopwatch watch = new Stopwatch().start();
     overrideFileResourcePathname =
         overrideFileResourcePathname == null
             ? CommonConstants.CONFIG_OVERRIDE_RESOURCE_PATHNAME
@@ -180,7 +156,7 @@ public final class DrillConfig extends NestedConfig{
       final URL url =
           classLoader.getResource(CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME);
       if (null != url) {
-        logger.info("Loading base configuration file at {}.", url);
+        logString.append("Base Configuration:\n\t- ").append(url).append("\n");
         fallback =
             ConfigFactory.load(classLoader,
                                CommonConstants.CONFIG_DEFAULT_RESOURCE_PATHNAME);
@@ -189,14 +165,13 @@ public final class DrillConfig extends NestedConfig{
     }
 
     // 2. Load per-module configuration files.
-    final Collection<URL> urls = PathScanner.getConfigURLs();
-    final String lineBrokenList =
-        urls.size() == 0 ? "" : "\n\t- " + Joiner.on("\n\t- ").join(urls);
-    logger.info("Loading {} module configuration files at: {}.",
-                urls.size(), lineBrokenList);
+    final Collection<URL> urls = ClassPathScanner.getConfigURLs();
+    logString.append("\nIntermediate Configuration and Plugin files, in order of precedence:\n");
     for (URL url : urls) {
+      logString.append("\t- ").append(url).append("\n");
       fallback = ConfigFactory.parseURL(url).withFallback(fallback);
     }
+    logString.append("\n");
 
     // 3. Load any specified overrides configuration file along with any
     //    overrides from JVM system properties (e.g., {-Dname=value").
@@ -205,19 +180,26 @@ public final class DrillConfig extends NestedConfig{
     final URL overrideFileUrl =
         Thread.currentThread().getContextClassLoader().getResource(overrideFileResourcePathname);
     if (null != overrideFileUrl ) {
-      logger.info("Loading override config. file at {}.", overrideFileUrl);
+      logString.append("Override File: ").append(overrideFileUrl).append("\n");
     }
     Config effectiveConfig =
         ConfigFactory.load(overrideFileResourcePathname).withFallback(fallback);
 
     // 4. Apply any overriding properties.
     if (overriderProps != null) {
-      logger.info("Loading override Properties parameter {}.", overriderProps);
+      logString.append("Overridden Properties:\n");
+      for(Entry<Object, Object> entry : overriderProps.entrySet()){
+        logString.append("\t-").append(entry.getKey()).append(" = ").append(entry.getValue()).append("\n");
+      }
+      logString.append("\n");
       effectiveConfig =
           ConfigFactory.parseProperties(overriderProps).withFallback(effectiveConfig);
     }
 
     // 5. Create DrillConfig object from Config object.
+    logger.info("Configuration and plugin file(s) identified in {}ms.\n{}",
+        watch.elapsed(TimeUnit.MILLISECONDS),
+        logString);
     return new DrillConfig(effectiveConfig.resolve(), enableServerConfigs);
   }
 
@@ -256,10 +238,6 @@ public final class DrillConfig extends NestedConfig{
     }
   }
 
-  public ObjectMapper getMapper() {
-    return mapper;
-  }
-
   @Override
   public String toString() {
     return this.root().render();

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java b/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
new file mode 100644
index 0000000..cd7a8d0
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/config/LogicalPlanPersistence.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.config;
+
+import java.util.Set;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.FormatPluginConfigBase;
+import org.apache.drill.common.logical.StoragePluginConfigBase;
+import org.apache.drill.common.logical.data.LogicalOperatorBase;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser.Feature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+
+public class LogicalPlanPersistence {
+  private ObjectMapper mapper;
+
+  public ObjectMapper getMapper() {
+    return mapper;
+  }
+
+  public LogicalPlanPersistence(DrillConfig conf, ScanResult scanResult) {
+    mapper = new ObjectMapper();
+
+    SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
+        .addDeserializer(LogicalExpression.class, new LogicalExpression.De(conf))
+        .addDeserializer(SchemaPath.class, new SchemaPath.De());
+
+    mapper.registerModule(deserModule);
+    mapper.enable(SerializationFeature.INDENT_OUTPUT);
+    mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
+    mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
+    mapper.configure(Feature.ALLOW_COMMENTS, true);
+    registerSubtypes(LogicalOperatorBase.getSubTypes(scanResult));
+    registerSubtypes(StoragePluginConfigBase.getSubTypes(scanResult));
+    registerSubtypes(FormatPluginConfigBase.getSubTypes(scanResult));
+  }
+
+  private <T> void registerSubtypes(Set<Class<? extends T>> types) {
+    for (Class<? extends T> type : types) {
+      mapper.registerSubtypes(type);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
index 02f3206..50de963 100644
--- a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
+++ b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
@@ -17,13 +17,9 @@
  */
 package org.apache.drill.common.logical;
 
-import java.util.List;
+import java.util.Set;
 
-import org.apache.drill.common.config.CommonConstants;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.PathScanner;
-
-import com.google.common.base.Joiner;
+import org.apache.drill.common.scanner.persistence.ScanResult;
 
 
 public abstract class FormatPluginConfigBase implements FormatPluginConfig{
@@ -31,21 +27,14 @@ public abstract class FormatPluginConfigBase implements FormatPluginConfig{
 
 
   /**
-   * Use reflection to scan for implementations of {@see FormatPlugin}.
+   * scan for implementations of {@see FormatPlugin}.
    *
-   * @param config - Drill configuration object, used to find the packages to scan
+   * @param classpathScan - Drill configuration object, used to find the packages to scan
    * @return - list of classes that implement the interface.
    */
-  public synchronized static Class<?>[] getSubTypes(final DrillConfig config) {
-    final List<String> packages =
-        config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES);
-    final Class<?>[] pluginClasses =
-        PathScanner.scanForImplementationsArr(FormatPluginConfig.class, packages);
-    final String lineBrokenList =
-        pluginClasses.length == 0
-        ? "" : "\n\t- " + Joiner.on("\n\t- ").join(pluginClasses);
-    logger.debug("Found {} format plugin configuration classes: {}.",
-                 pluginClasses.length, lineBrokenList);
+  public static Set<Class<? extends FormatPluginConfig>> getSubTypes(final ScanResult classpathScan) {
+    final Set<Class<? extends FormatPluginConfig>> pluginClasses = classpathScan.getImplementations(FormatPluginConfig.class);
+    logger.debug("Found {} format plugin configuration classes: {}.", pluginClasses.size(), pluginClasses);
     return pluginClasses;
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
index 313ed40..8a4a5ab 100644
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
+++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
@@ -23,6 +23,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.config.LogicalPlanPersistence;
 import org.apache.drill.common.graph.Graph;
 import org.apache.drill.common.graph.GraphAlgos;
 import org.apache.drill.common.logical.data.LogicalOperator;
@@ -80,11 +81,11 @@ public class LogicalPlan {
     return storageEngineMap;
   }
 
-  public String toJsonString(DrillConfig config) throws JsonProcessingException {
+  public String toJsonString(LogicalPlanPersistence config) throws JsonProcessingException {
     return config.getMapper().writeValueAsString(this);
   }
 
-  public String toJsonStringSafe(DrillConfig config){
+  public String toJsonStringSafe(LogicalPlanPersistence config){
     try{
       return toJsonString(config);
     }catch(JsonProcessingException e){
@@ -94,7 +95,7 @@ public class LogicalPlan {
   }
 
   /** Parses a logical plan. */
-  public static LogicalPlan parse(DrillConfig config, String planString) {
+  public static LogicalPlan parse(LogicalPlanPersistence config, String planString) {
     ObjectMapper mapper = config.getMapper();
     try {
       LogicalPlan plan = mapper.readValue(planString, LogicalPlan.class);
@@ -106,7 +107,7 @@ public class LogicalPlan {
   }
 
   /** Converts a logical plan to a string. (Opposite of {@link #parse}.) */
-  public String unparse(DrillConfig config) {
+  public String unparse(LogicalPlanPersistence config) {
     try {
       return config.getMapper().writeValueAsString(this);
     } catch (JsonProcessingException e) {

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
index 3ac858b..6075483 100644
--- a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
+++ b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
@@ -17,30 +17,18 @@
  */
 package org.apache.drill.common.logical;
 
-import java.util.List;
+import java.util.Set;
 
-import org.apache.drill.common.config.CommonConstants;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.PathScanner;
-
-import com.google.common.base.Joiner;
+import org.apache.drill.common.scanner.persistence.ScanResult;
 
 
 public abstract class StoragePluginConfigBase extends StoragePluginConfig {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginConfigBase.class);
 
-
-  public synchronized static Class<?>[] getSubTypes(final DrillConfig config) {
-    final List<String> packages =
-        config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES);
-    final Class<?>[] pluginClasses =
-        PathScanner.scanForImplementationsArr(StoragePluginConfig.class, packages);
-    final String lineBrokenList =
-        pluginClasses.length == 0
-        ? "" : "\n\t- " + Joiner.on("\n\t- ").join(pluginClasses);
-    logger.debug("Found {} storage plugin configuration classes: {}.",
-                 pluginClasses.length, lineBrokenList);
-    return pluginClasses;
+  public static Set<Class<? extends StoragePluginConfig>> getSubTypes(final ScanResult classpathScan) {
+    final Set<Class<? extends StoragePluginConfig>> packages = classpathScan.getImplementations(StoragePluginConfig.class);
+    logger.debug("Found {} logical operator classes: {}.", packages.size(), packages);
+    return packages;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java b/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
index e97ef19..a213c8d 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/LogicalOperatorBase.java
@@ -20,17 +20,15 @@ package org.apache.drill.common.logical.data;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
-import org.apache.drill.common.config.CommonConstants;
-import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.graph.GraphVisitor;
 import org.apache.drill.common.logical.ValidationError;
-import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.common.scanner.persistence.ScanResult;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Joiner;
 
 
 public abstract class LogicalOperatorBase implements LogicalOperator{
@@ -86,15 +84,9 @@ public abstract class LogicalOperatorBase implements LogicalOperator{
     this.memo = memo;
   }
 
-  public synchronized static Class<?>[] getSubTypes(final DrillConfig config) {
-    final List<String> packages =
-        config.getStringList(CommonConstants.LOGICAL_OPERATOR_SCAN_PACKAGES);
-    final Class<?>[] ops =
-        PathScanner.scanForImplementationsArr(LogicalOperator.class, packages);
-    final String lineBrokenList =
-        ops.length == 0 ? "" : "\n\t- " + Joiner.on("\n\t- ").join(ops);
-    logger.debug("Found {} logical operator classes: {}.", ops.length,
-                 lineBrokenList);
+  public static Set<Class<? extends LogicalOperator>> getSubTypes(final ScanResult classpathScan) {
+    final Set<Class<? extends LogicalOperator>> ops = classpathScan.getImplementations(LogicalOperator.class);
+    logger.debug("Found {} logical operator classes: {}.", ops.size(), ops);
     return ops;
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
new file mode 100644
index 0000000..ca4a37d
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/BuildTimeScan.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner;
+
+import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;
+import static java.lang.String.format;
+import static java.util.Arrays.asList;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+
+/**
+ * main class to integrate classpath scanning in the build.
+ * @see BuildTimeScan#main(String[])
+ */
+public class BuildTimeScan {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BuildTimeScan.class);
+  private static final String REGISTRY_FILE = "META-INF/drill-module-scan/registry.json";
+
+  private static final ObjectMapper mapper = new ObjectMapper().enable(INDENT_OUTPUT);
+  private static final ObjectReader reader = mapper.reader(ScanResult.class);
+  private static final ObjectWriter writer = mapper.writerWithType(ScanResult.class);
+
+  /**
+   * @return paths that have the prescanned registry file in them
+   */
+  static Set<URL> getPrescannedPaths() {
+    return ClassPathScanner.forResource(REGISTRY_FILE, true);
+  }
+
+  /**
+   * loads all the prescanned resources from classpath
+   * @return the result of the previous scan
+   */
+  static ScanResult load() {
+    return loadExcept(null);
+  }
+
+  /**
+   * loads all the prescanned resources from classpath
+   * (except for the target location in case it already exists)
+   * @return the result of the previous scan
+   */
+  private static ScanResult loadExcept(URL ignored) {
+    Set<URL> preScanned = ClassPathScanner.forResource(REGISTRY_FILE, false);
+    ScanResult result = null;
+    for (URL u : preScanned) {
+      if (ignored!= null && u.toString().startsWith(ignored.toString())) {
+        continue;
+      }
+      try (InputStream reflections = u.openStream()) {
+        ScanResult ref = reader.readValue(reflections);
+        if (result == null) {
+          result = ref;
+        } else {
+          result = result.merge(ref);
+        }
+      } catch (IOException e) {
+        throw new DrillRuntimeException("can't read function registry at " + u, e);
+      }
+    }
+    if (result != null) {
+      logger.info(format("Loaded prescanned packages %s from locations %s", result.getScannedPackages(), preScanned));
+      return result;
+    } else {
+      return ClassPathScanner.emptyResult();
+    }
+  }
+
+  private static void save(ScanResult scanResult, File file) {
+    try {
+      writer.writeValue(file, scanResult);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * to generate the prescan file during build
+   * @param args the root path for the classes where {@link BuildTimeScan#REGISTRY_FILE} is generated
+   * @throws Exception
+   */
+  public static void main(String[] args) throws Exception {
+    if (args.length != 1) {
+      throw new IllegalArgumentException("Usage: java {cp} " + ClassPathScanner.class.getName() + " path/to/scan");
+    }
+    String basePath = args[0];
+    System.out.println("Scanning: " + basePath);
+    File registryFile = new File(basePath, REGISTRY_FILE);
+    File dir = registryFile.getParentFile();
+    if ((!dir.exists() && !dir.mkdirs()) || !dir.isDirectory()) {
+      throw new IllegalArgumentException("could not create dir " + dir.getAbsolutePath());
+    }
+    DrillConfig config = DrillConfig.create();
+    // normalize
+    if (!basePath.endsWith("/")) {
+      basePath = basePath + "/";
+    }
+    URL url = new URL("file:" + basePath);
+    Set<URL> markedPaths = ClassPathScanner.getMarkedPaths();
+    if (!markedPaths.contains(url)) {
+      throw new IllegalArgumentException(url + " not in " + markedPaths);
+    }
+    List<String> packagePrefixes = ClassPathScanner.getPackagePrefixes(config);
+    List<String> baseClasses = ClassPathScanner.getScannedBaseClasses(config);
+    List<String> scannedAnnotations = ClassPathScanner.getScannedAnnotations(config);
+    ScanResult preScanned = loadExcept(url);
+    ScanResult scan = ClassPathScanner.scan(
+        asList(url),
+        packagePrefixes,
+        baseClasses,
+        scannedAnnotations,
+        preScanned);
+    save(scan, registryFile);
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java b/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java
new file mode 100644
index 0000000..92b2027
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/ClassPathScanner.java
@@ -0,0 +1,446 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner;
+
+import static java.lang.String.format;
+import static java.util.Collections.unmodifiableList;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.drill.common.config.CommonConstants;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.scanner.persistence.AnnotationDescriptor;
+import org.apache.drill.common.scanner.persistence.AttributeDescriptor;
+import org.apache.drill.common.scanner.persistence.ChildClassDescriptor;
+import org.apache.drill.common.scanner.persistence.FieldDescriptor;
+import org.apache.drill.common.scanner.persistence.AnnotatedClassDescriptor;
+import org.apache.drill.common.scanner.persistence.ParentClassDescriptor;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+import org.reflections.Reflections;
+import org.reflections.adapters.JavassistAdapter;
+import org.reflections.scanners.AbstractScanner;
+import org.reflections.util.ConfigurationBuilder;
+import org.reflections.util.FilterBuilder;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+
+import javassist.bytecode.AccessFlag;
+import javassist.bytecode.AnnotationsAttribute;
+import javassist.bytecode.ClassFile;
+import javassist.bytecode.FieldInfo;
+import javassist.bytecode.annotation.AnnotationMemberValue;
+import javassist.bytecode.annotation.ArrayMemberValue;
+import javassist.bytecode.annotation.BooleanMemberValue;
+import javassist.bytecode.annotation.ByteMemberValue;
+import javassist.bytecode.annotation.CharMemberValue;
+import javassist.bytecode.annotation.ClassMemberValue;
+import javassist.bytecode.annotation.DoubleMemberValue;
+import javassist.bytecode.annotation.EnumMemberValue;
+import javassist.bytecode.annotation.FloatMemberValue;
+import javassist.bytecode.annotation.IntegerMemberValue;
+import javassist.bytecode.annotation.LongMemberValue;
+import javassist.bytecode.annotation.MemberValue;
+import javassist.bytecode.annotation.MemberValueVisitor;
+import javassist.bytecode.annotation.ShortMemberValue;
+import javassist.bytecode.annotation.StringMemberValue;
+
+/**
+ * Classpath scanning utility.
+ * The classpath should be scanned once at startup from a DrillConfig instance. {@see ClassPathScanner#fromPrescan(DrillConfig)}
+ * The DrillConfig provides:
+ *  - the list of packages to scan. (drill.classpath.scanning.packages) {@see CommonConstants#IMPLEMENTATIONS_SCAN_PACKAGES}
+ *  - the list of base classes to scan for implementations. (drill.classpath.scanning.base.classes) {@see CommonConstants#IMPLEMENTATIONS_SCAN_CLASSES}
+ *  - the list of annotations to scan for. (drill.classpath.scanning.annotations) {@see CommonConstants#IMPLEMENTATIONS_SCAN_ANNOTATIONS}
+ * Only the class directories and jars containing a drill-module.conf will be scanned.
+ * Drill core packages are scanned at build time and the result is saved in a JSON file. {@see ClassPathScanner#FUNCTION_REGISTRY_FILE}
+ * At runtime only the locations that have not been scanned yet will be scanned.
+ */
+public final class ClassPathScanner {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassPathScanner.class);
+  private static final JavassistAdapter METADATA_ADAPTER = new JavassistAdapter();
+
+  /** Configuration pathname to list of names of packages to scan for implementations. */
+  private static final String IMPLEMENTATIONS_SCAN_PACKAGES = "drill.classpath.scanning.packages";
+
+  /** Configuration pathname to list of names of base classes to scan for implementations. */
+  private static final String IMPLEMENTATIONS_SCAN_CLASSES = "drill.classpath.scanning.base.classes";
+
+  /** Configuration pathname to list of names of annotations to scan for. */
+  private static final String IMPLEMENTATIONS_SCAN_ANNOTATIONS = "drill.classpath.scanning.annotations";
+
+  /** Configuration pathname to turn off build time caching. */
+  private static final String IMPLEMENTATIONS_SCAN_CACHE = "drill.classpath.scanning.cache.enabled";
+
+  /**
+   * scans the inheritance tree
+   */
+  private static class SubTypesScanner extends AbstractScanner {
+
+    private Multimap<String, ChildClassDescriptor> parentsChildren = HashMultimap.create();
+    private Multimap<String, ChildClassDescriptor> children = HashMultimap.create();
+
+    public SubTypesScanner(List<ParentClassDescriptor> parentImplementations) {
+      for (ParentClassDescriptor parentClassDescriptor : parentImplementations) {
+        parentsChildren.putAll(parentClassDescriptor.getName(), parentClassDescriptor.getChildren());
+      }
+    }
+
+    @Override
+    public void scan(final Object cls) {
+      final ClassFile classFile = (ClassFile)cls;
+      String className = classFile.getName();
+      String superclass = classFile.getSuperclass();
+      boolean isAbstract = (classFile.getAccessFlags() & (AccessFlag.INTERFACE | AccessFlag.ABSTRACT)) != 0;
+      ChildClassDescriptor scannedClass = new ChildClassDescriptor(className, isAbstract);
+      if (!superclass.equals(Object.class.getName())) {
+        children.put(superclass, scannedClass);
+      }
+      for (String anInterface : classFile.getInterfaces()) {
+        children.put(anInterface, scannedClass);
+      }
+    }
+
+    /**
+     * @param name the class name to get all the children of
+     * @return the class names of all children direct or indirect
+     */
+    public Set<ChildClassDescriptor> getChildrenOf(String name) {
+      Set<ChildClassDescriptor> result = new HashSet<>();
+      Collection<ChildClassDescriptor> scannedChildren = children.get(name);
+      // add all scanned children
+      for (ChildClassDescriptor child : scannedChildren) {
+        result.add(child);
+      }
+      // recursively add children's children
+      Collection<ChildClassDescriptor> allChildren = new ArrayList<>();
+      allChildren.addAll(scannedChildren);
+      allChildren.addAll(parentsChildren.get(name));
+      for (ChildClassDescriptor child : allChildren) {
+        result.addAll(getChildrenOf(child.getName()));
+      }
+      return result;
+    }
+
+  }
+
+  /**
+   * converts the annotation attribute value into a list of string to simplify
+   */
+  private static class ListingMemberValueVisitor implements MemberValueVisitor {
+    private final List<String> values;
+
+    private ListingMemberValueVisitor(List<String> values) {
+      this.values = values;
+    }
+
+    @Override
+    public void visitStringMemberValue(StringMemberValue node) {
+      values.add(node.getValue());
+    }
+
+    @Override
+    public void visitShortMemberValue(ShortMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+
+    @Override
+    public void visitLongMemberValue(LongMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+
+    @Override
+    public void visitIntegerMemberValue(IntegerMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+
+    @Override
+    public void visitFloatMemberValue(FloatMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+
+    @Override
+    public void visitEnumMemberValue(EnumMemberValue node) {
+      values.add(node.getValue());
+    }
+
+    @Override
+    public void visitDoubleMemberValue(DoubleMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+
+    @Override
+    public void visitClassMemberValue(ClassMemberValue node) {
+      values.add(node.getValue());
+    }
+
+    @Override
+    public void visitCharMemberValue(CharMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+
+    @Override
+    public void visitByteMemberValue(ByteMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+
+    @Override
+    public void visitBooleanMemberValue(BooleanMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+
+    @Override
+    public void visitArrayMemberValue(ArrayMemberValue node) {
+      MemberValue[] nestedValues = node.getValue();
+      for (MemberValue v : nestedValues) {
+        v.accept(new ListingMemberValueVisitor(values) {
+          @Override
+          public void visitArrayMemberValue(ArrayMemberValue node) {
+            values.add(Arrays.toString(node.getValue()));
+          }
+        });
+      }
+    }
+
+    @Override
+    public void visitAnnotationMemberValue(AnnotationMemberValue node) {
+      values.add(String.valueOf(node.getValue()));
+    }
+  }
+
+  /**
+   * scans functions annotated with configured annotations
+   *  and keeps track of its annotations and fields
+   */
+  private static final class AnnotationScanner extends AbstractScanner {
+
+    private final List<AnnotatedClassDescriptor> functions = new ArrayList<>();
+
+    private final Set<String> annotationsToScan;
+
+    AnnotationScanner(Collection<String> annotationsToScan) {
+      super();
+      this.annotationsToScan = Collections.unmodifiableSet(new HashSet<>(annotationsToScan));
+    }
+
+    public List<AnnotatedClassDescriptor> getAnnotatedClasses() {
+      return unmodifiableList(functions);
+    }
+
+    @Override
+    public void scan(final Object cls) {
+      final ClassFile classFile = (ClassFile)cls;
+      AnnotationsAttribute annotations = ((AnnotationsAttribute)classFile.getAttribute(AnnotationsAttribute.visibleTag));
+      if (annotations != null) {
+        boolean isAnnotated = false;
+        for (javassist.bytecode.annotation.Annotation a : annotations.getAnnotations()) {
+          if (annotationsToScan.contains(a.getTypeName())) {
+            isAnnotated = true;
+          }
+        }
+        if (isAnnotated) {
+          List<AnnotationDescriptor> classAnnotations = getAnnotationDescriptors(annotations);
+          @SuppressWarnings("unchecked")
+          List<FieldInfo> classFields = classFile.getFields();
+          List<FieldDescriptor> fieldDescriptors = new ArrayList<>(classFields.size());
+          for (FieldInfo field : classFields) {
+            String fieldName = field.getName();
+            AnnotationsAttribute fieldAnnotations = ((AnnotationsAttribute)field.getAttribute(AnnotationsAttribute.visibleTag));
+            fieldDescriptors.add(new FieldDescriptor(fieldName, field.getDescriptor(), getAnnotationDescriptors(fieldAnnotations)));
+          }
+          functions.add(new AnnotatedClassDescriptor(classFile.getName(), classAnnotations, fieldDescriptors));
+        }
+      }
+    }
+
+    private List<AnnotationDescriptor> getAnnotationDescriptors(AnnotationsAttribute annotationsAttr) {
+      List<AnnotationDescriptor> annotationDescriptors = new ArrayList<>(annotationsAttr.numAnnotations());
+      for (javassist.bytecode.annotation.Annotation annotation : annotationsAttr.getAnnotations()) {
+        // Sigh: javassist uses raw collections (is this 2002?)
+        @SuppressWarnings("unchecked")
+        Set<String> memberNames = annotation.getMemberNames();
+        List<AttributeDescriptor> attributes = new ArrayList<>();
+        if (memberNames != null) {
+          for (String name : memberNames) {
+            MemberValue memberValue = annotation.getMemberValue(name);
+            final List<String> values = new ArrayList<>();
+            memberValue.accept(new ListingMemberValueVisitor(values));
+            attributes.add(new AttributeDescriptor(name, values));
+          }
+        }
+        annotationDescriptors.add(new AnnotationDescriptor(annotation.getTypeName(), attributes));
+      }
+      return annotationDescriptors;
+    }
+  }
+
+  /**
+   * @return paths that have a drill config file in them
+   */
+  static Set<URL> getMarkedPaths() {
+    return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, true);
+  }
+
+  public static Collection<URL> getConfigURLs() {
+    return forResource(CommonConstants.DRILL_JAR_MARKER_FILE_RESOURCE_PATHNAME, false);
+  }
+
+  /**
+   * Gets URLs of any classpath resources with given resource pathname.
+   *
+   * @param  resourcePathname  resource pathname of classpath resource instances
+   *           to scan for (relative to specified class loaders' classpath roots)
+   * @param  returnRootPathname  whether to collect classpath root portion of
+   *           URL for each resource instead of full URL of each resource
+   * @param  classLoaders  set of class loaders in which to look up resource;
+   *           none (empty array) to specify to use current thread's context
+   *           class loader and {@link Reflections}'s class loader
+   * @returns  ...; empty set if none
+   */
+  public static Set<URL> forResource(final String resourcePathname, final boolean returnRootPathname) {
+    logger.debug("Scanning classpath for resources with pathname \"{}\".",
+                 resourcePathname);
+    final Set<URL> resultUrlSet = Sets.newHashSet();
+    final ClassLoader classLoader = ClassPathScanner.class.getClassLoader();
+    try {
+      final Enumeration<URL> resourceUrls = classLoader.getResources(resourcePathname);
+      while (resourceUrls.hasMoreElements()) {
+        final URL resourceUrl = resourceUrls.nextElement();
+        logger.trace("- found a(n) {} at {}.", resourcePathname, resourceUrl);
+        int index = resourceUrl.toExternalForm().lastIndexOf(resourcePathname);
+        if (index != -1 && returnRootPathname) {
+          final URL classpathRootUrl = new URL(resourceUrl.toExternalForm().substring(0, index));
+          resultUrlSet.add(classpathRootUrl);
+          logger.debug("- collected resource's classpath root URL {}.", classpathRootUrl);
+        } else {
+          resultUrlSet.add(resourceUrl);
+          logger.debug("- collected resource URL {}.", resourceUrl);
+        }
+      }
+    } catch (IOException e) {
+      throw new RuntimeException("Error scanning for resources named " + resourcePathname, e);
+    }
+    return resultUrlSet;
+  }
+
+  static List<String> getPackagePrefixes(DrillConfig config) {
+    return config.getStringList(IMPLEMENTATIONS_SCAN_PACKAGES);
+  }
+
+  static List<String> getScannedBaseClasses(DrillConfig config) {
+    return config.getStringList(IMPLEMENTATIONS_SCAN_CLASSES);
+  }
+
+  static List<String> getScannedAnnotations(DrillConfig config) {
+    if (config.hasPath(IMPLEMENTATIONS_SCAN_ANNOTATIONS)) {
+      return config.getStringList(IMPLEMENTATIONS_SCAN_ANNOTATIONS);
+    } else {
+      return Collections.emptyList();
+    }
+  }
+
+  static boolean isScanBuildTimeCacheEnabled(DrillConfig config) {
+    if (config.hasPath(IMPLEMENTATIONS_SCAN_CACHE)) {
+      return config.getBoolean(IMPLEMENTATIONS_SCAN_CACHE);
+    } else {
+      return true; // on by default
+    }
+  }
+
+  /**
+   *
+   * @param pathsToScan the locations to scan for .class files
+   * @param packagePrefixes the whitelist of package prefixes to scan
+   * @param parentResult if there was a prescan, its result
+   * @return the merged scan
+   */
+  static ScanResult scan(Collection<URL> pathsToScan, Collection<String> packagePrefixes, Collection<String> scannedClasses, Collection<String> scannedAnnotations, ScanResult parentResult) {
+    Stopwatch watch = new Stopwatch().start();
+    try {
+      AnnotationScanner annotationScanner = new AnnotationScanner(scannedAnnotations);
+      SubTypesScanner subTypesScanner = new SubTypesScanner(parentResult.getImplementations());
+      if (packagePrefixes.size() > 0) {
+        final FilterBuilder filter = new FilterBuilder();
+        for (String prefix : packagePrefixes) {
+          filter.include(FilterBuilder.prefix(prefix));
+        }
+        ConfigurationBuilder conf = new ConfigurationBuilder()
+            .setUrls(pathsToScan)
+            .setMetadataAdapter(METADATA_ADAPTER) // Scanners depend on this
+            .filterInputsBy(filter)
+            .setScanners(annotationScanner, subTypesScanner);
+
+        // scans stuff, but don't use the funky storage layer
+        new Reflections(conf);
+      }
+      List<ParentClassDescriptor> implementations = new ArrayList<>();
+      for (String baseTypeName: scannedClasses) {
+        implementations.add(
+            new ParentClassDescriptor(
+                baseTypeName,
+                new ArrayList<>(subTypesScanner.getChildrenOf(baseTypeName))));
+      }
+      List<AnnotatedClassDescriptor> annotated = annotationScanner.getAnnotatedClasses();
+      verifyClassUnicity(annotated);
+      return new ScanResult(
+          packagePrefixes,
+          scannedClasses,
+          scannedAnnotations,
+          annotated,
+          implementations);
+    } finally {
+      logger.info(
+          format("Scanning packages %s in locations %s took %dms",
+              packagePrefixes, pathsToScan, watch.elapsed(MILLISECONDS)));
+    }
+  }
+
+  private static void verifyClassUnicity(List<AnnotatedClassDescriptor> annotatedClasses) {
+    Set<String> scanned = new HashSet<>();
+    for (AnnotatedClassDescriptor annotated : annotatedClasses) {
+      if (!scanned.add(annotated.getClassName())) {
+        throw new RuntimeException("BUG: " + annotated.getClassName() + " scanned twice");
+      }
+    }
+  }
+
+  static ScanResult emptyResult() {
+    return new ScanResult(
+        Collections.<String>emptyList(),
+        Collections.<String>emptyList(),
+        Collections.<String>emptyList(),
+        Collections.<AnnotatedClassDescriptor>emptyList(),
+        Collections.<ParentClassDescriptor>emptyList());
+  }
+
+  public static ScanResult fromPrescan(DrillConfig config) {
+    return RunTimeScan.fromPrescan(config);
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java b/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java
new file mode 100644
index 0000000..1d95b04
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/RunTimeScan.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner;
+
+import java.net.URL;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.scanner.persistence.ScanResult;
+
+/**
+ * Utility to scan classpath at runtime
+ *
+ */
+public class RunTimeScan {
+
+  /** result of prescan */
+  private static final ScanResult PRESCANNED = BuildTimeScan.load();
+
+  /** urls of the locations (classes directory or jar) to scan that don't have a registry in them */
+  private static final Collection<URL> NON_PRESCANNED_MARKED_PATHS = getNonPrescannedMarkedPaths();
+
+  /**
+   * @return getMarkedPaths() sans getPrescannedPaths()
+   */
+  static Collection<URL> getNonPrescannedMarkedPaths() {
+    Collection<URL> markedPaths = ClassPathScanner.getMarkedPaths();
+    markedPaths.removeAll(BuildTimeScan.getPrescannedPaths());
+    return markedPaths;
+  }
+
+  /**
+   * loads prescanned classpath info and scans for extra ones based on configuration.
+   * (unless prescan is disabled with {@see ClassPathScanner#IMPLEMENTATIONS_SCAN_CACHE}=falses)
+   * @param config to retrieve the packages to scan
+   * @return the scan result
+   */
+  public static ScanResult fromPrescan(DrillConfig config) {
+    List<String> packagePrefixes = ClassPathScanner.getPackagePrefixes(config);
+    List<String> scannedBaseClasses = ClassPathScanner.getScannedBaseClasses(config);
+    List<String> scannedAnnotations = ClassPathScanner.getScannedAnnotations(config);
+    if (ClassPathScanner.isScanBuildTimeCacheEnabled(config)) {
+      // scan only locations that have not been scanned yet
+      ScanResult runtimeScan = ClassPathScanner.scan(
+          NON_PRESCANNED_MARKED_PATHS,
+          packagePrefixes,
+          scannedBaseClasses,
+          scannedAnnotations,
+          PRESCANNED);
+      return runtimeScan.merge(PRESCANNED);
+    } else {
+      // scan everything
+      return ClassPathScanner.scan(
+          ClassPathScanner.getMarkedPaths(),
+          packagePrefixes,
+          scannedBaseClasses,
+          scannedAnnotations,
+          ClassPathScanner.emptyResult());
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotatedClassDescriptor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotatedClassDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotatedClassDescriptor.java
new file mode 100644
index 0000000..13abbb5
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotatedClassDescriptor.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner.persistence;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * describe a class that was annotated with one of the configured annotations
+ */
+public final class AnnotatedClassDescriptor {
+
+  private final String className;
+  private final List<AnnotationDescriptor> annotations;
+  private final List<FieldDescriptor> fields;
+  private final Map<String, AnnotationDescriptor> annotationMap;
+
+  @JsonCreator public AnnotatedClassDescriptor(
+      @JsonProperty("className") String className,
+      @JsonProperty("annotations") List<AnnotationDescriptor> annotations,
+      @JsonProperty("fields") List<FieldDescriptor> fields) {
+    this.className = className;
+    this.annotations = Collections.unmodifiableList(annotations);
+    this.fields = Collections.unmodifiableList(fields);
+    this.annotationMap = AnnotationDescriptor.buildAnnotationsMap(annotations);
+  }
+
+  /**
+   * @return the fully qualified name of the annotated class
+   */
+  public String getClassName() {
+    return className;
+  }
+
+  /**
+   * @return the class level annotations
+   */
+  public List<AnnotationDescriptor> getAnnotations() {
+    return annotations;
+  }
+
+  /**
+   * @return the fields declared by the class
+   */
+  public List<FieldDescriptor> getFields() {
+    return fields;
+  }
+
+  /**
+   * @param clazz the annotation type
+   * @return the corresponding annotation descriptor
+   */
+  public AnnotationDescriptor getAnnotation(Class<?> clazz) {
+    return annotationMap.get(clazz.getName());
+  }
+
+  /**
+   * @param clazz the annotation type
+   * @return a bytecode based implementation of the annotation
+   */
+  public <T> T getAnnotationProxy(Class<T> clazz) {
+    final AnnotationDescriptor annotationDescriptor = getAnnotation(clazz);
+    if (annotationDescriptor == null) {
+      return null;
+    }
+    return annotationDescriptor.getProxy(clazz);
+  }
+
+  @Override
+  public String toString() {
+    return "Function [className=" + className + ", annotations=" + annotations
+        + ", fields=" + fields + "]";
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotationDescriptor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotationDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotationDescriptor.java
new file mode 100644
index 0000000..f76ad49
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/AnnotationDescriptor.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner.persistence;
+
+import static java.lang.String.format;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * a class annotation
+ */
+public final class AnnotationDescriptor {
+  private final String annotationType;
+  private final List<AttributeDescriptor> attributes;
+  private final Map<String, AttributeDescriptor> attributeMap;
+
+  @JsonCreator public AnnotationDescriptor(
+      @JsonProperty("annotationType") String annotationType,
+      @JsonProperty("attributes") List<AttributeDescriptor> attributes) {
+    this.annotationType = annotationType;
+    this.attributes = Collections.unmodifiableList(attributes);
+    ImmutableMap.Builder<String, AttributeDescriptor> mapBuilder = ImmutableMap.builder();
+    for (AttributeDescriptor att : attributes) {
+      mapBuilder.put(att.getName(), att);
+    }
+    this.attributeMap = mapBuilder.build();
+  }
+
+  /**
+   * @return the class name of the annotation
+   */
+  public String getAnnotationType() {
+    return annotationType;
+  }
+
+  public List<AttributeDescriptor> getAttributes() {
+    return attributes;
+  }
+
+  public boolean hasAttribute(String attributeName) {
+    return attributeMap.containsKey(attributeName);
+  }
+
+  public List<String> getValues(String attributeName) {
+    AttributeDescriptor desc = attributeMap.get(attributeName);
+    return desc == null ? Collections.<String>emptyList() : desc.getValues();
+  }
+
+  public String getSingleValue(String attributeName, String defaultValue) {
+    List<String> values = getValues(attributeName);
+    if (values.size() > 1) {
+      throw new IllegalStateException(String.format(
+          "Expected a single value for the attribute named %s but found %d (%s)",
+          attributeName, values.size(), values.toString()));
+    }
+    return values.isEmpty() ? defaultValue : values.get(0);
+  }
+
+  public String getSingleValue(String attributeName) {
+    String val = getSingleValue(attributeName, null);
+    if (val == null) {
+      throw new IllegalStateException(format("Attribute %s not found in %s", attributeName, attributes));
+    }
+    return val;
+  }
+
+  @Override
+  public String toString() {
+    return "Annotation[type=" + annotationType + ", attributes=" + attributes + "]";
+  }
+
+  static Map<String, AnnotationDescriptor> buildAnnotationsMap(List<AnnotationDescriptor> annotations) {
+    ImmutableMap.Builder<String, AnnotationDescriptor> annMapBuilder = ImmutableMap.builder();
+    for (AnnotationDescriptor ann : annotations) {
+      annMapBuilder.put(ann.getAnnotationType(), ann);
+    }
+    return annMapBuilder.build();
+  }
+
+  @SuppressWarnings("unchecked")
+  private <T> T proxy(Class<T> interfc, InvocationHandler ih) {
+    if (!interfc.isInterface()) {
+      throw new IllegalArgumentException("only proxying interfaces: " + interfc);
+    }
+    return (T)Proxy.newProxyInstance(this.getClass().getClassLoader(), new Class[]{ interfc }, ih);
+  }
+
+  public <T> T getProxy(Class<T> clazz) {
+    return proxy(clazz, new InvocationHandler() {
+      @Override
+      public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+        if (args != null && args.length > 0) {
+          // just property methods
+          throw new UnsupportedOperationException(method + " " + Arrays.toString(args));
+        }
+        String attributeName = method.getName();
+        if (!hasAttribute(attributeName)) {
+          return method.getDefaultValue();
+        }
+        Class<?> returnType = method.getReturnType();
+        if (returnType.isArray()) {
+          List<String> values = getValues(attributeName);
+          Class<?> componentType = returnType.getComponentType();
+          Object[] result = (Object[])Array.newInstance(componentType, values.size());
+          for (int i = 0; i < result.length; i++) {
+            String value = values.get(i);
+            result[i] = convertValue(componentType, value);
+          }
+          return result;
+        } else {
+          String value = getSingleValue(attributeName, null);
+          return convertValue(returnType, value);
+        }
+      }
+
+      private <U> Object convertValue(Class<U> c, String value) {
+        if (c.equals(String.class)) {
+          return value;
+        } else if (c.isEnum()) {
+          @SuppressWarnings("unchecked")
+          Enum<?> enumValue = Enum.valueOf(c.asSubclass(Enum.class), value);
+          return enumValue;
+        } else if (c.equals(boolean.class)) {
+          return Boolean.valueOf(value);
+        }
+        throw new UnsupportedOperationException(c.toString());
+      }
+    });
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/persistence/AttributeDescriptor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/AttributeDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/AttributeDescriptor.java
new file mode 100644
index 0000000..81a8097
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/AttributeDescriptor.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner.persistence;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * a field set in an annotation
+ * (to simplify we have a list of string representations of the values)
+ */
+public final class AttributeDescriptor {
+  private final String name;
+  private final List<String> values;
+
+  @JsonCreator public AttributeDescriptor(
+      @JsonProperty("name") String name,
+      @JsonProperty("values") List<String> values) {
+    this.name = name;
+    this.values = values;
+  }
+
+  /**
+   * @return field name
+   */
+  public String getName() {
+    return name;
+  }
+
+  public List<String> getValues() {
+    return values;
+  }
+
+  @Override
+  public String toString() {
+    return "Attribute[" + name + "=" + values + "]";
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/persistence/ChildClassDescriptor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/ChildClassDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/ChildClassDescriptor.java
new file mode 100644
index 0000000..7e81bd8
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/ChildClassDescriptor.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner.persistence;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * a class that implements a specific type
+ */
+public final class ChildClassDescriptor {
+
+  private final String name;
+  private final boolean isAbstract;
+
+  @JsonCreator
+  public ChildClassDescriptor(
+      @JsonProperty("name") String name,
+      @JsonProperty("abstract") boolean isAbstract) {
+    this.name = name;
+    this.isAbstract = isAbstract;
+  }
+
+  /**
+   * @return the class name
+   */
+  public String getName() {
+    return name;
+  }
+
+  /**
+   * @return whether the class is abstract
+   */
+  public boolean isAbstract() {
+    return isAbstract;
+  }
+
+  @Override
+  public String toString() {
+    return "ChildClassDescriptor [name=" + name + ", isAbstract=" + isAbstract + "]";
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/persistence/FieldDescriptor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/FieldDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/FieldDescriptor.java
new file mode 100644
index 0000000..dae2a74
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/FieldDescriptor.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner.persistence;
+
+import java.lang.reflect.Array;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * a class fields
+ */
+public final class FieldDescriptor {
+  private final String name;
+  private final String descriptor;
+  private final List<AnnotationDescriptor> annotations;
+  private final Map<String, AnnotationDescriptor> annotationMap;
+
+  @JsonCreator public FieldDescriptor(
+      @JsonProperty("name") String name,
+      @JsonProperty("descriptor") String descriptor,
+      @JsonProperty("annotations") List<AnnotationDescriptor> annotations) {
+    this.name = name;
+    this.descriptor = descriptor;
+    this.annotations = annotations;
+    this.annotationMap = AnnotationDescriptor.buildAnnotationsMap(annotations);
+    // validate the descriptor
+    getType();
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  /**
+   * @return the descriptor of the type of the field as defined in the bytecode
+   */
+  public String getDescriptor() {
+    return descriptor;
+  }
+
+  public List<AnnotationDescriptor> getAnnotations() {
+    return annotations;
+  }
+
+  public AnnotationDescriptor getAnnotation(Class<?> clazz) {
+    return annotationMap.get(clazz.getName());
+  }
+
+  public <T> T getAnnotationProxy(Class<T> clazz) {
+    final AnnotationDescriptor annotationDescriptor = getAnnotation(clazz);
+    if (annotationDescriptor == null) {
+      return null;
+    }
+    return annotationDescriptor.getProxy(clazz);
+  }
+
+  @JsonIgnore
+  public TypeDescriptor getType() {
+      int arrayDim = 0;
+      char c;
+      while ((c = descriptor.charAt(arrayDim)) == '[') {
+          ++arrayDim;
+      }
+      if (c == 'L') {
+        int lastIndex = descriptor.length() - 1;
+        if (descriptor.charAt(lastIndex) != ';') {
+          throw new DrillRuntimeException("Illegal descriptor: " + descriptor);
+        }
+        String className = descriptor.substring(arrayDim + 1, lastIndex).replace('/', '.');
+        return TypeDescriptor.forClass(className, arrayDim);
+      } else {
+        return TypeDescriptor.forPrimitive(c, arrayDim);
+      }
+  }
+
+  @JsonIgnore
+  public Class<?> getFieldClass() {
+    TypeDescriptor type = getType();
+    Class<?> elementClass = type.getType();
+    if (type.isArray()) {
+      return Array.newInstance(elementClass, new int[type.getArrayDim()]).getClass();
+    } else {
+      return elementClass;
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "Field[name=" + name + ", descriptor=" + descriptor + ", annotations=" + annotations + "]";
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/dbcab0fe/common/src/main/java/org/apache/drill/common/scanner/persistence/ParentClassDescriptor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/scanner/persistence/ParentClassDescriptor.java b/common/src/main/java/org/apache/drill/common/scanner/persistence/ParentClassDescriptor.java
new file mode 100644
index 0000000..210c4f5
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/scanner/persistence/ParentClassDescriptor.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.scanner.persistence;
+
+import static java.util.Collections.unmodifiableList;
+
+import java.util.List;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * a parent class and its implementations that was specifically searched for during scanning
+ */
+public final class ParentClassDescriptor {
+  private final String name;
+  private final List<ChildClassDescriptor> children;
+
+  @JsonCreator
+  public ParentClassDescriptor(
+      @JsonProperty("name") String name,
+      @JsonProperty("children") List<ChildClassDescriptor> children) {
+    this.name = name;
+    this.children = unmodifiableList(children);
+  }
+
+  /**
+   * @return the class name
+   */
+  public String getName() {
+    return name;
+  }
+
+ /**
+  * @return the implementations
+  */
+  public List<ChildClassDescriptor> getChildren() {
+    return children;
+  }
+
+  @Override
+  public String toString() {
+    return "ParentClassDescriptor [name=" + name + ", children=" + children + "]";
+  }
+
+}
\ No newline at end of file


Mime
View raw message