asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [3/3] asterixdb git commit: [ASTERIXDB-2154][COMP] Extensible built-in function management
Date Sun, 05 Nov 2017 18:06:32 GMT
[ASTERIXDB-2154][COMP] Extensible built-in function management

- user model changes: no
- storage format changes: no
- interface changes: yes

Details:
- Support for augmenting built-in functions in ILangExtension
- Move function type inferers from NonTaggedDataFormat to
  FunctionTypeInferers, exposed through IFunctionDescriptorFactory
  and IFunctionManager
- Cleanup: obtain IFunctionManager/IDataFormat from MetadataProvider

Change-Id: Iafbf7e4e298fb7af6cb84b402cb00cfa2e90b5ce
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2118
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/76d24d49
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/76d24d49
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/76d24d49

Branch: refs/heads/release-0.9.3-pre-rc
Commit: 76d24d49055d764fc03b19bce441e97a563622b4
Parents: 52e5134
Author: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
Authored: Fri Nov 3 13:28:21 2017 -0700
Committer: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
Committed: Sat Nov 4 11:56:19 2017 -0700

----------------------------------------------------------------------
 .../asterix/algebra/base/ILangExtension.java    |   5 +
 .../jobgen/QueryLogicalExpressionJobGen.java    |  62 +-
 .../optimizer/rules/ConstantFoldingRule.java    |   9 +-
 .../apache/asterix/api/common/APIFramework.java |   5 +-
 .../asterix/app/cc/CCExtensionManager.java      |  30 +-
 .../org/apache/asterix/utils/ExtensionUtil.java |  36 +-
 .../org/apache/asterix/runtime/ExceptionIT.java |   3 +-
 .../apache/asterix/runtime/NullMissingTest.java |   3 +-
 .../test/active/ActiveEventsListenerTest.java   |  12 +
 .../metadata/declared/DatasetDataSource.java    |   4 +-
 .../metadata/declared/FeedDataSource.java       |   3 +-
 .../metadata/declared/MetadataProvider.java     |  17 +-
 .../asterix/metadata/entities/Dataset.java      |   2 +-
 .../asterix/metadata/utils/DatasetUtil.java     |  18 +-
 .../utils/SecondaryBTreeOperationsHelper.java   |  17 +-
 ...econdaryCorrelatedBTreeOperationsHelper.java |  17 +-
 ...CorrelatedInvertedIndexOperationsHelper.java |  16 +-
 ...econdaryCorrelatedRTreeOperationsHelper.java |   6 +-
 ...daryCorrelatedTreeIndexOperationsHelper.java |   7 +-
 .../utils/SecondaryIndexOperationsHelper.java   |  17 +-
 .../SecondaryInvertedIndexOperationsHelper.java |  16 +-
 .../utils/SecondaryRTreeOperationsHelper.java   |   6 +-
 .../asterix/formats/base/IDataFormat.java       |  19 +-
 .../functions/IFunctionDescriptorFactory.java   |  10 +-
 .../om/functions/IFunctionExtensionManager.java |  32 +
 .../asterix/om/functions/IFunctionManager.java  |  12 +-
 .../om/functions/IFunctionTypeInferer.java      |  29 +
 .../collections/ListifyAggregateDescriptor.java |  10 +-
 .../AbstractScalarAggregateDescriptor.java      |  14 +-
 .../scalar/ScalarAvgAggregateDescriptor.java    |  10 +-
 .../scalar/ScalarCountAggregateDescriptor.java  |  10 +-
 .../scalar/ScalarMaxAggregateDescriptor.java    |  10 +-
 .../scalar/ScalarMinAggregateDescriptor.java    |  10 +-
 .../scalar/ScalarSqlAvgAggregateDescriptor.java |  10 +-
 .../ScalarSqlCountAggregateDescriptor.java      |  11 +-
 .../scalar/ScalarSqlMaxAggregateDescriptor.java |  10 +-
 .../scalar/ScalarSqlMinAggregateDescriptor.java |  10 +-
 .../scalar/ScalarSqlSumAggregateDescriptor.java |  10 +-
 .../scalar/ScalarSumAggregateDescriptor.java    |  10 +-
 .../ClosedRecordConstructorDescriptor.java      |  10 +-
 .../OpenRecordConstructorDescriptor.java        |   7 +
 .../OrderedListConstructorDescriptor.java       |  12 +-
 .../UnorderedListConstructorDescriptor.java     |  12 +-
 .../functions/CastTypeDescriptor.java           |   9 +-
 .../functions/CastTypeLaxDescriptor.java        |   7 +
 .../functions/DeepEqualityDescriptor.java       |   9 +-
 .../records/FieldAccessByIndexDescriptor.java   |  11 +-
 .../records/FieldAccessNestedDescriptor.java    |  11 +-
 .../records/GetRecordFieldValueDescriptor.java  |  11 +-
 .../records/GetRecordFieldsDescriptor.java      |  11 +-
 .../records/RecordAddFieldsDescriptor.java      |   8 +
 .../records/RecordMergeDescriptor.java          |   8 +
 .../records/RecordPairsDescriptor.java          |   7 +
 .../records/RecordRemoveFieldsDescriptor.java   |  18 +-
 .../runtime/formats/NonTaggedDataFormat.java    | 350 +---------
 .../runtime/functions/FunctionCollection.java   | 641 ++++++++++---------
 .../runtime/functions/FunctionManager.java      |  73 +++
 .../functions/FunctionManagerHolder.java        |  41 --
 .../runtime/functions/FunctionManagerImpl.java  |  68 --
 .../runtime/functions/FunctionTypeInferers.java | 271 ++++++++
 60 files changed, 1189 insertions(+), 944 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
index 4803691..fb74475 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
@@ -21,6 +21,7 @@ package org.apache.asterix.algebra.base;
 
 import org.apache.asterix.common.api.IExtension;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.om.functions.IFunctionManager;
 
 /**
  * An interface for language extensions
@@ -38,4 +39,8 @@ public interface ILangExtension extends IExtension {
     }
 
     ILangCompilationProvider getLangCompilationProvider(Language lang);
+
+    default IFunctionManager getFunctionManager() {
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
index 9f88bb6..812f3c9 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
@@ -23,11 +23,12 @@ import java.util.List;
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.functions.FunctionDescriptorTag;
 import org.apache.asterix.external.library.ExternalFunctionDescriptorProvider;
-import org.apache.asterix.formats.base.IDataFormat;
+import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.om.functions.BuiltinFunctions;
 import org.apache.asterix.om.functions.IExternalFunctionInfo;
 import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.runtime.formats.FormatUtils;
+import org.apache.asterix.om.functions.IFunctionManager;
+import org.apache.asterix.om.functions.IFunctionTypeInferer;
 import org.apache.commons.lang3.mutable.Mutable;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -40,6 +41,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir
 import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
 import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
@@ -51,9 +53,10 @@ import org.apache.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory;
 
 public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
 
-    public static final QueryLogicalExpressionJobGen INSTANCE = new QueryLogicalExpressionJobGen();
+    private final IFunctionManager functionManager;
 
-    private QueryLogicalExpressionJobGen() {
+    public QueryLogicalExpressionJobGen(IFunctionManager functionManager) {
+        this.functionManager = functionManager;
     }
 
     @Override
@@ -61,7 +64,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
             IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
             throws AlgebricksException {
         IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
-        IFunctionDescriptor fd = getFunctionDescriptor(expr, env, context);
+        IFunctionDescriptor fd = resolveFunction(expr, env);
         switch (fd.getFunctionDescriptorTag()) {
             case SERIALAGGREGATE:
                 return null;
@@ -79,7 +82,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
             IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
             throws AlgebricksException {
         IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
-        return getFunctionDescriptor(expr, env, context).createRunningAggregateEvaluatorFactory(args);
+        return resolveFunction(expr, env).createRunningAggregateEvaluatorFactory(args);
     }
 
     @Override
@@ -87,22 +90,22 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
             IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
             throws AlgebricksException {
         IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
-        return getFunctionDescriptor(expr, env, context).createUnnestingEvaluatorFactory(args);
+        return resolveFunction(expr, env).createUnnestingEvaluatorFactory(args);
     }
 
     @Override
     public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
             IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
-        IScalarEvaluatorFactory copyEvaluatorFactory = null;
+        IScalarEvaluatorFactory copyEvaluatorFactory;
         switch (expr.getExpressionTag()) {
             case VARIABLE: {
                 VariableReferenceExpression v = (VariableReferenceExpression) expr;
-                copyEvaluatorFactory = createVariableEvaluatorFactory(v, inputSchemas, context);
+                copyEvaluatorFactory = createVariableEvaluatorFactory(v, inputSchemas);
                 return copyEvaluatorFactory;
             }
             case CONSTANT: {
                 ConstantExpression c = (ConstantExpression) expr;
-                copyEvaluatorFactory = createConstantEvaluatorFactory(c, inputSchemas, context);
+                copyEvaluatorFactory = createConstantEvaluatorFactory(c, context);
                 return copyEvaluatorFactory;
             }
             case FUNCTION_CALL: {
@@ -117,7 +120,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
     }
 
     private IScalarEvaluatorFactory createVariableEvaluatorFactory(VariableReferenceExpression expr,
-            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
+            IOperatorSchema[] inputSchemas) throws AlgebricksException {
         LogicalVariable variable = expr.getVariableReference();
         for (IOperatorSchema scm : inputSchemas) {
             int pos = scm.findVariable(variable);
@@ -132,22 +135,17 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
             IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
             throws AlgebricksException {
         IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
-        IFunctionDescriptor fd = null;
-        if (!(expr.getFunctionInfo() instanceof IExternalFunctionInfo)) {
-            IDataFormat format = FormatUtils.getDefaultFormat();
-            fd = format.resolveFunction(expr, env);
-        } else {
-            ICcApplicationContext appCtx = (ICcApplicationContext) context.getAppContext();
-            fd = ExternalFunctionDescriptorProvider
-                    .getExternalFunctionDescriptor((IExternalFunctionInfo) expr.getFunctionInfo(), appCtx);
-        }
+        IFunctionDescriptor fd = expr.getFunctionInfo() instanceof IExternalFunctionInfo
+                ? ExternalFunctionDescriptorProvider.getExternalFunctionDescriptor(
+                        (IExternalFunctionInfo) expr.getFunctionInfo(), (ICcApplicationContext) context.getAppContext())
+                : resolveFunction(expr, env);
         return fd.createEvaluatorFactory(args);
     }
 
-    private IScalarEvaluatorFactory createConstantEvaluatorFactory(ConstantExpression expr,
-            IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
-        IDataFormat format = FormatUtils.getDefaultFormat();
-        return format.getConstantEvalFactory(expr.getValue());
+    private IScalarEvaluatorFactory createConstantEvaluatorFactory(ConstantExpression expr, JobGenContext context)
+            throws AlgebricksException {
+        MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
+        return metadataProvider.getDataFormat().getConstantEvalFactory(expr.getValue());
     }
 
     private IScalarEvaluatorFactory[] codegenArguments(AbstractFunctionCallExpression expr,
@@ -168,14 +166,14 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
             AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
             JobGenContext context) throws AlgebricksException {
         IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context);
-        IFunctionDescriptor fd = getFunctionDescriptor(expr, env, context);
+        IFunctionDescriptor fd = resolveFunction(expr, env);
 
         switch (fd.getFunctionDescriptorTag()) {
             case AGGREGATE: {
                 if (BuiltinFunctions.isAggregateFunctionSerializable(fd.getIdentifier())) {
                     AggregateFunctionCallExpression serialAggExpr = BuiltinFunctions
                             .makeSerializableAggregateFunctionExpression(fd.getIdentifier(), expr.getArguments());
-                    IFunctionDescriptor afdd = getFunctionDescriptor(serialAggExpr, env, context);
+                    IFunctionDescriptor afdd = resolveFunction(serialAggExpr, env);
                     return afdd.createSerializableAggregateEvaluatorFactory(args);
                 } else {
                     throw new AlgebricksException(
@@ -194,10 +192,14 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen {
         }
     }
 
-    private IFunctionDescriptor getFunctionDescriptor(AbstractFunctionCallExpression expr, IVariableTypeEnvironment env,
-            JobGenContext context) throws AlgebricksException {
-        IFunctionDescriptor fd = FormatUtils.getDefaultFormat().resolveFunction(expr, env);
+    private IFunctionDescriptor resolveFunction(ILogicalExpression expr, IVariableTypeEnvironment env)
+            throws AlgebricksException {
+        FunctionIdentifier fnId = ((AbstractFunctionCallExpression) expr).getFunctionIdentifier();
+        IFunctionDescriptor fd = functionManager.lookupFunction(fnId);
+        IFunctionTypeInferer fnTypeInfer = functionManager.lookupFunctionTypeInferer(fnId);
+        if (fnTypeInfer != null) {
+            fnTypeInfer.infer(expr, fd, env);
+        }
         return fd;
     }
-
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index 5ef41c4..f7695ea 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -36,9 +36,11 @@ import org.apache.asterix.formats.nontagged.BinaryIntegerInspector;
 import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
 import org.apache.asterix.formats.nontagged.TypeTraitProvider;
 import org.apache.asterix.jobgen.QueryLogicalExpressionJobGen;
+import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.om.base.IAObject;
 import org.apache.asterix.om.constants.AsterixConstantValue;
 import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionExtensionManager;
 import org.apache.asterix.om.typecomputer.base.TypeCastUtils;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.ATypeTag;
@@ -125,12 +127,13 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule {
     private static final IOperatorSchema[] _emptySchemas = new IOperatorSchema[] {};
 
     public ConstantFoldingRule(ICcApplicationContext appCtx) {
-        jobGenCtx = new JobGenContext(null, null, appCtx, SerializerDeserializerProvider.INSTANCE,
+        MetadataProvider metadataProvider = new MetadataProvider(appCtx, null);
+        jobGenCtx = new JobGenContext(null, metadataProvider, appCtx, SerializerDeserializerProvider.INSTANCE,
                 BinaryHashFunctionFactoryProvider.INSTANCE, BinaryHashFunctionFamilyProvider.INSTANCE,
                 BinaryComparatorFactoryProvider.INSTANCE, TypeTraitProvider.INSTANCE, BinaryBooleanInspector.FACTORY,
                 BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null,
-                new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE), ExpressionTypeComputer.INSTANCE,
-                null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null);
+                new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())),
+                ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 8290446..2078288 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -263,7 +263,7 @@ public class APIFramework {
         builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig());
         builder.setLogicalRewrites(ruleSetFactory.getLogicalRewrites(metadataProvider.getApplicationContext()));
         builder.setPhysicalRewrites(ruleSetFactory.getPhysicalRewrites(metadataProvider.getApplicationContext()));
-        IDataFormat format = metadataProvider.getFormat();
+        IDataFormat format = metadataProvider.getDataFormat();
         ICompilerFactory compilerFactory = builder.create();
         builder.setExpressionEvalSizeComputer(format.getExpressionEvalSizeComputer());
         builder.setIMergeAggregationExpressionFactory(new MergeAggregationExpressionFactory());
@@ -321,7 +321,8 @@ public class APIFramework {
         builder.setBinaryBooleanInspectorFactory(format.getBinaryBooleanInspectorFactory());
         builder.setBinaryIntegerInspectorFactory(format.getBinaryIntegerInspectorFactory());
         builder.setComparatorFactoryProvider(format.getBinaryComparatorFactoryProvider());
-        builder.setExpressionRuntimeProvider(new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE));
+        builder.setExpressionRuntimeProvider(
+                new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())));
         builder.setHashFunctionFactoryProvider(format.getBinaryHashFunctionFactoryProvider());
         builder.setHashFunctionFamilyProvider(format.getBinaryHashFunctionFamilyProvider());
         builder.setMissingWriterFactory(format.getMissingWriterFactory());

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
index 0c6b2cc..768416d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java
@@ -18,9 +18,9 @@
  */
 package org.apache.asterix.app.cc;
 
-import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 
 import org.apache.asterix.algebra.base.ILangExtension;
@@ -35,22 +35,25 @@ import org.apache.asterix.common.exceptions.RuntimeDataException;
 import org.apache.asterix.compiler.provider.AqlCompilationProvider;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
+import org.apache.asterix.om.functions.IFunctionExtensionManager;
+import org.apache.asterix.om.functions.IFunctionManager;
+import org.apache.asterix.runtime.functions.FunctionCollection;
+import org.apache.asterix.runtime.functions.FunctionManager;
 import org.apache.asterix.translator.IStatementExecutorFactory;
 import org.apache.asterix.utils.ExtensionUtil;
 import org.apache.hyracks.algebricks.common.utils.Pair;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 
 /**
- * AsterixDB's implementation of {@code IAlgebraExtensionManager} which takes care of
- * initializing extensions for App and Compilation purposes
+ * AsterixDB's implementation of {@code IAlgebraExtensionManager} and {@code IFunctionExtensionManager}
+ * which takes care of initializing extensions for App and Compilation purposes
  */
-public class CCExtensionManager implements IAlgebraExtensionManager {
-
-    private final Map<ExtensionId, IExtension> extensions = new HashMap<>();
+public class CCExtensionManager implements IAlgebraExtensionManager, IFunctionExtensionManager {
 
     private final IStatementExecutorExtension statementExecutorExtension;
     private final ILangCompilationProvider aqlCompilationProvider;
     private final ILangCompilationProvider sqlppCompilationProvider;
+    private final IFunctionManager functionManager;
     private transient IStatementExecutorFactory statementExecutorFactory;
 
     /**
@@ -67,15 +70,16 @@ public class CCExtensionManager implements IAlgebraExtensionManager {
             throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException {
         Pair<ExtensionId, ILangCompilationProvider> aqlcp = null;
         Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null;
+        Pair<ExtensionId, IFunctionManager> fm = null;
         IStatementExecutorExtension see = null;
         if (list != null) {
+            Set<ExtensionId> extensionIds = new HashSet<>();
             for (AsterixExtension extensionConf : list) {
                 IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance();
                 extension.configure(extensionConf.getArgs());
-                if (extensions.containsKey(extension.getId())) {
+                if (!extensionIds.add(extension.getId())) {
                     throw new RuntimeDataException(ErrorCode.EXTENSION_ID_CONFLICT, extension.getId());
                 }
-                extensions.put(extension.getId(), extension);
                 switch (extension.getExtensionKind()) {
                     case STATEMENT_EXECUTOR:
                         see = ExtensionUtil.extendStatementExecutor(see, (IStatementExecutorExtension) extension);
@@ -84,6 +88,7 @@ public class CCExtensionManager implements IAlgebraExtensionManager {
                         ILangExtension le = (ILangExtension) extension;
                         aqlcp = ExtensionUtil.extendLangCompilationProvider(Language.AQL, aqlcp, le);
                         sqlppcp = ExtensionUtil.extendLangCompilationProvider(Language.SQLPP, sqlppcp, le);
+                        fm = ExtensionUtil.extendFunctionManager(fm, le);
                         break;
                     default:
                         break;
@@ -93,6 +98,8 @@ public class CCExtensionManager implements IAlgebraExtensionManager {
         this.statementExecutorExtension = see;
         this.aqlCompilationProvider = aqlcp == null ? new AqlCompilationProvider() : aqlcp.second;
         this.sqlppCompilationProvider = sqlppcp == null ? new SqlppCompilationProvider() : sqlppcp.second;
+        this.functionManager =
+                fm == null ? new FunctionManager(FunctionCollection.createDefaultFunctionCollection()) : fm.second;
     }
 
     /** @deprecated use getStatementExecutorFactory instead */
@@ -117,4 +124,9 @@ public class CCExtensionManager implements IAlgebraExtensionManager {
             default: throw new IllegalArgumentException(String.valueOf(lang));
         }
     }
+
+    @Override
+    public IFunctionManager getFunctionManager() {
+        return functionManager;
+    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
index 07eed0d..380c5a9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java
@@ -26,6 +26,7 @@ import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.metadata.api.IMetadataExtension;
+import org.apache.asterix.om.functions.IFunctionManager;
 import org.apache.asterix.translator.IStatementExecutorFactory;
 import org.apache.hyracks.algebricks.common.utils.Pair;
 
@@ -52,18 +53,39 @@ public class ExtensionUtil {
      */
     public static Pair<ExtensionId, ILangCompilationProvider> extendLangCompilationProvider(Language lang,
             Pair<ExtensionId, ILangCompilationProvider> cp, ILangExtension le) throws RuntimeDataException {
-        if (cp != null && le.getLangCompilationProvider(lang) != null) {
+        ILangCompilationProvider lecp = le.getLangCompilationProvider(lang);
+        if (cp != null && lecp != null) {
             throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, le.getId(), cp.first,
                     lang.toString());
         }
-        return (le.getLangCompilationProvider(lang) != null)
-                ? new Pair<>(le.getId(), le.getLangCompilationProvider(lang)) : cp;
+        return lecp != null ? new Pair<>(le.getId(), lecp) : cp;
+    }
+
+    /**
+     * Validate no extension conflict and return function manager extension
+     *
+     * @param fm
+     *            place holder for function manager extension
+     * @param le
+     *            user defined extension
+     * @return the user defined extension
+     * @throws RuntimeDataException
+     *             if extension conflict was detected
+     */
+    public static Pair<ExtensionId, IFunctionManager> extendFunctionManager(Pair<ExtensionId, IFunctionManager> fm,
+            ILangExtension le) throws RuntimeDataException {
+        IFunctionManager lefm = le.getFunctionManager();
+        if (fm != null && lefm != null) {
+            throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, le.getId(), fm.first,
+                    IFunctionManager.class.getSimpleName());
+        }
+        return lefm != null ? new Pair<>(le.getId(), lefm) : fm;
     }
 
     /**
      * Validate no extension conflict and return statement executor extension
      *
-     * @param qte
+     * @param see
      *            place holder for statement executor extension
      * @param extension
      *            user defined extension
@@ -71,10 +93,10 @@ public class ExtensionUtil {
      * @throws RuntimeDataException
      *             if extension conflict was detected
      */
-    public static IStatementExecutorExtension extendStatementExecutor(IStatementExecutorExtension qte,
+    public static IStatementExecutorExtension extendStatementExecutor(IStatementExecutorExtension see,
             IStatementExecutorExtension extension) throws RuntimeDataException {
-        if (qte != null) {
-            throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, qte.getId(), extension.getId(),
+        if (see != null) {
+            throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, see.getId(), extension.getId(),
                     IStatementExecutorFactory.class.getSimpleName());
         }
         return extension;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
index 3c54c7c..4bcee04 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java
@@ -43,7 +43,8 @@ public class ExceptionIT {
 
     @Test
     public void test() throws Exception {
-        List<IFunctionDescriptorFactory> functions = FunctionCollection.getFunctionDescriptorFactories();
+        List<IFunctionDescriptorFactory> functions =
+                FunctionCollection.createDefaultFunctionCollection().getFunctionDescriptorFactories();
         int testedFunctions = 0;
         for (IFunctionDescriptorFactory func : functions) {
             String className = func.getClass().getName();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
index f5e7402..3b33868 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
@@ -43,7 +43,8 @@ public class NullMissingTest {
 
     @Test
     public void test() throws Exception {
-        List<IFunctionDescriptorFactory> functions = FunctionCollection.getFunctionDescriptorFactories();
+        List<IFunctionDescriptorFactory> functions =
+                FunctionCollection.createDefaultFunctionCollection().getFunctionDescriptorFactories();
         int testedFunctions = 0;
         for (IFunctionDescriptorFactory func : functions) {
             String className = func.getClass().getName();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
index b1191ec..40d4f6a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
@@ -45,6 +45,9 @@ import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Feed;
 import org.apache.asterix.metadata.lock.MetadataLockManager;
+import org.apache.asterix.om.functions.IFunctionExtensionManager;
+import org.apache.asterix.runtime.functions.FunctionCollection;
+import org.apache.asterix.runtime.functions.FunctionManager;
 import org.apache.asterix.runtime.utils.CcApplicationContext;
 import org.apache.asterix.test.active.TestEventsListener.Behavior;
 import org.apache.asterix.test.base.TestMethodTracer;
@@ -82,6 +85,7 @@ public class ActiveEventsListenerTest {
     static CcApplicationContext appCtx;
     static IStatementExecutor statementExecutor;
     static IHyracksClientConnection hcc;
+    static IFunctionExtensionManager functionExtensionManager;
     static MetadataProvider metadataProvider;
     static IStorageComponentProvider componentProvider;
     static JobIdFactory jobIdFactory;
@@ -121,6 +125,10 @@ public class ActiveEventsListenerTest {
         Mockito.when(ccServiceCtx.getControllerService()).thenReturn(ccService);
         Mockito.when(ccService.getExecutor()).thenReturn(executor);
         locations = new AlgebricksAbsolutePartitionConstraint(nodes);
+        functionExtensionManager = Mockito.mock(IFunctionExtensionManager.class);
+        Mockito.when(functionExtensionManager.getFunctionManager())
+                .thenReturn(new FunctionManager(FunctionCollection.createDefaultFunctionCollection()));
+        Mockito.when(appCtx.getExtensionManager()).thenReturn(functionExtensionManager);
         metadataProvider = new MetadataProvider(appCtx, null);
         clusterController = new TestClusterControllerActor("CC", handler, allDatasets);
         nodeControllers = new TestNodeControllerActor[2];
@@ -1449,6 +1457,10 @@ public class ActiveEventsListenerTest {
             CcApplicationContext ccAppCtx = Mockito.mock(CcApplicationContext.class);
             IStatementExecutor statementExecutor = Mockito.mock(IStatementExecutor.class);
             IHyracksClientConnection hcc = Mockito.mock(IHyracksClientConnection.class);
+            IFunctionExtensionManager functionExtensionManager = Mockito.mock(IFunctionExtensionManager.class);
+            Mockito.when(functionExtensionManager.getFunctionManager())
+                    .thenReturn(new FunctionManager(FunctionCollection.createDefaultFunctionCollection()));
+            Mockito.when(ccAppCtx.getExtensionManager()).thenReturn(functionExtensionManager);
             Mockito.when(ccAppCtx.getActiveNotificationHandler()).thenReturn(handler);
             Mockito.when(ccAppCtx.getMetadataLockManager()).thenReturn(lockManager);
             Mockito.when(ccAppCtx.getServiceContext()).thenReturn(ccServiceCtx);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index e2b1761..3f3a27f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -31,7 +31,6 @@ import org.apache.asterix.metadata.entities.InternalDatasetDetails;
 import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -107,8 +106,7 @@ public class DatasetDataSource extends DataSource {
                 ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
                 IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset,
                         edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null);
-                return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
-                        NonTaggedDataFormat.INSTANCE);
+                return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory);
             case INTERNAL:
                 DataSourceId id = getId();
                 String dataverseName = id.getDataverseName();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
index 0b6608c..325d23b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
@@ -33,7 +33,6 @@ import org.apache.asterix.metadata.entities.FeedPolicyEntity;
 import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -170,7 +169,7 @@ public class FeedDataSource extends DataSource implements IMutationDataSource {
         try {
             ARecordType feedOutputType = (ARecordType) itemType;
             ISerializerDeserializer payloadSerde =
-                    NonTaggedDataFormat.INSTANCE.getSerdeProvider().getSerializerDeserializer(feedOutputType);
+                    metadataProvider.getDataFormat().getSerdeProvider().getSerializerDeserializer(feedOutputType);
             ArrayList<ISerializerDeserializer> serdes = new ArrayList<>();
             serdes.add(payloadSerde);
             if (metaItemType != null) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 9271f33..2f94ad7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -57,6 +57,7 @@ import org.apache.asterix.external.provider.AdapterFactoryProvider;
 import org.apache.asterix.external.util.ExternalDataConstants;
 import org.apache.asterix.external.util.FeedConstants;
 import org.apache.asterix.formats.base.IDataFormat;
+import org.apache.asterix.om.functions.IFunctionExtensionManager;
 import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
 import org.apache.asterix.formats.nontagged.LinearizeComparatorFactoryProvider;
 import org.apache.asterix.formats.nontagged.TypeTraitProvider;
@@ -78,6 +79,7 @@ import org.apache.asterix.metadata.utils.DatasetUtil;
 import org.apache.asterix.metadata.utils.MetadataConstants;
 import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
 import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionManager;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.ATypeTag;
 import org.apache.asterix.om.types.IAType;
@@ -143,6 +145,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
     private final ICcApplicationContext appCtx;
     private final IStorageComponentProvider storageComponentProvider;
     private final StorageProperties storageProperties;
+    private final IFunctionManager functionManager;
     private final Dataverse defaultDataverse;
     private final LockList locks;
     private final Map<String, String> config;
@@ -164,6 +167,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
         this.defaultDataverse = defaultDataverse == null ? MetadataBuiltinEntities.DEFAULT_DATAVERSE : defaultDataverse;
         this.storageComponentProvider = appCtx.getStorageComponentProvider();
         storageProperties = appCtx.getStorageProperties();
+        functionManager = ((IFunctionExtensionManager) appCtx.getExtensionManager()).getFunctionManager();
         locks = new LockList();
         config = new HashMap<>();
     }
@@ -259,7 +263,11 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
         return isTemporaryDatasetWriteJob;
     }
 
-    public IDataFormat getFormat() {
+    public IFunctionManager getFunctionManager() {
+        return functionManager;
+    }
+
+    public IDataFormat getDataFormat() {
         return FormatUtils.getDefaultFormat();
     }
 
@@ -868,13 +876,14 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
     }
 
     public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDatasetDataScannerRuntime(
-            JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory, IDataFormat format)
+            JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory)
             throws AlgebricksException {
         if (itemType.getTypeTag() != ATypeTag.OBJECT) {
             throw new AlgebricksException("Can only scan datasets of records.");
         }
 
-        ISerializerDeserializer<?> payloadSerde = format.getSerdeProvider().getSerializerDeserializer(itemType);
+        ISerializerDeserializer<?> payloadSerde =
+                getDataFormat().getSerdeProvider().getSerializerDeserializer(itemType);
         RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { payloadSerde });
 
         ExternalScanOperatorDescriptor dataScanner =
@@ -1501,7 +1510,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
             // Generate Output Record format
             ISerializerDeserializer<?>[] tokenKeyPairFields = new ISerializerDeserializer[numTokenKeyPairFields];
             ITypeTraits[] tokenKeyPairTypeTraits = new ITypeTraits[numTokenKeyPairFields];
-            ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
+            ISerializerDeserializerProvider serdeProvider = getDataFormat().getSerdeProvider();
 
             // The order of the output record: propagated variables (including
             // PK and SK), token, and number of token.

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 9156b0f..8e1c34d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -730,7 +730,7 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset {
         ISerializerDeserializer[] primaryRecFields =
                 new ISerializerDeserializer[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)];
         ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)];
-        ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
+        ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
         List<Integer> indicators = null;
         if (hasMetaPart()) {
             indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index e4a6ca8..53c3dc0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -41,6 +41,7 @@ import org.apache.asterix.common.exceptions.MetadataException;
 import org.apache.asterix.common.transactions.IRecoveryManager;
 import org.apache.asterix.common.transactions.JobId;
 import org.apache.asterix.external.indexing.IndexingConstants;
+import org.apache.asterix.formats.base.IDataFormat;
 import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
 import org.apache.asterix.formats.nontagged.TypeTraitProvider;
 import org.apache.asterix.metadata.MetadataManager;
@@ -57,7 +58,6 @@ import org.apache.asterix.om.base.AString;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.runtime.formats.FormatUtils;
 import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
 import org.apache.asterix.runtime.operators.LSMPrimaryUpsertOperatorDescriptor;
 import org.apache.asterix.runtime.utils.RuntimeUtils;
@@ -419,16 +419,16 @@ public class DatasetUtil {
                     + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
             ISerializerDeserializer<?>[] outputSerDes = new ISerializerDeserializer[inputRecordDesc.getFieldCount()
                     + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
+            IDataFormat dataFormat = metadataProvider.getDataFormat();
 
             // add the previous record first
             int f = 0;
-            outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType);
+            outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(itemType);
             f++;
             // add the previous meta second
             if (dataset.hasMetaPart()) {
-                outputSerDes[f] =
-                        FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(metaItemType);
-                outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(metaItemType);
+                outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(metaItemType);
+                outputTypeTraits[f] = dataFormat.getTypeTraitProvider().getTypeTrait(metaItemType);
                 f++;
             }
             // add the previous filter third
@@ -443,10 +443,10 @@ public class DatasetUtil {
                     }
                 }
                 fieldIdx = i;
-                outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider()
-                        .getTypeTrait(itemType.getFieldTypes()[fieldIdx]);
-                outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider()
-                        .getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]);
+                outputTypeTraits[f] =
+                        dataFormat.getTypeTraitProvider().getTypeTrait(itemType.getFieldTypes()[fieldIdx]);
+                outputSerDes[f] =
+                        dataFormat.getSerdeProvider().getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]);
                 f++;
             }
             for (int j = 0; j < inputRecordDesc.getFieldCount(); j++) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 445fbd8..78d9c19 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -212,10 +212,10 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations
         ITypeTraits[] enforcedTypeTraits =
                 new ITypeTraits[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields];
         secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
-        ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
-        ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider();
+        ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
+        ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider();
         IBinaryComparatorFactoryProvider comparatorFactoryProvider =
-                metadataProvider.getFormat().getBinaryComparatorFactoryProvider();
+                metadataProvider.getDataFormat().getBinaryComparatorFactoryProvider();
         // Record column is 0 for external datasets, numPrimaryKeys for internal ones
         int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
         boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes();
@@ -230,9 +230,9 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations
                 sourceType = metaType;
                 sourceColumn = recordColumn + 1;
             }
-            secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(
-                    isOverridingKeyFieldTypes ? enforcedItemType : sourceType, index.getKeyFieldNames().get(i),
-                    sourceColumn);
+            secondaryFieldAccessEvalFactories[i] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+                    metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : sourceType,
+                    index.getKeyFieldNames().get(i), sourceColumn);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i),
                     index.getKeyFieldNames().get(i), sourceType);
             IAType keyType = keyTypePair.first;
@@ -270,8 +270,9 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations
         }
 
         if (numFilterFields > 0) {
-            secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getFormat()
-                    .getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys);
+            secondaryFieldAccessEvalFactories[numSecondaryKeys] =
+                    metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+                            metadataProvider.getFunctionManager(), itemType, filterFieldName, numPrimaryKeys);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
             IAType type = keyTypePair.first;
             ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
index 96fdf27..8ef5f34 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
@@ -137,10 +137,10 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate
         ITypeTraits[] enforcedTypeTraits =
                 new ITypeTraits[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields];
         secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
-        ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
-        ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider();
+        ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
+        ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider();
         IBinaryComparatorFactoryProvider comparatorFactoryProvider =
-                metadataProvider.getFormat().getBinaryComparatorFactoryProvider();
+                metadataProvider.getDataFormat().getBinaryComparatorFactoryProvider();
         // Record column is 0 for external datasets, numPrimaryKeys for internal ones
         int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
         boolean isOverridingKeyTypes = index.isOverridingKeyFieldTypes();
@@ -155,9 +155,9 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate
                 sourceType = metaType;
                 sourceColumn = recordColumn + 1;
             }
-            secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(
-                    isOverridingKeyTypes ? enforcedItemType : sourceType, index.getKeyFieldNames().get(i),
-                    sourceColumn);
+            secondaryFieldAccessEvalFactories[i] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+                    metadataProvider.getFunctionManager(), isOverridingKeyTypes ? enforcedItemType : sourceType,
+                    index.getKeyFieldNames().get(i), sourceColumn);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i),
                     index.getKeyFieldNames().get(i), sourceType);
             IAType keyType = keyTypePair.first;
@@ -185,8 +185,9 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate
         }
 
         if (numFilterFields > 0) {
-            secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getFormat()
-                    .getFieldAccessEvaluatorFactory(itemType, filterFieldName, recordColumn);
+            secondaryFieldAccessEvalFactories[numSecondaryKeys] =
+                    metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+                            metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
             IAType type = keyTypePair.first;
             ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
index 15f8a23..262b259 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
@@ -29,7 +29,6 @@ import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.om.utils.NonTaggedFormatUtil;
 import org.apache.asterix.om.utils.RecordUtil;
-import org.apache.asterix.runtime.formats.FormatUtils;
 import org.apache.asterix.runtime.utils.RuntimeUtils;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -108,13 +107,13 @@ public class SecondaryCorrelatedInvertedIndexOperationsHelper extends SecondaryC
         ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields];
         secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
         ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys];
-        ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
-        ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider();
+        ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
+        ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider();
         int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
         if (numSecondaryKeys > 0) {
-            secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(
-                    isOverridingKeyFieldTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0),
-                    recordColumn);
+            secondaryFieldAccessEvalFactories[0] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+                    metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+                    index.getKeyFieldNames().get(0), recordColumn);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
                     index.getKeyFieldNames().get(0), itemType);
             secondaryKeyType = keyTypePair.first;
@@ -124,8 +123,9 @@ public class SecondaryCorrelatedInvertedIndexOperationsHelper extends SecondaryC
             secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
         }
         if (numFilterFields > 0) {
-            secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat()
-                    .getFieldAccessEvaluatorFactory(itemType, filterFieldName, recordColumn);
+            secondaryFieldAccessEvalFactories[numSecondaryKeys] =
+                    metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+                            metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
             IAType type = keyTypePair.first;
             ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
index 8fd8a7a..b663b42 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
@@ -89,9 +89,9 @@ public class SecondaryCorrelatedRTreeOperationsHelper extends SecondaryCorrelate
         int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
         numNestedSecondaryKeyFields = numDimensions * 2;
         int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys;
-        secondaryFieldAccessEvalFactories =
-                metadataProvider.getFormat().createMBRFactory(isOverridingKeyFieldTypes ? enforcedItemType : itemType,
-                        secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
+        secondaryFieldAccessEvalFactories = metadataProvider.getDataFormat().createMBRFactory(
+                metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+                secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
         secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
         valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
         ISerializerDeserializer[] secondaryRecFields =

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
index 171d72e..b99ae2f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java
@@ -32,7 +32,6 @@ import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.om.functions.BuiltinFunctions;
 import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.runtime.functions.FunctionManagerHolder;
 import org.apache.asterix.runtime.operators.LSMSecondaryIndexBulkLoadOperatorDescriptor;
 import org.apache.asterix.runtime.operators.LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor;
 import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory;
@@ -155,10 +154,6 @@ public abstract class SecondaryCorrelatedTreeIndexOperationsHelper extends Secon
     @Override
     protected AlgebricksMetaOperatorDescriptor createCastOp(JobSpecification spec, DatasetType dsType,
             boolean strictCast) throws AlgebricksException {
-        IFunctionDescriptor castFuncDesc = FunctionManagerHolder.getFunctionManager()
-                .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX);
-        castFuncDesc.setImmutableStates(enforcedItemType, itemType);
-
         int[] outColumns = new int[1];
 
         // tags(2) + primary keys + record + meta part(?)
@@ -185,7 +180,7 @@ public abstract class SecondaryCorrelatedTreeIndexOperationsHelper extends Secon
         IScalarEvaluatorFactory[] castEvalFact =
                 new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) };
         IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1];
-        sefs[0] = castFuncDesc.createEvaluatorFactory(castEvalFact);
+        sefs[0] = createCastFunction(strictCast).createEvaluatorFactory(castEvalFact);
         AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
         return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign },
                 new RecordDescriptor[] { getTaggedRecordDescriptor(enforcedRecDesc) });

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index 98c47a2..5dac407 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -22,7 +22,6 @@ package org.apache.asterix.metadata.utils;
 import static org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption;
 
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -52,7 +51,6 @@ import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.runtime.evaluators.functions.AndDescriptor;
 import org.apache.asterix.runtime.evaluators.functions.IsUnknownDescriptor;
 import org.apache.asterix.runtime.evaluators.functions.NotDescriptor;
-import org.apache.asterix.runtime.functions.FunctionManagerHolder;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -239,7 +237,7 @@ public abstract class SecondaryIndexOperationsHelper {
         ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)];
         primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys];
         primaryBloomFilterKeyFields = new int[numPrimaryKeys];
-        ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
+        ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
         List<Integer> indicators = null;
         if (dataset.hasMetaPart()) {
             indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
@@ -298,10 +296,6 @@ public abstract class SecondaryIndexOperationsHelper {
 
     protected AlgebricksMetaOperatorDescriptor createCastOp(JobSpecification spec, DatasetType dsType,
             boolean strictCast) throws AlgebricksException {
-        IFunctionDescriptor castFuncDesc = FunctionManagerHolder.getFunctionManager()
-                .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX);
-        castFuncDesc.setImmutableStates(enforcedItemType, itemType);
-
         int[] outColumns = new int[1];
         int[] projectionList = new int[(dataset.hasMetaPart() ? 2 : 1) + numPrimaryKeys];
         int recordIdx;
@@ -322,12 +316,19 @@ public abstract class SecondaryIndexOperationsHelper {
         IScalarEvaluatorFactory[] castEvalFact =
                 new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) };
         IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1];
-        sefs[0] = castFuncDesc.createEvaluatorFactory(castEvalFact);
+        sefs[0] = createCastFunction(strictCast).createEvaluatorFactory(castEvalFact);
         AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
         return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign },
                 new RecordDescriptor[] { enforcedRecDesc });
     }
 
+    protected IFunctionDescriptor createCastFunction(boolean strictCast) throws AlgebricksException {
+        IFunctionDescriptor castFuncDesc = metadataProvider.getFunctionManager()
+                .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX);
+        castFuncDesc.setImmutableStates(enforcedItemType, itemType);
+        return castFuncDesc;
+    }
+
     protected ExternalSortOperatorDescriptor createSortOp(JobSpecification spec,
             IBinaryComparatorFactory[] secondaryComparatorFactories, RecordDescriptor secondaryRecDesc) {
         int[] sortFields = new int[secondaryComparatorFactories.length];

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
index b4d8a22..077e431 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
@@ -30,7 +30,6 @@ import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.om.utils.NonTaggedFormatUtil;
 import org.apache.asterix.om.utils.RecordUtil;
-import org.apache.asterix.runtime.formats.FormatUtils;
 import org.apache.asterix.runtime.utils.RuntimeUtils;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -111,12 +110,12 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryTreeIndexOp
         ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields];
         secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
         ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys];
-        ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
-        ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider();
+        ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider();
+        ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider();
         if (numSecondaryKeys > 0) {
-            secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(
-                    isOverridingKeyFieldTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0),
-                    numPrimaryKeys);
+            secondaryFieldAccessEvalFactories[0] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+                    metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+                    index.getKeyFieldNames().get(0), numPrimaryKeys);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
                     index.getKeyFieldNames().get(0), itemType);
             secondaryKeyType = keyTypePair.first;
@@ -126,8 +125,9 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryTreeIndexOp
             secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
         }
         if (numFilterFields > 0) {
-            secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat()
-                    .getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys);
+            secondaryFieldAccessEvalFactories[numSecondaryKeys] =
+                    metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
+                            metadataProvider.getFunctionManager(), itemType, filterFieldName, numPrimaryKeys);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
             IAType type = keyTypePair.first;
             ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
index 08ee0aa..1726470 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
@@ -99,9 +99,9 @@ public class SecondaryRTreeOperationsHelper extends SecondaryTreeIndexOperations
         int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
         numNestedSecondaryKeyFields = numDimensions * 2;
         int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
-        secondaryFieldAccessEvalFactories =
-                metadataProvider.getFormat().createMBRFactory(isOverridingKeyFieldTypes ? enforcedItemType : itemType,
-                        secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
+        secondaryFieldAccessEvalFactories = metadataProvider.getDataFormat().createMBRFactory(
+                metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType,
+                secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR);
         secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
         valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
         ISerializerDeserializer[] secondaryRecFields =

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java
index 7624a8a..678a864 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java
@@ -20,16 +20,13 @@ package org.apache.asterix.formats.base;
 
 import java.util.List;
 
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.functions.IFunctionManager;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Triple;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
-import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
 import org.apache.hyracks.algebricks.data.IBinaryBooleanInspectorFactory;
 import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
@@ -69,18 +66,16 @@ public interface IDataFormat {
     public IMissingWriterFactory getMissingWriterFactory();
 
     public Triple<IScalarEvaluatorFactory, ScalarFunctionCallExpression, IAType> partitioningEvaluatorFactory(
-            ARecordType recType, List<String> fldName) throws AlgebricksException;
+            IFunctionManager functionManager, ARecordType recType, List<String> fldName) throws AlgebricksException;
 
-    public IScalarEvaluatorFactory getFieldAccessEvaluatorFactory(ARecordType recType, List<String> fldName,
-            int recordColumn) throws AlgebricksException;
-
-    public IFunctionDescriptor resolveFunction(ILogicalExpression expr, IVariableTypeEnvironment typeEnvironment)
-            throws AlgebricksException;
+    public IScalarEvaluatorFactory getFieldAccessEvaluatorFactory(IFunctionManager functionManager, ARecordType recType,
+            List<String> fldName, int recordColumn) throws AlgebricksException;
 
     public IScalarEvaluatorFactory getConstantEvalFactory(IAlgebricksConstantValue value) throws AlgebricksException;
 
-    public IScalarEvaluatorFactory[] createMBRFactory(ARecordType recType, List<String> fldName, int recordColumn,
-            int dimension, List<String> filterFieldName, boolean isPointMBR) throws AlgebricksException;
+    public IScalarEvaluatorFactory[] createMBRFactory(IFunctionManager functionManager, ARecordType recType,
+            List<String> fldName, int recordColumn, int dimension, List<String> filterFieldName, boolean isPointMBR)
+            throws AlgebricksException;
 
     public IExpressionEvalSizeComputer getExpressionEvalSizeComputer();
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java
index 526eab6..0e746f9 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java
@@ -26,5 +26,13 @@ public interface IFunctionDescriptorFactory {
      *
      * @return a new IFunctionDescriptor instance
      */
-    public IFunctionDescriptor createFunctionDescriptor();
+    IFunctionDescriptor createFunctionDescriptor();
+
+    /**
+     * Creates type inferer for this function
+     * @return an new instance of the type inferer or {@code null} if not provided
+     */
+    default IFunctionTypeInferer createFunctionTypeInferer() {
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java
new file mode 100644
index 0000000..fb8155a
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.functions;
+
+/**
+ * An extension point for built-in functions
+ */
+public interface IFunctionExtensionManager {
+    /**
+     * Returns function manager
+     *
+     * @return a new {@link IFunctionManager} instance
+     */
+    IFunctionManager getFunctionManager();
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
index f7c11ff..e1657ff 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java
@@ -21,11 +21,13 @@ package org.apache.asterix.om.functions;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 
-public interface IFunctionManager extends Iterable<IFunctionDescriptorFactory> {
-
-    public void registerFunction(IFunctionDescriptorFactory descriptorFactory);
+/**
+ * A registry of {@link IFunctionDescriptor} and {@link IFunctionTypeInferer}
+ * for built-in functions
+ */
+public interface IFunctionManager {
 
-    public void unregisterFunction(IFunctionDescriptorFactory descriptorFactory);
+    IFunctionDescriptor lookupFunction(FunctionIdentifier fid) throws AlgebricksException;
 
-    public IFunctionDescriptor lookupFunction(FunctionIdentifier fid) throws AlgebricksException;
+    IFunctionTypeInferer lookupFunctionTypeInferer(FunctionIdentifier fid);
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java
new file mode 100644
index 0000000..b8e27e1
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.functions;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public interface IFunctionTypeInferer {
+    void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context)
+            throws AlgebricksException;
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java
index f117d14..4e35950 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java
@@ -21,22 +21,29 @@ package org.apache.asterix.runtime.aggregates.collections;
 import org.apache.asterix.om.functions.BuiltinFunctions;
 import org.apache.asterix.om.functions.IFunctionDescriptor;
 import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.functions.IFunctionTypeInferer;
 import org.apache.asterix.om.types.AOrderedListType;
 import org.apache.asterix.runtime.aggregates.base.AbstractAggregateFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.functions.FunctionTypeInferers;
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory;
 import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
 
 public class ListifyAggregateDescriptor extends AbstractAggregateFunctionDynamicDescriptor {
 
-    private static final long serialVersionUID = 1L;
     public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
         @Override
         public IFunctionDescriptor createFunctionDescriptor() {
             return new ListifyAggregateDescriptor();
         }
+
+        @Override
+        public IFunctionTypeInferer createFunctionTypeInferer() {
+            return FunctionTypeInferers.SET_EXPRESSION_TYPE;
+        }
     };
 
+    private static final long serialVersionUID = 1L;
     private AOrderedListType oltype;
 
     @Override
@@ -53,5 +60,4 @@ public class ListifyAggregateDescriptor extends AbstractAggregateFunctionDynamic
     public IAggregateEvaluatorFactory createAggregateEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
         return new ListifyAggregateFunctionEvalFactory(args, oltype);
     }
-
 }


Mime
View raw message