asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ti...@apache.org
Subject [2/6] incubator-asterixdb git commit: ASTERIXDB-1187, ASTERIXDB-1162 and ASTERIXDB-1252 fixes, plus new internal functions
Date Mon, 11 Jan 2016 16:20:47 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddFieldsTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddFieldsTypeComputer.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddFieldsTypeComputer.java
new file mode 100644
index 0000000..4f8d1f2
--- /dev/null
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordAddFieldsTypeComputer.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.typecomputer.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeHelper;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public class RecordAddFieldsTypeComputer implements IResultTypeComputer {
+    public static final RecordAddFieldsTypeComputer INSTANCE = new RecordAddFieldsTypeComputer();
+
+    private static final String FIELD_NAME_NAME = "field-name";
+    private static final String FIELD_VALUE_VALUE = "field-value";
+
+    private RecordAddFieldsTypeComputer() {
+    }
+
+    @Override
+    public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
+            IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
+
+        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
+        IAType type0 = (IAType) env.getType(funcExpr.getArguments().get(0).getValue());
+
+        ARecordType inputRecordType = TypeComputerUtils.extractRecordType(type0);
+        if (inputRecordType == null) {
+            throw new AlgebricksException("Input record cannot be null");
+        }
+
+        AbstractLogicalExpression arg1 = (AbstractLogicalExpression) funcExpr.getArguments().get(1).getValue();
+        IAType type1 = (IAType) env.getType(arg1);
+        AOrderedListType inputOrderedListType = TypeComputerUtils.extractOrderedListType(type1);
+        if (inputOrderedListType == null) {
+            return inputRecordType;
+        }
+
+        boolean nullable = TypeHelper.canBeNull(type0) || TypeHelper.canBeNull(type1);
+        Map<String, IAType> additionalFields = new HashMap<>();
+        List<String> resultFieldNames = new ArrayList<>();
+        List<IAType> resultFieldTypes = new ArrayList<>();
+
+        resultFieldNames.addAll(Arrays.asList(inputRecordType.getFieldNames()));
+        Collections.sort(resultFieldNames);
+
+        for (String fieldName : resultFieldNames) {
+            try {
+                if (inputRecordType.getFieldType(fieldName).getTypeTag() == ATypeTag.RECORD) {
+                    ARecordType nestedType = (ARecordType) inputRecordType.getFieldType(fieldName);
+                    //Deep Copy prevents altering of input types
+                    resultFieldTypes.add(nestedType.deepCopy(nestedType));
+                } else {
+                    resultFieldTypes.add(inputRecordType.getFieldType(fieldName));
+                }
+            } catch (IOException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+
+        if (!containsVariable(arg1)) {
+            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) arg1;
+            List<Mutable<ILogicalExpression>> args = f.getArguments();
+
+            AString fieldName = null;
+            IAType fieldType = null;
+
+            // Iterating through the orderlist input
+            for (Mutable<ILogicalExpression> arg : args) {
+                AbstractFunctionCallExpression recConsExpr = (AbstractFunctionCallExpression) arg.getValue();
+                ARecordType rtype = TypeComputerUtils.extractRecordType((IAType) env.getType(recConsExpr));
+                if (rtype != null) {
+                    String[] fn = rtype.getFieldNames();
+                    IAType[] ft = rtype.getFieldTypes();
+                    for (int j = 0; j < fn.length; j++) {
+                        if (fn[j].equals(FIELD_NAME_NAME)) {
+                            ILogicalExpression fieldNameExpr = recConsExpr.getArguments().get(j).getValue();
+                            switch (fieldNameExpr.getExpressionTag()) {
+                                case CONSTANT: // Top fields only
+                                    IAObject object = ((AsterixConstantValue) ((ConstantExpression) fieldNameExpr)
+                                            .getValue()).getObject();
+                                    if (object.getType().getTypeTag() == ATypeTag.STRING) {
+                                        // Get the actual "field-name" string
+                                        ILogicalExpression recFieldExpr = recConsExpr.getArguments().get(j + 1)
+                                                .getValue();
+                                        if (recFieldExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+                                            fieldName = (AString) ((AsterixConstantValue) ((ConstantExpression) recFieldExpr)
+                                                    .getValue()).getObject();
+                                        }
+                                    }
+                                    break;
+                                default:
+                                    throw new AlgebricksException(fieldNameExpr + " is not supported.");
+                            }
+                        } else if (fn[j].equals(FIELD_VALUE_VALUE)) {
+                            fieldType = ft[j];
+                        }
+                    }
+                    if (fieldName != null) {
+                        additionalFields.put(fieldName.getStringValue(), fieldType);
+                    }
+                }
+            }
+
+            if (!additionalFields.isEmpty()) {
+                Iterator<Map.Entry<String, IAType>> it = additionalFields.entrySet().iterator();
+                while (it.hasNext()) {
+                    Map.Entry<String, IAType> entry = it.next();
+                    resultFieldNames.add(entry.getKey());
+                    resultFieldTypes.add(entry.getValue());
+                }
+            }
+        } // If variable ignore, deal with the addition at runtime
+
+        String resultTypeName = "appended(" + inputRecordType.getTypeName() + ")";
+        int n = resultFieldNames.size();
+        IAType resultType = new ARecordType(resultTypeName, resultFieldNames.toArray(new String[n]),
+                resultFieldTypes.toArray(new IAType[n]), true);
+        if (nullable) {
+            resultType = AUnionType.createNullableType(resultType);
+        }
+        return resultType;
+    }
+
+    // Handle variable as input
+    private boolean containsVariable(ILogicalExpression expression) {
+        if (expression.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expression;
+            List<Mutable<ILogicalExpression>> args = f.getArguments();
+            for (Mutable<ILogicalExpression> arg : args) {
+                ILogicalExpression subExpression = arg.getValue();
+                switch (subExpression.getExpressionTag()) {
+                    case VARIABLE:
+                        return true;
+                    case CONSTANT:
+                        return false;
+                    default: //FUNCTION_CALL
+                        return containsVariable(subExpression);
+                }
+            }
+        }
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java
index 83f4f44..653b26e 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordMergeTypeComputer.java
@@ -23,7 +23,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
 import org.apache.asterix.om.types.ARecordType;
@@ -44,21 +43,6 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
     private RecordMergeTypeComputer() {
     }
 
-    public static ARecordType extractRecordType(IAType t) {
-        if (t.getTypeTag() == ATypeTag.RECORD) {
-            return (ARecordType) t;
-        }
-
-        if (t.getTypeTag() == ATypeTag.UNION) {
-            IAType innerType = ((AUnionType) t).getNullableType();
-            if (innerType.getTypeTag() == ATypeTag.RECORD) {
-                return (ARecordType) innerType;
-            }
-        }
-
-        return null;
-    }
-
     @Override
     public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
             IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
@@ -66,12 +50,12 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
         IAType t0 = (IAType) env.getType(f.getArguments().get(0).getValue());
         IAType t1 = (IAType) env.getType(f.getArguments().get(1).getValue());
         boolean nullable = TypeHelper.canBeNull(t0) || TypeHelper.canBeNull(t1);
-        ARecordType recType0 = extractRecordType(t0);
-        ARecordType recType1 = extractRecordType(t1);
+        ARecordType recType0 = TypeComputerUtils.extractRecordType(t0);
+        ARecordType recType1 = TypeComputerUtils.extractRecordType(t1);
 
         if (recType0 == null || recType1 == null) {
-            throw new AlgebricksException(
-                    "record-merge expects possibly NULL records as arguments, but got (" + t0 + ", " + t1 + ")");
+            throw new AlgebricksException("record-merge expects possibly NULL records as arguments, but got (" + t0
+                    + ", " + t1 + ")");
         }
 
         List<String> resultFieldNames = new ArrayList<>();
@@ -79,6 +63,7 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
             resultFieldNames.add(fieldName);
         }
         Collections.sort(resultFieldNames);
+
         List<IAType> resultFieldTypes = new ArrayList<>();
         for (String fieldName : resultFieldNames) {
             try {
@@ -96,20 +81,27 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
 
         List<String> additionalFieldNames = new ArrayList<>();
         List<IAType> additionalFieldTypes = new ArrayList<>();
-        for (int i = 0; i < recType1.getFieldNames().length; ++i) {
-            String fieldName = recType1.getFieldNames()[i];
-            IAType fieldType = recType1.getFieldTypes()[i];
-            int pos = Collections.binarySearch(resultFieldNames, fieldName);
+        String fieldNames[] = recType1.getFieldNames();
+        IAType fieldTypes[] = recType1.getFieldTypes();
+        for (int i = 0; i < fieldNames.length; ++i) {
+            int pos = Collections.binarySearch(resultFieldNames, fieldNames[i]);
             if (pos >= 0) {
+                IAType resultFieldType = resultFieldTypes.get(pos);
+                if (resultFieldType.getTypeTag() != fieldTypes[i].getTypeTag()) {
+                    throw new AlgebricksException("Duplicate field " + fieldNames[i] + " encountered");
+                }
                 try {
-                    resultFieldTypes.set(pos, mergedNestedType(fieldType, resultFieldTypes.get(pos)));
+                    // Assuming fieldTypes[i].getTypeTag() = resultFieldType.getTypeTag()
+                    if (fieldTypes[i].getTypeTag() == ATypeTag.RECORD) {
+                        resultFieldTypes.set(pos, mergedNestedType(fieldTypes[i], resultFieldType));
+                    }
                 } catch (AsterixException e) {
                     throw new AlgebricksException(e);
                 }
 
             } else {
-                additionalFieldNames.add(fieldName);
-                additionalFieldTypes.add(fieldType);
+                additionalFieldNames.add(fieldNames[i]);
+                additionalFieldTypes.add(fieldTypes[i]);
             }
         }
 
@@ -117,9 +109,9 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
         resultFieldTypes.addAll(additionalFieldTypes);
         String resultTypeName = "merged(" + recType0.getTypeName() + ", " + recType1.getTypeName() + ")";
         boolean isOpen = recType0.isOpen() || recType1.isOpen();
-        IAType resultType = null;
-        resultType = new ARecordType(resultTypeName, resultFieldNames.toArray(new String[] {}),
-                resultFieldTypes.toArray(new IAType[] {}), isOpen);
+
+        IAType resultType = new ARecordType(resultTypeName, resultFieldNames.toArray(new String[] {}),
+                    resultFieldTypes.toArray(new IAType[] {}), isOpen);
 
         if (nullable) {
             resultType = AUnionType.createNullableType(resultType);
@@ -127,32 +119,32 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
         return resultType;
     }
 
-    IAType mergedNestedType(IAType fieldType1, IAType fieldType0) throws AlgebricksException, AsterixException {
+    private IAType mergedNestedType(IAType fieldType1, IAType fieldType0) throws AlgebricksException, AsterixException {
         if (fieldType1.getTypeTag() != ATypeTag.RECORD || fieldType0.getTypeTag() != ATypeTag.RECORD) {
-            throw new AlgebricksException("Duplicate field \"" + fieldType1.getTypeName() + "\" encountered");
+            throw new AlgebricksException("Duplicate field " + fieldType1.getTypeName() + " encountered");
         }
 
-        ARecordType returnType = (ARecordType) fieldType0;
+        ARecordType resultType = (ARecordType) fieldType0;
         ARecordType fieldType1Copy = (ARecordType) fieldType1;
 
         for (int i = 0; i < fieldType1Copy.getFieldTypes().length; i++) {
             try {
-                int pos = returnType.getFieldIndex(fieldType1Copy.getFieldNames()[i]);
+                int pos = resultType.getFieldIndex(fieldType1Copy.getFieldNames()[i]);
                 if (pos >= 0) {
-                    if (fieldType1Copy.getFieldTypes()[i].getTypeTag() != ATypeTag.RECORD) {
-                        break;
+                    // If a sub-record do merge, else ignore and let the values decide what to do
+                    if (fieldType1Copy.getFieldTypes()[i].getTypeTag() == ATypeTag.RECORD) {
+                        IAType[] oldTypes = resultType.getFieldTypes();
+                        oldTypes[pos] = mergedNestedType(fieldType1Copy.getFieldTypes()[i],
+                                resultType.getFieldTypes()[pos]);
+                        resultType = new ARecordType(resultType.getTypeName(), resultType.getFieldNames(), oldTypes,
+                                resultType.isOpen());
                     }
-                    IAType[] oldTypes = returnType.getFieldTypes();
-                    oldTypes[pos] = mergedNestedType(fieldType1Copy.getFieldTypes()[i],
-                            returnType.getFieldTypes()[pos]);
-                    returnType = new ARecordType(returnType.getTypeName(), returnType.getFieldNames(), oldTypes,
-                            returnType.isOpen());
                 } else {
-                    IAType[] combinedFieldTypes = ArrayUtils.addAll(returnType.getFieldTypes().clone(),
+                    IAType[] combinedFieldTypes = ArrayUtils.addAll(resultType.getFieldTypes().clone(),
                             fieldType1Copy.getFieldTypes()[i]);
-                    returnType = new ARecordType(returnType.getTypeName(),
-                            ArrayUtils.addAll(returnType.getFieldNames(), fieldType1Copy.getFieldNames()[i]),
-                            combinedFieldTypes, returnType.isOpen());
+                    resultType = new ARecordType(resultType.getTypeName(), ArrayUtils.addAll(
+                            resultType.getFieldNames(), fieldType1Copy.getFieldNames()[i]), combinedFieldTypes,
+                            resultType.isOpen());
                 }
 
             } catch (IOException | AsterixException e) {
@@ -160,6 +152,6 @@ public class RecordMergeTypeComputer implements IResultTypeComputer {
             }
         }
 
-        return returnType;
+        return resultType;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
new file mode 100644
index 0000000..9f4d155
--- /dev/null
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/RecordRemoveFieldsTypeComputer.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.om.typecomputer.impl;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.om.base.AOrderedList;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.constants.AsterixConstantValue;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+/**
+ * Cases to support:
+ * remove-fields($record, ["foo", ["bar", "access"]]),
+ * where ["bar", "access"] is equivalent to the path bar->access
+ */
+public class RecordRemoveFieldsTypeComputer implements IResultTypeComputer {
+
+    public static final RecordRemoveFieldsTypeComputer INSTANCE = new RecordRemoveFieldsTypeComputer();
+
+    private RecordRemoveFieldsTypeComputer() {
+    }
+
+    private void getPathFromConstantExpression(ILogicalExpression expression, Set<String> fieldNameSet,
+            List<List<String>> pathList) throws AlgebricksException {
+        ConstantExpression ce = (ConstantExpression) expression;
+        if (!(ce.getValue() instanceof AsterixConstantValue)) {
+            throw new AlgebricksException("Expecting a list of strings and found " + ce.getValue() + " instead.");
+        }
+        IAObject item = ((AsterixConstantValue) ce.getValue()).getObject();
+        ATypeTag type = item.getType().getTypeTag();
+
+        switch (type) {
+            case STRING:
+                String fn = ((AString) item).getStringValue();
+                fieldNameSet.add(fn);
+                break;
+            case ORDEREDLIST:
+                AOrderedList pathOrdereList = (AOrderedList) item;
+                String fieldName = ((AString) pathOrdereList.getItem(0)).getStringValue();
+                fieldNameSet.add(fieldName);
+                List<String> path = new ArrayList<>();
+                for (int i = 0; i < pathOrdereList.size(); i++) {
+                    path.add(((AString) pathOrdereList.getItem(i)).getStringValue());
+                }
+                pathList.add(path);
+                break;
+            default:
+                throw new AlgebricksException("Unsupport type: " + type);
+        }
+    }
+
+    private List<String> getListFromExpression(ILogicalExpression expression) throws AlgebricksException {
+        AbstractFunctionCallExpression funcExp = (AbstractFunctionCallExpression) expression;
+        List<Mutable<ILogicalExpression>> args = funcExp.getArguments();
+
+        List<String> list = new ArrayList<>();
+        for (Mutable<ILogicalExpression> arg : args) {
+            // At this point all elements has to be a constant
+            // Input list has only one level of nesting (list of list or list of strings)
+            ConstantExpression ce = (ConstantExpression) arg.getValue();
+            if (!(ce.getValue() instanceof AsterixConstantValue)) {
+                throw new AlgebricksException("Expecting a list of strings and found " + ce.getValue() + " instead.");
+            }
+            IAObject item = ((AsterixConstantValue) ce.getValue()).getObject();
+            ATypeTag type = item.getType().getTypeTag();
+            if (type == ATypeTag.STRING) {
+                list.add(((AString) item).getStringValue());
+            } else {
+                throw new AlgebricksException(type + " is currently not supported. Please check your function call.");
+            }
+        }
+
+        return list;
+    }
+
+    private void getPathFromFunctionExpression(ILogicalExpression expression, Set<String> fieldNameSet,
+            List<List<String>> pathList) throws AlgebricksException {
+
+        List<String> path = getListFromExpression(expression);
+        // Add the path head to remove set
+        fieldNameSet.add(path.get(0));
+        pathList.add(path);
+
+    }
+
+    private void computeTypeFromNonConstantExpression(ILogicalExpression expression, Set<String> fieldNameSet,
+            List<List<String>> pathList) throws AlgebricksException {
+        AbstractFunctionCallExpression funcExp = (AbstractFunctionCallExpression) expression;
+        List<Mutable<ILogicalExpression>> args = funcExp.getArguments();
+
+        for (Mutable<ILogicalExpression> arg : args) {
+            ILogicalExpression le = arg.getValue();
+            switch (le.getExpressionTag()) {
+                case CONSTANT:
+                    getPathFromConstantExpression(le, fieldNameSet, pathList);
+                    break;
+                case FUNCTION_CALL:
+                    getPathFromFunctionExpression(le, fieldNameSet, pathList);
+                    break;
+                default:
+                    throw new AlgebricksException("Unsupported expression: " + le);
+            }
+        }
+    }
+
+    @Override
+    public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
+            IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
+
+        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
+        IAType type0 = (IAType) env.getType(funcExpr.getArguments().get(0).getValue());
+
+        List<List<String>> pathList = new ArrayList<>();
+        Set<String> fieldNameSet = new HashSet<>();
+        Deque<String> fieldPathStack = new ArrayDeque<>();
+
+        ARecordType inputRecordType = NonTaggedFieldAccessByNameResultType.getRecordTypeFromType(type0, expression);
+        if (inputRecordType == null) {
+            return BuiltinType.ANY;
+        }
+
+        AbstractLogicalExpression arg1 = (AbstractLogicalExpression) funcExpr.getArguments().get(1).getValue();
+        IAType inputListType = (IAType) env.getType(arg1);
+        AOrderedListType inputOrderedListType = TypeComputerUtils.extractOrderedListType(inputListType);
+        if (inputOrderedListType == null) {
+            throw new AlgebricksException(
+                    "The function 'remove-fields' expects an ordered list as the second argument, but got "
+                            + inputListType);
+        }
+
+        ATypeTag tt = inputOrderedListType.getItemType().getTypeTag();
+        if (tt == ATypeTag.STRING) { // If top-fieldlist
+            if (setFieldNameSet(arg1, fieldNameSet)) {
+                return buildOutputType(fieldPathStack, inputRecordType, fieldNameSet, pathList);
+            } else {
+                return DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+            }
+        } else { // tt == ATypeTag.ANY, meaning the list is nested
+            computeTypeFromNonConstantExpression(arg1, fieldNameSet, pathList);
+            IAType resultType = buildOutputType(fieldPathStack, inputRecordType, fieldNameSet, pathList);
+            return resultType;
+        }
+    }
+
+    private boolean setFieldNameSet(ILogicalExpression expr, Set<String> fieldNameSet) {
+        if (expr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+            AOrderedList orderedList = (AOrderedList) (((AsterixConstantValue) ((ConstantExpression) expr).getValue())
+                    .getObject());
+            for (int i = 0; i < orderedList.size(); i++) {
+                AString as = (AString) orderedList.getItem(i);
+                fieldNameSet.add(as.getStringValue());
+            }
+            return true; // Success
+        }
+        return false;
+    }
+
+    private void addField(ARecordType inputRecordType,  String fieldName, List<String> resultFieldNames, List<IAType>
+            resultFieldTypes)
+            throws AlgebricksException {
+        try {
+            resultFieldNames.add(fieldName);
+            if (inputRecordType.getFieldType(fieldName).getTypeTag() == ATypeTag.RECORD) {
+                ARecordType nestedType = (ARecordType) inputRecordType.getFieldType(fieldName);
+                //Deep Copy prevents altering of input types
+                resultFieldTypes.add(nestedType.deepCopy(nestedType));
+            } else {
+                resultFieldTypes.add(inputRecordType.getFieldType(fieldName));
+            }
+
+        } catch (IOException e) {
+            throw new AlgebricksException(e);
+        }
+    }
+
+    private IAType buildOutputType(Deque<String> fieldPathStack, ARecordType inputRecordType, Set<String> fieldNameSet,
+            List<List<String>> pathList) throws AlgebricksException {
+        IAType resultType;
+        List<String> resultFieldNames = new ArrayList<>();
+        List<IAType> resultFieldTypes = new ArrayList<>();
+
+        String[] fieldNames = inputRecordType.getFieldNames();
+        IAType[] fieldTypes = inputRecordType.getFieldTypes();
+
+        for (int i = 0; i < fieldNames.length; i++) {
+            if (!fieldNameSet.contains(fieldNames[i])) { // The main field is to be kept
+                addField(inputRecordType, fieldNames[i], resultFieldNames, resultFieldTypes);
+            } else if (!pathList.isEmpty()) { // Further check needed for nested fields
+                if (fieldTypes[i].getTypeTag() == ATypeTag.RECORD) {
+                    ARecordType subRecord = (ARecordType) fieldTypes[i];
+
+                    fieldPathStack.push(fieldNames[i]);
+                    subRecord = deepCheckAndCopy(fieldPathStack, subRecord, pathList, inputRecordType.isOpen());
+                    fieldPathStack.pop();
+                    if (subRecord != null) {
+                        resultFieldNames.add(fieldNames[i]);
+                        resultFieldTypes.add(subRecord);
+                    }
+                }
+            }
+        }
+
+        int n = resultFieldNames.size();
+        String resultTypeName = "result-record(" + inputRecordType.getTypeName() + ")";
+
+        return new ARecordType(resultTypeName, resultFieldNames.toArray(new String[n]),
+                    resultFieldTypes.toArray(new IAType[n]), true); // Make the output type open always
+
+    }
+
+    /**
+     * Comparison elements of two paths
+     * Note: l2 uses a LIFO insert and removal.
+     */
+    private <E> boolean isEqualPaths(List<E> l1, Deque<E> l2) {
+        if ((l1 == null) || (l2 == null))
+            return false;
+
+        if (l1.size() != l2.size())
+            return false;
+
+        Iterator<E> it2 = l2.iterator();
+
+        int len = l1.size();
+        for (int i = len - 1; i >= 0; i--) {
+            E o1 = l1.get(i);
+            E o2 = it2.next();
+            if (!o1.equals(o2))
+                return false;
+        }
+        return true;
+    }
+
+    private boolean isRemovePath(Deque<String> fieldPath, List<List<String>> pathList) {
+        for (List<String> removePath : pathList) {
+            if (isEqualPaths(removePath, fieldPath)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /*
+        A method to deep copy a record the path validation
+             i.e., keep only fields that are valid
+     */
+    private ARecordType deepCheckAndCopy(Deque<String> fieldPath, ARecordType srcRecType, List<List<String>>
+            pathList, boolean isOpen)
+            throws AlgebricksException {
+        // Make sure the current path is valid before going further
+        if (isRemovePath(fieldPath, pathList)) {
+            return null;
+        }
+
+        String srcFieldNames[] = srcRecType.getFieldNames();
+        IAType srcFieldTypes[] = srcRecType.getFieldTypes();
+
+        List<IAType> destFieldTypes = new ArrayList<>();
+        List<String> destFieldNames = new ArrayList<>();
+
+        for (int i = 0; i < srcFieldNames.length; i++) {
+            fieldPath.push(srcFieldNames[i]);
+            if (!isRemovePath(fieldPath, pathList)) {
+                if (srcFieldTypes[i].getTypeTag() == ATypeTag.RECORD) {
+                    ARecordType subRecord = (ARecordType) srcFieldTypes[i];
+                    subRecord = deepCheckAndCopy(fieldPath, subRecord, pathList, isOpen);
+                    if (subRecord != null) {
+                        destFieldNames.add(srcFieldNames[i]);
+                        destFieldTypes.add(subRecord);
+                    }
+                } else {
+                    destFieldNames.add(srcFieldNames[i]);
+                    destFieldTypes.add(srcFieldTypes[i]);
+                }
+            }
+            fieldPath.pop();
+        }
+
+        int n = destFieldNames.size();
+        if (n == 0) {
+            return null;
+        }
+        return new ARecordType(srcRecType.getTypeName(), destFieldNames.toArray(new String[n]),
+                    destFieldTypes.toArray(new IAType[n]), isOpen);
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
new file mode 100644
index 0000000..cba988b
--- /dev/null
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/TypeComputerUtils.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.asterix.om.types.IAType;
+
+public class TypeComputerUtils {
+
+    public static ARecordType extractRecordType(IAType t) {
+        if (t.getTypeTag() == ATypeTag.RECORD) {
+            return (ARecordType) t;
+        }
+
+        if (t.getTypeTag() == ATypeTag.UNION) {
+            IAType innerType = ((AUnionType) t).getUnionList().get(1);
+            if (innerType.getTypeTag() == ATypeTag.RECORD) {
+                return (ARecordType) innerType;
+            }
+        }
+
+        return null;
+    }
+
+    public static AOrderedListType extractOrderedListType(IAType t) {
+        if (t.getTypeTag() == ATypeTag.ORDEREDLIST) {
+            return (AOrderedListType) t;
+        }
+
+        if (t.getTypeTag() == ATypeTag.UNION) {
+            IAType innerType = ((AUnionType) t).getUnionList().get(1);
+            if (innerType.getTypeTag() == ATypeTag.ORDEREDLIST) {
+                return (AOrderedListType) innerType;
+            }
+        }
+
+        return null;
+    }
+
+    public static AUnorderedListType extractUnorderedListType(IAType t) {
+        if (t.getTypeTag() == ATypeTag.UNORDEREDLIST) {
+            return (AUnorderedListType) t;
+        }
+
+        if (t.getTypeTag() == ATypeTag.UNION) {
+            IAType innerType = ((AUnionType) t).getUnionList().get(1);
+            if (innerType.getTypeTag() == ATypeTag.UNORDEREDLIST) {
+                return (AUnorderedListType) innerType;
+            }
+        }
+
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java b/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java
index 6524930..0b6b45d 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/types/hierachy/ATypeHierarchy.java
@@ -45,13 +45,6 @@ import org.apache.hyracks.data.std.primitive.ShortPointable;
 
 public class ATypeHierarchy {
 
-    public static enum Domain {
-        SPATIAL,
-        NUMERIC,
-        LIST,
-        ANY
-    }
-
     private static BitSet typePromotionHierachyMap = new BitSet(ATypeTag.TYPE_COUNT * ATypeTag.TYPE_COUNT);
     private static BitSet typeDemotionHierachyMap = new BitSet(ATypeTag.TYPE_COUNT * ATypeTag.TYPE_COUNT);
     private static HashMap<Integer, ITypeConvertComputer> promoteComputerMap = new HashMap<Integer, ITypeConvertComputer>();
@@ -125,6 +118,10 @@ public class ATypeHierarchy {
         hierarchyDomains.put(ATypeTag.UNORDEREDLIST, Domain.LIST);
     }
 
+    public static Domain getTypeDomain(ATypeTag tag) {
+        return hierarchyDomains.get(tag);
+    }
+
     public static boolean isSameTypeDomain(ATypeTag tag1, ATypeTag tag2, boolean useListDomain) {
         Domain tagHierarchy1 = hierarchyDomains.get(tag1);
         Domain tagHierarchy2 = hierarchyDomains.get(tag2);
@@ -805,4 +802,11 @@ public class ATypeHierarchy {
         return value;
     }
 
+    public static enum Domain {
+        SPATIAL,
+        NUMERIC,
+        LIST,
+        ANY
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/comparisons/DeepEqualAssessor.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/comparisons/DeepEqualAssessor.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/comparisons/DeepEqualAssessor.java
new file mode 100644
index 0000000..b4e0375
--- /dev/null
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/comparisons/DeepEqualAssessor.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.comparisons;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy.Domain;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
+import org.apache.asterix.runtime.evaluators.visitors.DeepEqualityVisitor;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Use {@link DeepEqualityVisitor} to assess the deep equality between two
+ * pointable values, including oredered and unordered lists, record values, etc.
+ * Example: Let IVisitablePointable leftPointable, IVisitablePointable rightPointable be two
+ * value references. To assess their equality, simply use
+ * DeepEqualAssessor dea = new DeepEqualAssessor();
+ * boolean isEqual = dea.isEqual(leftPointable, rightPointable);
+ */
+
+public class DeepEqualAssessor {
+    private final DeepEqualityVisitor equalityVisitor = new DeepEqualityVisitor();
+
+    public boolean isEqual(IVisitablePointable leftPointable, IVisitablePointable rightPointable)
+            throws AlgebricksException, AsterixException {
+
+        if (leftPointable == null || rightPointable == null) {
+            return false;
+        }
+
+        if (leftPointable.equals(rightPointable)) {
+            return true;
+        }
+
+        ATypeTag leftTypeTag = PointableHelper.getTypeTag(leftPointable);
+        ATypeTag rightTypeTag = PointableHelper.getTypeTag(rightPointable);
+
+        if (leftTypeTag != rightTypeTag) {
+            // If types are numeric compare their real values instead
+            if (ATypeHierarchy.isSameTypeDomain(leftTypeTag, rightTypeTag, false)
+                    && ATypeHierarchy.getTypeDomain(leftTypeTag) == Domain.NUMERIC) {
+                try {
+                    double leftVal = ATypeHierarchy.getDoubleValue(leftPointable.getByteArray(),
+                            leftPointable.getStartOffset());
+                    double rightVal = ATypeHierarchy.getDoubleValue(rightPointable.getByteArray(),
+                            rightPointable.getStartOffset());
+                    return (leftVal == rightVal);
+                } catch (HyracksDataException e) {
+                    throw new AlgebricksException(e);
+                }
+
+            } else {
+                return false;
+            }
+        }
+
+        Pair<IVisitablePointable, Boolean> arg = new Pair<IVisitablePointable, Boolean>(rightPointable, Boolean.FALSE);
+        // Assess the nested equality
+        leftPointable.accept(equalityVisitor, arg);
+
+        return arg.second;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DeepEqualityDescriptor.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DeepEqualityDescriptor.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DeepEqualityDescriptor.java
new file mode 100644
index 0000000..e6a4232
--- /dev/null
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/DeepEqualityDescriptor.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.comparisons.DeepEqualAssessor;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.data.std.api.IDataOutputProvider;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class DeepEqualityDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+    public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+        public IFunctionDescriptor createFunctionDescriptor() {
+            return new DeepEqualityDescriptor();
+        }
+    };
+
+    private static final long serialVersionUID = 1L;
+    private IAType inputTypeLeft;
+    private IAType inputTypeRight;
+
+    public void reset(IAType inTypeLeft, IAType inTypeRight) {
+        this.inputTypeLeft = inTypeLeft;
+        this.inputTypeRight = inTypeRight;
+    }
+
+    @Override
+    public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args) {
+        final ICopyEvaluatorFactory evalFactoryLeft = args[0];
+        final ICopyEvaluatorFactory evalFactoryRight = args[1];
+
+        return new ICopyEvaluatorFactory() {
+            private static final long serialVersionUID = 1L;
+            private final ISerializerDeserializer boolSerde = AqlSerializerDeserializerProvider.INSTANCE
+                    .getSerializerDeserializer(BuiltinType.ABOOLEAN);
+
+            @Override
+            public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+                final DataOutput out = output.getDataOutput();
+                final ArrayBackedValueStorage abvsLeft = new ArrayBackedValueStorage();
+                final ICopyEvaluator evalLeft = evalFactoryLeft.createEvaluator(abvsLeft);
+
+                final ArrayBackedValueStorage abvsRight = new ArrayBackedValueStorage();
+                final ICopyEvaluator evalRight = evalFactoryRight.createEvaluator(abvsRight);
+                final DeepEqualAssessor deepEqualAssessor = new DeepEqualAssessor();
+
+                return new ICopyEvaluator() {
+                    private final PointableAllocator allocator = new PointableAllocator();
+                    private final IVisitablePointable pointableLeft = allocator.allocateFieldValue(inputTypeLeft);
+                    private final IVisitablePointable pointableRight = allocator.allocateFieldValue(inputTypeRight);
+
+                    @Override
+                    public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+                        try {
+                            abvsLeft.reset();
+                            abvsRight.reset();
+                            evalLeft.evaluate(tuple);
+                            evalRight.evaluate(tuple);
+                            pointableLeft.set(abvsLeft);
+                            pointableRight.set(abvsRight);
+
+                            // Using deep equality assessment to assess the equality of the two values
+                            boolean isEqual = deepEqualAssessor.isEqual(pointableLeft, pointableRight);
+                            ABoolean result = isEqual ? ABoolean.TRUE : ABoolean.FALSE;
+
+                            boolSerde.serialize(result, out);
+                        } catch (Exception ioe) {
+                            throw new AlgebricksException(ioe);
+                        }
+                    }
+                };
+            }
+        };
+    }
+
+    @Override
+    public FunctionIdentifier getIdentifier() {
+        return AsterixBuiltinFunctions.DEEP_EQUAL;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
new file mode 100644
index 0000000..24e66f2
--- /dev/null
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import org.apache.hyracks.data.std.api.IMutableValueStorage;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+/**
+ * An utility class for some frequently used methods like checking the equality between two pointables (binary values)
+ * (e.g., field names), string value of a fieldname pointable, getting the typetag of a pointable, etc.
+ * Note: To get the typetag of a fieldvalue (i) in a record, it is recommended to use the getFieldTypeTags().get(i)
+ * method rather than getting it from fhe field value itself.
+ */
+
+public class PointableHelper {
+    private static final IBinaryComparator STRING_BINARY_COMPARATOR = PointableBinaryComparatorFactory.of(
+            UTF8StringPointable.FACTORY).createBinaryComparator();
+    private final UTF8StringWriter utf8Writer;
+
+    public PointableHelper() {
+        utf8Writer = new UTF8StringWriter();
+    }
+
+    public static int compareStringBinValues(IValueReference a, IValueReference b) throws HyracksDataException {
+        // start+1 and len-1 due to type tag ignore (only interested in String value)
+        return STRING_BINARY_COMPARATOR.compare(a.getByteArray(), a.getStartOffset() + 1, a.getLength() - 1,
+                b.getByteArray(), b.getStartOffset() + 1, b.getLength() - 1);
+    }
+
+    public static boolean isEqual(IValueReference a, IValueReference b) throws HyracksDataException {
+        return (compareStringBinValues(a, b) == 0);
+    }
+
+    public static boolean byteArrayEqual(IValueReference valueRef1, IValueReference valueRef2) {
+        return byteArrayEqual(valueRef1, valueRef2, 3);
+    }
+
+    public static boolean byteArrayEqual(IValueReference valueRef1, IValueReference valueRef2, int dataOffset) {
+        if (valueRef1 == null || valueRef2 == null) {
+            return false;
+        }
+        if (valueRef1 == valueRef2) {
+            return true;
+        }
+
+        int length1 = valueRef1.getLength();
+        int length2 = valueRef2.getLength();
+
+        if (length1 != length2) {
+            return false;
+        }
+
+        byte[] bytes1 = valueRef1.getByteArray();
+        byte[] bytes2 = valueRef2.getByteArray();
+        int start1 = valueRef1.getStartOffset() + dataOffset;
+        int start2 = valueRef2.getStartOffset() + dataOffset;
+
+        int end = start1 + length1 - dataOffset;
+
+        for (int i = start1, j = start2; i < end; i++, j++) {
+            if (bytes1[i] != bytes2[j]) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    public static boolean sameType(ATypeTag typeTag, IVisitablePointable visitablePointable) {
+        return (getTypeTag(visitablePointable) == typeTag);
+    }
+
+    public static ATypeTag getTypeTag(IValueReference visitablePointable) {
+        byte[] bytes = visitablePointable.getByteArray();
+        int s = visitablePointable.getStartOffset();
+        return EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes[s]);
+    }
+
+    /**
+     * @param str
+     *            The input string
+     * @param vs
+     *            The storage buffer
+     * @param writeTag
+     *            Specifying whether a tag for the string should also be written
+     * @throws AlgebricksException
+     */
+    public void serializeString(String str, IMutableValueStorage vs, boolean writeTag) throws AsterixException {
+        vs.reset();
+        try {
+            DataOutput output = vs.getDataOutput();
+            if (writeTag) {
+                output.write(ATypeTag.STRING.serialize());
+            }
+            utf8Writer.writeUTF8(str, output);
+        } catch (IOException e) {
+            throw new AsterixException("Could not serialize " + str);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/4053f390/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddFieldsDescriptor.java
----------------------------------------------------------------------
diff --git a/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddFieldsDescriptor.java b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddFieldsDescriptor.java
new file mode 100644
index 0000000..c26daca
--- /dev/null
+++ b/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/records/RecordAddFieldsDescriptor.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions.records;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.dataflow.data.nontagged.comparators.ListItemBinaryComparatorFactory;
+import org.apache.asterix.dataflow.data.nontagged.hash.ListItemBinaryHashFunctionFactory;
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.om.base.ANull;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.pointables.AListVisitablePointable;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.PointableAllocator;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.typecomputer.impl.TypeComputerUtils;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.runtime.RuntimeRecordTypeInfo;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.BinaryHashMap;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IDataOutputProvider;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class RecordAddFieldsDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+    public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+        public IFunctionDescriptor createFunctionDescriptor() {
+            return new RecordAddFieldsDescriptor();
+        }
+    };
+    private static final long serialVersionUID = 1L;
+    private ARecordType outRecType;
+    private ARecordType inRecType;
+    private AOrderedListType inListType;
+    private IAType inputFieldListItemType;
+
+    public void reset(IAType outType, IAType inType0, IAType inType1) {
+        outRecType = TypeComputerUtils.extractRecordType(outType);
+        inRecType = TypeComputerUtils.extractRecordType(inType0);
+        inListType = TypeComputerUtils.extractOrderedListType(inType1);
+        inputFieldListItemType = inListType.getItemType();
+        if (inputFieldListItemType == null || inputFieldListItemType.getTypeTag() == ATypeTag.ANY) {
+            inputFieldListItemType = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+        }
+    }
+
+    @Override
+    public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args) throws AlgebricksException {
+        return new ICopyEvaluatorFactory() {
+
+            private static final long serialVersionUID = 1L;
+            private final byte SER_NULL_TYPE_TAG = ATypeTag.NULL.serialize();
+            private final byte SER_ORDERED_TYPE_TAG = ATypeTag.ORDEREDLIST.serialize();
+            private final byte SER_RECORD_TYPE_TAG = ATypeTag.RECORD.serialize();
+
+            @SuppressWarnings("unchecked")
+            private final ISerializerDeserializer<ANull> nullSerDe = AqlSerializerDeserializerProvider.INSTANCE
+                    .getSerializerDeserializer(BuiltinType.ANULL);
+
+            @Override
+            public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+                final PointableAllocator allocator = new PointableAllocator();
+                final IVisitablePointable vp0 = allocator.allocateRecordValue(inRecType);
+                final IVisitablePointable vp1 = allocator.allocateListValue(inListType);
+
+                final ArrayBackedValueStorage abvs0 = new ArrayBackedValueStorage();
+                final ArrayBackedValueStorage abvs1 = new ArrayBackedValueStorage();
+
+                final ICopyEvaluator eval0 = args[0].createEvaluator(abvs0);
+                final ICopyEvaluator eval1 = args[1].createEvaluator(abvs1);
+
+                final ArrayBackedValueStorage fieldNamePointable = new ArrayBackedValueStorage();
+                final ArrayBackedValueStorage fieldValuePointer = new ArrayBackedValueStorage();
+                final PointableHelper pointableHelper = new PointableHelper();
+                try {
+                    pointableHelper.serializeString("field-name", fieldNamePointable, true);
+                    pointableHelper.serializeString("field-value", fieldValuePointer, true);
+                } catch (AsterixException e) {
+                    throw new AlgebricksException(e);
+                }
+
+                return new ICopyEvaluator() {
+                    public static final int TABLE_FRAME_SIZE = 32768; // the default 32k frame size
+                    public static final int TABLE_SIZE = 100; // the default 32k frame size
+                    private final RecordBuilder recordBuilder = new RecordBuilder();
+                    private final RuntimeRecordTypeInfo requiredRecordTypeInfo = new RuntimeRecordTypeInfo();
+
+                    private final IBinaryHashFunction putHashFunc = ListItemBinaryHashFunctionFactory.INSTANCE
+                            .createBinaryHashFunction();
+                    private final IBinaryHashFunction getHashFunc = ListItemBinaryHashFunctionFactory.INSTANCE
+                            .createBinaryHashFunction();
+                    private final BinaryHashMap.BinaryEntry keyEntry = new BinaryHashMap.BinaryEntry();
+                    private final BinaryHashMap.BinaryEntry valEntry = new BinaryHashMap.BinaryEntry();
+                    private final IVisitablePointable tempValReference = allocator.allocateEmpty();
+                    private final IBinaryComparator cmp = ListItemBinaryComparatorFactory.INSTANCE
+                            .createBinaryComparator();
+                    private BinaryHashMap hashMap = new BinaryHashMap(TABLE_SIZE, TABLE_FRAME_SIZE, putHashFunc,
+                            getHashFunc, cmp);
+
+                    @Override
+                    public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+                        recordBuilder.reset(outRecType);
+                        requiredRecordTypeInfo.reset(outRecType);
+                        abvs0.reset();
+                        abvs1.reset();
+
+                        eval0.evaluate(tuple);
+                        eval1.evaluate(tuple);
+
+                        if (abvs0.getByteArray()[0] == SER_NULL_TYPE_TAG
+                                || abvs1.getByteArray()[0] == SER_NULL_TYPE_TAG) {
+                            try {
+                                nullSerDe.serialize(ANull.NULL, output.getDataOutput());
+                            } catch (HyracksDataException e) {
+                                throw new AlgebricksException(e);
+                            }
+                            return;
+                        }
+
+                        // Make sure we get a valid record
+                        if (abvs0.getByteArray()[0] != SER_RECORD_TYPE_TAG) {
+                            throw new AlgebricksException("Expected an ordederlist of type " + inRecType + " but "
+                                    + "got "
+                                    + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(abvs0.getByteArray()[0]));
+                        }
+
+                        // Make sure we get a valid list
+                        if (abvs1.getByteArray()[0] != SER_ORDERED_TYPE_TAG) {
+                            throw new AlgebricksException("Expected an ordederlist of type " + inListType + " but "
+                                    + "got "
+                                    + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(abvs1.getByteArray()[0]));
+                        }
+
+                        vp0.set(abvs0);
+                        vp1.set(abvs1);
+
+                        try {
+                            ARecordVisitablePointable recordPointable = (ARecordVisitablePointable) vp0;
+                            AListVisitablePointable listPointable = (AListVisitablePointable) vp1;
+
+                            // Initialize our hashmap
+                            int tableSize = recordPointable.getFieldNames().size() + listPointable.getItems().size();
+                            // Construct a new hash table only if table size is larger than the default
+                            // Thus avoiding unnecessary object construction
+                            if (hashMap == null || tableSize > TABLE_SIZE) {
+                                hashMap = new BinaryHashMap(tableSize, TABLE_FRAME_SIZE, putHashFunc, getHashFunc, cmp);
+                            } else {
+                                hashMap.clear();
+                            }
+                            addFields(recordPointable, listPointable);
+                            recordBuilder.write(output.getDataOutput(), true);
+                        } catch (IOException | AsterixException e) {
+                            throw new AlgebricksException(e);
+                        }
+                    }
+
+                    private void addFields(ARecordVisitablePointable inputRecordPointer,
+                            AListVisitablePointable listPointable) throws AlgebricksException {
+                        List<IVisitablePointable> inputRecordFieldNames = inputRecordPointer.getFieldNames();
+                        List<IVisitablePointable> inputRecordFieldValues = inputRecordPointer.getFieldValues();
+                        List<IVisitablePointable> inputFields = listPointable.getItems();
+                        IVisitablePointable namePointable = null;
+                        IVisitablePointable valuePointable = null;
+                        int numInputRecordFields = inputRecordFieldNames.size();
+
+                        try {
+                            // Add original record without duplicate checking
+                            for (int i = 0; i < numInputRecordFields; ++i) {
+                                IVisitablePointable fnp = inputRecordFieldNames.get(i);
+                                IVisitablePointable fvp = inputRecordFieldValues.get(i);
+                                int pos = requiredRecordTypeInfo.getFieldIndex(fnp.getByteArray(),
+                                        fnp.getStartOffset() + 1, fnp.getLength() - 1);
+                                if (pos >= 0) {
+                                    recordBuilder.addField(pos, fvp);
+                                } else {
+                                    recordBuilder.addField(fnp, fvp);
+                                }
+                                keyEntry.set(fnp.getByteArray(), fnp.getStartOffset(), fnp.getLength());
+                                valEntry.set(fvp.getByteArray(), fvp.getStartOffset(), fvp.getLength());
+                                hashMap.put(keyEntry, valEntry);
+                            }
+
+                            // Get the fields from a list of records
+                            for (int i = 0; i < inputFields.size(); i++) {
+                                if (!PointableHelper.sameType(ATypeTag.RECORD, inputFields.get(i))) {
+                                    throw new AsterixException("Expected list of record, got "
+                                            + PointableHelper.getTypeTag(inputFields.get(i)));
+                                }
+                                List<IVisitablePointable> names = ((ARecordVisitablePointable) inputFields.get(i))
+                                        .getFieldNames();
+                                List<IVisitablePointable> values = ((ARecordVisitablePointable) inputFields.get(i))
+                                        .getFieldValues();
+
+                                // Get name and value of the field to be added
+                                // Use loop to account for the cases where users switches the order of the fields
+                                IVisitablePointable fieldName;
+                                for (int j = 0; j < names.size(); j++) {
+                                    fieldName = names.get(j);
+                                    // if fieldName is "field-name" then read the name
+                                    if (PointableHelper.byteArrayEqual(fieldNamePointable, fieldName)) {
+                                        namePointable = values.get(j);
+                                    } else { // otherwise the fieldName is "field-value". Thus, read the value
+                                        valuePointable = values.get(j);
+                                    }
+                                }
+
+                                if (namePointable == null || valuePointable == null) {
+                                    throw new AlgebricksException("Trying to add a null field name or field value");
+                                }
+
+                                // Check that the field being added is a valid field
+                                int pos = requiredRecordTypeInfo.getFieldIndex(namePointable.getByteArray(),
+                                        namePointable.getStartOffset() + 1, namePointable.getLength() - 1);
+
+                                keyEntry.set(namePointable.getByteArray(), namePointable.getStartOffset(),
+                                        namePointable.getLength());
+                                // Check if already in our built record
+                                BinaryHashMap.BinaryEntry entry = hashMap.get(keyEntry);
+                                if (entry != null) {
+                                    tempValReference.set(entry.buf, entry.off, entry.len);
+                                    // If value is not equal throw conflicting duplicate field, otherwise ignore
+                                    if (!PointableHelper.byteArrayEqual(valuePointable, tempValReference)) {
+                                        throw new AlgebricksException("Conflicting duplicate field found.");
+                                    }
+                                } else {
+                                    if (pos > -1) {
+                                        recordBuilder.addField(pos, valuePointable);
+                                    } else {
+                                        recordBuilder.addField(namePointable, valuePointable);
+                                    }
+                                    valEntry.set(valuePointable.getByteArray(), valuePointable.getStartOffset(),
+                                            valuePointable.getLength());
+                                    hashMap.put(keyEntry, valEntry);
+                                }
+                            }
+                        } catch (AsterixException | HyracksDataException e) {
+                            throw new AlgebricksException(e);
+                        }
+                    }
+                };
+            }
+        };
+    }
+
+    @Override
+    public FunctionIdentifier getIdentifier() {
+        return AsterixBuiltinFunctions.ADD_FIELDS;
+    }
+}


Mime
View raw message