asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [3/9] incubator-asterixdb git commit: Changed metadata storage format for nullable field types. Moved field name generation to the client out of metadata node code. Changed naming scheme for autogenerated types. Moved GroupName, CompactionPolicy & Compac
Date Thu, 20 Aug 2015 02:27:48 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/AsterixBuiltinTypeMap.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/AsterixBuiltinTypeMap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/AsterixBuiltinTypeMap.java
index 81502ca..41ba9a7 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/AsterixBuiltinTypeMap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/AsterixBuiltinTypeMap.java
@@ -22,6 +22,7 @@ import java.util.Map;
 import edu.uci.ics.asterix.common.transactions.JobId;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataNode;
+import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
 
@@ -65,15 +66,18 @@ public class AsterixBuiltinTypeMap {
     }
 
     public static IAType getTypeFromTypeName(MetadataNode metadataNode, JobId jobId, String dataverseName,
-            String typeName) throws MetadataException {
+            String typeName, boolean isNullable) throws MetadataException {
         IAType type = AsterixBuiltinTypeMap.getBuiltinTypes().get(typeName);
         if (type == null) {
             try {
-                type = metadataNode.getDatatype(jobId, dataverseName, typeName).getDatatype();
+                Datatype dt = metadataNode.getDatatype(jobId, dataverseName, typeName);
+                type = dt.getDatatype();
             } catch (RemoteException e) {
                 throw new MetadataException(e);
             }
         }
+        if (isNullable)
+            type = AUnionType.createNullableType(type);
         return type;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Dataset.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Dataset.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Dataset.java
index 1f45c6a..b4e6426 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Dataset.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Dataset.java
@@ -34,6 +34,9 @@ public class Dataset implements IMetadataEntity {
     private final String datasetName;
     // Type of items stored in this dataset.
     private final String itemTypeName;
+    private final String nodeGroupName;
+    private final String compactionPolicy;
+    private final Map<String, String> compactionPolicyProperties;
     private final DatasetType datasetType;
     private final IDatasetDetails datasetDetails;
     // Hints related to cardinatlity of dataset, avg size of tuples etc.
@@ -42,11 +45,15 @@ public class Dataset implements IMetadataEntity {
     // Type of pending operations with respect to atomic DDL operation
     private int pendingOp;
 
-    public Dataset(String dataverseName, String datasetName, String itemTypeName, IDatasetDetails datasetDetails,
+    public Dataset(String dataverseName, String datasetName, String itemTypeName, String nodeGroupName,
+            String compactionPolicy, Map<String, String> compactionPolicyProperties, IDatasetDetails datasetDetails,
             Map<String, String> hints, DatasetType datasetType, int datasetId, int pendingOp) {
         this.dataverseName = dataverseName;
         this.datasetName = datasetName;
         this.itemTypeName = itemTypeName;
+        this.nodeGroupName = nodeGroupName;
+        this.compactionPolicy = compactionPolicy;
+        this.compactionPolicyProperties = compactionPolicyProperties;
         this.datasetType = datasetType;
         this.datasetDetails = datasetDetails;
         this.datasetId = datasetId;
@@ -66,6 +73,18 @@ public class Dataset implements IMetadataEntity {
         return itemTypeName;
     }
 
+    public String getNodeGroupName() {
+        return nodeGroupName;
+    }
+
+    public String getCompactionPolicy() {
+        return compactionPolicy;
+    }
+
+    public Map<String, String> getCompactionPolicyProperties() {
+        return compactionPolicyProperties;
+    }
+
     public DatasetType getDatasetType() {
         return datasetType;
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
index 5f7fb74..e7078c5 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
@@ -29,12 +29,12 @@ import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
 import edu.uci.ics.asterix.metadata.IDatasetDetails;
 import edu.uci.ics.asterix.metadata.bootstrap.MetadataRecordTypes;
+import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.base.ADateTime;
 import edu.uci.ics.asterix.om.base.AInt32;
 import edu.uci.ics.asterix.om.base.AMutableString;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.types.AOrderedListType;
-import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -45,24 +45,17 @@ public class ExternalDatasetDetails implements IDatasetDetails {
     private static final long serialVersionUID = 1L;
     private final String adapter;
     private final Map<String, String> properties;
-    private final String nodeGroupName;
     private final long addToCacheTime;
     private Date lastRefreshTime;
     private ExternalDatasetTransactionState state;
-    protected String compactionPolicy;
-    protected Map<String, String> compactionPolicyProperties;
 
-    public ExternalDatasetDetails(String adapter, Map<String, String> properties, String nodeGroupName,
-            Date lastRefreshTime, ExternalDatasetTransactionState state, String compactionPolicy,
-            Map<String, String> compactionPolicyProperties) {
+    public ExternalDatasetDetails(String adapter, Map<String, String> properties, Date lastRefreshTime,
+            ExternalDatasetTransactionState state) {
         this.properties = properties;
         this.adapter = adapter;
-        this.nodeGroupName = nodeGroupName;
         this.addToCacheTime = System.currentTimeMillis();
         this.lastRefreshTime = lastRefreshTime;
         this.state = state;
-        this.compactionPolicy = compactionPolicy;
-        this.compactionPolicyProperties = compactionPolicyProperties;
     }
 
     public String getAdapter() {
@@ -108,7 +101,7 @@ public class ExternalDatasetDetails implements IDatasetDetails {
             String name = property.getKey();
             String value = property.getValue();
             itemValue.reset();
-            writePropertyTypeRecord(name, value, itemValue.getDataOutput(),
+            DatasetUtils.writePropertyTypeRecord(name, value, itemValue.getDataOutput(),
                     MetadataRecordTypes.DATASOURCE_ADAPTER_PROPERTIES_RECORDTYPE);
             listBuilder.addItem(itemValue);
         }
@@ -118,44 +111,15 @@ public class ExternalDatasetDetails implements IDatasetDetails {
 
         // write field 2
         fieldValue.reset();
-        aString.setValue(getNodeGroupName());
-        stringSerde.serialize(aString, fieldValue.getDataOutput());
-        externalRecordBuilder.addField(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_GROUPNAME_FIELD_INDEX, fieldValue);
-
-        // write field 3
-        fieldValue.reset();
         dateTimeSerde.serialize(new ADateTime(lastRefreshTime.getTime()), fieldValue.getDataOutput());
         externalRecordBuilder.addField(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_LAST_REFRESH_TIME_FIELD_INDEX,
                 fieldValue);
 
-        // write field 4
+        // write field 3
         fieldValue.reset();
         intSerde.serialize(new AInt32(state.ordinal()), fieldValue.getDataOutput());
         externalRecordBuilder.addField(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_TRANSACTION_STATE_FIELD_INDEX,
                 fieldValue);
-
-        // write field 6
-        fieldValue.reset();
-        aString.setValue(getCompactionPolicy().toString());
-        stringSerde.serialize(aString, fieldValue.getDataOutput());
-        externalRecordBuilder.addField(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_FIELD_INDEX,
-                fieldValue);
-
-        // write field 7
-        listBuilder
-                .reset((AOrderedListType) MetadataRecordTypes.EXTERNAL_DETAILS_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX]);
-        for (Map.Entry<String, String> property : compactionPolicyProperties.entrySet()) {
-            String name = property.getKey();
-            String value = property.getValue();
-            itemValue.reset();
-            writePropertyTypeRecord(name, value, itemValue.getDataOutput(),
-                    MetadataRecordTypes.COMPACTION_POLICY_PROPERTIES_RECORDTYPE);
-            listBuilder.addItem(itemValue);
-        }
-        fieldValue.reset();
-        listBuilder.write(fieldValue.getDataOutput(), true);
-        externalRecordBuilder.addField(
-                MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX, fieldValue);
         try {
             externalRecordBuilder.write(out, true);
         } catch (IOException | AsterixException e) {
@@ -164,40 +128,6 @@ public class ExternalDatasetDetails implements IDatasetDetails {
 
     }
 
-    @SuppressWarnings("unchecked")
-    protected void writePropertyTypeRecord(String name, String value, DataOutput out, ARecordType recordType)
-            throws HyracksDataException {
-        IARecordBuilder propertyRecordBuilder = new RecordBuilder();
-        ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
-        propertyRecordBuilder.reset(recordType);
-        AMutableString aString = new AMutableString("");
-        ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
-                .getSerializerDeserializer(BuiltinType.ASTRING);
-
-        // write field 0
-        fieldValue.reset();
-        aString.setValue(name);
-        stringSerde.serialize(aString, fieldValue.getDataOutput());
-        propertyRecordBuilder.addField(0, fieldValue);
-
-        // write field 1
-        fieldValue.reset();
-        aString.setValue(value);
-        stringSerde.serialize(aString, fieldValue.getDataOutput());
-        propertyRecordBuilder.addField(1, fieldValue);
-
-        try {
-            propertyRecordBuilder.write(out, true);
-        } catch (IOException | AsterixException e) {
-            throw new HyracksDataException(e);
-        }
-    }
-
-    @Override
-    public String getNodeGroupName() {
-        return nodeGroupName;
-    }
-
     @Override
     public boolean isTemp() {
         return false;
@@ -223,14 +153,4 @@ public class ExternalDatasetDetails implements IDatasetDetails {
     public void setState(ExternalDatasetTransactionState state) {
         this.state = state;
     }
-
-    @Override
-    public String getCompactionPolicy() {
-        return compactionPolicy;
-    }
-
-    @Override
-    public Map<String, String> getCompactionPolicyProperties() {
-        return compactionPolicyProperties;
-    }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Index.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Index.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Index.java
index 425dad9..72e0b8c 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Index.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/Index.java
@@ -22,9 +22,9 @@ import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
 import edu.uci.ics.asterix.metadata.MetadataCache;
 import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
 import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
 
@@ -130,13 +130,9 @@ public class Index implements IMetadataEntity, Comparable<Index> {
 
     public static Pair<IAType, Boolean> getNonNullableType(IAType keyType) throws AlgebricksException {
         boolean nullable = false;
-        if (keyType.getTypeTag() == ATypeTag.UNION) {
-            AUnionType unionType = (AUnionType) keyType;
-            if (unionType.isNullableType()) {
-                // The non-null type is always at index 1.
-                keyType = unionType.getUnionList().get(1);
-                nullable = true;
-            }
+        if (NonTaggedFormatUtil.isOptional(keyType)) {
+            keyType = ((AUnionType) keyType).getNullableType();
+            nullable = true;
         }
         return new Pair<IAType, Boolean>(keyType, nullable);
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java
index 68b158a..5342ce8 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/InternalDatasetDetails.java
@@ -18,7 +18,6 @@ package edu.uci.ics.asterix.metadata.entities;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.util.List;
-import java.util.Map;
 
 import edu.uci.ics.asterix.builders.IARecordBuilder;
 import edu.uci.ics.asterix.builders.OrderedListBuilder;
@@ -56,10 +55,7 @@ public class InternalDatasetDetails implements IDatasetDetails {
     protected final List<List<String>> partitioningKeys;
     protected final List<List<String>> primaryKeys;
     protected final List<IAType> primaryKeyTypes;
-    protected final String nodeGroupName;
     protected final boolean autogenerated;
-    protected final String compactionPolicy;
-    protected final Map<String, String> compactionPolicyProperties;
     protected final boolean temp;
     protected long lastAccessTime;
     protected final List<String> filterField;
@@ -67,27 +63,18 @@ public class InternalDatasetDetails implements IDatasetDetails {
 
     public InternalDatasetDetails(FileStructure fileStructure, PartitioningStrategy partitioningStrategy,
             List<List<String>> partitioningKey, List<List<String>> primaryKey, List<IAType> primaryKeyType,
-            String groupName, boolean autogenerated, String compactionPolicy,
-            Map<String, String> compactionPolicyProperties, List<String> filterField, boolean temp) {
+            boolean autogenerated, List<String> filterField, boolean temp) {
         this.fileStructure = fileStructure;
         this.partitioningStrategy = partitioningStrategy;
         this.partitioningKeys = partitioningKey;
         this.primaryKeys = primaryKey;
         this.primaryKeyTypes = primaryKeyType;
         this.autogenerated = autogenerated;
-        this.nodeGroupName = groupName;
-        this.compactionPolicy = compactionPolicy;
-        this.compactionPolicyProperties = compactionPolicyProperties;
         this.filterField = filterField;
         this.temp = temp;
         this.lastAccessTime = System.currentTimeMillis();
     }
 
-    @Override
-    public String getNodeGroupName() {
-        return nodeGroupName;
-    }
-
     public List<List<String>> getPartitioningKey() {
         return partitioningKeys;
     }
@@ -112,16 +99,6 @@ public class InternalDatasetDetails implements IDatasetDetails {
         return partitioningStrategy;
     }
 
-    @Override
-    public String getCompactionPolicy() {
-        return compactionPolicy;
-    }
-
-    @Override
-    public Map<String, String> getCompactionPolicyProperties() {
-        return compactionPolicyProperties;
-    }
-
     public List<String> getFilterField() {
         return filterField;
     }
@@ -215,42 +192,11 @@ public class InternalDatasetDetails implements IDatasetDetails {
 
         // write field 4
         fieldValue.reset();
-        aString.setValue(getNodeGroupName());
-        stringSerde.serialize(aString, fieldValue.getDataOutput());
-        internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_GROUPNAME_FIELD_INDEX, fieldValue);
-
-        // write field 5
-        fieldValue.reset();
         ABoolean b = isAutogenerated() ? ABoolean.TRUE : ABoolean.FALSE;
         booleanSerde.serialize(b, fieldValue.getDataOutput());
         internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_AUTOGENERATED_FIELD_INDEX,
                 fieldValue);
 
-        // write field 6
-        fieldValue.reset();
-        aString.setValue(getCompactionPolicy().toString());
-        stringSerde.serialize(aString, fieldValue.getDataOutput());
-        internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_FIELD_INDEX,
-                fieldValue);
-
-        // write field 7
-        listBuilder
-                .reset((AOrderedListType) MetadataRecordTypes.INTERNAL_DETAILS_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX]);
-        if (compactionPolicyProperties != null) {
-            for (Map.Entry<String, String> property : compactionPolicyProperties.entrySet()) {
-                String name = property.getKey();
-                String value = property.getValue();
-                itemValue.reset();
-                writePropertyTypeRecord(name, value, itemValue.getDataOutput(),
-                        MetadataRecordTypes.COMPACTION_POLICY_PROPERTIES_RECORDTYPE);
-                listBuilder.addItem(itemValue);
-            }
-        }
-        fieldValue.reset();
-        listBuilder.write(fieldValue.getDataOutput(), true);
-        internalRecordBuilder.addField(
-                MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX, fieldValue);
-
         List<String> filterField = getFilterField();
         if (filterField != null) {
             listBuilder.reset(stringList);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index c16a794..6ed504f 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Map;
 
 import edu.uci.ics.asterix.builders.IARecordBuilder;
+import edu.uci.ics.asterix.builders.OrderedListBuilder;
 import edu.uci.ics.asterix.builders.RecordBuilder;
 import edu.uci.ics.asterix.builders.UnorderedListBuilder;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
@@ -44,6 +45,7 @@ import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails.FileStructure;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
+import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.base.ABoolean;
 import edu.uci.ics.asterix.om.base.ADateTime;
 import edu.uci.ics.asterix.om.base.AInt32;
@@ -54,6 +56,7 @@ import edu.uci.ics.asterix.om.base.ARecord;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.base.AUnorderedList;
 import edu.uci.ics.asterix.om.base.IACursor;
+import edu.uci.ics.asterix.om.types.AOrderedListType;
 import edu.uci.ics.asterix.om.types.AUnorderedListType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
@@ -112,6 +115,22 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
                 .getValueByPos(MetadataRecordTypes.DATASET_ARECORD_DATASETID_FIELD_INDEX)).getIntegerValue();
         int pendingOp = ((AInt32) datasetRecord
                 .getValueByPos(MetadataRecordTypes.DATASET_ARECORD_PENDINGOP_FIELD_INDEX)).getIntegerValue();
+        String nodeGroupName = ((AString) datasetRecord
+                .getValueByPos(MetadataRecordTypes.DATASET_ARECORD_GROUPNAME_FIELD_INDEX)).getStringValue();
+        String compactionPolicy = ((AString) datasetRecord
+                .getValueByPos(MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_FIELD_INDEX)).getStringValue();
+        IACursor cursor = ((AOrderedList) datasetRecord
+                .getValueByPos(MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX))
+                .getCursor();
+        Map<String, String> compactionPolicyProperties = new LinkedHashMap<String, String>();
+        String key;
+        String value;
+        while (cursor.next()) {
+            ARecord field = (ARecord) cursor.get();
+            key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX)).getStringValue();
+            value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX)).getStringValue();
+            compactionPolicyProperties.put(key, value);
+        }
         switch (datasetType) {
             case INTERNAL: {
                 ARecord datasetDetailsRecord = (ARecord) datasetRecord
@@ -123,7 +142,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
                         .valueOf(((AString) datasetDetailsRecord
                                 .getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_PARTITIONSTRATEGY_FIELD_INDEX))
                                 .getStringValue());
-                IACursor cursor = ((AOrderedList) datasetDetailsRecord
+                cursor = ((AOrderedList) datasetDetailsRecord
                         .getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_PARTITIONKEY_FIELD_INDEX))
                         .getCursor();
                 List<List<String>> partitioningKey = new ArrayList<List<String>>();
@@ -141,29 +160,9 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
                     partitioningKeyType.add(BuiltinType.ASTRING);
                 }
 
-                String groupName = ((AString) datasetDetailsRecord
-                        .getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_GROUPNAME_FIELD_INDEX))
-                        .getStringValue();
                 boolean autogenerated = ((ABoolean) datasetDetailsRecord
                         .getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_AUTOGENERATED_FIELD_INDEX))
                         .getBoolean();
-                String compactionPolicy = ((AString) datasetDetailsRecord
-                        .getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_FIELD_INDEX))
-                        .getStringValue();
-                cursor = ((AOrderedList) datasetDetailsRecord
-                        .getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX))
-                        .getCursor();
-                Map<String, String> compactionPolicyProperties = new LinkedHashMap<String, String>();
-                String key;
-                String value;
-                while (cursor.next()) {
-                    ARecord field = (ARecord) cursor.get();
-                    key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX))
-                            .getStringValue();
-                    value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX))
-                            .getStringValue();
-                    compactionPolicyProperties.put(key, value);
-                }
 
                 // Check if there is a filter field.
                 List<String> filterField = null;
@@ -180,8 +179,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
                 // Temporary dataset only lives in the compiler therefore the temp field is false.
                 //  DatasetTupleTranslator always read from the metadata node, so the temp flag should be always false.
                 datasetDetails = new InternalDatasetDetails(fileStructure, partitioningStrategy, partitioningKey,
-                        partitioningKey, partitioningKeyType, groupName, autogenerated, compactionPolicy,
-                        compactionPolicyProperties, filterField, false);
+                        partitioningKey, partitioningKeyType, autogenerated, filterField, false);
                 break;
             }
 
@@ -191,12 +189,10 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
                 String adapter = ((AString) datasetDetailsRecord
                         .getValueByPos(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_DATASOURCE_ADAPTER_FIELD_INDEX))
                         .getStringValue();
-                IACursor cursor = ((AOrderedList) datasetDetailsRecord
+                cursor = ((AOrderedList) datasetDetailsRecord
                         .getValueByPos(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_PROPERTIES_FIELD_INDEX))
                         .getCursor();
                 Map<String, String> properties = new HashMap<String, String>();
-                String key;
-                String value;
                 while (cursor.next()) {
                     ARecord field = (ARecord) cursor.get();
                     key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX))
@@ -205,9 +201,6 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
                             .getStringValue();
                     properties.put(key, value);
                 }
-                String nodeGroupName = ((AString) datasetDetailsRecord
-                        .getValueByPos(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_GROUPNAME_FIELD_INDEX))
-                        .getStringValue();
 
                 // Timestamp
                 Date timestamp = new Date(
@@ -218,36 +211,20 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
                 ExternalDatasetTransactionState state = ExternalDatasetTransactionState.values()[((AInt32) datasetDetailsRecord
                         .getValueByPos(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_TRANSACTION_STATE_FIELD_INDEX))
                         .getIntegerValue()];
-                // Compaction Policy
-                String compactionPolicy = ((AString) datasetDetailsRecord
-                        .getValueByPos(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_FIELD_INDEX))
-                        .getStringValue();
-                // Compaction Policy Properties
-                cursor = ((AOrderedList) datasetDetailsRecord
-                        .getValueByPos(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX))
-                        .getCursor();
-                Map<String, String> compactionPolicyProperties = new LinkedHashMap<String, String>();
-                while (cursor.next()) {
-                    ARecord field = (ARecord) cursor.get();
-                    key = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_NAME_FIELD_INDEX))
-                            .getStringValue();
-                    value = ((AString) field.getValueByPos(MetadataRecordTypes.PROPERTIES_VALUE_FIELD_INDEX))
-                            .getStringValue();
-                    compactionPolicyProperties.put(key, value);
-                }
 
-                datasetDetails = new ExternalDatasetDetails(adapter, properties, nodeGroupName, timestamp, state,
-                        compactionPolicy, compactionPolicyProperties);
+                datasetDetails = new ExternalDatasetDetails(adapter, properties, timestamp, state);
         }
 
         Map<String, String> hints = getDatasetHints(datasetRecord);
 
-        return new Dataset(dataverseName, datasetName, typeName, datasetDetails, hints, datasetType, datasetId,
-                pendingOp);
+        return new Dataset(dataverseName, datasetName, typeName, nodeGroupName, compactionPolicy,
+                compactionPolicyProperties, datasetDetails, hints, datasetType, datasetId, pendingOp);
     }
 
     @Override
     public ITupleReference getTupleFromMetadataEntity(Dataset dataset) throws IOException, MetadataException {
+        OrderedListBuilder listBuilder = new OrderedListBuilder();
+        ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
         // write the key in the first 2 fields of the tuple
         tupleBuilder.reset();
         aString.setValue(dataset.getDataverseName());
@@ -285,39 +262,68 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
         stringSerde.serialize(aString, fieldValue.getDataOutput());
         recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_DATASETTYPE_FIELD_INDEX, fieldValue);
 
-        // write field 4/5/6
+        // write field 4
         fieldValue.reset();
-        writeDatasetDetailsRecordType(recordBuilder, dataset, fieldValue.getDataOutput());
+        aString.setValue(dataset.getNodeGroupName());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_GROUPNAME_FIELD_INDEX, fieldValue);
+
+        // write field 5
+        fieldValue.reset();
+        aString.setValue(dataset.getCompactionPolicy());
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_FIELD_INDEX, fieldValue);
 
-        // write field 7
-        UnorderedListBuilder listBuilder = new UnorderedListBuilder();
+        // write field 6
         listBuilder
+                .reset((AOrderedListType) MetadataRecordTypes.DATASET_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX]);
+        if (dataset.getCompactionPolicyProperties() != null) {
+            for (Map.Entry<String, String> property : dataset.getCompactionPolicyProperties().entrySet()) {
+                String name = property.getKey();
+                String value = property.getValue();
+                itemValue.reset();
+                DatasetUtils.writePropertyTypeRecord(name, value, itemValue.getDataOutput(),
+                        MetadataRecordTypes.COMPACTION_POLICY_PROPERTIES_RECORDTYPE);
+                listBuilder.addItem(itemValue);
+            }
+        }
+        fieldValue.reset();
+        listBuilder.write(fieldValue.getDataOutput(), true);
+        recordBuilder
+                .addField(MetadataRecordTypes.DATASET_ARECORD_COMPACTION_POLICY_PROPERTIES_FIELD_INDEX, fieldValue);
+
+        // write field 7/8
+        fieldValue.reset();
+        writeDatasetDetailsRecordType(recordBuilder, dataset, fieldValue.getDataOutput());
+
+        // write field 9
+        UnorderedListBuilder uListBuilder = new UnorderedListBuilder();
+        uListBuilder
                 .reset((AUnorderedListType) MetadataRecordTypes.DATASET_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.DATASET_ARECORD_HINTS_FIELD_INDEX]);
-        ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
         for (Map.Entry<String, String> property : dataset.getHints().entrySet()) {
             String name = property.getKey();
             String value = property.getValue();
             itemValue.reset();
             writeDatasetHintRecord(name, value, itemValue.getDataOutput());
-            listBuilder.addItem(itemValue);
+            uListBuilder.addItem(itemValue);
         }
         fieldValue.reset();
-        listBuilder.write(fieldValue.getDataOutput(), true);
+        uListBuilder.write(fieldValue.getDataOutput(), true);
         recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_HINTS_FIELD_INDEX, fieldValue);
 
-        // write field 8
+        // write field 10
         fieldValue.reset();
         aString.setValue(Calendar.getInstance().getTime().toString());
         stringSerde.serialize(aString, fieldValue.getDataOutput());
         recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_TIMESTAMP_FIELD_INDEX, fieldValue);
 
-        // write field 9
+        // write field 11
         fieldValue.reset();
         aInt32.setValue(dataset.getDatasetId());
         aInt32Serde.serialize(aInt32, fieldValue.getDataOutput());
         recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_DATASETID_FIELD_INDEX, fieldValue);
 
-        // write field 10
+        // write field 12
         fieldValue.reset();
         aInt32.setValue(dataset.getPendingOp());
         aInt32Serde.serialize(aInt32, fieldValue.getDataOutput());

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java
index 718d533..bd7d24c 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java
@@ -20,9 +20,7 @@ import java.io.DataInput;
 import java.io.DataInputStream;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Calendar;
-import java.util.List;
 
 import edu.uci.ics.asterix.builders.IARecordBuilder;
 import edu.uci.ics.asterix.builders.OrderedListBuilder;
@@ -48,9 +46,8 @@ import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.AUnorderedListType;
 import edu.uci.ics.asterix.om.types.AbstractCollectionType;
 import edu.uci.ics.asterix.om.types.AbstractComplexType;
-import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
+import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
 import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
@@ -70,9 +67,7 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
     public static final int DATATYPE_PAYLOAD_TUPLE_FIELD_INDEX = 2;
 
     public enum DerivedTypeTag {
-        ENUM,
         RECORD,
-        UNION,
         UNORDEREDLIST,
         ORDEREDLIST
     };
@@ -115,8 +110,6 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
             boolean isAnonymous = ((ABoolean) derivedTypeRecord
                     .getValueByPos(MetadataRecordTypes.DERIVEDTYPE_ARECORD_ISANONYMOUS_FIELD_INDEX)).getBoolean();
             switch (tag) {
-                case ENUM:
-                    throw new NotImplementedException("Enum type");
                 case RECORD: {
                     ARecord recordType = (ARecord) derivedTypeRecord
                             .getValueByPos(MetadataRecordTypes.DERIVEDTYPE_ARECORD_RECORD_FIELD_INDEX);
@@ -139,8 +132,11 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
                         fieldTypeName = ((AString) field
                                 .getValueByPos(MetadataRecordTypes.FIELD_ARECORD_FIELDTYPE_FIELD_INDEX))
                                 .getStringValue();
+                        boolean isNullable = ((ABoolean) field
+                                .getValueByPos(MetadataRecordTypes.FIELD_ARECORD_ISNULLABLE_FIELD_INDEX)).getBoolean()
+                                .booleanValue();
                         fieldTypes[fieldId] = AsterixBuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId,
-                                dataverseName, fieldTypeName);
+                                dataverseName, fieldTypeName, isNullable);
                         fieldId++;
                     }
                     try {
@@ -150,26 +146,13 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
                         throw new MetadataException(e);
                     }
                 }
-                case UNION: {
-                    IACursor cursor = ((AOrderedList) derivedTypeRecord
-                            .getValueByPos(MetadataRecordTypes.DERIVEDTYPE_ARECORD_UNION_FIELD_INDEX)).getCursor();
-                    List<IAType> unionList = new ArrayList<IAType>();
-                    String itemTypeName;
-                    while (cursor.next()) {
-                        itemTypeName = ((AString) cursor.get()).getStringValue();
-                        unionList.add(AsterixBuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId, dataverseName,
-                                itemTypeName));
-                    }
-                    return new Datatype(dataverseName, datatypeName, new AUnionType(unionList, datatypeName),
-                            isAnonymous);
-                }
                 case UNORDEREDLIST: {
                     String unorderedlistTypeName = ((AString) derivedTypeRecord
                             .getValueByPos(MetadataRecordTypes.DERIVEDTYPE_ARECORD_UNORDEREDLIST_FIELD_INDEX))
                             .getStringValue();
                     return new Datatype(dataverseName, datatypeName, new AUnorderedListType(
                             AsterixBuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId, dataverseName,
-                                    unorderedlistTypeName), datatypeName), isAnonymous);
+                                    unorderedlistTypeName, false), datatypeName), isAnonymous);
                 }
                 case ORDEREDLIST: {
                     String orderedlistTypeName = ((AString) derivedTypeRecord
@@ -177,7 +160,7 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
                             .getStringValue();
                     return new Datatype(dataverseName, datatypeName, new AOrderedListType(
                             AsterixBuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId, dataverseName,
-                                    orderedlistTypeName), datatypeName), isAnonymous);
+                                    orderedlistTypeName, false), datatypeName), isAnonymous);
                 }
                 default:
                     throw new UnsupportedOperationException("Unsupported derived type: " + tag);
@@ -212,19 +195,24 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
         stringSerde.serialize(aString, fieldValue.getDataOutput());
         recordBuilder.addField(MetadataRecordTypes.DATATYPE_ARECORD_DATATYPENAME_FIELD_INDEX, fieldValue);
 
-        // write field 2
-        ATypeTag tag = dataType.getDatatype().getTypeTag();
-        if (tag.isDerivedType()) {
+        IAType fieldType = dataType.getDatatype();
+        //unwrap nullable type out of the union
+        if (fieldType.getTypeTag() == ATypeTag.UNION) {
+            fieldType = ((AUnionType) dataType.getDatatype()).getNullableType();
+        }
+
+        // write field 3
+        if (fieldType.getTypeTag().isDerivedType()) {
             fieldValue.reset();
             try {
-                writeDerivedTypeRecord(dataType, fieldValue.getDataOutput());
+                writeDerivedTypeRecord(dataType, (AbstractComplexType) fieldType, fieldValue.getDataOutput());
             } catch (AsterixException e) {
                 throw new MetadataException(e);
             }
             recordBuilder.addField(MetadataRecordTypes.DATATYPE_ARECORD_DERIVED_FIELD_INDEX, fieldValue);
         }
 
-        // write field 3
+        // write field 4
         fieldValue.reset();
         aString.setValue(Calendar.getInstance().getTime().toString());
         stringSerde.serialize(aString, fieldValue.getDataOutput());
@@ -242,14 +230,12 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
         return tuple;
     }
 
-    private void writeDerivedTypeRecord(Datatype type, DataOutput out) throws IOException, AsterixException {
-        DerivedTypeTag tag;
+    private void writeDerivedTypeRecord(Datatype type, AbstractComplexType derivedDatatype, DataOutput out)
+            throws IOException, AsterixException {
+        DerivedTypeTag tag = null;
         IARecordBuilder derivedRecordBuilder = new RecordBuilder();
         ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
-        switch (type.getDatatype().getTypeTag()) {
-            case UNION:
-                tag = DerivedTypeTag.UNION;
-                break;
+        switch (derivedDatatype.getTypeTag()) {
             case ORDEREDLIST:
                 tag = DerivedTypeTag.ORDEREDLIST;
                 break;
@@ -260,8 +246,8 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
                 tag = DerivedTypeTag.RECORD;
                 break;
             default:
-                throw new UnsupportedOperationException("No metadata record Type for"
-                        + type.getDatatype().getDisplayName());
+                throw new UnsupportedOperationException("No metadata record Type for "
+                        + derivedDatatype.getDisplayName());
         }
 
         derivedRecordBuilder.reset(MetadataRecordTypes.DERIVEDTYPE_RECORDTYPE);
@@ -278,27 +264,20 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
         derivedRecordBuilder.addField(MetadataRecordTypes.DERIVEDTYPE_ARECORD_ISANONYMOUS_FIELD_INDEX, fieldValue);
 
         switch (tag) {
-            case ENUM:
-                break;
             case RECORD:
                 fieldValue.reset();
-                writeRecordType(type, fieldValue.getDataOutput());
+                writeRecordType(type, derivedDatatype, fieldValue.getDataOutput());
                 derivedRecordBuilder.addField(MetadataRecordTypes.DERIVEDTYPE_ARECORD_RECORD_FIELD_INDEX, fieldValue);
                 break;
-            case UNION:
-                fieldValue.reset();
-                writeUnionType(type, fieldValue.getDataOutput());
-                derivedRecordBuilder.addField(MetadataRecordTypes.DERIVEDTYPE_ARECORD_UNION_FIELD_INDEX, fieldValue);
-                break;
             case UNORDEREDLIST:
                 fieldValue.reset();
-                writeCollectionType(type, fieldValue.getDataOutput());
+                writeCollectionType(type, derivedDatatype, fieldValue.getDataOutput());
                 derivedRecordBuilder.addField(MetadataRecordTypes.DERIVEDTYPE_ARECORD_UNORDEREDLIST_FIELD_INDEX,
                         fieldValue);
                 break;
             case ORDEREDLIST:
                 fieldValue.reset();
-                writeCollectionType(type, fieldValue.getDataOutput());
+                writeCollectionType(type, derivedDatatype, fieldValue.getDataOutput());
                 derivedRecordBuilder.addField(MetadataRecordTypes.DERIVEDTYPE_ARECORD_ORDEREDLIST_FIELD_INDEX,
                         fieldValue);
                 break;
@@ -306,80 +285,40 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
         derivedRecordBuilder.write(out, true);
     }
 
-    private void writeCollectionType(Datatype instance, DataOutput out) throws HyracksDataException {
-        AbstractCollectionType listType = (AbstractCollectionType) instance.getDatatype();
-        String itemTypeName = listType.getItemType().getTypeName();
-        if (listType.getItemType().getTypeTag().isDerivedType()) {
-            try {
-                itemTypeName = handleNestedDerivedType(itemTypeName, instance.getDatatypeName() + "_ItemType",
-                        (AbstractComplexType) listType.getItemType(), instance);
-            } catch (Exception e) {
-                // TODO: This should not be a HyracksDataException. Can't
-                // fix this currently because of BTree exception model whose
-                // fixes must get in.
-                throw new HyracksDataException(e);
-            }
-        }
-        aString.setValue(itemTypeName);
+    private void writeCollectionType(Datatype instance, AbstractComplexType type, DataOutput out)
+            throws HyracksDataException {
+        AbstractCollectionType listType = (AbstractCollectionType) type;
+        IAType itemType = listType.getItemType();
+        if (itemType.getTypeTag().isDerivedType())
+            handleNestedDerivedType(itemType.getTypeName(), (AbstractComplexType) itemType, instance,
+                    instance.getDataverseName(), instance.getDatatypeName());
+        aString.setValue(listType.getItemType().getTypeName());
         stringSerde.serialize(aString, out);
     }
 
-    private void writeUnionType(Datatype instance, DataOutput dataOutput) throws HyracksDataException {
-        List<IAType> unionList = ((AUnionType) instance.getDatatype()).getUnionList();
-        OrderedListBuilder listBuilder = new OrderedListBuilder();
-        listBuilder.reset(new AOrderedListType(BuiltinType.ASTRING, null));
-        ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
-        String typeName = null;
-
-        int i = 0;
-        for (IAType t : unionList) {
-            typeName = t.getTypeName();
-            if (t.getTypeTag().isDerivedType()) {
-                try {
-                    typeName = handleNestedDerivedType(typeName,
-                            "Type_#" + i + "_UnionType_" + instance.getDatatypeName(), (AbstractComplexType) t,
-                            instance);
-                } catch (Exception e) {
-                    // TODO: This should not be a HyracksDataException. Can't
-                    // fix this currently because of BTree exception model whose
-                    // fixes must get in.
-                    throw new HyracksDataException(e);
-                }
-            }
-            itemValue.reset();
-            aString.setValue(typeName);
-            stringSerde.serialize(aString, itemValue.getDataOutput());
-            listBuilder.addItem(itemValue);
-            i++;
-        }
-        listBuilder.write(dataOutput, true);
-    }
-
-    private void writeRecordType(Datatype instance, DataOutput out) throws IOException, AsterixException {
+    private void writeRecordType(Datatype instance, AbstractComplexType type, DataOutput out) throws IOException,
+            AsterixException {
 
         ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
         ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
         IARecordBuilder recordRecordBuilder = new RecordBuilder();
         IARecordBuilder fieldRecordBuilder = new RecordBuilder();
 
-        ARecordType recType = (ARecordType) instance.getDatatype();
+        ARecordType recType = (ARecordType) type;
         OrderedListBuilder listBuilder = new OrderedListBuilder();
         listBuilder.reset(new AOrderedListType(MetadataRecordTypes.FIELD_RECORDTYPE, null));
-        String fieldTypeName = null;
+        IAType fieldType = null;
+
         for (int i = 0; i < recType.getFieldNames().length; i++) {
-            fieldTypeName = recType.getFieldTypes()[i].getTypeName();
-            if (recType.getFieldTypes()[i].getTypeTag().isDerivedType()) {
-                try {
-                    fieldTypeName = handleNestedDerivedType(fieldTypeName, "Field_" + recType.getFieldNames()[i]
-                            + "_in_" + instance.getDatatypeName(), (AbstractComplexType) recType.getFieldTypes()[i],
-                            instance);
-                } catch (Exception e) {
-                    // TODO: This should not be a HyracksDataException. Can't
-                    // fix this currently because of BTree exception model whose
-                    // fixes must get in.
-                    throw new HyracksDataException(e);
-                }
+            fieldType = recType.getFieldTypes()[i];
+            boolean fieldIsNullable = false;
+            if (NonTaggedFormatUtil.isOptional(fieldType)) {
+                fieldIsNullable = true;
+                fieldType = ((AUnionType) fieldType).getNullableType();
             }
+            if (fieldType.getTypeTag().isDerivedType())
+                handleNestedDerivedType(fieldType.getTypeName(), (AbstractComplexType) fieldType, instance,
+                        instance.getDataverseName(), instance.getDatatypeName());
 
             itemValue.reset();
             fieldRecordBuilder.reset(MetadataRecordTypes.FIELD_RECORDTYPE);
@@ -392,10 +331,15 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
 
             // write field 1
             fieldValue.reset();
-            aString.setValue(fieldTypeName);
+            aString.setValue(fieldType.getTypeName());
             stringSerde.serialize(aString, fieldValue.getDataOutput());
             fieldRecordBuilder.addField(MetadataRecordTypes.FIELD_ARECORD_FIELDTYPE_FIELD_INDEX, fieldValue);
 
+            // write field 2
+            fieldValue.reset();
+            booleanSerde.serialize(fieldIsNullable ? ABoolean.TRUE : ABoolean.FALSE, fieldValue.getDataOutput());
+            fieldRecordBuilder.addField(MetadataRecordTypes.FIELD_ARECORD_ISNULLABLE_FIELD_INDEX, fieldValue);
+
             // write record
             fieldRecordBuilder.write(itemValue.getDataOutput(), true);
 
@@ -418,25 +362,22 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
         recordRecordBuilder.write(out, true);
     }
 
-    private String handleNestedDerivedType(String typeName, String suggestedTypeName, AbstractComplexType nestedType,
-            Datatype topLevelType) throws Exception {
-        MetadataNode mn = MetadataNode.INSTANCE;
+    private String handleNestedDerivedType(String typeName, AbstractComplexType nestedType, Datatype topLevelType,
+            String dataverseName, String datatypeName) throws HyracksDataException {
         try {
-            if (typeName == null) {
-                typeName = suggestedTypeName;
-                nestedType.setTypeName(typeName);
-                metadataNode.addDatatype(jobId, new Datatype(topLevelType.getDataverseName(), typeName, nestedType,
-                        true));
+            metadataNode.addDatatype(jobId, new Datatype(dataverseName, typeName, nestedType, true));
 
-            }
-            mn.insertIntoDatatypeSecondaryIndex(jobId, topLevelType.getDataverseName(), typeName,
-                    topLevelType.getDatatypeName());
-
-        } catch (TreeIndexDuplicateKeyException e) {
-            // The key may have been inserted by a previous DDL statement or by
+        } catch (MetadataException e) {
+            // The nested record type may have been inserted by a previous DDL statement or by
             // a previous nested type.
+            if (!e.getCause().getClass().equals(TreeIndexDuplicateKeyException.class))
+                throw new HyracksDataException(e);
+        } catch (Exception e) {
+            // TODO: This should not be a HyracksDataException. Can't
+            // fix this currently because of BTree exception model whose
+            // fixes must get in.
+            throw new HyracksDataException(e);
         }
         return typeName;
     }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index 0831c7f..f85bf45 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -42,6 +42,7 @@ import edu.uci.ics.asterix.om.base.ARecord;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.base.IACursor;
 import edu.uci.ics.asterix.om.types.AOrderedListType;
+import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -124,12 +125,17 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
         List<IAType> searchKeyType = new ArrayList<IAType>(searchKey.size());
         while (fieldTypeCursor.next()) {
             String typeName = ((AString) fieldTypeCursor.get()).getStringValue();
-            IAType fieldType = AsterixBuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId, dvName, typeName);
+            IAType fieldType = AsterixBuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId, dvName, typeName, false);
             searchKeyType.add(fieldType);
         }
+        // index key type information is not persisted, thus we extract type information from the record metadata
         if (searchKeyType.isEmpty()) {
-            for (int i = 0; i < searchKey.size(); i++)
-                searchKeyType.add(BuiltinType.ANULL);
+            String datatypeName = metadataNode.getDataset(jobId, dvName, dsName).getItemTypeName();
+            ARecordType recordDt = (ARecordType) metadataNode.getDatatype(jobId, dvName, datatypeName).getDatatype();
+            for (int i = 0; i < searchKey.size(); i++) {
+                IAType fieldType = recordDt.getSubFieldType(searchKey.get(i));
+                searchKeyType.add(fieldType);
+            }
         }
         int isEnforcedFieldPos = rec.getType().findFieldPosition(INDEX_ISENFORCED_FIELD_NAME);
         Boolean isEnforcingKeys = false;
@@ -243,10 +249,10 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
             }
         }
 
-        // write optional field 9
-        OrderedListBuilder typeListBuilder = new OrderedListBuilder();
-        typeListBuilder.reset(new AOrderedListType(BuiltinType.ASTRING, null));
-        if (instance.getKeyFieldTypes() != null) {
+        if (instance.isEnforcingKeyFileds()) {
+            // write optional field 9
+            OrderedListBuilder typeListBuilder = new OrderedListBuilder();
+            typeListBuilder.reset(new AOrderedListType(BuiltinType.ASTRING, null));
             ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
             nameValue.reset();
             aString.setValue(INDEX_SEARCHKEY_TYPE_FIELD_NAME);
@@ -267,12 +273,9 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
             } catch (AsterixException e) {
                 throw new MetadataException(e);
             }
-        }
 
-        // write optional field 10
-        if (instance.isEnforcingKeyFileds()) {
+            // write optional field 10
             fieldValue.reset();
-            ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
             nameValue.reset();
             aString.setValue(INDEX_ISENFORCED_FIELD_NAME);
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/utils/DatasetUtils.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/utils/DatasetUtils.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/utils/DatasetUtils.java
index f651085..f7c7b6a 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/utils/DatasetUtils.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/utils/DatasetUtils.java
@@ -15,13 +15,17 @@
 
 package edu.uci.ics.asterix.metadata.utils;
 
+import java.io.DataOutput;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
+import edu.uci.ics.asterix.builders.IARecordBuilder;
+import edu.uci.ics.asterix.builders.RecordBuilder;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
 import edu.uci.ics.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
 import edu.uci.ics.asterix.formats.nontagged.AqlTypeTraitProvider;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataManager;
@@ -31,7 +35,10 @@ import edu.uci.ics.asterix.metadata.entities.CompactionPolicy;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
 import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
 import edu.uci.ics.asterix.metadata.external.IndexingConstants;
+import edu.uci.ics.asterix.om.base.AMutableString;
+import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.BuiltinType;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
@@ -39,7 +46,10 @@ import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
 import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
 import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
 import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
 import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
 
 public class DatasetUtils {
@@ -122,8 +132,6 @@ public class DatasetUtils {
         typeTraits[numKeys] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(itemType);
         return typeTraits;
     }
-    
-    
 
     public static List<List<String>> getPartitioningKeys(Dataset dataset) {
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
@@ -132,10 +140,6 @@ public class DatasetUtils {
         return ((InternalDatasetDetails) dataset.getDatasetDetails()).getPartitioningKey();
     }
 
-    public static String getNodegroupName(Dataset dataset) {
-        return (((InternalDatasetDetails) dataset.getDatasetDetails())).getNodeGroupName();
-    }
-
     public static List<String> getFilterField(Dataset dataset) {
         return (((InternalDatasetDetails) dataset.getDatasetDetails())).getFilterField();
     }
@@ -220,7 +224,7 @@ public class DatasetUtils {
     public static int getPositionOfPartitioningKeyField(Dataset dataset, String fieldExpr) {
         List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
         for (int i = 0; i < partitioningKeys.size(); i++) {
-            if (partitioningKeys.get(i).size() == 1 &&  partitioningKeys.get(i).get(0).equals(fieldExpr)) {
+            if (partitioningKeys.get(i).size() == 1 && partitioningKeys.get(i).get(0).equals(fieldExpr)) {
                 return i;
             }
         }
@@ -229,7 +233,7 @@ public class DatasetUtils {
 
     public static Pair<ILSMMergePolicyFactory, Map<String, String>> getMergePolicyFactory(Dataset dataset,
             MetadataTransactionContext mdTxnCtx) throws AlgebricksException, MetadataException {
-        String policyName = dataset.getDatasetDetails().getCompactionPolicy();
+        String policyName = dataset.getCompactionPolicy();
         CompactionPolicy compactionPolicy = MetadataManager.INSTANCE.getCompactionPolicy(mdTxnCtx,
                 MetadataConstants.METADATA_DATAVERSE_NAME, policyName);
         String compactionPolicyFactoryClassName = compactionPolicy.getClassName();
@@ -242,7 +246,36 @@ public class DatasetUtils {
         } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
             throw new AlgebricksException(e);
         }
-        Map<String, String> properties = dataset.getDatasetDetails().getCompactionPolicyProperties();
+        Map<String, String> properties = dataset.getCompactionPolicyProperties();
         return new Pair<ILSMMergePolicyFactory, Map<String, String>>(mergePolicyFactory, properties);
     }
+
+    @SuppressWarnings("unchecked")
+    public static void writePropertyTypeRecord(String name, String value, DataOutput out, ARecordType recordType)
+            throws HyracksDataException {
+        IARecordBuilder propertyRecordBuilder = new RecordBuilder();
+        ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
+        propertyRecordBuilder.reset(recordType);
+        AMutableString aString = new AMutableString("");
+        ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
+                .getSerializerDeserializer(BuiltinType.ASTRING);
+
+        // write field 0
+        fieldValue.reset();
+        aString.setValue(name);
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        propertyRecordBuilder.addField(0, fieldValue);
+
+        // write field 1
+        fieldValue.reset();
+        aString.setValue(value);
+        stringSerde.serialize(aString, fieldValue.getDataOutput());
+        propertyRecordBuilder.addField(1, fieldValue);
+
+        try {
+            propertyRecordBuilder.write(out, true);
+        } catch (IOException | AsterixException e) {
+            throw new HyracksDataException(e);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilder.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilder.java b/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilder.java
index 845d201..b61fe41 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilder.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/builders/RecordBuilder.java
@@ -104,7 +104,10 @@ public class RecordBuilder implements IARecordBuilder {
     public void reset(ARecordType recType) {
         this.recType = recType;
         this.closedPartOutputStream.reset();
+        this.openPartOutputStream.reset();
         this.numberOfClosedFields = 0;
+        this.numberOfOpenFields = 0;
+        this.offsetPosition = 0;
         if (recType != null) {
             this.isOpen = recType.isOpen();
             this.isNullable = NonTaggedFormatUtil.hasNullableField(recType);
@@ -155,7 +158,7 @@ public class RecordBuilder implements IARecordBuilder {
             nullBitMap[id / 8] |= (byte) (1 << (7 - (id % 8)));
         }
     }
-    
+
     public void addField(int id, byte[] value) {
         closedPartOffsets[id] = closedPartOutputStream.size();
         // We assume the tag is not included (closed field)

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlExpressionTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlExpressionTypeComputer.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlExpressionTypeComputer.java
index bf1601b..5abcbc7 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlExpressionTypeComputer.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlExpressionTypeComputer.java
@@ -14,9 +14,6 @@
  */
 package edu.uci.ics.asterix.dataflow.data.common;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
 import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
@@ -69,10 +66,7 @@ public class AqlExpressionTypeComputer implements IExpressionTypeComputer {
         FunctionIdentifier fi = expr.getFunctionIdentifier();
         ComparisonKind ck = AlgebricksBuiltinFunctions.getComparisonType(fi);
         if (ck != null) {
-            List<IAType> unionList = new ArrayList<IAType>();
-            unionList.add(BuiltinType.ANULL);
-            unionList.add(BuiltinType.ABOOLEAN);
-            return new AUnionType(unionList, "OptionalBoolean");
+            return AUnionType.createNullableType(BuiltinType.ABOOLEAN, "OptionalBoolean");
         }
         // Note: built-in functions + udfs
         IResultTypeComputer rtc = null;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlPartialAggregationTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlPartialAggregationTypeComputer.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlPartialAggregationTypeComputer.java
index 3f1c38e..bd8b6de 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlPartialAggregationTypeComputer.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/common/AqlPartialAggregationTypeComputer.java
@@ -14,13 +14,9 @@
  */
 package edu.uci.ics.asterix.dataflow.data.common;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
 import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
-import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -52,10 +48,7 @@ public class AqlPartialAggregationTypeComputer implements IPartialAggregationTyp
         FunctionIdentifier fi = expr.getFunctionIdentifier();
         ComparisonKind ck = AlgebricksBuiltinFunctions.getComparisonType(fi);
         if (ck != null) {
-            List<IAType> unionList = new ArrayList<IAType>();
-            unionList.add(BuiltinType.ANULL);
-            unionList.add(BuiltinType.ABOOLEAN);
-            return new AUnionType(unionList, "OptionalBoolean");
+            return AUnionType.createNullableType(BuiltinType.ABOOLEAN, "OptionalBoolean");
         }
         return AsterixBuiltinFunctions.getResultTypeComputer(fi).computeType(expr, env, metadataProvider);
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/ANullableFieldPrinterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/ANullableFieldPrinterFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/ANullableFieldPrinterFactory.java
index b7f491d..22c6361 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/ANullableFieldPrinterFactory.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/ANullableFieldPrinterFactory.java
@@ -43,7 +43,7 @@ public class ANullableFieldPrinterFactory implements IPrinterFactory {
             @Override
             public void init() throws AlgebricksException {
                 nullPrinter = (AqlPrinterFactoryProvider.INSTANCE.getPrinterFactory(BuiltinType.ANULL)).createPrinter();
-                fieldPrinter = (AqlPrinterFactoryProvider.INSTANCE.getPrinterFactory(unionType.getUnionList().get(1)))
+                fieldPrinter = (AqlPrinterFactoryProvider.INSTANCE.getPrinterFactory(unionType.getNullableType()))
                         .createPrinter();
             }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/csv/ANullableFieldPrinterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/csv/ANullableFieldPrinterFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/csv/ANullableFieldPrinterFactory.java
index 924e16b..03170eb 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/csv/ANullableFieldPrinterFactory.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/csv/ANullableFieldPrinterFactory.java
@@ -44,8 +44,8 @@ public class ANullableFieldPrinterFactory implements IPrinterFactory {
             public void init() throws AlgebricksException {
                 nullPrinter = (AqlCSVPrinterFactoryProvider.INSTANCE.getPrinterFactory(BuiltinType.ANULL))
                         .createPrinter();
-                fieldPrinter = (AqlCSVPrinterFactoryProvider.INSTANCE.getPrinterFactory(unionType.getUnionList()
-                        .get(1))).createPrinter();
+                fieldPrinter = (AqlCSVPrinterFactoryProvider.INSTANCE.getPrinterFactory(unionType.getNullableType()))
+                        .createPrinter();
             }
 
             @Override

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/json/ANullableFieldPrinterFactory.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/json/ANullableFieldPrinterFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/json/ANullableFieldPrinterFactory.java
index 74ca54f..c6b68f3 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/json/ANullableFieldPrinterFactory.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/printers/json/ANullableFieldPrinterFactory.java
@@ -44,8 +44,8 @@ public class ANullableFieldPrinterFactory implements IPrinterFactory {
             public void init() throws AlgebricksException {
                 nullPrinter = (AqlJSONPrinterFactoryProvider.INSTANCE.getPrinterFactory(BuiltinType.ANULL))
                         .createPrinter();
-                fieldPrinter = (AqlJSONPrinterFactoryProvider.INSTANCE.getPrinterFactory(unionType.getUnionList()
-                        .get(1))).createPrinter();
+                fieldPrinter = (AqlJSONPrinterFactoryProvider.INSTANCE.getPrinterFactory(unionType.getNullableType()))
+                        .createPrinter();
             }
 
             @Override

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
index ac619d6..b694320 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
@@ -67,12 +67,11 @@ public class ARecordSerializerDeserializer implements ISerializerDeserializer<AR
                 IAType t = recordType.getFieldTypes()[i];
                 IAType t2;
                 if (t.getTypeTag() == ATypeTag.UNION) {
-                    if (NonTaggedFormatUtil.isOptionalField((AUnionType) t)) {
-                        t2 = ((AUnionType) recordType.getFieldTypes()[i]).getUnionList().get(
-                                AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
+                    if (((AUnionType) t).isNullableType()) {
+                        t2 = ((AUnionType) recordType.getFieldTypes()[i]).getNullableType();
                         serializers[i] = AqlSerializerDeserializerProvider.INSTANCE
-                                .getSerializerDeserializer(((AUnionType) recordType.getFieldTypes()[i]).getUnionList()
-                                        .get(AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST));
+                                .getSerializerDeserializer(((AUnionType) recordType.getFieldTypes()[i])
+                                        .getNullableType());
                     } else {
                         // union .. the general case
                         throw new NotImplementedException();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlCSVPrinterFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlCSVPrinterFactoryProvider.java b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlCSVPrinterFactoryProvider.java
index 2f120cc..c67689d 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlCSVPrinterFactoryProvider.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlCSVPrinterFactoryProvider.java
@@ -45,7 +45,6 @@ import edu.uci.ics.asterix.dataflow.data.nontagged.printers.csv.AYearMonthDurati
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
 import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
@@ -114,7 +113,7 @@ public class AqlCSVPrinterFactoryProvider implements IPrinterFactoryProvider {
                 case UNORDEREDLIST:
                     throw new AlgebricksException("'Unorderedlist' type unsupported for CSV output");
                 case UNION: {
-                    if (NonTaggedFormatUtil.isOptionalField((AUnionType) aqlType))
+                    if (((AUnionType) aqlType).isNullableType())
                         return new ANullableFieldPrinterFactory((AUnionType) aqlType);
                     else
                         return new AUnionPrinterFactory((AUnionType) aqlType);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlJSONPrinterFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlJSONPrinterFactoryProvider.java b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlJSONPrinterFactoryProvider.java
index 15bab45..601f51d 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlJSONPrinterFactoryProvider.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlJSONPrinterFactoryProvider.java
@@ -50,7 +50,6 @@ import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.AUnorderedListType;
 import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
 import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
@@ -121,7 +120,7 @@ public class AqlJSONPrinterFactoryProvider implements IPrinterFactoryProvider {
                 case UNORDEREDLIST:
                     return new AUnorderedlistPrinterFactory((AUnorderedListType) aqlType);
                 case UNION: {
-                    if (NonTaggedFormatUtil.isOptionalField((AUnionType) aqlType))
+                    if (((AUnionType) aqlType).isNullableType())
                         return new ANullableFieldPrinterFactory((AUnionType) aqlType);
                     else
                         return new AUnionPrinterFactory((AUnionType) aqlType);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlPrinterFactoryProvider.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlPrinterFactoryProvider.java b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlPrinterFactoryProvider.java
index 24a0561..6737837 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlPrinterFactoryProvider.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlPrinterFactoryProvider.java
@@ -50,7 +50,6 @@ import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.AUnorderedListType;
 import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.data.IPrinterFactory;
 import edu.uci.ics.hyracks.algebricks.data.IPrinterFactoryProvider;
@@ -121,7 +120,7 @@ public class AqlPrinterFactoryProvider implements IPrinterFactoryProvider {
                 case UNORDEREDLIST:
                     return new AUnorderedlistPrinterFactory((AUnorderedListType) aqlType);
                 case UNION: {
-                    if (NonTaggedFormatUtil.isOptionalField((AUnionType) aqlType))
+                    if (((AUnionType) aqlType).isNullableType())
                         return new ANullableFieldPrinterFactory((AUnionType) aqlType);
                     else
                         return new AUnionPrinterFactory((AUnionType) aqlType);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/ARecordVisitablePointable.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/ARecordVisitablePointable.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/ARecordVisitablePointable.java
index 9ea28d7..d5efb1b 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/ARecordVisitablePointable.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/ARecordVisitablePointable.java
@@ -94,11 +94,9 @@ public class ARecordVisitablePointable extends AbstractVisitablePointable {
             for (int i = 0; i < numberOfSchemaFields; i++) {
                 ATypeTag ftypeTag = fieldTypes[i].getTypeTag();
 
-                if (fieldTypes[i].getTypeTag() == ATypeTag.UNION
-                        && NonTaggedFormatUtil.isOptionalField((AUnionType) fieldTypes[i]))
+                if (NonTaggedFormatUtil.isOptional(fieldTypes[i]))
                     // optional field: add the embedded non-null type tag
-                    ftypeTag = ((AUnionType) fieldTypes[i]).getUnionList()
-                            .get(AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST).getTypeTag();
+                    ftypeTag = ((AUnionType) fieldTypes[i]).getNullableType().getTypeTag();
 
                 // add type tag Reference
                 int tagStart = typeBos.size();
@@ -204,9 +202,8 @@ public class ARecordVisitablePointable extends AbstractVisitablePointable {
 
                     IAType fieldType = fieldTypes[fieldNumber];
                     if (fieldTypes[fieldNumber].getTypeTag() == ATypeTag.UNION) {
-                        if (NonTaggedFormatUtil.isOptionalField((AUnionType) fieldTypes[fieldNumber])) {
-                            fieldType = ((AUnionType) fieldTypes[fieldNumber]).getUnionList().get(
-                                    AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
+                        if (((AUnionType) fieldTypes[fieldNumber]).isNullableType()) {
+                            fieldType = ((AUnionType) fieldTypes[fieldNumber]).getNullableType();
                             typeTag = fieldType.getTypeTag();
                             fieldValueLength = NonTaggedFormatUtil.getFieldValueLength(b, fieldOffsets[fieldNumber],
                                     typeTag, false);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/cast/ARecordCaster.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/cast/ARecordCaster.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/cast/ARecordCaster.java
index 66df700..77b6dd6 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/cast/ARecordCaster.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/cast/ARecordCaster.java
@@ -163,11 +163,9 @@ class ARecordCaster {
             String fname = fieldNames[i];
 
             // add type tag pointable
-            if (fieldTypes[i].getTypeTag() == ATypeTag.UNION
-                    && NonTaggedFormatUtil.isOptionalField((AUnionType) fieldTypes[i])) {
+            if (NonTaggedFormatUtil.isOptional(fieldTypes[i])) {
                 // optional field: add the embedded non-null type tag
-                ftypeTag = ((AUnionType) fieldTypes[i]).getUnionList()
-                        .get(AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST).getTypeTag();
+                ftypeTag = ((AUnionType) fieldTypes[i]).getNullableType().getTypeTag();
                 optionalFields[i] = true;
             }
             int tagStart = bos.size();
@@ -266,7 +264,7 @@ class ARecordCaster {
         for (int i = 0; i < fieldPermutation.length; i++) {
             if (fieldPermutation[i] < 0) {
                 IAType t = cachedReqType.getFieldTypes()[i];
-                if (!(t.getTypeTag() == ATypeTag.UNION && NonTaggedFormatUtil.isOptionalField((AUnionType) t))) {
+                if (!NonTaggedFormatUtil.isOptional(t)) {
                     // no matched field in the input for a required closed field
                     throw new IllegalStateException("type mismatch: missing a required closed field "
                             + cachedReqType.getFieldNames()[i] + ":" + t.getTypeName());
@@ -301,8 +299,7 @@ class ARecordCaster {
                     //the field is optional in the input record
                     nestedVisitorArg.second = ((AUnionType) fType).getUnionList().get(0);
                 } else {
-                    nestedVisitorArg.second = ((AUnionType) fType).getUnionList().get(
-                            AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
+                    nestedVisitorArg.second = ((AUnionType) fType).getNullableType();
                 }
             }
             field.accept(visitor, nestedVisitorArg);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java
index 7e97efc..61a21a7 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/pointables/nonvisitor/ARecordPointable.java
@@ -236,9 +236,9 @@ public class ARecordPointable extends AbstractPointable {
 
     public IAType getClosedFieldType(ARecordType recordType, int fieldId) {
         IAType aType = recordType.getFieldTypes()[fieldId];
-        if (aType.getTypeTag() == ATypeTag.UNION && NonTaggedFormatUtil.isOptionalField((AUnionType) aType)) {
+        if (NonTaggedFormatUtil.isOptional(aType)) {
             // optional field: add the embedded non-null type tag
-            aType = ((AUnionType) aType).getUnionList().get(AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
+            aType = ((AUnionType) aType).getNullableType();
         }
         return aType;
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a5895308/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/GetOverlappingInvervalTypeComputer.java
----------------------------------------------------------------------
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/GetOverlappingInvervalTypeComputer.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/GetOverlappingInvervalTypeComputer.java
index 9d95dbf..d3f50ce 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/GetOverlappingInvervalTypeComputer.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/GetOverlappingInvervalTypeComputer.java
@@ -14,9 +14,6 @@
  */
 package edu.uci.ics.asterix.om.typecomputer.impl;
 
-import java.util.ArrayList;
-import java.util.List;
-
 import edu.uci.ics.asterix.om.typecomputer.base.IResultTypeComputer;
 import edu.uci.ics.asterix.om.types.AUnionType;
 import edu.uci.ics.asterix.om.types.BuiltinType;
@@ -36,10 +33,7 @@ public class GetOverlappingInvervalTypeComputer implements IResultTypeComputer {
 
     public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
             IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
-        List<IAType> unionList = new ArrayList<IAType>();
-        unionList.add(BuiltinType.ANULL);
-        unionList.add(BuiltinType.AINTERVAL);
-        return new AUnionType(unionList, "IntervalOrNullResult");
+        return AUnionType.createNullableType(BuiltinType.AINTERVAL, "IntervalOrNullResult");
     }
 
 }



Mime
View raw message