asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [1/2] incubator-asterixdb git commit: Add an integer ordered list to dataset "Dataset" and "Index" optinally.
Date Tue, 01 Mar 2016 20:52:44 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master 06e06b052 -> c9bfe259e


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
----------------------------------------------------------------------
diff --git a/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 6259faa..ec915e9 100644
--- a/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -421,14 +421,14 @@ DatasetDecl DatasetSpecification() throws ParseException:
   Map<String,String> properties = null;
   Map<String,String> compactionPolicyProperties = null;
   FunctionSignature appliedFunction = null;
-  List<List<String>> primaryKeyFields = null;
+  Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
   String nodeGroupName = null;
   Map<String,String> hints = new HashMap<String,String>();
   DatasetDecl dsetDecl = null;
   boolean autogenerated = false;
   String compactionPolicy = null;
   boolean temp = false;
-  List<String> filterField = null;
+  Pair<Integer, List<String>> filterField = null;
   Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null,
null);
 }
 {
@@ -481,9 +481,13 @@ DatasetDecl DatasetSpecification() throws ParseException:
     ( <USING> <COMPACTION> <POLICY> compactionPolicy = CompactionPolicy()
(LOOKAHEAD(1) compactionPolicyProperties = Configuration())? )?
     ( LOOKAHEAD(2) <WITH> <FILTER> <ON>  filterField = NestedField() )?
       {
-        InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields,
+        if(filterField!=null && filterField.first!=0){
+          throw new ParseException("A filter field can only be a field in the main record
of the dataset.");
+        }
+        InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second,
+                                                          primaryKeyFields.first,
                                                           autogenerated,
-                                                          filterField,
+                                                          filterField == null? null : filterField.second,
                                                           temp);
         dsetDecl = new DatasetDecl(nameComponents.first,
                                    nameComponents.second,
@@ -547,7 +551,7 @@ CreateIndexStatement IndexSpecification() throws ParseException:
   String indexName = null;
   boolean ifNotExists = false;
   Pair<Identifier,Identifier> nameComponents = null;
-  Pair<List<String>, TypeExpression> fieldPair = null;
+  Pair<Integer, Pair<List<String>, TypeExpression>> fieldPair = null;
   IndexParams indexType = null;
   boolean enforced = false;
 }
@@ -557,11 +561,13 @@ CreateIndexStatement IndexSpecification() throws ParseException:
   <ON> nameComponents = QualifiedName()
   <LEFTPAREN> ( fieldPair = OpenField()
     {
-      cis.addFieldExprPair(fieldPair);
+      cis.addFieldExprPair(fieldPair.second);
+      cis.addFieldIndexIndicator(fieldPair.first);
     }
   ) (<COMMA> fieldPair = OpenField()
     {
-      cis.addFieldExprPair(fieldPair);
+      cis.addFieldExprPair(fieldPair.second);
+      cis.addFieldIndexIndicator(fieldPair.first);
     }
   )* <RIGHTPAREN> ( <TYPE> indexType = IndexType() )? ( <ENFORCED> { enforced
= true; } )?
     {
@@ -825,23 +831,26 @@ FunctionSignature FunctionSignature() throws ParseException:
     }
 }
 
-List<List<String>> PrimaryKey() throws ParseException:
+Pair<List<Integer>, List<List<String>>> PrimaryKey() throws ParseException:
 {
-  List<String> tmp = null;
+  Pair<Integer, List<String>> tmp = null;
+  List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
   List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
 }
 {
   <PRIMARY> <KEY> tmp = NestedField()
     {
-      primaryKeyFields.add(tmp);
+      keyFieldSourceIndicators.add(tmp.first);
+      primaryKeyFields.add(tmp.second);
     }
   ( <COMMA> tmp = NestedField()
     {
-      primaryKeyFields.add(tmp);
+      keyFieldSourceIndicators.add(tmp.first);
+      primaryKeyFields.add(tmp.second);
     }
   )*
     {
-      return primaryKeyFields;
+      return new Pair<List<Integer>, List<List<String>>> (keyFieldSourceIndicators,
primaryKeyFields);
     }
 }
 
@@ -1380,23 +1389,25 @@ String Identifier() throws ParseException:
   )
 }
 
-Pair<List<String>, TypeExpression> OpenField() throws ParseException:
+Pair<Integer, Pair<List<String>, TypeExpression>> OpenField() throws ParseException:
 {
   TypeExpression fieldType = null;
-  List<String> fieldList = null;
+  Pair<Integer, List<String>> fieldList = null;
 }
 {
   fieldList = NestedField()
   ( <COLON> fieldType =  IndexedTypeExpr() )?
   {
-    return new Pair<List<String>, TypeExpression>(fieldList, fieldType);
+    return new Pair<Integer, Pair<List<String>, TypeExpression>>
+          (fieldList.first, new Pair<List<String>, TypeExpression>(fieldList.second,
fieldType));
   }
 }
 
-List<String> NestedField() throws ParseException:
+Pair<Integer, List<String>> NestedField() throws ParseException:
 {
   List<String> exprList = new ArrayList<String>();
   String lit = null;
+  int source = 0;
 }
 {
   lit = Identifier()
@@ -1404,14 +1415,14 @@ List<String> NestedField() throws ParseException:
     boolean meetParens = false;
   }
   (
+    LOOKAHEAD(1)
     <LEFTPAREN><RIGHTPAREN>
     {
-        if(lit.toLowerCase().equals("meta")){
-            exprList.add(lit.toLowerCase() + "()");
-        }else{
+        if(!lit.toLowerCase().equals("meta")){
             throw new ParseException("The string before () has to be \"meta\".");
         }
         meetParens = true;
+        source = 1;
     }
   )?
   {
@@ -1426,7 +1437,7 @@ List<String> NestedField() throws ParseException:
     }
   )*
   {
-    return exprList;
+    return new Pair<Integer, List<String>>(source, exprList);
   }
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
index b1eaf0f..78a7d6f 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
@@ -156,8 +156,8 @@ public class MetadataCache {
                 if (dataset.getDatasetType() == DatasetType.INTERNAL) {
                     InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
                     Index index = new Index(dataset.getDataverseName(), dataset.getDatasetName(),
-                            dataset.getDatasetName(), IndexType.BTREE, id.getPartitioningKey(),
id.getPrimaryKeyType(),
-                            false, true, dataset.getPendingOp());
+                            dataset.getDatasetName(), IndexType.BTREE, id.getPartitioningKey(),
+                            id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, true,
dataset.getPendingOp());
                     addIndexIfNotExistsInternal(index);
                 }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 088a85b..542c12d 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -174,7 +174,7 @@ public class MetadataNode implements IMetadataNode {
         } catch (TreeIndexDuplicateKeyException e) {
             throw new MetadataException(
                     "A dataverse with this name " + dataverse.getDataverseName() + " already
exists.", e);
-        } catch (ACIDException|IndexException|IOException e) {
+        } catch (ACIDException | IndexException | IOException e) {
             throw new MetadataException(e);
         }
     }
@@ -191,15 +191,15 @@ public class MetadataNode implements IMetadataNode {
                 // Add the primary index for the dataset.
                 InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
                 Index primaryIndex = new Index(dataset.getDataverseName(), dataset.getDatasetName(),
-                        dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getPrimaryKeyType(),
false,
-                        true, dataset.getPendingOp());
+                        dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getKeySourceIndicator(),
+                        id.getPrimaryKeyType(), false, true, dataset.getPendingOp());
 
                 addIndex(jobId, primaryIndex);
             }
         } catch (TreeIndexDuplicateKeyException e) {
             throw new MetadataException("A dataset with this name " + dataset.getDatasetName()
                     + " already exists in dataverse '" + dataset.getDataverseName() + "'.",
e);
-        } catch (ACIDException|IndexException|IOException e) {
+        } catch (ACIDException | IndexException | IOException e) {
             throw new MetadataException(e);
         }
     }
@@ -212,7 +212,7 @@ public class MetadataNode implements IMetadataNode {
             insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.INDEX_DATASET, tuple);
         } catch (TreeIndexDuplicateKeyException e) {
             throw new MetadataException("An index with name '" + index.getIndexName() + "'
already exists.", e);
-        } catch (ACIDException|IndexException|IOException e) {
+        } catch (ACIDException | IndexException | IOException e) {
             throw new MetadataException(e);
         }
     }
@@ -225,7 +225,7 @@ public class MetadataNode implements IMetadataNode {
             insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.NODE_DATASET, tuple);
         } catch (TreeIndexDuplicateKeyException e) {
             throw new MetadataException("A node with name '" + node.getNodeName() + "' already
exists.", e);
-        } catch (ACIDException|IndexException|IOException e) {
+        } catch (ACIDException | IndexException | IOException e) {
             throw new MetadataException(e);
         }
     }
@@ -239,7 +239,7 @@ public class MetadataNode implements IMetadataNode {
         } catch (TreeIndexDuplicateKeyException e) {
             throw new MetadataException("A nodegroup with name '" + nodeGroup.getNodeGroupName()
+ "' already exists.",
                     e);
-        } catch (ACIDException|IndexException|IOException e) {
+        } catch (ACIDException | IndexException | IOException e) {
             throw new MetadataException(e);
         }
     }
@@ -252,7 +252,7 @@ public class MetadataNode implements IMetadataNode {
             insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATATYPE_DATASET, tuple);
         } catch (TreeIndexDuplicateKeyException e) {
             throw new MetadataException("A datatype with name '" + datatype.getDatatypeName()
+ "' already exists.", e);
-        } catch (ACIDException|IndexException|IOException e) {
+        } catch (ACIDException | IndexException | IOException e) {
             throw new MetadataException(e);
         }
     }
@@ -268,7 +268,7 @@ public class MetadataNode implements IMetadataNode {
         } catch (TreeIndexDuplicateKeyException e) {
             throw new MetadataException("A function with this name " + function.getName()
+ " and arity "
                     + function.getArity() + " already exists in dataverse '" + function.getDataverseName()
+ "'.", e);
-        } catch (ACIDException|IndexException|IOException e) {
+        } catch (ACIDException | IndexException | IOException e) {
             throw new MetadataException(e);
         }
     }
@@ -1082,6 +1082,7 @@ public class MetadataNode implements IMetadataNode {
     // TODO: Can use Hyrack's TupleUtils for this, once we switch to a newer
     // Hyracks version.
     public ITupleReference createTuple(String... fields) throws HyracksDataException {
+        @SuppressWarnings("unchecked")
         ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
                 .getSerializerDeserializer(BuiltinType.ASTRING);
         AMutableString aString = new AMutableString("");

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
index 84a75ef..a8c3e44 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
@@ -128,7 +128,7 @@ public class MetadataTransactionContext extends MetadataCache {
     }
 
     public void dropIndex(String dataverseName, String datasetName, String indexName) {
-        Index index = new Index(dataverseName, datasetName, indexName, null, null, null,
false, false,
+        Index index = new Index(dataverseName, datasetName, indexName, null, null, null,
null, false, false,
                 IMetadataEntity.PENDING_NO_OP);
         droppedCache.addIndexIfNotExists(index);
         logAndApply(new MetadataLogicalOperation(index, false));

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 3e709bf..ec0d21f 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -226,7 +226,7 @@ public class MetadataBootstrap {
     public static void insertInitialDatasets(MetadataTransactionContext mdTxnCtx) throws
Exception {
         for (int i = 0; i < primaryIndexes.length; i++) {
             IDatasetDetails id = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH,
-                    primaryIndexes[i].getPartitioningExpr(), primaryIndexes[i].getPartitioningExpr(),
+                    primaryIndexes[i].getPartitioningExpr(), primaryIndexes[i].getPartitioningExpr(),
null,
                     primaryIndexes[i].getPartitioningExprType(), false, null, false);
             MetadataManager.INSTANCE.addDataset(mdTxnCtx, new Dataset(primaryIndexes[i].getDataverseName(),
                     primaryIndexes[i].getIndexedDatasetName(), primaryIndexes[i].getDataverseName(),

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
index d401864..db3ccc1 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
@@ -46,6 +46,7 @@ public class Index implements IMetadataEntity<Index>, Comparable<Index>
{
     private final String indexName;
     private final IndexType indexType;
     private final List<List<String>> keyFieldNames;
+    private final List<Integer> keyFieldSourceIndicators;
     private final List<IAType> keyFieldTypes;
     private final boolean enforceKeyFields;
     private final boolean isPrimaryIndex;
@@ -55,13 +56,14 @@ public class Index implements IMetadataEntity<Index>, Comparable<Index>
{
     private int pendingOp;
 
     public Index(String dataverseName, String datasetName, String indexName, IndexType indexType,
-            List<List<String>> keyFieldNames, List<IAType> keyFieldTypes,
int gramLength, boolean enforceKeyFields,
-            boolean isPrimaryIndex, int pendingOp) {
+            List<List<String>> keyFieldNames, List<Integer> keyFieldSourceIndicators,
List<IAType> keyFieldTypes,
+            int gramLength, boolean enforceKeyFields, boolean isPrimaryIndex, int pendingOp)
{
         this.dataverseName = dataverseName;
         this.datasetName = datasetName;
         this.indexName = indexName;
         this.indexType = indexType;
         this.keyFieldNames = keyFieldNames;
+        this.keyFieldSourceIndicators = keyFieldSourceIndicators;
         this.keyFieldTypes = keyFieldTypes;
         this.gramLength = gramLength;
         this.enforceKeyFields = enforceKeyFields;
@@ -70,18 +72,10 @@ public class Index implements IMetadataEntity<Index>, Comparable<Index>
{
     }
 
     public Index(String dataverseName, String datasetName, String indexName, IndexType indexType,
-            List<List<String>> keyFieldNames, List<IAType> keyFieldTypes,
boolean enforceKeyFields,
-            boolean isPrimaryIndex, int pendingOp) {
-        this.dataverseName = dataverseName;
-        this.datasetName = datasetName;
-        this.indexName = indexName;
-        this.indexType = indexType;
-        this.keyFieldNames = keyFieldNames;
-        this.keyFieldTypes = keyFieldTypes;
-        this.gramLength = -1;
-        this.enforceKeyFields = enforceKeyFields;
-        this.isPrimaryIndex = isPrimaryIndex;
-        this.pendingOp = pendingOp;
+            List<List<String>> keyFieldNames, List<Integer> keyFieldSourceIndicators,
List<IAType> keyFieldTypes,
+            boolean enforceKeyFields, boolean isPrimaryIndex, int pendingOp) {
+        this(dataverseName, datasetName, indexName, indexType, keyFieldNames, keyFieldSourceIndicators,
keyFieldTypes,
+                -1, enforceKeyFields, isPrimaryIndex, pendingOp);
     }
 
     public String getDataverseName() {
@@ -100,6 +94,10 @@ public class Index implements IMetadataEntity<Index>, Comparable<Index>
{
         return keyFieldNames;
     }
 
+    public List<Integer> getKeyFieldSourceIndicators() {
+        return keyFieldSourceIndicators;
+    }
+
     public List<IAType> getKeyFieldTypes() {
         return keyFieldTypes;
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
index a772c39..6971037 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
@@ -30,6 +30,8 @@ import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
 import org.apache.asterix.metadata.IDatasetDetails;
 import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
 import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.base.AInt8;
+import org.apache.asterix.om.base.AMutableInt8;
 import org.apache.asterix.om.base.AMutableString;
 import org.apache.asterix.om.base.AString;
 import org.apache.asterix.om.types.AOrderedListType;
@@ -52,24 +54,28 @@ public class InternalDatasetDetails implements IDatasetDetails {
         HASH
     };
 
-    protected final FileStructure fileStructure;
-    protected final PartitioningStrategy partitioningStrategy;
-    protected final List<List<String>> partitioningKeys;
-    protected final List<List<String>> primaryKeys;
-    protected final List<IAType> primaryKeyTypes;
-    protected final boolean autogenerated;
-    protected final boolean temp;
-    protected long lastAccessTime;
-    protected final List<String> filterField;
+    private final FileStructure fileStructure;
+    private final PartitioningStrategy partitioningStrategy;
+    private final List<List<String>> partitioningKeys;
+    private final List<List<String>> primaryKeys;
+    private final List<IAType> primaryKeyTypes;
+    private final boolean autogenerated;
+    private final boolean temp;
+    private long lastAccessTime;
+    private final List<String> filterField;
+    private final List<Integer> keySourceIndicators;
+
     public static final String FILTER_FIELD_NAME = "FilterField";
+    public static final String KEY_FILD_SOURCE_INDICATOR_FIELD_NAME = "KeySourceIndicator";
 
     public InternalDatasetDetails(FileStructure fileStructure, PartitioningStrategy partitioningStrategy,
-            List<List<String>> partitioningKey, List<List<String>>
primaryKey, List<IAType> primaryKeyType,
-            boolean autogenerated, List<String> filterField, boolean temp) {
+            List<List<String>> partitioningKey, List<List<String>>
primaryKey, List<Integer> keyFieldIndicators,
+            List<IAType> primaryKeyType, boolean autogenerated, List<String>
filterField, boolean temp) {
         this.fileStructure = fileStructure;
         this.partitioningStrategy = partitioningStrategy;
         this.partitioningKeys = partitioningKey;
         this.primaryKeys = primaryKey;
+        this.keySourceIndicators = keyFieldIndicators;
         this.primaryKeyTypes = primaryKeyType;
         this.autogenerated = autogenerated;
         this.filterField = filterField;
@@ -89,6 +95,10 @@ public class InternalDatasetDetails implements IDatasetDetails {
         return primaryKeys;
     }
 
+    public List<Integer> getKeySourceIndicator() {
+        return keySourceIndicators;
+    }
+
     public List<IAType> getPrimaryKeyType() {
         return primaryKeyTypes;
     }
@@ -130,14 +140,19 @@ public class InternalDatasetDetails implements IDatasetDetails {
         ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
         OrderedListBuilder primaryKeyListBuilder = new OrderedListBuilder();
         AOrderedListType stringList = new AOrderedListType(BuiltinType.ASTRING, null);
+        AOrderedListType int8List = new AOrderedListType(BuiltinType.AINT8, null);
         internalRecordBuilder.reset(MetadataRecordTypes.INTERNAL_DETAILS_RECORDTYPE);
         AMutableString aString = new AMutableString("");
+        AMutableInt8 aInt8 = new AMutableInt8((byte) 0);
         @SuppressWarnings("unchecked")
         ISerializerDeserializer<ABoolean> booleanSerde = AqlSerializerDeserializerProvider.INSTANCE
                 .getSerializerDeserializer(BuiltinType.ABOOLEAN);
         @SuppressWarnings("unchecked")
         ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
                 .getSerializerDeserializer(BuiltinType.ASTRING);
+        @SuppressWarnings("unchecked")
+        ISerializerDeserializer<AInt8> int8Serde = AqlSerializerDeserializerProvider.INSTANCE
+                .getSerializerDeserializer(BuiltinType.AINT8);
 
         // write field 0
         fieldValue.reset();
@@ -199,6 +214,7 @@ public class InternalDatasetDetails implements IDatasetDetails {
         internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_AUTOGENERATED_FIELD_INDEX,
                 fieldValue);
 
+        // write filter fields if any
         List<String> filterField = getFilterField();
         if (filterField != null) {
             listBuilder.reset(stringList);
@@ -217,6 +233,34 @@ public class InternalDatasetDetails implements IDatasetDetails {
             internalRecordBuilder.addField(nameValue, fieldValue);
         }
 
+        // write key source indicators if any
+        List<Integer> keySourceIndicator = getKeySourceIndicator();
+        boolean needSerialization = false;
+        if (keySourceIndicator != null) {
+            for (int source : keySourceIndicator) {
+                if (source != 0) {
+                    needSerialization = true;
+                    break;
+                }
+            }
+        }
+        if (needSerialization) {
+            listBuilder.reset(int8List);
+            ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
+            nameValue.reset();
+            aString.setValue(KEY_FILD_SOURCE_INDICATOR_FIELD_NAME);
+            stringSerde.serialize(aString, nameValue.getDataOutput());
+            for (int source : keySourceIndicator) {
+                itemValue.reset();
+                aInt8.setValue((byte) source);
+                int8Serde.serialize(aInt8, itemValue.getDataOutput());
+                listBuilder.addItem(itemValue);
+            }
+            fieldValue.reset();
+            listBuilder.write(fieldValue.getDataOutput(), true);
+            internalRecordBuilder.addField(nameValue, fieldValue);
+        }
+
         internalRecordBuilder.write(out, true);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 12543c3..0825aa4 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -52,6 +52,7 @@ import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.om.base.ABoolean;
 import org.apache.asterix.om.base.ADateTime;
 import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AInt8;
 import org.apache.asterix.om.base.AMutableInt32;
 import org.apache.asterix.om.base.AMutableString;
 import org.apache.asterix.om.base.AOrderedList;
@@ -183,10 +184,27 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset>
{
                     }
                 }
 
+                // Read a field-source-indicator field.
+                List<Integer> keyFieldSourceIndicator = new ArrayList<>();
+                int keyFieldSourceIndicatorIndex = datasetDetailsRecord.getType()
+                        .getFieldIndex(InternalDatasetDetails.KEY_FILD_SOURCE_INDICATOR_FIELD_NAME);
+                if (keyFieldSourceIndicatorIndex >= 0) {
+                    cursor = ((AOrderedList) datasetDetailsRecord.getValueByPos(keyFieldSourceIndicatorIndex))
+                            .getCursor();
+                    while (cursor.next()) {
+                        keyFieldSourceIndicator.add((int) ((AInt8) cursor.get()).getByteValue());
+                    }
+                } else {
+                    for (int index = 0; index < partitioningKey.size(); ++index) {
+                        keyFieldSourceIndicator.add(0);
+                    }
+                }
+
                 // Temporary dataset only lives in the compiler therefore the temp field
is false.
                 //  DatasetTupleTranslator always read from the metadata node, so the temp
flag should be always false.
                 datasetDetails = new InternalDatasetDetails(fileStructure, partitioningStrategy,
partitioningKey,
-                        partitioningKey, partitioningKeyType, autogenerated, filterField,
false);
+                        partitioningKey, keyFieldSourceIndicator, partitioningKeyType, autogenerated,
filterField,
+                        false);
                 break;
             }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index 37f0e48..f310a03 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -41,6 +41,8 @@ import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.om.base.ABoolean;
 import org.apache.asterix.om.base.ACollectionCursor;
 import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AInt8;
+import org.apache.asterix.om.base.AMutableInt8;
 import org.apache.asterix.om.base.AOrderedList;
 import org.apache.asterix.om.base.ARecord;
 import org.apache.asterix.om.base.AString;
@@ -70,18 +72,24 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index>
{
     public static final String GRAM_LENGTH_FIELD_NAME = "GramLength";
     public static final String INDEX_SEARCHKEY_TYPE_FIELD_NAME = "SearchKeyType";
     public static final String INDEX_ISENFORCED_FIELD_NAME = "IsEnforced";
+    public static final String INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME = "SearchKeySourceIndicator";
 
     private OrderedListBuilder listBuilder = new OrderedListBuilder();
     private OrderedListBuilder primaryKeyListBuilder = new OrderedListBuilder();
     private AOrderedListType stringList = new AOrderedListType(BuiltinType.ASTRING, null);
+    private AOrderedListType int8List = new AOrderedListType(BuiltinType.AINT8, null);
     private ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
     private ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
     private List<List<String>> searchKey;
     private List<IAType> searchKeyType;
+    private AMutableInt8 aInt8 = new AMutableInt8((byte) 0);
     @SuppressWarnings("unchecked")
-    protected ISerializerDeserializer<AInt32> intSerde = AqlSerializerDeserializerProvider.INSTANCE
+    private ISerializerDeserializer<AInt32> intSerde = AqlSerializerDeserializerProvider.INSTANCE
             .getSerializerDeserializer(BuiltinType.AINT32);
     @SuppressWarnings("unchecked")
+    private ISerializerDeserializer<AInt8> int8Serde = AqlSerializerDeserializerProvider.INSTANCE
+            .getSerializerDeserializer(BuiltinType.AINT8);
+    @SuppressWarnings("unchecked")
     private ISerializerDeserializer<ARecord> recordSerde = AqlSerializerDeserializerProvider.INSTANCE
             .getSerializerDeserializer(MetadataRecordTypes.INDEX_RECORDTYPE);
     private final MetadataNode metadataNode;
@@ -161,8 +169,22 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index>
{
         if (gramLenPos >= 0) {
             gramLength = ((AInt32) rec.getValueByPos(gramLenPos)).getIntegerValue();
         }
-        return new Index(dvName, dsName, indexName, indexStructure, searchKey, searchKeyType,
gramLength,
-                isEnforcingKeys, isPrimaryIndex, pendingOp);
+
+        // Read a field-source-indicator field.
+        List<Integer> keyFieldSourceIndicator = new ArrayList<>();
+        int keyFieldSourceIndicatorIndex = rec.getType().getFieldIndex(INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME);
+        if (keyFieldSourceIndicatorIndex >= 0) {
+            IACursor cursor = ((AOrderedList) rec.getValueByPos(keyFieldSourceIndicatorIndex)).getCursor();
+            while (cursor.next()) {
+                keyFieldSourceIndicator.add((int) ((AInt8) cursor.get()).getByteValue());
+            }
+        } else {
+            for (int index = 0; index < searchKey.size(); ++index) {
+                keyFieldSourceIndicator.add(0);
+            }
+        }
+        return new Index(dvName, dsName, indexName, indexStructure, searchKey, keyFieldSourceIndicator,
searchKeyType,
+                gramLength, isEnforcingKeys, isPrimaryIndex, pendingOp);
     }
 
     @Override
@@ -288,6 +310,33 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index>
{
             recordBuilder.addField(nameValue, fieldValue);
         }
 
+        List<Integer> keySourceIndicator = instance.getKeyFieldSourceIndicators();
+        boolean needSerialization = false;
+        if (keySourceIndicator != null) {
+            for (int source : keySourceIndicator) {
+                if (source != 0) {
+                    needSerialization = true;
+                    break;
+                }
+            }
+        }
+        if (needSerialization) {
+            listBuilder.reset(int8List);
+            ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
+            nameValue.reset();
+            aString.setValue(INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME);
+            stringSerde.serialize(aString, nameValue.getDataOutput());
+            for (int source : keySourceIndicator) {
+                itemValue.reset();
+                aInt8.setValue((byte) source);
+                int8Serde.serialize(aInt8, itemValue.getDataOutput());
+                listBuilder.addItem(itemValue);
+            }
+            fieldValue.reset();
+            listBuilder.write(fieldValue.getDataOutput(), true);
+            recordBuilder.addField(nameValue, fieldValue);
+        }
+
         // write record
         recordBuilder.write(tupleBuilder.getDataOutput(), true);
         tupleBuilder.addFieldEndOffset();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
b/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
index 9328d30..182f296 100644
--- a/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
+++ b/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
@@ -40,24 +40,37 @@ public class DatasetTupleTranslatorTest {
 
     @Test
     public void test() throws MetadataException, IOException {
-        Map<String, String> compactionPolicyProperties = new HashMap<>();
-        compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
-        compactionPolicyProperties.put("max-tolerance-component-count", "3");
+        Integer[] indicators = { 0, 1, null };
+        for (Integer indicator : indicators) {
+            Map<String, String> compactionPolicyProperties = new HashMap<>();
+            compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
+            compactionPolicyProperties.put("max-tolerance-component-count", "3");
 
-        InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE,
PartitioningStrategy.HASH,
-                Collections.singletonList(Collections.singletonList("row_id")),
-                Collections.singletonList(Collections.singletonList("row_id")),
-                Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(),
false);
+            InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE,
PartitioningStrategy.HASH,
+                    Collections.singletonList(Collections.singletonList("row_id")),
+                    Collections.singletonList(Collections.singletonList("row_id")),
+                    indicator == null ? null : Collections.singletonList(indicator),
+                    Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(),
false);
 
-        Dataset dataset = new Dataset("test", "log", "foo", "LogType", "CB", "MetaType",
"DEFAULT_NG_ALL_NODES",
-                "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL,
115, 0);
+            Dataset dataset = new Dataset("test", "log", "foo", "LogType", "CB", "MetaType",
"DEFAULT_NG_ALL_NODES",
+                    "prefix", compactionPolicyProperties, details, Collections.emptyMap(),
DatasetType.INTERNAL, 115,
+                    0);
 
-        MetadataRecordTypes.init();
-        MetadataPrimaryIndexes.init();
-        DatasetTupleTranslator dtTranslator = new DatasetTupleTranslator(true);
-        ITupleReference tuple = dtTranslator.getTupleFromMetadataEntity(dataset);
-        Dataset deserializedDataset = dtTranslator.getMetadataEntityFromTuple(tuple);
-        Assert.assertEquals(dataset.getMetaItemTypeDataverseName(), deserializedDataset.getMetaItemTypeDataverseName());
-        Assert.assertEquals(dataset.getMetaItemTypeName(), deserializedDataset.getMetaItemTypeName());
+            MetadataRecordTypes.init();
+            MetadataPrimaryIndexes.init();
+            DatasetTupleTranslator dtTranslator = new DatasetTupleTranslator(true);
+            ITupleReference tuple = dtTranslator.getTupleFromMetadataEntity(dataset);
+            Dataset deserializedDataset = dtTranslator.getMetadataEntityFromTuple(tuple);
+            Assert.assertEquals(dataset.getMetaItemTypeDataverseName(),
+                    deserializedDataset.getMetaItemTypeDataverseName());
+            Assert.assertEquals(dataset.getMetaItemTypeName(), deserializedDataset.getMetaItemTypeName());
+            if (indicator == null) {
+                Assert.assertEquals(Collections.singletonList(new Integer(0)),
+                        ((InternalDatasetDetails) deserializedDataset.getDatasetDetails()).getKeySourceIndicator());
+            } else {
+                Assert.assertEquals(((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator(),
+                        ((InternalDatasetDetails) deserializedDataset.getDatasetDetails()).getKeySourceIndicator());
+            }
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/c9bfe259/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
----------------------------------------------------------------------
diff --git a/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
b/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
new file mode 100644
index 0000000..de548e2
--- /dev/null
+++ b/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.entitytupletranslators;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.metadata.MetadataException;
+import org.apache.asterix.metadata.MetadataNode;
+import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
+import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Datatype;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails.FileStructure;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class IndexTupleTranslatorTest {
+
+    @Test
+    public void test() throws MetadataException, IOException {
+        Integer[] indicators = { 0, 1, null };
+        for (Integer indicator : indicators) {
+            Map<String, String> compactionPolicyProperties = new HashMap<>();
+            compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
+            compactionPolicyProperties.put("max-tolerance-component-count", "3");
+
+            InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE,
PartitioningStrategy.HASH,
+                    Collections.singletonList(Collections.singletonList("row_id")),
+                    Collections.singletonList(Collections.singletonList("row_id")),
+                    indicator == null ? null : Collections.singletonList(indicator),
+                    Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(),
false);
+
+            Dataset dataset = new Dataset("test", "d1", "foo", "LogType", "CB", "MetaType",
"DEFAULT_NG_ALL_NODES",
+                    "prefix", compactionPolicyProperties, details, Collections.emptyMap(),
DatasetType.INTERNAL, 115,
+                    0);
+
+            Index index = new Index("test", "d1", "i1", IndexType.BTREE,
+                    Collections.singletonList(Collections.singletonList("row_id")),
+                    indicator == null ? null : Collections.singletonList(indicator),
+                    Collections.singletonList(BuiltinType.AINT64), -1, false, false, 0);
+
+            MetadataRecordTypes.init();
+            MetadataPrimaryIndexes.init();
+
+            MetadataNode mockMetadataNode = mock(MetadataNode.class);
+            when(mockMetadataNode.getDatatype(any(), anyString(), anyString())).thenReturn(new
Datatype("test", "d1",
+                    new ARecordType("", new String[] { "row_id" }, new IAType[] { BuiltinType.AINT64
}, true), true));
+            when(mockMetadataNode.getDataset(any(), anyString(), anyString())).thenReturn(dataset);
+
+            IndexTupleTranslator idxTranslator = new IndexTupleTranslator(null, mockMetadataNode,
true);
+            ITupleReference tuple = idxTranslator.getTupleFromMetadataEntity(index);
+            Index deserializedIndex = idxTranslator.getMetadataEntityFromTuple(tuple);
+            if (indicator == null) {
+                Assert.assertEquals(Collections.singletonList(new Integer(0)),
+                        deserializedIndex.getKeyFieldSourceIndicators());
+            } else {
+                Assert.assertEquals(index.getKeyFieldSourceIndicators(),
+                        deserializedIndex.getKeyFieldSourceIndicators());
+            }
+        }
+    }
+}



Mime
View raw message