asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [4/4] asterixdb git commit: [ASTERIXDB-1946][STO][IDX] Create BTreeIndex for Correlated Datasets
Date Tue, 20 Jun 2017 16:17:28 GMT
[ASTERIXDB-1946][STO][IDX] Create BTreeIndex for Correlated Datasets

Implemented create seconary BTree index for datasets using
correlated merge policy. Instead of creating one component
for the new index, this change creates one component for each
component of the primary index to maintain the correlation.
The current implementation assumes when a secondary index is being
created, the dataset is locked with no modifications.

Change-Id: I2a3435e6720f07bd6a5092d4d9ce42e8d4b7894c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1813
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a7fa05bb
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a7fa05bb
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a7fa05bb

Branch: refs/heads/master
Commit: a7fa05bb014503609d1b6287fa08b0a296098088
Parents: 86cbec5
Author: luochen01 <cluo8@uci.edu>
Authored: Mon Jun 19 22:47:49 2017 -0700
Committer: Yingyi Bu <buyingyi@gmail.com>
Committed: Tue Jun 20 09:16:37 2017 -0700

----------------------------------------------------------------------
 .../asterix/app/translator/QueryTranslator.java |   5 +
 ...orrelated-secondary-index-nullable.1.ddl.aql |  57 +++
 ...elated-secondary-index-nullable.2.update.aql |  35 ++
 ...orrelated-secondary-index-nullable.3.ddl.aql |  21 ++
 ...related-secondary-index-nullable.4.query.aql |  32 ++
 ...ee-correlated-secondary-index-open.1.ddl.aql |  61 ++++
 ...correlated-secondary-index-open.2.update.aql |  38 ++
 ...ee-correlated-secondary-index-open.3.ddl.aql |  21 ++
 ...-correlated-secondary-index-open.4.query.aql |  25 ++
 ...orrelated-secondary-index-nullable.1.ddl.aql |  54 +++
 ...elated-secondary-index-nullable.2.update.aql |  45 +++
 ...orrelated-secondary-index-nullable.3.ddl.aql |  22 ++
 ...related-secondary-index-nullable.4.query.aql |  33 ++
 ...ee-correlated-secondary-index-open.1.ddl.aql |  62 ++++
 ...correlated-secondary-index-open.2.update.aql |  45 +++
 ...ee-correlated-secondary-index-open.3.ddl.aql |  22 ++
 ...-correlated-secondary-index-open.4.query.aql |  32 ++
 ...ated-prefix-merge-policy-with-feed.1.ddl.aql |  61 ++++
 ...d-prefix-merge-policy-with-feed.2.update.aql |  28 ++
 ...ated-prefix-merge-policy-with-feed.3.ddl.aql |  22 ++
 ...ed-prefix-merge-policy-with-feed.4.query.aql |  25 ++
 ...rt-with-correlated-secondary-btree.1.ddl.aql |  47 +++
 ...with-correlated-secondary-btree.2.update.aql |  27 ++
 ...rt-with-correlated-secondary-btree.3.ddl.aql |  21 ++
 ...-with-correlated-secondary-btree.4.query.aql |  25 ++
 ...scan-dataset-with-correlated-index.1.ddl.aql |  39 +++
 ...n-dataset-with-correlated-index.2.update.aql |  42 +++
 ...scan-dataset-with-correlated-index.3.ddl.aql |  28 ++
 ...an-dataset-with-correlated-index.4.query.aql |  30 ++
 ...primary-correlated-secondary-btree.1.ddl.aql |  42 +++
 ...mary-correlated-secondary-btree.2.update.aql |  42 +++
 ...primary-correlated-secondary-btree.3.ddl.aql |  27 ++
 ...imary-correlated-secondary-btree.4.query.aql |  29 ++
 ...related-secondary-index-nullable.1.ddl.sqlpp |  65 ++++
 ...ated-secondary-index-nullable.2.update.sqlpp |  36 ++
 ...related-secondary-index-nullable.3.ddl.sqlpp |  24 ++
 ...lated-secondary-index-nullable.4.query.sqlpp |  33 ++
 ...-correlated-secondary-index-open.1.ddl.sqlpp |  73 ++++
 ...rrelated-secondary-index-open.2.update.sqlpp |  31 ++
 ...-correlated-secondary-index-open.3.ddl.sqlpp |  24 ++
 ...orrelated-secondary-index-open.4.query.sqlpp |  27 ++
 ...related-secondary-index-nullable.1.ddl.sqlpp |  62 ++++
 ...ated-secondary-index-nullable.2.update.sqlpp |  37 ++
 ...related-secondary-index-nullable.3.ddl.sqlpp |  22 ++
 ...lated-secondary-index-nullable.4.query.sqlpp |  34 ++
 ...-correlated-secondary-index-open.1.ddl.sqlpp |  72 ++++
 ...rrelated-secondary-index-open.2.update.sqlpp |  35 ++
 ...-correlated-secondary-index-open.3.ddl.sqlpp |  24 ++
 ...orrelated-secondary-index-open.4.query.sqlpp |  33 ++
 ...ed-prefix-merge-policy-with-feed.1.ddl.sqlpp |  61 ++++
 ...prefix-merge-policy-with-feed.2.update.sqlpp |  29 ++
 ...ed-prefix-merge-policy-with-feed.3.ddl.sqlpp |  25 ++
 ...-prefix-merge-policy-with-feed.5.query.sqlpp |  25 ++
 ...-with-correlated-secondary-btree.1.ddl.sqlpp |  48 +++
 ...th-correlated-secondary-btree.2.update.sqlpp |  28 ++
 ...-with-correlated-secondary-btree.3.ddl.sqlpp |  28 ++
 ...ith-correlated-secondary-btree.4.query.sqlpp |  26 ++
 .../btree-index-01/btree-index-01.1.ddl.sqlpp   |  35 ++
 .../btree-index-01.10.query.sqlpp               |  25 ++
 .../btree-index-01.11.query.sqlpp               |  25 ++
 .../btree-index-01.2.update.sqlpp               |  79 +++++
 .../btree-index-01/btree-index-01.3.ddl.sqlpp   |  28 ++
 .../btree-index-01/btree-index-01.4.query.sqlpp |  25 ++
 .../btree-index-01/btree-index-01.5.query.sqlpp |  25 ++
 .../btree-index-01/btree-index-01.6.query.sqlpp |  25 ++
 .../btree-index-01/btree-index-01.7.query.sqlpp |  25 ++
 .../btree-index-01/btree-index-01.8.query.sqlpp |  25 ++
 .../btree-index-01/btree-index-01.9.query.sqlpp |  25 ++
 ...an-dataset-with-correlated-index.1.ddl.sqlpp |  42 +++
 ...dataset-with-correlated-index.2.update.sqlpp |  35 ++
 ...an-dataset-with-correlated-index.3.ddl.sqlpp |  30 ++
 ...-dataset-with-correlated-index.4.query.sqlpp |  33 ++
 .../primary-secondary-btree.1.ddl.sqlpp         |  42 +++
 .../primary-secondary-btree.2.update.sqlpp      |  42 +++
 .../primary-secondary-btree.3.ddl.sqlpp         |  27 ++
 .../primary-secondary-btree.4.query.sqlpp       |  31 ++
 .../btree-index-01/btree-index-01.10.adm        |   3 +
 .../btree-index-01/btree-index-01.11.adm        |   2 +
 .../btree-index-01/btree-index-01.4.adm         |   2 +
 .../btree-index-01/btree-index-01.5.adm         |   2 +
 .../btree-index-01/btree-index-01.6.adm         |   2 +
 .../btree-index-01/btree-index-01.7.adm         |   2 +
 .../btree-index-01/btree-index-01.8.adm         |   5 +
 .../btree-index-01/btree-index-01.9.adm         |   4 +
 .../src/test/resources/runtimets/testsuite.xml  |  43 +++
 .../resources/runtimets/testsuite_sqlpp.xml     |  46 +++
 ...ault_correlated_secondary_index.1.script.aql |  19 +
 ...ult_correlated_secondary_index.10.script.aql |  19 +
 ...default_correlated_secondary_index.2.ddl.aql |  71 ++++
 ...ault_correlated_secondary_index.3.update.aql |  29 ++
 ...fault_correlated_secondary_index.4.txneu.aql |  43 +++
 ...default_correlated_secondary_index.5.ddl.aql |  29 ++
 ...ault_correlated_secondary_index.6.txnqbc.aql |  28 ++
 ...ault_correlated_secondary_index.7.script.aql |  19 +
 ...ault_correlated_secondary_index.8.script.aql |  19 +
 ...ault_correlated_secondary_index.9.txnqar.aql |  28 ++
 .../create_and_start.sh                         |  18 +
 .../kill_cc_and_nc.sh                           |  18 +
 .../stop_and_delete.sh                          |  20 ++
 .../stop_and_start.sh                           |  20 ++
 .../test/resources/transactionts/testsuite.xml  |   6 +
 .../asterix/metadata/entities/Dataset.java      |   5 +
 .../asterix/metadata/utils/IndexUtil.java       |  35 +-
 ...econdaryCorrelatedBTreeOperationsHelper.java | 201 +++++++++++
 ...daryCorrelatedTreeIndexOperationsHelper.java | 324 ++++++++++++++++++
 ...ctLSMSecondaryIndexCreationNodePushable.java |  86 +++++
 .../LSMSecondaryIndexBulkLoadNodePushable.java  | 250 ++++++++++++++
 ...econdaryIndexBulkLoadOperatorDescriptor.java |  66 ++++
 ...IndexCreationTupleProcessorNodePushable.java | 298 ++++++++++++++++
 ...reationTupleProcessorOperatorDescriptor.java |  63 ++++
 ...econdaryIndexCreationTupleProcessorTest.java | 343 +++++++++++++++++++
 .../hyracks-storage-am-lsm-btree/pom.xml        |   5 +
 ...TreeDiskComponentScanOperatorDescriptor.java |  53 +++
 ...eeDiskComponentScanOperatorNodePushable.java |  69 ++++
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |   3 +-
 .../storage/am/lsm/btree/impls/LSMBTree.java    |   3 +-
 .../impls/LSMBTreeDiskComponentScanCursor.java  |  23 +-
 .../storage/am/lsm/common/api/ILSMIndex.java    |   7 +
 .../invertedindex/impls/LSMInvertedIndex.java   |  13 +-
 .../storage/am/lsm/rtree/impls/LSMRTree.java    |   3 +-
 .../impls/LSMRTreeWithAntiMatterTuples.java     |  11 +-
 121 files changed, 4989 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 65268b1..f7327b4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -952,6 +952,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
             // #. create the index artifact in NC.
             runJob(hcc, spec);
 
+            // #. flush the internal dataset for correlated policy
+            if (ds.isCorrelated() && ds.getDatasetType() == DatasetType.INTERNAL) {
+                FlushDatasetUtil.flushDataset(hcc, metadataProvider, dataverseName, datasetName, datasetName);
+            }
+
             mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
             bActiveTxn = true;
             metadataProvider.setMetadataTxnContext(mdTxnCtx);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..e4a402e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-secondary-index-nullable.aql
+ * Description     : This test is intended to test deletion from correlated secondary btree indexes that
+ * are built on nullable fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as closed {
+  number: int64,
+  street: string,
+  city: string
+}
+
+create type CustomerType as closed {
+  cid: int64,
+  name: string,
+  age: int64?,
+  address: AddressType?,
+  interests: {{string}},
+  children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create feed CustomerFeed
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),
+("format"="adm"),
+("type-name"="CustomerType"),
+("tuple-interval"="10"));
+

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..4fcc858
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.aql
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-secondary-index-nullable.aql
+ * Description     : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+use dataverse test;
+
+set "wait-for-completion-feed" "true";
+
+connect feed CustomerFeed to dataset Customers;
+
+start feed CustomerFeed;
+
+delete $c from dataset Customers where $c.cid>=200;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..c19ef10
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index age_index on Customers(age);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.aql
new file mode 100644
index 0000000..4b8ff9a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.4.query.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-secondary-index-nullable.aql
+ * Description     : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+use dataverse test;
+
+for $c in dataset('Customers')
+where $c.age < 20
+order by $c.cid
+return $c

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.aql
new file mode 100644
index 0000000..c540b1f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.1.ddl.aql
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description     : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on open fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as closed {
+  number: int64,
+  street: string,
+  city: string
+}
+
+create type CustomerType as closed {
+  cid: int64,
+  name: string,
+  age: int64?,
+  address: AddressType?,
+  interests: {{string}},
+  children: [ { name: string, age: int64? } ]
+}
+
+create type CustomerOpenType as open {
+  cid: int64,
+  name: string,
+  address: AddressType?,
+  interests: {{string}},
+  children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create dataset CustomersOpen(CustomerOpenType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.aql
new file mode 100644
index 0000000..1c24546
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.2.update.aql
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+load dataset Customers
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),("format"="adm"));
+
+insert into dataset test.CustomersOpen (
+    for $x in dataset test.Customers
+        return {
+            "cid": $x.cid,
+            "name": $x.name,
+            "age": $x.age,
+            "address": $x.address,
+            "interests": $x.interests,
+            "children": $x.children
+        }
+);
+
+delete $c from dataset CustomersOpen where $c.cid>=200;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.aql
new file mode 100644
index 0000000..7c358d2
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.3.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index age_index on CustomersOpen(age:int32?) enforced;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.aql
new file mode 100644
index 0000000..33d6039
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-delete-btree-correlated-secondary-index-open/scan-delete-btree-correlated-secondary-index-open.4.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+for $c in dataset('CustomersOpen')
+where $c.age < 20
+order by $c.cid
+return $c

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.aql
new file mode 100644
index 0000000..c846c06
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.1.ddl.aql
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description     : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as closed {
+  number: int64,
+  street: string,
+  city: string
+}
+
+create type CustomerType as closed {
+  cid: int64,
+  name: string,
+  age: int64?,
+  address: AddressType?,
+  interests: {{string}},
+  children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create dataset CustomersMini(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.aql
new file mode 100644
index 0000000..833c400
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.2.update.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description     : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+use dataverse test;
+
+load dataset Customers
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),("format"="adm"));
+
+insert into dataset CustomersMini
+(
+    for $c in dataset('Customers')
+    where $c.cid < 200
+    return {
+      "cid": $c.cid,
+        "name": $c.name,
+        "age": $c.age,
+        "address": $c.address,
+        "interests": $c.interests,
+        "children": $c.children
+    }
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.aql
new file mode 100644
index 0000000..f54c11d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.3.ddl.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index age_index on CustomersMini(age);
+

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.aql
new file mode 100644
index 0000000..6b70584
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-nullable/scan-insert-btree-correlated-secondary-index-nullable.4.query.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description     : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+
+use dataverse test;
+
+for $c in dataset('CustomersMini')
+where $c.age < 20
+order by $c.cid
+return $c

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.aql
new file mode 100644
index 0000000..09e4d94
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.1.ddl.aql
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description     : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on open fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type AddressType as closed {
+  number: int64,
+  street: string,
+  city: string
+}
+
+create type CustomerType as closed {
+  cid: int64,
+  name: string,
+  age: int64?,
+  address: AddressType?,
+  interests: {{string}},
+  children: [ { name: string, age: int64? } ]
+}
+
+create type CustomerOpenType as open {
+  cid: int64,
+  name: string,
+  address: AddressType?,
+  interests: {{string}},
+  children: [ { name: string, age: int64? } ]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create dataset CustomersOpen(CustomerOpenType) primary key cid
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.aql
new file mode 100644
index 0000000..8883bbf
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.2.update.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description     : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on open fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+use dataverse test;
+
+load dataset Customers
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),("format"="adm"));
+
+insert into dataset CustomersOpen
+(
+    for $c in dataset('Customers')
+    where $c.cid < 200
+    return {
+      "cid": $c.cid,
+        "name": $c.name,
+        "age": $c.age,
+        "address": $c.address,
+        "interests": $c.interests,
+        "children": $c.children
+    }
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.aql
new file mode 100644
index 0000000..38baed5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.3.ddl.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index age_index on CustomersOpen(age:int32?) enforced;
+

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.aql
new file mode 100644
index 0000000..a0ffb86
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/scan-insert-btree-correlated-secondary-index-open/scan-insert-btree-correlated-secondary-index-open.4.query.aql
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-open.aql
+ * Description     : This test is intended to test insertion into correlated secondary btree indexes
+ * that are built on open fields
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+use dataverse test;
+
+for $c in dataset('CustomersOpen')
+where $c.age < 20
+order by $c.cid
+return $c

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql
new file mode 100644
index 0000000..ab50500
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.1.ddl.aql
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : using-correlated-prefix-merge-policy-with-feed.aql
+ * Description     : This test is inteded to test the correlated prefix merge policy, and create secondary
+ * index for datasets using this policy
+ * Expected Result : Success
+ * Date            : June 9 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type LineItemType as closed {
+  l_orderkey: int64,
+  l_partkey: int64,
+  l_suppkey: int64,
+  l_linenumber: int64,
+  l_quantity: int64,
+  l_extendedprice: double,
+  l_discount: double,
+  l_tax: double,
+  l_returnflag: string,
+  l_linestatus: string,
+  l_shipdate: string,
+  l_commitdate: string,
+  l_receiptdate: string,
+  l_shipinstruct: string,
+  l_shipmode: string,
+  l_comment: string
+}
+
+create dataset LineItem(LineItemType)
+  primary key l_orderkey, l_linenumber using compaction policy correlated-prefix
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create feed LineItemFeed
+using localfs
+(("path"="asterix_nc1://data/tpch0.001/lineitem.tbl"),
+("format"="delimited-text"),
+("delimiter"="|"),
+("type-name"="LineItemType"),
+("tuple-interval"="10"));
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql
new file mode 100644
index 0000000..f5cf29b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.2.update.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+set wait-for-completion-feed "true";
+
+connect feed LineItemFeed to dataset LineItem;
+
+start feed LineItemFeed;
+
+delete $l from dataset LineItem where $l.l_suppkey>=2 or $l.l_linenumber>1;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.aql
new file mode 100644
index 0000000..4a96c91
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.3.ddl.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index idx_LineItem_partkey on LineItem(l_linenumber);
+create index idx_LineItem_suppkey on LineItem(l_suppkey);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql
new file mode 100644
index 0000000..e4c670a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/dml/using-correlated-prefix-merge-policy-with-feed/using-correlated-prefix-merge-policy-with-feed.4.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+for $c in dataset('LineItem')
+where $c.l_suppkey<150
+order by $c.l_orderkey, $c.l_linenumber
+return $c

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.aql
new file mode 100644
index 0000000..511775d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.1.ddl.aql
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Test filters with insert pipeline in the existence of a correlated secondary b-tree
+ * Expected Res : Success
+ * Date         : 16 June 2017
+ */
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+create type FacebookMessageType as closed {
+        message-id: int64,
+        author-id: int64,
+        in-response-to: int64?,
+        sender-location: point?,
+        message: string,
+        send-time: datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType)
+primary key message-id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+
+create dataset FacebookMessages2(FacebookMessageType)
+primary key message-id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"))
+with filter on send-time;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.aql
new file mode 100644
index 0000000..0e0575a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.2.update.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+load dataset FacebookMessages using localfs
+(("path"="asterix_nc1://data/fbm-with-send-time.adm"),("format"="adm"));
+
+insert into dataset FacebookMessages2 (
+for $m in dataset('FacebookMessages')
+    return $m
+);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.aql
new file mode 100644
index 0000000..1bfb269
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.3.ddl.aql
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+create index fbAuthorIdx on FacebookMessages2(author-id) type btree;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.aql
new file mode 100644
index 0000000..df3092c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/filters/insert-with-correlated-secondary-btree/insert-with-correlated-secondary-btree.4.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+use dataverse test;
+
+for $m in dataset('FacebookMessages2')
+where $m.author-id = 1
+and $m.send-time > datetime("2012-08-20T10:10:00")
+and $m.send-time < datetime("2012-11-20T10:10:00")
+return $m

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.aql
new file mode 100644
index 0000000..6834cde
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.1.ddl.aql
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description     : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data.
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+create type test.Emp as closed {
+id: int64,
+fname: string,
+lname: string,
+age: int64,
+dept: string
+}
+
+create temporary dataset test.employee(Emp) primary key id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.aql
new file mode 100644
index 0000000..c68a59e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.2.update.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description     : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data.
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+use dataverse test;
+
+load dataset test.employee
+using localfs
+(("path"="asterix_nc1://data/names.adm"),("format"="delimited-text"),("delimiter"="|"));
+
+
+insert into dataset test.employee (
+for $x in dataset test.employee
+return {
+    "id": $x.id + 10000,
+    "fname": $x.fname,
+    "lname": $x.lname,
+    "age": $x.age,
+    "dept": $x.dept
+}
+);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.aql
new file mode 100644
index 0000000..00b0850
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.3.ddl.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description     : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data.
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+use dataverse test;
+
+create index idx_employee_first_name on test.employee(fname);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.aql
new file mode 100644
index 0000000..195f11e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/temp-dataset/insert-and-scan-dataset-with-correlated-index/insert-and-scan-dataset-with-correlated-index.4.query.aql
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description     : This test is intended to test inserting into a temporary dataset that has a correlated
+ * secondary index and scan the data.
+ * Expected Result : Success
+ * Date            : June 8 2017
+ */
+
+use dataverse test;
+
+for $l in dataset('test.employee')
+order by $l.id
+return $l

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.1.ddl.aql
new file mode 100644
index 0000000..5bb3985
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.1.ddl.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which has a b-tree correlated secondary index
+ * Expected Res : Success
+ * Date         : June 8 2017
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TestType as closed{
+id:int32,
+age:int32,
+name:string,
+salary:double
+};
+
+create dataset UpsertTo("TestType") primary key id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));
+
+create dataset UpsertFrom("TestType") primary key id
+using compaction policy "correlated-prefix"
+(("max-mergable-component-size"="16384"),("max-tolerance-component-count"="3"));

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.2.update.aql
new file mode 100644
index 0000000..2f9ac1e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.2.update.aql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which has a b-tree correlated secondary index
+ * Expected Res : Success
+ * Date         : June 8 2017
+ */
+
+use dataverse test;
+// load first dataset
+load dataset UpsertTo using
+localfs(("format"="delimited-text"),
+  ("path"="asterix_nc1://data/upsert/raw-data/overlapping.data"),
+  ("delimiter"=","));
+// load second dataset
+load dataset UpsertFrom using
+localfs(("format"="delimited-text"),
+  ("path"="asterix_nc1://data/upsert/raw-data/test-data.txt,asterix_nc1://data/upsert/raw-data/more-data.txt"),
+  ("delimiter"=","));
+
+// upsert UpsertFrom into UpsertTo
+use dataverse test;
+upsert into dataset UpsertTo(
+ for $x in dataset UpsertFrom
+ return $x
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.3.ddl.aql
new file mode 100644
index 0000000..e46c96b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.3.ddl.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which has a b-tree correlated secondary index
+ * Expected Res : Success
+ * Date         : June 08 2017
+ */
+
+use dataverse test;
+
+create index ageindex on UpsertTo('age');

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.4.query.aql
new file mode 100644
index 0000000..35711c1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/upsert/primary-correlated-secondary-btree/primary-correlated-secondary-btree.4.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Upsert into a dataset which has a b-tree correlated secondary index
+ * Expected Res : Success
+ * Date         : June 8 2017
+ */
+
+ // So far this one doesn't use the btree index, need another query
+use dataverse test;
+for $x in dataset UpsertTo
+where $x.age >5
+return $x;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.sqlpp
new file mode 100644
index 0000000..64a9d9b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.1.ddl.sqlpp
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description     : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date            : June 7 2017
+ */
+
+drop  dataverse test if exists;
+create  dataverse test;
+
+use test;
+
+
+create type test.AddressType as
+ closed {
+  number : bigint,
+  street : string,
+  city : string
+}
+
+create type test.CustomerType as
+ closed {
+  cid : bigint,
+  name : string,
+  age : bigint?,
+  address : AddressType?,
+  interests : {{string}},
+  children : [{
+          name : string,
+          age : bigint?
+      }
+]
+}
+
+create dataset Customers(CustomerType) primary key cid
+using compaction policy `correlated-prefix`
+((`max-mergable-component-size`=`16384`),(`max-tolerance-component-count`=`3`));
+
+
+create feed CustomerFeed
+using localfs
+(("path"="asterix_nc1://data/semistructured/co1k/customer.adm"),
+("format"="adm"),
+("type-name"="CustomerType"),
+("tuple-interval"="10"));
+

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.sqlpp
new file mode 100644
index 0000000..57b7529
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.2.update.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : scan-delete-btree-correlated-secondary-index-nullable.aql
+ * Description     : This test is intended to test deletion from correlated secondary btree indexes
+ * that are built on nullable fields
+ * Expected Result : Success
+ * Date            : June 07 2017
+ */
+
+use test;
+
+set `wait-for-completion-feed` "true";
+
+connect feed CustomerFeed to dataset Customers;
+
+start feed CustomerFeed;
+
+delete from Customers
+ where cid >= 200;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a7fa05bb/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.sqlpp
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.sqlpp
new file mode 100644
index 0000000..8934610
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/dml/scan-delete-btree-correlated-secondary-index-nullable/scan-delete-btree-correlated-secondary-index-nullable.3.ddl.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+create  index age_index  on Customers (age) type btree;
+


Mime
View raw message