asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [2/5] asterixdb git commit: Fix Indexing on Open fields and Meta fields
Date Wed, 22 Jun 2016 21:05:33 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.1.ddl.aql
new file mode 100644
index 0000000..6d475bd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.1.ddl.aql
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, ingest data then create an index
+ * on a field which sometimes is missing and sometimes is null
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+
+drop dataverse KeyVerse if exists;
+create dataverse KeyVerse;
+use dataverse KeyVerse;
+
+create type DocumentType as open{
+};
+
+create type KVMetaType as open{
+"key":string,
+vbucket:int32,
+seq:int64,
+cas:int64,
+expiration:int32,
+flags:int32,
+revSeq:int64,
+lockTime:int32
+};
+
+create dataset KVStore(DocumentType) with meta(KVMetaType)primary key meta()."key";
+
+create feed KVChangeStream using adapter(
+    ("type-name"="DocumentType"),
+    ("meta-type-name"="KVMetaType"),
+    ("reader"="org.apache.asterix.external.input.record.reader.kv.KVTestReaderFactory"),
+    ("parser"="record-with-metadata"),
+    ("format"="dcp"),
+    ("record-format"="json"),
+    ("change-feed"="true"),
+    ("key-indexes"="0"),
+    ("key-indicators"="1"),
+    ("num-of-records"="1000")
+);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.2.update.aql
new file mode 100644
index 0000000..37d67d0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.2.update.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, ingest data then create an index
+ * on a field which sometimes is missing and sometimes is null
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+set wait-for-completion-feed "true";
+connect feed KVChangeStream to dataset KVStore;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.3.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.3.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.3.ddl.aql
new file mode 100644
index 0000000..5bfc09b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.3.ddl.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, ingest data then create an index
+ * on a field which sometimes is missing and sometimes is null
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+create index VBucketIndex on KVStore(exp:int32) enforced;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.4.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.4.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.4.query.aql
new file mode 100644
index 0000000..883be64
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.4.query.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, ingest data then create an index
+ * on a field which sometimes is missing and sometimes is null
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+count(
+    for $d in dataset KVStore
+    where $d.exp = 15
+    return $d
+);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.5.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.5.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.5.ddl.aql
new file mode 100644
index 0000000..2f043ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.5.ddl.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, ingest data then create an index
+ * on a field which sometimes is missing and sometimes is null
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+drop dataverse KeyVerse;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.1.ddl.aql
new file mode 100644
index 0000000..0ae91fa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.1.ddl.aql
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create an index
+ * on an open field in the record part then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+
+drop dataverse KeyVerse if exists;
+create dataverse KeyVerse;
+use dataverse KeyVerse;
+
+create type DocumentType as open{
+};
+
+create type KVMetaType as open{
+"key":string,
+vbucket:int32,
+seq:int64,
+cas:int64,
+expiration:int32,
+flags:int32,
+revSeq:int64,
+lockTime:int32
+};
+
+create dataset KVStore(DocumentType) with meta(KVMetaType)primary key meta()."key";
+
+create feed KVChangeStream using adapter(
+    ("type-name"="DocumentType"),
+    ("meta-type-name"="KVMetaType"),
+    ("reader"="org.apache.asterix.external.input.record.reader.kv.KVTestReaderFactory"),
+    ("parser"="record-with-metadata"),
+    ("format"="dcp"),
+    ("record-format"="json"),
+    ("change-feed"="true"),
+    ("key-indexes"="0"),
+    ("key-indicators"="1"),
+    ("num-of-records"="1000")
+);
+
+create index OpenIndex on KVStore(id:int32) enforced;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.2.update.aql
new file mode 100644
index 0000000..ae5390b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.2.update.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create an index
+ * on an open field in the record part then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+set wait-for-completion-feed "true";
+connect feed KVChangeStream to dataset KVStore;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.3.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.3.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.3.query.aql
new file mode 100644
index 0000000..5e9b3f4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.3.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create an index
+ * on an open field in the record part then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+for $d in dataset KVStore
+where $d.id = 5
+return $d;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.4.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.4.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.4.ddl.aql
new file mode 100644
index 0000000..004cebe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.4.ddl.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create an index
+ * on an open field in the record part then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+drop dataverse KeyVerse;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.1.ddl.aql
new file mode 100644
index 0000000..3cf9374
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.1.ddl.aql
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create an index
+ * on a field which sometimes is missing and sometimes is null and then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+
+drop dataverse KeyVerse if exists;
+create dataverse KeyVerse;
+use dataverse KeyVerse;
+
+create type DocumentType as open{
+};
+
+create type KVMetaType as open{
+"key":string,
+vbucket:int32,
+seq:int64,
+cas:int64,
+expiration:int32,
+flags:int32,
+revSeq:int64,
+lockTime:int32
+};
+
+create dataset KVStore(DocumentType) with meta(KVMetaType)primary key meta()."key";
+
+create feed KVChangeStream using adapter(
+    ("type-name"="DocumentType"),
+    ("meta-type-name"="KVMetaType"),
+    ("reader"="org.apache.asterix.external.input.record.reader.kv.KVTestReaderFactory"),
+    ("parser"="record-with-metadata"),
+    ("format"="dcp"),
+    ("record-format"="json"),
+    ("change-feed"="true"),
+    ("key-indexes"="0"),
+    ("key-indicators"="1"),
+    ("num-of-records"="1000")
+);
+
+create index OpenIndex on KVStore(exp:int32) enforced;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.2.update.aql
new file mode 100644
index 0000000..9b34e70
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.2.update.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create an index
+ * on a field which sometimes is missing and sometimes is null and then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+set wait-for-completion-feed "true";
+connect feed KVChangeStream to dataset KVStore;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.3.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.3.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.3.query.aql
new file mode 100644
index 0000000..8c887f1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.3.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create an index
+ * on a field which sometimes is missing and sometimes is null and then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+for $d in dataset KVStore
+where $d.exp = 15
+return $d;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.4.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.4.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.4.ddl.aql
new file mode 100644
index 0000000..2fcc497
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.4.ddl.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create an index
+ * on a field which sometimes is missing and sometimes is null and then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+drop dataverse KeyVerse;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.1.ddl.aql
new file mode 100644
index 0000000..2471f79
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.1.ddl.aql
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create a composite index
+ * on a field from meta and a field from the value and then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+
+drop dataverse KeyVerse if exists;
+create dataverse KeyVerse;
+use dataverse KeyVerse;
+
+create type DocumentType as open{
+id:int32
+};
+
+create type KVMetaType as open{
+"key":string,
+vbucket:int32,
+seq:int64,
+cas:int64,
+expiration:int32,
+flags:int32,
+revSeq:int64,
+lockTime:int32
+};
+
+create dataset KVStore(DocumentType) with meta(KVMetaType)primary key meta()."key";
+
+create feed KVChangeStream using adapter(
+    ("type-name"="DocumentType"),
+    ("meta-type-name"="KVMetaType"),
+    ("reader"="org.apache.asterix.external.input.record.reader.kv.KVTestReaderFactory"),
+    ("parser"="record-with-metadata"),
+    ("format"="dcp"),
+    ("record-format"="json"),
+    ("change-feed"="true"),
+    ("key-indexes"="0"),
+    ("key-indicators"="1"),
+    ("num-of-records"="1000")
+);
+
+create index MixedIndex on KVStore(meta().vbucket,id);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.2.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.2.update.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.2.update.aql
new file mode 100644
index 0000000..9fa0c3d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.2.update.aql
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create a composite index
+ * on a field from meta and a field from the value and then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+set wait-for-completion-feed "true";
+connect feed KVChangeStream to dataset KVStore;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.3.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.3.query.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.3.query.aql
new file mode 100644
index 0000000..5089770
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.3.query.aql
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create a composite index
+ * on a field from meta and a field from the value and then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+use dataverse KeyVerse;
+
+for $d in dataset KVStore
+where $d.id = 5
+and meta($d).vbucket<100
+return $d;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.4.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.4.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.4.ddl.aql
new file mode 100644
index 0000000..8f688b1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.4.ddl.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Create a change feed with meta-data, create a composite index
+ * on a field from meta and a field from the value and then ingest data
+ * Expected Res : Success
+ * Date         : 18th Jun 2016
+ */
+drop dataverse KeyVerse;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-after-ingest/change-feed-with-meta-pk-in-meta-index-after-ingest.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-after-ingest/change-feed-with-meta-pk-in-meta-index-after-ingest.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-after-ingest/change-feed-with-meta-pk-in-meta-index-after-ingest.4.adm
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-after-ingest/change-feed-with-meta-pk-in-meta-index-after-ingest.4.adm
@@ -0,0 +1 @@
+1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-in-meta/change-feed-with-meta-pk-in-meta-index-in-meta.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-in-meta/change-feed-with-meta-pk-in-meta-index-in-meta.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-in-meta/change-feed-with-meta-pk-in-meta-index-in-meta.4.adm
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-in-meta/change-feed-with-meta-pk-in-meta-index-in-meta.4.adm
@@ -0,0 +1 @@
+1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.4.adm
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest/change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest.4.adm
@@ -0,0 +1 @@
+1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.4.adm
new file mode 100644
index 0000000..77d1727
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-open-index-in-value/change-feed-with-meta-pk-in-meta-open-index-in-value.4.adm
@@ -0,0 +1 @@
+{ "id": 5, "name": "Ian Maxon", "exp": 15 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.4.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.4.adm
new file mode 100644
index 0000000..77d1727
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-pk-in-meta-open-index-with-missing/change-feed-with-meta-pk-in-meta-open-index-with-missing.4.adm
@@ -0,0 +1 @@
+{ "id": 5, "name": "Ian Maxon", "exp": 15 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.3.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.3.adm
new file mode 100644
index 0000000..8402b39
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/change-feed-with-meta-with-mixed-index/change-feed-with-meta-with-mixed-index.3.adm
@@ -0,0 +1 @@
+{ "id": 5i32, "name": "Ian Maxon", "exp": 15 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
index be663e3..3907677 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -92,6 +92,42 @@
   </test-group>
   <test-group name="feeds">
     <test-case FilePath="feeds">
+      <compilation-unit name="change-feed-with-meta-with-mixed-index">
+        <output-dir compare="Text">change-feed-with-meta-with-mixed-index</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="feeds">
+      <compilation-unit name="change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest">
+        <output-dir compare="Text">change-feed-with-meta-pk-in-meta-index-with-missing-after-ingest</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="feeds">
+      <compilation-unit name="change-feed-with-meta-pk-in-meta-open-index-with-missing">
+        <output-dir compare="Text">change-feed-with-meta-pk-in-meta-open-index-with-missing</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="feeds">
+      <compilation-unit name="change-feed-with-meta-open-index-in-meta">
+        <output-dir compare="Text">change-feed-with-meta-open-index-in-meta</output-dir>
+        <expected-error>Typed open index can only be created on the record part</expected-error>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="feeds">
+      <compilation-unit name="change-feed-with-meta-pk-in-meta-open-index-in-value">
+        <output-dir compare="Text">change-feed-with-meta-pk-in-meta-open-index-in-value</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="feeds">
+      <compilation-unit name="change-feed-with-meta-pk-in-meta-index-after-ingest">
+        <output-dir compare="Text">change-feed-with-meta-pk-in-meta-index-after-ingest</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="feeds">
+      <compilation-unit name="change-feed-with-meta-pk-in-meta-index-in-meta">
+        <output-dir compare="Text">change-feed-with-meta-pk-in-meta-index-in-meta</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="feeds">
       <compilation-unit name="drop-nonexistent-feed">
         <output-dir compare="Text">drop-nonexistent-feed</output-dir>
       </compilation-unit>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
index f7fe77f..57f37da 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/kv/KVTestReader.java
@@ -153,8 +153,21 @@ public class KVTestReader implements IRecordReader<DCPRequest> {
         // reset the string
         strBuilder.setLength(0);
         strBuilder.append("{\"id\":" + (counter + upsertCounter) + ",\"name\":\""
-                + names[(counter + upsertCounter) % names.length] + "\",\"exp\":" + ((counter + upsertCounter) * 3)
-                + "}");
+                + names[(counter + upsertCounter) % names.length] + "\"");
+        switch (counter % 3) {
+            case 0:
+                // Missing
+                break;
+            case 1:
+                strBuilder.append(",\"exp\":null");
+                break;
+            case 2:
+                strBuilder.append(",\"exp\":" + ((counter + upsertCounter) * 3));
+                break;
+            default:
+                break;
+        }
+        strBuilder.append("}");
         byteBuff.clear();
         byteBuff.writeBytes(strBuilder.toString().getBytes(StandardCharsets.UTF_8));
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
index d3d681a..44c249f 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
@@ -123,6 +123,17 @@ public class CreateIndexStatement implements Statement {
         return Kind.CREATE_INDEX;
     }
 
+    public boolean hasMetaField() {
+        if (fieldIndexIndicators != null) {
+            for (Integer indicator : fieldIndexIndicators) {
+                if (indicator.intValue() != 0) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     @Override
     public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
         return visitor.visit(this, arg);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java
index c13ee0e..d809f4f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java
@@ -209,19 +209,12 @@ public abstract class AqlDataSource implements IDataSource<AqlSourceId> {
     }
 
     public LogicalVariable getDataRecordVariable(List<LogicalVariable> dataScanVariables) {
-        if (hasMeta()) {
-            return dataScanVariables.get(dataScanVariables.size() - 2);
-        } else {
-            return dataScanVariables.get(dataScanVariables.size() - 1);
-        }
+        return hasMeta() ? dataScanVariables.get(dataScanVariables.size() - 2)
+                : dataScanVariables.get(dataScanVariables.size() - 1);
     }
 
     public List<LogicalVariable> getPrimaryKeyVariables(List<LogicalVariable> dataScanVariables) {
-        if (hasMeta()) {
-            return new ArrayList<>(dataScanVariables.subList(0, dataScanVariables.size() - 2));
-        } else {
-            return new ArrayList<>(dataScanVariables.subList(0, dataScanVariables.size() - 1));
-        }
+        return new ArrayList<>(dataScanVariables.subList(0, dataScanVariables.size() - (hasMeta() ? 2 : 1)));
     }
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index f2e0424..7557b79 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -581,6 +581,14 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
 
             ARecordType itemType =
                     (ARecordType) this.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
+            ARecordType metaType = null;
+            List<Integer> primaryKeyIndicators = null;
+            if (dataset.hasMetaPart()) {
+                metaType =
+                        (ARecordType) findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+                primaryKeyIndicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
+            }
+
             ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, itemType);
             IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
                     itemType, context.getBinaryComparatorFactoryProvider());
@@ -598,7 +606,9 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
                 Pair<IBinaryComparatorFactory[], ITypeTraits[]> comparatorFactoriesAndTypeTraits =
                         getComparatorFactoriesAndTypeTraitsOfSecondaryBTreeIndex(secondaryIndex.getIndexType(),
                                 secondaryIndex.getKeyFieldNames(), secondaryIndex.getKeyFieldTypes(),
-                                DatasetUtils.getPartitioningKeys(dataset), itemType, dataset.getDatasetType());
+                                DatasetUtils.getPartitioningKeys(dataset), itemType, dataset.getDatasetType(),
+                                dataset.hasMetaPart(), primaryKeyIndicators,
+                                secondaryIndex.getKeyFieldSourceIndicators(), metaType);
                 comparatorFactories = comparatorFactoriesAndTypeTraits.first;
                 typeTraits = comparatorFactoriesAndTypeTraits.second;
                 if (filterTypeTraits != null) {
@@ -618,7 +628,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
                 // get meta item type
                 ARecordType metaItemType = DatasetUtils.getMetaType(this, dataset);
                 typeTraits = DatasetUtils.computeTupleTypeTraits(dataset, itemType, metaItemType);
-                comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset, itemType,
+                comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset, itemType, metaItemType,
                         context.getBinaryComparatorFactoryProvider());
                 filterFields = DatasetUtils.createFilterFields(dataset);
                 btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
@@ -699,7 +709,9 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
 
     private Pair<IBinaryComparatorFactory[], ITypeTraits[]> getComparatorFactoriesAndTypeTraitsOfSecondaryBTreeIndex(
             IndexType indexType, List<List<String>> sidxKeyFieldNames, List<IAType> sidxKeyFieldTypes,
-            List<List<String>> pidxKeyFieldNames, ARecordType recType, DatasetType dsType) throws AlgebricksException {
+            List<List<String>> pidxKeyFieldNames, ARecordType recType, DatasetType dsType, boolean hasMeta,
+            List<Integer> primaryIndexKeyIndicators, List<Integer> secondaryIndexIndicators, ARecordType metaType)
+            throws AlgebricksException {
 
         IBinaryComparatorFactory[] comparatorFactories;
         ITypeTraits[] typeTraits;
@@ -711,7 +723,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
         int i = 0;
         for (; i < sidxKeyFieldCount; ++i) {
             Pair<IAType, Boolean> keyPairType =
-                    Index.getNonNullableOpenFieldType(sidxKeyFieldTypes.get(i), sidxKeyFieldNames.get(i), recType);
+                    Index.getNonNullableOpenFieldType(sidxKeyFieldTypes.get(i), sidxKeyFieldNames.get(i),
+                            (hasMeta && secondaryIndexIndicators.get(i).intValue() == 1) ? metaType : recType);
             IAType keyType = keyPairType.first;
             comparatorFactories[i] =
                     AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
@@ -723,7 +736,9 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
             try {
                 switch (dsType) {
                     case INTERNAL:
-                        keyType = recType.getSubFieldType(pidxKeyFieldNames.get(j));
+                        keyType = (hasMeta && primaryIndexKeyIndicators.get(j).intValue() == 1)
+                                ? metaType.getSubFieldType(pidxKeyFieldNames.get(j))
+                                : recType.getSubFieldType(pidxKeyFieldNames.get(j));
                         break;
                     case EXTERNAL:
                         keyType = IndexingConstants.getFieldType(j);
@@ -794,9 +809,14 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
             Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc =
                     splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), dataset.getDatasetName(),
                             indexName, temp);
+            ARecordType metaType = null;
+            if (dataset.hasMetaPart()) {
+                metaType =
+                        (ARecordType) findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+            }
 
             IBinaryComparatorFactory[] primaryComparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(
-                    dataset, recType, context.getBinaryComparatorFactoryProvider());
+                    dataset, recType, metaType, context.getBinaryComparatorFactoryProvider());
             int[] btreeFields = new int[primaryComparatorFactories.length];
             for (int i = 0; i < btreeFields.length; i++) {
                 btreeFields[i] = i + numNestedSecondaryKeyFields;
@@ -1015,12 +1035,18 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
                     dataset.getDatasetName(), dataset.getDatasetName());
             String indexName = primaryIndex.getIndexName();
 
+            ARecordType metaType = null;
+            if (dataset.hasMetaPart()) {
+                metaType =
+                        (ARecordType) findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+            }
+
             String itemTypeName = dataset.getItemTypeName();
             ARecordType itemType = (ARecordType) MetadataManager.INSTANCE
                     .getDatatype(mdTxnCtx, dataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
             ITypeTraits[] typeTraits = DatasetUtils.computeTupleTypeTraits(dataset, itemType, null);
             IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
-                    itemType, context.getBinaryComparatorFactoryProvider());
+                    itemType, metaType, context.getBinaryComparatorFactoryProvider());
 
             Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
                     splitProviderAndPartitionConstraintsForDataset(dataSource.getId().getDataverseName(), datasetName,
@@ -1111,7 +1137,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
 
             IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
             IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
-                    itemType, context.getBinaryComparatorFactoryProvider());
+                    itemType, metaItemType, context.getBinaryComparatorFactoryProvider());
             Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
                     splitProviderAndPartitionConstraintsForDataset(dataSource.getId().getDataverseName(), datasetName,
                             indexName, temp);
@@ -1749,11 +1775,12 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
                 numTokenFields = secondaryKeys.size() + 1;
             }
 
+            ARecordType metaItemType = DatasetUtils.getMetaType(this, dataset);
             ITypeTraits[] tokenTypeTraits = new ITypeTraits[numTokenFields];
             ITypeTraits[] invListsTypeTraits = new ITypeTraits[primaryKeys.size()];
             IBinaryComparatorFactory[] tokenComparatorFactories = new IBinaryComparatorFactory[numTokenFields];
             IBinaryComparatorFactory[] invListComparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(
-                    dataset, recType, context.getBinaryComparatorFactoryProvider());
+                    dataset, recType, metaItemType, context.getBinaryComparatorFactoryProvider());
 
             IAType secondaryKeyType = null;
 
@@ -1936,9 +1963,9 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
                 typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
                 ++i;
             }
-
+            ARecordType metaItemType = DatasetUtils.getMetaType(this, dataset);
             IBinaryComparatorFactory[] primaryComparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(
-                    dataset, recType, context.getBinaryComparatorFactoryProvider());
+                    dataset, recType, metaItemType, context.getBinaryComparatorFactoryProvider());
             IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
             Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
                     splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
@@ -2276,7 +2303,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
             ITypeTraits[] typeTraits = DatasetUtils.computeTupleTypeTraits(dataset, itemType, metaItemType);
             IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
             IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
-                    itemType, context.getBinaryComparatorFactoryProvider());
+                    itemType, metaItemType, context.getBinaryComparatorFactoryProvider());
             Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
                     splitProviderAndPartitionConstraintsForDataset(dataSource.getId().getDataverseName(), datasetName,
                             indexName, temp);
@@ -2315,17 +2342,27 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
                     btreeFields, filterFields, !temp);
             AsterixLSMTreeUpsertOperatorDescriptor op;
 
-            ITypeTraits[] outputTypeTraits = new ITypeTraits[recordDesc.getFieldCount() + 1 + numFilterFields];
+            ITypeTraits[] outputTypeTraits =
+                    new ITypeTraits[recordDesc.getFieldCount() + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
             ISerializerDeserializer[] outputSerDes =
-                    new ISerializerDeserializer[recordDesc.getFieldCount() + 1 + numFilterFields];
+                    new ISerializerDeserializer[recordDesc.getFieldCount() + (dataset.hasMetaPart() ? 2 : 1)
+                            + numFilterFields];
             for (int j = 0; j < recordDesc.getFieldCount(); j++) {
                 outputTypeTraits[j] = recordDesc.getTypeTraits()[j];
                 outputSerDes[j] = recordDesc.getFields()[j];
             }
-            outputSerDes[outputSerDes.length - 1 - numFilterFields] =
+            outputSerDes[outputSerDes.length - (dataset.hasMetaPart() ? 2 : 1) - numFilterFields] =
                     FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType);
-            outputTypeTraits[outputTypeTraits.length - 1 - numFilterFields] =
+            outputTypeTraits[outputTypeTraits.length - (dataset.hasMetaPart() ? 2 : 1) - numFilterFields] =
                     FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(itemType);
+
+            if (dataset.hasMetaPart()) {
+                outputSerDes[outputSerDes.length - 1 - numFilterFields] =
+                        FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(metaItemType);
+                outputTypeTraits[outputTypeTraits.length - 1 - numFilterFields] =
+                        FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(metaItemType);
+            }
+
             int fieldIdx = -1;
             if (numFilterFields > 0) {
                 String filterField = DatasetUtils.getFilterField(dataset).get(0);
@@ -2531,11 +2568,12 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
                 numTokenFields = secondaryKeys.size() + 1;
             }
 
+            ARecordType metaItemType = DatasetUtils.getMetaType(this, dataset);
             ITypeTraits[] tokenTypeTraits = new ITypeTraits[numTokenFields];
             ITypeTraits[] invListsTypeTraits = new ITypeTraits[primaryKeys.size()];
             IBinaryComparatorFactory[] tokenComparatorFactories = new IBinaryComparatorFactory[numTokenFields];
             IBinaryComparatorFactory[] invListComparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(
-                    dataset, recType, context.getBinaryComparatorFactoryProvider());
+                    dataset, recType, metaItemType, context.getBinaryComparatorFactoryProvider());
 
             IAType secondaryKeyType = null;
 
@@ -2736,8 +2774,9 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
                 ++i;
             }
 
+            ARecordType metaItemType = DatasetUtils.getMetaType(this, dataset);
             IBinaryComparatorFactory[] primaryComparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(
-                    dataset, recType, context.getBinaryComparatorFactoryProvider());
+                    dataset, recType, metaItemType, context.getBinaryComparatorFactoryProvider());
             IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
             Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
                     splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
index 949e6ca..bdf9ed0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
@@ -238,4 +238,15 @@ public class Index implements IMetadataEntity<Index>, Comparable<Index> {
         }
         return dataverseName.compareTo(otherIndex.getDataverseName());
     }
+
+    public boolean hasMetaFields() {
+        if (keyFieldSourceIndicators != null) {
+            for (Integer indicator : keyFieldSourceIndicators) {
+                if (indicator.intValue() != 0) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
index 0ac4f56..d0af744 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
@@ -59,7 +59,8 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
 
 public class DatasetUtils {
     public static IBinaryComparatorFactory[] computeKeysBinaryComparatorFactories(Dataset dataset, ARecordType itemType,
-            IBinaryComparatorFactoryProvider comparatorFactoryProvider) throws AlgebricksException {
+            ARecordType metaItemType, IBinaryComparatorFactoryProvider comparatorFactoryProvider)
+            throws AlgebricksException {
         List<List<String>> partitioningKeys = getPartitioningKeys(dataset);
         IBinaryComparatorFactory[] bcfs = new IBinaryComparatorFactory[partitioningKeys.size()];
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
@@ -72,8 +73,11 @@ public class DatasetUtils {
                 }
             }
         } else {
+            InternalDatasetDetails dsd = (InternalDatasetDetails) dataset.getDatasetDetails();
             for (int i = 0; i < partitioningKeys.size(); i++) {
-                IAType keyType = itemType.getSubFieldType(partitioningKeys.get(i));
+                IAType keyType = (dataset.hasMetaPart() && dsd.getKeySourceIndicator().get(i).intValue() == 1)
+                        ? metaItemType.getSubFieldType(partitioningKeys.get(i))
+                        : itemType.getSubFieldType(partitioningKeys.get(i));
                 bcfs[i] = comparatorFactoryProvider.getBinaryComparatorFactory(keyType, true);
             }
         }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtils.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtils.java
index 60ca42f..108cd33 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtils.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/KeyFieldTypeUtils.java
@@ -204,6 +204,6 @@ public class KeyFieldTypeUtils {
      */
     public static ARecordType chooseSource(List<Integer> keySourceIndicators, int index, ARecordType recordType,
             ARecordType metaRecordType) {
-        return keySourceIndicators.get(0) == 0 ? recordType : metaRecordType;
+        return keySourceIndicators.get(index) == 0 ? recordType : metaRecordType;
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastRecordDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastRecordDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastRecordDescriptor.java
index 8f21ffc..81e132a 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastRecordDescriptor.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CastRecordDescriptor.java
@@ -39,6 +39,9 @@ import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
 
 public class CastRecordDescriptor extends AbstractScalarFunctionDynamicDescriptor {
 
+    private CastRecordDescriptor() {
+    }
+
     public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
         @Override
         public IFunctionDescriptor createFunctionDescriptor() {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
index 24988d4..857f1e2 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/AsterixLSMPrimaryUpsertOperatorNodePushable.java
@@ -74,29 +74,31 @@ public class AsterixLSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertU
     private int presetFieldIndex = -1;
     private ARecordPointable recPointable;
     private DataOutput prevDos;
+    private final boolean hasMeta;
 
     public AsterixLSMPrimaryUpsertOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
             int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, int numOfPrimaryKeys,
             ARecordType recordType, int filterFieldIndex) {
         super(opDesc, ctx, partition, fieldPermutation, recordDescProvider, IndexOperation.UPSERT);
-        // initialize missingWriter
-        this.missingWriter = opDesc.getMissingWriterFactory().createMissingWriter();
-        // The search key should only have the primary index and not use the permutations.
         this.key = new PermutingFrameTupleReference();
+        this.numOfPrimaryKeys = numOfPrimaryKeys;
+        missingWriter = opDesc.getMissingWriterFactory().createMissingWriter();
         int[] searchKeyPermutations = new int[numOfPrimaryKeys];
         for (int i = 0; i < searchKeyPermutations.length; i++) {
             searchKeyPermutations[i] = fieldPermutation[i];
         }
         key.setFieldPermutation(searchKeyPermutations);
-        this.numOfPrimaryKeys = numOfPrimaryKeys;
+        hasMeta = (fieldPermutation.length > numOfPrimaryKeys + 1) && (filterFieldIndex < 0
+                || (filterFieldIndex >= 0 && (fieldPermutation.length > numOfPrimaryKeys + 2)));
         if (filterFieldIndex >= 0) {
             isFiltered = true;
             this.recordType = recordType;
             this.presetFieldIndex = filterFieldIndex;
             this.recPointable = (ARecordPointable) ARecordPointable.FACTORY.createPointable();
-            this.prevRecWithPKWithFilterValue = new ArrayTupleBuilder(fieldPermutation.length);
+            this.prevRecWithPKWithFilterValue = new ArrayTupleBuilder(fieldPermutation.length + (hasMeta ? 1 : 0));
             this.prevDos = prevRecWithPKWithFilterValue.getDataOutput();
         }
+
     }
 
     // we have the permutation which has [pk locations, record location, optional:filter-location]
@@ -129,7 +131,6 @@ public class AsterixLSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertU
             modCallback = opDesc.getModificationOpCallbackFactory().createModificationOperationCallback(
                     indexHelper.getResourcePath(), indexHelper.getResourceID(), indexHelper.getResourcePartition(),
                     index, ctx, this);
-
             indexAccessor = index.createAccessor(modCallback, opDesc.getSearchOpCallbackFactory()
                     .createSearchOperationCallback(indexHelper.getResourceID(), ctx, this));
             cursor = indexAccessor.createSearchCursor(false);
@@ -161,14 +162,24 @@ public class AsterixLSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertU
             dos.write(prevTuple.getFieldData(numOfPrimaryKeys), prevTuple.getFieldStart(numOfPrimaryKeys),
                     prevTuple.getFieldLength(numOfPrimaryKeys));
             tb.addFieldEndOffset();
-            // if with filters, append the filter
-            if (isFiltered) {
+            // if has meta, then append meta
+            if (hasMeta) {
                 dos.write(prevTuple.getFieldData(numOfPrimaryKeys + 1), prevTuple.getFieldStart(numOfPrimaryKeys + 1),
                         prevTuple.getFieldLength(numOfPrimaryKeys + 1));
                 tb.addFieldEndOffset();
             }
+            // if with filters, append the filter
+            if (isFiltered) {
+                dos.write(prevTuple.getFieldData(numOfPrimaryKeys + (hasMeta ? 2 : 1)),
+                        prevTuple.getFieldStart(numOfPrimaryKeys + (hasMeta ? 2 : 1)),
+                        prevTuple.getFieldLength(numOfPrimaryKeys + (hasMeta ? 2 : 1)));
+                tb.addFieldEndOffset();
+            }
         } else {
             addNullField();
+            if (hasMeta) {
+                addNullField();
+            }
             // if with filters, append null
             if (isFiltered) {
                 addNullField();
@@ -191,7 +202,6 @@ public class AsterixLSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertU
     //TODO: use tryDelete/tryInsert in order to prevent deadlocks
     @Override
     public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-
         accessor.reset(buffer);
         LSMTreeIndexAccessor lsmAccessor = (LSMTreeIndexAccessor) indexAccessor;
         int tupleCount = accessor.getTupleCount();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java
index c06cc18..d47492c 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteUpsertOperator.java
@@ -49,16 +49,19 @@ public class IndexInsertDeleteUpsertOperator extends AbstractLogicalOperator {
     // used for upsert operations
     private List<Mutable<ILogicalExpression>> prevSecondaryKeyExprs;
     private Mutable<ILogicalExpression> prevAdditionalFilteringExpression;
+    private final int numberOfAdditionalNonFilteringFields;
 
     public IndexInsertDeleteUpsertOperator(IDataSourceIndex<?, ?> dataSourceIndex,
             List<Mutable<ILogicalExpression>> primaryKeyExprs, List<Mutable<ILogicalExpression>> secondaryKeyExprs,
-            Mutable<ILogicalExpression> filterExpr, Kind operation, boolean bulkload) {
+            Mutable<ILogicalExpression> filterExpr, Kind operation, boolean bulkload,
+            int numberOfAdditionalNonFilteringFields) {
         this.dataSourceIndex = dataSourceIndex;
         this.primaryKeyExprs = primaryKeyExprs;
         this.secondaryKeyExprs = secondaryKeyExprs;
         this.filterExpr = filterExpr;
         this.operation = operation;
         this.bulkload = bulkload;
+        this.numberOfAdditionalNonFilteringFields = numberOfAdditionalNonFilteringFields;
     }
 
     @Override
@@ -181,4 +184,8 @@ public class IndexInsertDeleteUpsertOperator extends AbstractLogicalOperator {
     public void setPrevAdditionalFilteringExpression(Mutable<ILogicalExpression> prevAdditionalFilteringExpression) {
         this.prevAdditionalFilteringExpression = prevAdditionalFilteringExpression;
     }
+
+    public int getNumberOfAdditionalNonFilteringFields() {
+        return numberOfAdditionalNonFilteringFields;
+    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteUpsertOperator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteUpsertOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteUpsertOperator.java
index 6a74eb9..7d6c299 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteUpsertOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteUpsertOperator.java
@@ -56,6 +56,9 @@ public class InsertDeleteUpsertOperator extends AbstractLogicalOperator {
     // previous filter (for UPSERT)
     private LogicalVariable prevFilterVar;
     private Object prevFilterType;
+    // previous additional fields (for UPSERT)
+    private List<LogicalVariable> prevAdditionalNonFilteringVars;
+    private List<Object> prevAdditionalNonFilteringTypes;
 
     public InsertDeleteUpsertOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payloadExpr,
             List<Mutable<ILogicalExpression>> primaryKeyExprs,
@@ -85,6 +88,9 @@ public class InsertDeleteUpsertOperator extends AbstractLogicalOperator {
         if (operation == Kind.UPSERT) {
             // The upsert case also produces the previous record
             schema.add(prevRecordVar);
+            if (additionalNonFilteringExpressions != null) {
+                schema.addAll(prevAdditionalNonFilteringVars);
+            }
             if (prevFilterVar != null) {
                 schema.add(prevFilterVar);
             }
@@ -95,6 +101,9 @@ public class InsertDeleteUpsertOperator extends AbstractLogicalOperator {
         if (prevRecordVar != null) {
             producedVariables.add(prevRecordVar);
         }
+        if (prevAdditionalNonFilteringVars != null) {
+            producedVariables.addAll(prevAdditionalNonFilteringVars);
+        }
         if (prevFilterVar != null) {
             producedVariables.add(prevFilterVar);
         }
@@ -140,6 +149,11 @@ public class InsertDeleteUpsertOperator extends AbstractLogicalOperator {
                 target.addAllVariables(sources[0]);
                 if (operation == Kind.UPSERT) {
                     target.addVariable(prevRecordVar);
+                    if (prevAdditionalNonFilteringVars != null) {
+                        for (LogicalVariable var : prevAdditionalNonFilteringVars) {
+                            target.addVariable(var);
+                        }
+                    }
                     if (prevFilterVar != null) {
                         target.addVariable(prevFilterVar);
                     }
@@ -158,6 +172,11 @@ public class InsertDeleteUpsertOperator extends AbstractLogicalOperator {
         PropagatingTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
         if (operation == Kind.UPSERT) {
             env.setVarType(prevRecordVar, prevRecordType);
+            if (prevAdditionalNonFilteringVars != null) {
+                for (int i = 0; i < prevAdditionalNonFilteringVars.size(); i++) {
+                    env.setVarType(prevAdditionalNonFilteringVars.get(i), prevAdditionalNonFilteringTypes.get(i));
+                }
+            }
             if (prevFilterVar != null) {
                 env.setVarType(prevFilterVar, prevFilterType);
             }
@@ -224,4 +243,20 @@ public class InsertDeleteUpsertOperator extends AbstractLogicalOperator {
     public void setPrevFilterType(Object prevFilterType) {
         this.prevFilterType = prevFilterType;
     }
+
+    public List<LogicalVariable> getPrevAdditionalNonFilteringVars() {
+        return prevAdditionalNonFilteringVars;
+    }
+
+    public void setPrevAdditionalNonFilteringVars(List<LogicalVariable> prevAdditionalNonFilteringVars) {
+        this.prevAdditionalNonFilteringVars = prevAdditionalNonFilteringVars;
+    }
+
+    public List<Object> getPrevAdditionalNonFilteringTypes() {
+        return prevAdditionalNonFilteringTypes;
+    }
+
+    public void setPrevAdditionalNonFilteringTypes(List<Object> prevAdditionalNonFilteringTypes) {
+        this.prevAdditionalNonFilteringTypes = prevAdditionalNonFilteringTypes;
+    }
 }


Mime
View raw message