asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject incubator-asterixdb git commit: ASTERIXDB-1333 Fix Feed Ingestion for Dataset with Filters
Date Mon, 14 Mar 2016 10:45:40 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master fb431ffea -> f707c0ed2


ASTERIXDB-1333 Fix Feed Ingestion for Dataset with Filters

This change fixes a bug in the compilation of feed connection
when the dataset has filters. The bug is caused by not setting
the filter expressions during initial plan generation.

Change-Id: Ic323c9a74504921b984f68790bff2d3a7140b85b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/716
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/f707c0ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/f707c0ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/f707c0ed

Branch: refs/heads/master
Commit: f707c0ed2053e62d9b862e0cef4c17f2939533aa
Parents: fb431ff
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Mon Mar 14 10:59:52 2016 +0300
Committer: abdullah alamoudi <bamousaa@gmail.com>
Committed: Mon Mar 14 03:40:24 2016 -0700

----------------------------------------------------------------------
 .../LangExpressionToPlanTranslator.java         |  3 +-
 .../feed-with-filtered-dataset.1.ddl.aql        | 45 ++++++++++++++++++++
 .../feed-with-filtered-dataset.2.update.aql     | 27 ++++++++++++
 .../feed-with-filtered-dataset.3.query.aql      | 29 +++++++++++++
 .../feed-with-filtered-dataset.4.ddl.aql        | 25 +++++++++++
 .../feed-with-filtered-dataset.1.adm            |  7 +++
 .../src/test/resources/runtimets/testsuite.xml  |  5 +++
 7 files changed, 140 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index 28de56b..4739b71 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -398,9 +398,10 @@ class LangExpressionToPlanTranslator
                     break;
                 }
                 case SUBSCRIBE_FEED: {
-                    ILogicalOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource,
varRef,
+                    InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource,
varRef,
                             varRefsForLoading, InsertDeleteUpsertOperator.Kind.INSERT, false);
                     insertOp.getInputs().add(new MutableObject<ILogicalOperator>(assign));
+                    insertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
                     leafOperator = new SinkOperator();
                     leafOperator.getInputs().add(new MutableObject<ILogicalOperator>(insertOp));
                     break;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql
new file mode 100644
index 0000000..260dd1c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.ddl.aql
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Test feed ingestion on a filtered dataset
+ * Expected Res : Success
+ * Date         : 14th Mar 2016
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type FacebookMessageType as closed {
+        message-id: int64,
+        author-id: int64,
+        in-response-to: int64?,
+        sender-location: point?,
+        message: string,
+        send-time: datetime
+}
+
+create dataset FacebookMessages(FacebookMessageType)
+primary key message-id with filter on send-time;
+
+create feed MessageFeed using push_localfs(
+("path"="asterix_nc1://data/fbm-with-send-time.adm"),
+("format"="adm"),
+("type-name"="FacebookMessageType"));

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql
new file mode 100644
index 0000000..36564a3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.2.update.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Test feed ingestion on a filtered dataset
+ * Expected Res : Success
+ * Date         : 14th Mar 2016
+ */
+
+use dataverse test;
+set wait-for-completion-feed "true";
+connect feed MessageFeed to dataset FacebookMessages;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql
new file mode 100644
index 0000000..daf4882
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.3.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Test feed ingestion on a filtered dataset
+ * Expected Res : Success
+ * Date         : 14th Mar 2016
+ */
+
+use dataverse test;
+
+for $m in dataset('FacebookMessages')
+where $m.send-time > datetime("2012-08-20T10:10:00")
+return $m;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql
new file mode 100644
index 0000000..cc50807
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.4.ddl.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description  : Test feed ingestion on a filtered dataset
+ * Expected Res : Success
+ * Date         : 14th Mar 2016
+ */
+
+drop dataverse test;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm
b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm
new file mode 100644
index 0000000..7a4df99
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/feeds/feed-with-filtered-dataset/feed-with-filtered-dataset.1.adm
@@ -0,0 +1,7 @@
+{ "message-id": 11, "author-id": 1, "in-response-to": 1, "sender-location": point("38.97,77.49"),
"message": " can't stand at&t its plan is terrible", "send-time": datetime("2012-11-20T10:10:00.000Z")
}
+{ "message-id": 12, "author-id": 10, "in-response-to": 6, "sender-location": point("42.26,77.76"),
"message": " can't stand t-mobile its voicemail-service is OMG:(", "send-time": datetime("2012-12-20T10:10:00.000Z")
}
+{ "message-id": 14, "author-id": 9, "in-response-to": 12, "sender-location": point("41.33,85.28"),
"message": " love at&t its 3G is good:)", "send-time": datetime("2013-09-20T10:10:00.000Z")
}
+{ "message-id": 13, "author-id": 10, "in-response-to": 4, "sender-location": point("42.77,78.92"),
"message": " dislike iphone the voice-command is bad:(", "send-time": datetime("2013-08-20T10:10:00.000Z")
}
+{ "message-id": 15, "author-id": 7, "in-response-to": 11, "sender-location": point("44.47,67.11"),
"message": " like iphone the voicemail-service is awesome", "send-time": datetime("2014-01-20T10:10:00.000Z")
}
+{ "message-id": 9, "author-id": 3, "in-response-to": 12, "sender-location": point("34.45,96.48"),
"message": " love verizon its wireless is good", "send-time": datetime("2012-09-20T10:10:00.000Z")
}
+{ "message-id": 10, "author-id": 1, "in-response-to": 12, "sender-location": point("42.5,70.01"),
"message": " can't stand motorola the touch-screen is terrible", "send-time": datetime("2012-10-20T10:10:00.000Z")
}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/f707c0ed/asterix-app/src/test/resources/runtimets/testsuite.xml
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index b46b75b..d50a0a6 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -36,6 +36,11 @@
           </compilation-unit>
         </test-case> -->
         <test-case FilePath="feeds">
+            <compilation-unit name="feed-with-filtered-dataset">
+                <output-dir compare="Text">feed-with-filtered-dataset</output-dir>
+            </compilation-unit>
+        </test-case>
+        <test-case FilePath="feeds">
             <compilation-unit name="feed-push-socket">
                 <output-dir compare="Text">feed-push-socket</output-dir>
             </compilation-unit>


Mime
View raw message