asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Ian Maxon (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: Make LSN recording in components compatible with append-only
Date Mon, 23 Nov 2015 17:00:41 GMT
Ian Maxon has submitted this change and it was merged.

Change subject: Make LSN recording in components compatible with append-only
......................................................................


Make LSN recording in components compatible with append-only

Change-Id: I5610e903be3347893e676c915b98316b06073c25
Reviewed-on: https://asterix-gerrit.ics.uci.edu/265
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>
---
A asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
A asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
A asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
A asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
A asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
M asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
7 files changed, 174 insertions(+), 33 deletions(-)

Approvals:
  Murtadha Hubail: Looks good to me, approved
  Jenkins: Verified



diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
new file mode 100644
index 0000000..6365678
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Comparison of dataset scan results to secondary index point lookups for
BTree and RTree
+ * Success        : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type TwitterUserType as {
+    screen-name: string,
+    lang: string,
+    friends_count: int32,
+    statuses_count: int32,
+    name: string,
+    followers_count: int32
+}
+
+create type TweetMessageType as {
+    tweetid: int64,
+    user: string,
+    sender-location: point,
+    send-time: datetime,
+    forward-from: int64,
+    retweet-from: int64,
+    referred-topics: {{ string }},
+    message-text: string
+}
+
+create type FacebookMessageType as {
+    message-id: int64,
+    author-id: int64,
+    in-response-to: int64?,
+    sender-location: point,
+    send-time: datetime,
+    message: string
+}
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+create dataset FacebookMessages(FacebookMessageType) primary key message-id;
+
+create index fbmAutIdIx on FacebookMessages(author-id);
+
+create index twmSndTmIx on TweetMessages(send-time);
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
new file mode 100644
index 0000000..d1f883d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description    : Index Nested Loop Join on three datasets. Two index nested loop joins
should be nested properly.
+ * Success        : Yes
+ */
+ use dataverse test;
+
+load dataset TweetMessages
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/index-join/tw_messages.adm"),("format"="adm"));
+
+load dataset FacebookMessages
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/fbm-with-send-time.adm"),("format"="adm"));
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
new file mode 100644
index 0000000..be4ed69
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description    : BTree selection compared with dataset scan
+ * Success        : Yes
+ */
+use dataverse test;
+
+let $a := (for $n in dataset FacebookMessages return $n)
+
+let $b :=  (for $n in dataset FacebookMessages where $n.author-id >= 0 and
+$n.author-id <= 10 return $n
+)
+
+for $x in $a
+where every $y in $b satisfies $y.message-id != $x.message-id
+return $x;
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
new file mode 100644
index 0000000..b161db6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description    : BTree selection compared with dataset scan
+ * Success        : Yes
+ */
+use dataverse test;
+
+let $a :=count (
+let $pts := for $t in dataset TweetMessages return $t.sender-location
+
+for $t in dataset TweetMessages
+for $p in $pts
+where spatial-intersect($t.sender-location,$p)
+return $t
+)
+
+let $b := count(
+for $t in dataset TweetMessages return $t
+)
+
+
+return $a = $b
diff --git a/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
new file mode 100644
index 0000000..27ba77d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
@@ -0,0 +1 @@
+true
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
b/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
index 76a11d1..b3cae06 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
@@ -24,12 +24,9 @@
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 
 // A single LSMIOOperationCallback per LSM index used to perform actions around Flush and
Merge operations
@@ -96,35 +93,11 @@
     protected void putLSNIntoMetadata(ITreeIndex treeIndex, List<ILSMComponent> oldComponents)
             throws HyracksDataException {
         long componentLSN = getComponentLSN(oldComponents);
-        int fileId = treeIndex.getFileId();
-        IBufferCache bufferCache = treeIndex.getBufferCache();
-        ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
-        int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
-        ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId,
metadataPageId), false);
-        metadataPage.acquireWriteLatch();
-        try {
-            metadataFrame.setPage(metadataPage);
-            metadataFrame.setLSN(componentLSN);
-        } finally {
-            metadataPage.releaseWriteLatch(true);
-            bufferCache.unpin(metadataPage);
-        }
+        treeIndex.getMetaManager().setLSN(componentLSN);
     }
 
     protected long getTreeIndexLSN(ITreeIndex treeIndex) throws HyracksDataException {
-        int fileId = treeIndex.getFileId();
-        IBufferCache bufferCache = treeIndex.getBufferCache();
-        ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
-        int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
-        ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId,
metadataPageId), false);
-        metadataPage.acquireReadLatch();
-        try {
-            metadataFrame.setPage(metadataPage);
-            return metadataFrame.getLSN();
-        } finally {
-            metadataPage.releaseReadLatch();
-            bufferCache.unpin(metadataPage);
-        }
+        return treeIndex.getMetaManager().getLSN();
     }
 
     public void updateLastLSN(long lastLSN) {
@@ -133,7 +106,7 @@
 
     public void setFirstLSN(long firstLSN) throws AsterixException {
         // We make sure that this method is only called on an empty component so the first
LSN is not set incorrectly
-          firstLSNs[writeIndex] = firstLSN;
+        firstLSNs[writeIndex] = firstLSN;
     }
 
     public synchronized long getFirstLSN() {
@@ -142,10 +115,10 @@
         return firstLSNs[readIndex];
     }
 
-    public synchronized boolean hasPendingFlush(){
+    public synchronized boolean hasPendingFlush() {
 
-        for(int i=0; i<flushRequested.length; i++){
-            if(flushRequested[i]){
+        for (int i = 0; i < flushRequested.length; i++) {
+            if (flushRequested[i]) {
                 return true;
             }
         }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/265
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I5610e903be3347893e676c915b98316b06073c25
Gerrit-PatchSet: 32
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Ian Maxon <imaxon@apache.org>
Gerrit-Reviewer: Ian Maxon <imaxon@apache.org>
Gerrit-Reviewer: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hubailmor@gmail.com>
Gerrit-Reviewer: Yingyi Bu <buyingyi@gmail.com>
Gerrit-Reviewer: Young-Seok Kim <kisskys@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <bamousaa@gmail.com>

Mime
View raw message