asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject incubator-asterixdb git commit: Make LSN recording in components compatible with append-only
Date Mon, 23 Nov 2015 17:08:13 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master 18a9dca53 -> fbbb42244


Make LSN recording in components compatible with append-only

Change-Id: I5610e903be3347893e676c915b98316b06073c25
Reviewed-on: https://asterix-gerrit.ics.uci.edu/265
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/fbbb4224
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/fbbb4224
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/fbbb4224

Branch: refs/heads/master
Commit: fbbb422441498dd9cd31c387ce6fcfff0d9c7ab0
Parents: 18a9dca
Author: Ian Maxon <imaxon@apache.org>
Authored: Thu Nov 19 16:37:11 2015 -0800
Committer: Ian Maxon <imaxon@apache.org>
Committed: Mon Nov 23 09:00:41 2015 -0800

----------------------------------------------------------------------
 .../verify/index-verify.1.ddl.aql               | 64 ++++++++++++++++++++
 .../verify/index-verify.2.update.aql            | 31 ++++++++++
 .../verify/index-verify.3.query.aql             | 33 ++++++++++
 .../verify/index-verify.4.query.aql             | 39 ++++++++++++
 .../results/index-selection/verify/verify.1.adm |  0
 .../results/index-selection/verify/verify.2.adm |  1 +
 .../AbstractLSMIOOperationCallback.java         | 39 ++----------
 7 files changed, 174 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fbbb4224/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
new file mode 100644
index 0000000..6365678
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.1.ddl.aql
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description    : Comparison of dataset scan results to secondary index point lookups for
BTree and RTree
+ * Success        : Yes
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type TwitterUserType as {
+    screen-name: string,
+    lang: string,
+    friends_count: int32,
+    statuses_count: int32,
+    name: string,
+    followers_count: int32
+}
+
+create type TweetMessageType as {
+    tweetid: int64,
+    user: string,
+    sender-location: point,
+    send-time: datetime,
+    forward-from: int64,
+    retweet-from: int64,
+    referred-topics: {{ string }},
+    message-text: string
+}
+
+create type FacebookMessageType as {
+    message-id: int64,
+    author-id: int64,
+    in-response-to: int64?,
+    sender-location: point,
+    send-time: datetime,
+    message: string
+}
+
+create dataset TweetMessages(TweetMessageType) primary key tweetid;
+create dataset FacebookMessages(FacebookMessageType) primary key message-id;
+
+create index fbmAutIdIx on FacebookMessages(author-id);
+
+create index twmSndTmIx on TweetMessages(send-time);
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fbbb4224/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
new file mode 100644
index 0000000..d1f883d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.2.update.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description    : Index Nested Loop Join on three datasets. Two index nested loop joins
should be nested properly.
+ * Success        : Yes
+ */
+ use dataverse test;
+
+load dataset TweetMessages
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/index-join/tw_messages.adm"),("format"="adm"));
+
+load dataset FacebookMessages
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/fbm-with-send-time.adm"),("format"="adm"));

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fbbb4224/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
new file mode 100644
index 0000000..be4ed69
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.3.query.aql
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description    : BTree selection compared with dataset scan
+ * Success        : Yes
+ */
+use dataverse test;
+
+let $a := (for $n in dataset FacebookMessages return $n)
+
+let $b :=  (for $n in dataset FacebookMessages where $n.author-id >= 0 and
+$n.author-id <= 10 return $n
+)
+
+for $x in $a
+where every $y in $b satisfies $y.message-id != $x.message-id
+return $x;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fbbb4224/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
new file mode 100644
index 0000000..b161db6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/index-selection/verify/index-verify.4.query.aql
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ /*
+ * Description    : BTree selection compared with dataset scan
+ * Success        : Yes
+ */
+use dataverse test;
+
+let $a :=count (
+let $pts := for $t in dataset TweetMessages return $t.sender-location
+
+for $t in dataset TweetMessages
+for $p in $pts
+where spatial-intersect($t.sender-location,$p)
+return $t
+)
+
+let $b := count(
+for $t in dataset TweetMessages return $t
+)
+
+
+return $a = $b

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fbbb4224/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.1.adm
new file mode 100644
index 0000000..e69de29

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fbbb4224/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
new file mode 100644
index 0000000..27ba77d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/index-selection/verify/verify.2.adm
@@ -0,0 +1 @@
+true

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/fbbb4224/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
----------------------------------------------------------------------
diff --git a/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
b/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
index 76a11d1..b3cae06 100644
--- a/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
+++ b/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/AbstractLSMIOOperationCallback.java
@@ -24,12 +24,9 @@ import java.util.List;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 
 // A single LSMIOOperationCallback per LSM index used to perform actions around Flush and
Merge operations
@@ -96,35 +93,11 @@ public abstract class AbstractLSMIOOperationCallback implements ILSMIOOperationC
     protected void putLSNIntoMetadata(ITreeIndex treeIndex, List<ILSMComponent> oldComponents)
             throws HyracksDataException {
         long componentLSN = getComponentLSN(oldComponents);
-        int fileId = treeIndex.getFileId();
-        IBufferCache bufferCache = treeIndex.getBufferCache();
-        ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
-        int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
-        ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId,
metadataPageId), false);
-        metadataPage.acquireWriteLatch();
-        try {
-            metadataFrame.setPage(metadataPage);
-            metadataFrame.setLSN(componentLSN);
-        } finally {
-            metadataPage.releaseWriteLatch(true);
-            bufferCache.unpin(metadataPage);
-        }
+        treeIndex.getMetaManager().setLSN(componentLSN);
     }
 
     protected long getTreeIndexLSN(ITreeIndex treeIndex) throws HyracksDataException {
-        int fileId = treeIndex.getFileId();
-        IBufferCache bufferCache = treeIndex.getBufferCache();
-        ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
-        int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
-        ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId,
metadataPageId), false);
-        metadataPage.acquireReadLatch();
-        try {
-            metadataFrame.setPage(metadataPage);
-            return metadataFrame.getLSN();
-        } finally {
-            metadataPage.releaseReadLatch();
-            bufferCache.unpin(metadataPage);
-        }
+        return treeIndex.getMetaManager().getLSN();
     }
 
     public void updateLastLSN(long lastLSN) {
@@ -133,7 +106,7 @@ public abstract class AbstractLSMIOOperationCallback implements ILSMIOOperationC
 
     public void setFirstLSN(long firstLSN) throws AsterixException {
         // We make sure that this method is only called on an empty component so the first
LSN is not set incorrectly
-          firstLSNs[writeIndex] = firstLSN;
+        firstLSNs[writeIndex] = firstLSN;
     }
 
     public synchronized long getFirstLSN() {
@@ -142,10 +115,10 @@ public abstract class AbstractLSMIOOperationCallback implements ILSMIOOperationC
         return firstLSNs[readIndex];
     }
 
-    public synchronized boolean hasPendingFlush(){
+    public synchronized boolean hasPendingFlush() {
 
-        for(int i=0; i<flushRequested.length; i++){
-            if(flushRequested[i]){
+        for (int i = 0; i < flushRequested.length; i++) {
+            if (flushRequested[i]) {
                 return true;
             }
         }


Mime
View raw message