Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E264A200B49 for ; Tue, 14 Jun 2016 04:31:57 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id E1042160A3C; Tue, 14 Jun 2016 02:31:57 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C624A160A5B for ; Tue, 14 Jun 2016 04:31:55 +0200 (CEST) Received: (qmail 6841 invoked by uid 500); 14 Jun 2016 02:31:55 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 6790 invoked by uid 99); 14 Jun 2016 02:31:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 14 Jun 2016 02:31:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C3834E01E2; Tue, 14 Jun 2016 02:31:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mblow@apache.org To: commits@asterixdb.apache.org Date: Tue, 14 Jun 2016 02:31:55 -0000 Message-Id: <401e78cd89ea42f5a1b469e7f87f739c@git.apache.org> In-Reply-To: <353e34f26f3a4a0bbbb27157d9193334@git.apache.org> References: <353e34f26f3a4a0bbbb27157d9193334@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] asterixdb git commit: ASTERIXDB-1474: Cannot query dataset with big obj archived-at: Tue, 14 Jun 2016 02:31:58 -0000 ASTERIXDB-1474: Cannot query dataset with big obj Fixes ASTERIXDB-1474: Cannot query a dataset with big object after restart - large page information managed by buffer cache, outside of page - size header written out a prefix to pages Change-Id: I2f47c551a053e193519a4337c338934f3e171afd Reviewed-on: https://asterix-gerrit.ics.uci.edu/920 Tested-by: Jenkins Reviewed-by: abdullah alamoudi Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/171ff2ff Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/171ff2ff Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/171ff2ff Branch: refs/heads/master Commit: 171ff2ffedb9b6c8ffea8c5e0087277c0a9946c0 Parents: 809be1b Author: Michael Blow Authored: Mon Jun 13 17:43:39 2016 -0400 Committer: Michael Blow Committed: Mon Jun 13 19:31:24 2016 -0700 ---------------------------------------------------------------------- .../common/config/AsterixStorageProperties.java | 4 +- .../big_object_20M/big_object_20M.1.script.aql | 1 + .../big_object_20M/big_object_20M.2.ddl.aql | 36 +++++ .../big_object_20M/big_object_20M.3.update.aql | 30 ++++ .../big_object_20M/big_object_20M.4.script.aql | 1 + .../big_object_20M/big_object_20M.5.query.aql | 23 +++ .../big_object_20M/big_object_20M.6.script.aql | 1 + .../big_object_20M.1.adm.template | 1 + .../big_object_20M/create_and_start.sh | 2 + .../big_object_20M/stop_and_delete.sh | 3 + .../big_object_20M/stop_and_start.sh | 2 + .../test/resources/transactionts/testsuite.xml | 5 + .../org/apache/hyracks/api/io/IIOManager.java | 2 + .../apache/hyracks/control/nc/io/IOManager.java | 40 ++++- .../storage/am/btree/api/IBTreeFrame.java | 2 + .../storage/am/btree/api/IBTreeLeafFrame.java | 13 +- .../frames/BTreeFieldPrefixNSMLeafFrame.java | 27 +--- .../BTreeFieldPrefixNSMLeafFrameFactory.java | 5 - .../am/btree/frames/BTreeLargeFrameHelper.java | 44 ------ .../am/btree/frames/BTreeNSMInteriorFrame.java | 7 +- .../frames/BTreeNSMInteriorFrameFactory.java | 5 - .../am/btree/frames/BTreeNSMLeafFrame.java | 94 +++-------- .../btree/frames/BTreeNSMLeafFrameFactory.java | 7 +- .../hyracks/storage/am/btree/impls/BTree.java | 61 +++----- .../btree/impls/BTreeCountingSearchCursor.java | 4 +- .../storage/am/btree/impls/BTreeOpContext.java | 26 +++- .../am/btree/impls/BTreeRangeSearchCursor.java | 4 +- .../storage/am/btree/util/BTreeUtils.java | 3 - .../storage/am/common/api/ITreeIndexFrame.java | 6 +- .../am/common/api/ITreeIndexFrameFactory.java | 3 - .../am/common/frames/TreeIndexNSMFrame.java | 15 +- .../am/common/impls/AbstractTreeIndex.java | 29 ++-- .../impls/TreeIndexDiskOrderScanCursor.java | 9 +- .../common/util/TreeIndexBufferCacheWarmup.java | 6 +- .../am/common/util/TreeIndexStatsGatherer.java | 3 +- .../impls/MultitenantVirtualBufferCache.java | 17 +- .../am/lsm/common/impls/VirtualBufferCache.java | 25 +-- .../storage/am/rtree/frames/RTreeNSMFrame.java | 7 +- .../frames/RTreeNSMInteriorFrameFactory.java | 5 - .../rtree/frames/RTreeNSMLeafFrameFactory.java | 5 - .../hyracks/storage/am/rtree/impls/RTree.java | 8 +- .../storage/am/rtree/impls/RTreeOpContext.java | 25 ++- .../storage/common/buffercache/BufferCache.java | 154 ++++++++++++++----- .../storage/common/buffercache/CachedPage.java | 36 +++-- .../ClockPageReplacementStrategy.java | 84 ++++++---- .../common/buffercache/DebugBufferCache.java | 17 +- .../common/buffercache/IBufferCache.java | 14 +- .../storage/common/buffercache/ICachedPage.java | 19 ++- .../common/buffercache/ICachedPageInternal.java | 5 + .../buffercache/IExtraPageBlockHelper.java | 26 ++++ .../common/buffercache/ILargePageHelper.java | 25 --- .../buffercache/IPageReplacementStrategy.java | 8 +- .../storage/am/btree/FieldPrefixNSMTest.java | 4 +- .../storage/am/btree/StorageManagerTest.java | 4 +- .../common/BufferCacheRegressionTest.java | 16 +- 55 files changed, 576 insertions(+), 452 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java index 7d92ce2..9fb4e94 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixStorageProperties.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.common.config; +import org.apache.hyracks.storage.common.buffercache.IBufferCache; + public class AsterixStorageProperties extends AbstractAsterixProperties { private static final String STORAGE_BUFFERCACHE_PAGESIZE_KEY = "storage.buffercache.pagesize"; @@ -62,7 +64,7 @@ public class AsterixStorageProperties extends AbstractAsterixProperties { } public int getBufferCacheNumPages() { - return (int) (getBufferCacheSize() / getBufferCachePageSize()); + return (int) (getBufferCacheSize() / (getBufferCachePageSize() + IBufferCache.RESERVED_HEADER_BYTES)); } public int getBufferCacheMaxOpenFiles() { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql new file mode 100644 index 0000000..cc46136 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.1.script.aql @@ -0,0 +1 @@ +create_and_start.sh http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql new file mode 100644 index 0000000..77c8179 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.2.ddl.aql @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/* + * Test case Name : big_object_bulkload.aql + * Description : bulkload insert of large objects + * Expected Result : Success + * Date : 20th April 2016 + */ + +drop dataverse testdv2 if exists; +create dataverse testdv2; +use dataverse testdv2; + +create type testtype as closed { + id: int64, + name: string, + hobbies: {{string}} +} + +create dataset testds(testtype) primary key id; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql new file mode 100644 index 0000000..f1b3819 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.3.update.aql @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * + * Big object (20 MB) loading test + * Expected result: success + * + */ + +use dataverse testdv2; + +load dataset testds +using localfs +(("path"="asterix_nc1://../../../../../../asterix-app/target/data/big-object/big_object_20M.adm"),("format"="adm")) pre-sorted; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql new file mode 100644 index 0000000..3ba1dc0 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.4.script.aql @@ -0,0 +1 @@ +stop_and_start.sh http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql new file mode 100644 index 0000000..23c1bed --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.5.query.aql @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use dataverse testdv2; + +for $d in dataset("testds") +where $d.id = 1 +return $d http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql new file mode 100644 index 0000000..10e1a51 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/big_object_20M/big_object_20M.6.script.aql @@ -0,0 +1 @@ +stop_and_delete.sh http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template new file mode 100644 index 0000000..5acf078 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/big_object_20M/big_object_20M.1.adm.template @@ -0,0 +1 @@ +{ "id": 1, "name": "Person One", "hobbies": {{ "%lorembytes:20971520%" }} } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh new file mode 100755 index 0000000..37bc7b0 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/create_and_start.sh @@ -0,0 +1,2 @@ +jps | awk '{if ($2 == "NCDriver" || $2 == "CCDriver") print $1;}' | xargs -n 1 kill -9; +$MANAGIX_HOME/bin/managix create -n asterix -c $MANAGIX_HOME/clusters/local/local.xml; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh new file mode 100755 index 0000000..5f247db --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_delete.sh @@ -0,0 +1,3 @@ +$MANAGIX_HOME/bin/managix stop -n asterix; +$MANAGIX_HOME/bin/managix delete -n asterix; + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh new file mode 100755 index 0000000..e2b0583 --- /dev/null +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/big_object_20M/stop_and_start.sh @@ -0,0 +1,2 @@ +$MANAGIX_HOME/bin/managix stop -n asterix; +$MANAGIX_HOME/bin/managix start -n asterix; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml index 08b8e87..75b2457 100644 --- a/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml +++ b/asterixdb/asterix-installer/src/test/resources/transactionts/testsuite.xml @@ -28,6 +28,11 @@ external_index + + + big_object_20M + + http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java index 47eac7e..63709ed 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java @@ -43,6 +43,8 @@ public interface IIOManager { public int syncWrite(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException; + public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException; + public int syncRead(IFileHandle fHandle, long offset, ByteBuffer data) throws HyracksDataException; public IIOFuture asyncWrite(IFileHandle fHandle, long offset, ByteBuffer data); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java index 519a9d7..6588f3b 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -49,7 +50,7 @@ public class IOManager implements IIOManager { public IOManager(List devices) throws HyracksException { this.ioDevices = Collections.unmodifiableList(devices); - workAreaIODevices = new ArrayList(); + workAreaIODevices = new ArrayList<>(); for (IODeviceHandle d : ioDevices) { if (d.getWorkAreaPath() != null) { new File(d.getPath(), d.getWorkAreaPath()).mkdirs(); @@ -109,13 +110,48 @@ public class IOManager implements IIOManager { } } + @Override + public long syncWrite(IFileHandle fHandle, long offset, ByteBuffer[] dataArray) throws HyracksDataException { + try { + if (fHandle == null) { + throw new IllegalStateException("Trying to write to a deleted file."); + } + int n = 0; + int remaining = 0; + for (ByteBuffer buf : dataArray) { + remaining += buf.remaining(); + } + final FileChannel fileChannel = ((FileHandle) fHandle).getFileChannel(); + while (remaining > 0) { + long len; + synchronized (fileChannel) { + fileChannel.position(offset); + len = fileChannel.write(dataArray); + } + if (len < 0) { + throw new HyracksDataException( + "Error writing to file: " + ((FileHandle) fHandle).getFileReference().toString()); + } + remaining -= len; + offset += len; + n += len; + } + return n; + } catch (HyracksDataException e) { + throw e; + } catch (IOException e) { + throw new HyracksDataException(e); + } + } + /** * Please do check the return value of this read! * * @param fHandle * @param offset * @param data - * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's current size + * @return The number of bytes read, possibly zero, or -1 if the given offset is greater than or equal to the file's + * current size * @throws HyracksDataException */ @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java index 1081dc8..e0abb1c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeFrame.java @@ -38,5 +38,7 @@ public interface IBTreeFrame extends ITreeIndexFrame { public void setLargeFlag(boolean largePage); + public boolean getLargeFlag(); + public void validate(PageValidationInfo pvi) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java index 7648ee6..2e072ce 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/api/IBTreeLeafFrame.java @@ -21,14 +21,13 @@ package org.apache.hyracks.storage.am.btree.api; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.TreeIndexException; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleMode; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public interface IBTreeLeafFrame extends IBTreeFrame { public int findTupleIndex(ITupleReference searchKey, ITreeIndexTupleReference pageTuple, MultiComparator cmp, @@ -53,12 +52,6 @@ public interface IBTreeLeafFrame extends IBTreeFrame { public int getNextLeaf(); - public void configureLargePage(int supplementalPages, int supplementalBlockPageId); - - public boolean isLargePage(); - - public int getSupplementalNumPages(); - - void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, - IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException; + void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, IExtraPageBlockHelper extraPageBlockHelper) + throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java index 701f44f..258a5d6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java @@ -33,11 +33,9 @@ import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationIn import org.apache.hyracks.storage.am.btree.impls.FieldPrefixPrefixTupleReference; import org.apache.hyracks.storage.am.btree.impls.FieldPrefixSlotManager; import org.apache.hyracks.storage.am.btree.impls.FieldPrefixTupleReference; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.ISplitKey; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameCompressor; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.api.TreeIndexException; @@ -51,7 +49,7 @@ import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; /** * WARNING: only works when tupleWriter is an instance of TypeAwareTupleWriter @@ -537,25 +535,15 @@ public class BTreeFieldPrefixNSMLeafFrame implements IBTreeLeafFrame { } @Override - public void configureLargePage(int supplementalPages, int supplementalBlockPageId) { - throw new IllegalStateException(); - } - - @Override public void setLargeFlag(boolean largePage) { throw new IllegalStateException(); } @Override - public boolean isLargePage() { + public boolean getLargeFlag() { return false; } - @Override - public int getSupplementalNumPages() { - throw new IllegalStateException(); - } - public int getPrefixTupleCount() { return buf.getInt(prefixTupleCountOff); } @@ -596,7 +584,8 @@ public class BTreeFieldPrefixNSMLeafFrame implements IBTreeLeafFrame { } @Override - public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) + public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException { BTreeFieldPrefixNSMLeafFrame rf = (BTreeFieldPrefixNSMLeafFrame) rightFrame; @@ -804,12 +793,8 @@ public class BTreeFieldPrefixNSMLeafFrame implements IBTreeLeafFrame { } @Override - public ILargePageHelper getLargePageHelper() { - return null; - } - - @Override - public void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException { + public void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, IExtraPageBlockHelper helper) + throws HyracksDataException { throw new IllegalStateException("nyi"); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java index c558d30..f7b270c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java @@ -22,7 +22,6 @@ package org.apache.hyracks.storage.am.btree.frames; import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public class BTreeFieldPrefixNSMLeafFrameFactory implements ITreeIndexFrameFactory { @@ -44,8 +43,4 @@ public class BTreeFieldPrefixNSMLeafFrameFactory implements ITreeIndexFrameFacto return tupleWriterFactory; } - @Override - public ILargePageHelper getLargePageHelper() { - return null; - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java deleted file mode 100644 index 79f7f53..0000000 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeLargeFrameHelper.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.hyracks.storage.am.btree.frames; - -import java.nio.ByteBuffer; - -import org.apache.hyracks.storage.common.buffercache.CachedPage; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; - -public class BTreeLargeFrameHelper implements ILargePageHelper { - - public static final BTreeLargeFrameHelper INSTANCE = new BTreeLargeFrameHelper(); - - private BTreeLargeFrameHelper() { - } - - @Override - public int getSupplementalBlockNumPages(CachedPage cPage) { - ByteBuffer buffer = cPage.getBuffer(); - return BTreeNSMLeafFrame.isLargePage(buffer) ? BTreeNSMLeafFrame.getSupplementalNumPages(buffer) : 0; - } - - @Override - public int getSupplementalBlockPageId(CachedPage cPage) { - ByteBuffer buffer = cPage.getBuffer(); - return BTreeNSMLeafFrame.isLargePage(buffer) ? BTreeNSMLeafFrame.getSupplementalPageId(buffer) : -1; - } -} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java index c2b18b7..9505f8a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java @@ -29,10 +29,8 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame; import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo; import org.apache.hyracks.storage.am.btree.impls.RangePredicate; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.ISplitKey; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.api.TreeIndexException; @@ -43,6 +41,7 @@ import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolic import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeInteriorFrame { @@ -55,7 +54,7 @@ public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeIn private MultiComparator cmp; public BTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter) { - super(tupleWriter, new OrderedSlotManager(), null); + super(tupleWriter, new OrderedSlotManager()); cmpFrameTuple = tupleWriter.createTupleReference(); previousFt = tupleWriter.createTupleReference(); } @@ -201,7 +200,7 @@ public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeIn @Override public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, - IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException, TreeIndexException { ByteBuffer right = rightFrame.getBuffer(); int tupleCount = getTupleCount(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java index 3737486..4ecb83d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java @@ -22,7 +22,6 @@ package org.apache.hyracks.storage.am.btree.frames; import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public class BTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory { @@ -44,8 +43,4 @@ public class BTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory { return tupleWriterFactory; } - @Override - public ILargePageHelper getLargePageHelper() { - return null; - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java index 8f560fe..406b494 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java @@ -25,10 +25,8 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo; -import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.ISplitKey; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; -import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.api.TreeIndexException; @@ -40,25 +38,23 @@ import org.apache.hyracks.storage.am.common.ophelpers.FindTupleMode; import org.apache.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFrame { protected static final int nextLeafOff = flagOff + 1; // 22 - protected static final int supplementalNumPagesOff = nextLeafOff + 4; // 26 - protected static final int supplementalPageIdOff = supplementalNumPagesOff + 4; // 30 private MultiComparator cmp; private final ITreeIndexTupleReference previousFt; - public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter, ILargePageHelper largePageHelper) { - super(tupleWriter, new OrderedSlotManager(), largePageHelper); + public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) { + super(tupleWriter, new OrderedSlotManager()); previousFt = tupleWriter.createTupleReference(); } @Override public int getPageHeaderSize() { - return supplementalPageIdOff + 4; + return nextLeafOff + 4; } @Override @@ -70,8 +66,6 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr public void initBuffer(byte level) { super.initBuffer(level); buf.putInt(nextLeafOff, -1); - buf.putInt(supplementalNumPagesOff, 0); - buf.putInt(supplementalPageIdOff, -1); } @Override @@ -84,28 +78,6 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr return buf.getInt(nextLeafOff); } - public static int getSupplementalNumPages(ByteBuffer buf) { - return buf.getInt(supplementalNumPagesOff); - } - - public int getSupplementalNumPages() { - return getSupplementalNumPages(buf); - } - - public static int getSupplementalPageId(ByteBuffer buf) { - return buf.getInt(supplementalPageIdOff); - } - - public int getSupplementalPageId() { - return getSupplementalPageId(buf); - } - - public void configureLargePage(int supplementalPages, int supplementalBlockPageId) { - setLargeFlag(true); - buf.putInt(supplementalNumPagesOff, supplementalPages); - buf.putInt(supplementalPageIdOff, supplementalBlockPageId); - } - @Override public int findInsertTupleIndex(ITupleReference tuple) throws TreeIndexException { int tupleIndex; @@ -204,7 +176,7 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr boolean isLargeTuple(int tupleSize) { // TODO(mblow): make page size available to avoid calculating it - int pageSize = isLargePage() ? buf.capacity() / (getSupplementalNumPages() + 1) : buf.capacity(); + int pageSize = getLargeFlag() ? buf.capacity() / page.getFrameSizeMultiplier() : buf.capacity(); return tupleSize > getMaxTupleSize(pageSize); } @@ -228,7 +200,7 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr int oldTupleBytes = frameTuple.getTupleSize(); int newTupleBytes = tupleWriter.bytesRequired(newTuple); FrameOpSpaceStatus status = hasSpaceUpdate(oldTupleBytes, newTupleBytes); - if (status == FrameOpSpaceStatus.INSUFFICIENT_SPACE && (isLargePage() || getTupleCount() == 1) + if (status == FrameOpSpaceStatus.INSUFFICIENT_SPACE && (getLargeFlag() || getTupleCount() == 1) && isLargeTuple(newTupleBytes)) { return FrameOpSpaceStatus.EXPAND; } @@ -237,8 +209,8 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr @Override public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, - IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) - throws HyracksDataException { + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) + throws HyracksDataException { int tupleSize = getBytesRequiredToWriteTuple(tuple); @@ -249,7 +221,7 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr // Find split point, and determine into which frame the new tuple should // be inserted into. - BTreeNSMLeafFrame targetFrame = null; + BTreeNSMLeafFrame targetFrame; frameTuple.resetByTupleIndex(this, tupleCount - 1); if (cmp.compare(tuple, frameTuple) > 0) { // This is a special optimization case when the tuple to be inserted is the largest key on the page. @@ -275,22 +247,14 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr targetFrame = this; } int tuplesToRight = tupleCount - tuplesToLeft; - int supplementalPages = 0; - int supplementalPageId = -1; - if (isLargePage()) { - ((BTreeNSMLeafFrame) rightFrame).growCapacity(freePageManager, metaFrame, bufferCache, + if (getLargeFlag()) { + ((BTreeNSMLeafFrame) rightFrame).growCapacity(extraPageBlockHelper, bufferCache, buf.capacity() - rightFrame.getBuffer().capacity()); - supplementalPages = ((BTreeNSMLeafFrame) rightFrame).getSupplementalNumPages(); - supplementalPageId = ((BTreeNSMLeafFrame) rightFrame).getSupplementalPageId(); } ByteBuffer right = rightFrame.getBuffer(); // Copy entire page. System.arraycopy(buf.array(), 0, right.array(), 0, buf.capacity()); - if (isLargePage()) { - // restore the supplemental page metadata - ((BTreeNSMLeafFrame) rightFrame).configureLargePage(supplementalPages, supplementalPageId); - } // On the right page we need to copy rightmost slots to the left. int src = rightFrame.getSlotManager().getSlotEndOff(); @@ -309,7 +273,7 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr } if (tupleLarge) { - targetFrame.ensureCapacity(freePageManager, metaFrame, bufferCache, tuple); + targetFrame.ensureCapacity(bufferCache, tuple, extraPageBlockHelper); } // Insert the new tuple. @@ -317,7 +281,7 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr // it's safe to catch this exception since it will have been caught // before reaching here try { - targetTupleIndex = ((BTreeNSMLeafFrame) targetFrame).findInsertTupleIndex(tuple); + targetTupleIndex = targetFrame.findInsertTupleIndex(tuple); } catch (TreeIndexException e) { throw new IllegalStateException(e); } @@ -332,13 +296,13 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0); } - public void ensureCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, - IBufferCache bufferCache, ITupleReference tuple) throws HyracksDataException { + public void ensureCapacity(IBufferCache bufferCache, ITupleReference tuple, + IExtraPageBlockHelper extraPageBlockHelper) throws HyracksDataException { int gapBytes = getBytesRequiredToWriteTuple(tuple) - getFreeContiguousSpace(); - growCapacity(freePageManager, metaFrame, bufferCache, gapBytes); + growCapacity(extraPageBlockHelper, bufferCache, gapBytes); } - public void growCapacity(IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, + public void growCapacity(IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache, int delta) throws HyracksDataException { if (delta <= 0) { setLargeFlag(true); @@ -346,35 +310,21 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr } int deltaPages = (int) Math.ceil((double) delta / bufferCache.getPageSize()); int framePagesOld = getBuffer().capacity() / bufferCache.getPageSize(); - int oldSupplementalPages = 0; - int oldSupplementalPageId = -1; - if (isLargePage()) { - oldSupplementalPages = getSupplementalNumPages(); - oldSupplementalPageId = getSupplementalPageId(); - } - - configureLargePage(framePagesOld + deltaPages - 1, - freePageManager.getFreePageBlock(metaFrame, framePagesOld + deltaPages - 1)); - - int pageDelta = (framePagesOld + deltaPages) - 1 - oldSupplementalPages; + int newMultiplier = framePagesOld + deltaPages; // we need to get the old slot offsets before we grow int oldSlotEnd = slotManager.getSlotEndOff(); int oldSlotStart = slotManager.getSlotStartOff() + slotManager.getSlotSize(); - bufferCache.resizePage(getPage(), framePagesOld + deltaPages); - buf = getPage().getBuffer(); + bufferCache.resizePage(getPage(), newMultiplier, extraPageBlockHelper); - // return the dropped supplemental pages to the page manager... - if (oldSupplementalPages > 0) { - freePageManager.addFreePageBlock(metaFrame, oldSupplementalPageId, oldSupplementalPages); - } + buf = getPage().getBuffer(); // fixup the slots System.arraycopy(buf.array(), oldSlotEnd, buf.array(), slotManager.getSlotEndOff(), oldSlotStart - oldSlotEnd); // fixup total free space counter - buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + (bufferCache.getPageSize() * pageDelta)); + buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + (bufferCache.getPageSize() * deltaPages)); } @Override @@ -416,8 +366,6 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr public String printHeader() { StringBuilder strBuilder = new StringBuilder(super.printHeader()); strBuilder.append("nextLeafOff: " + nextLeafOff + "\n"); - strBuilder.append("supplementalNumPagesOff: " + supplementalNumPagesOff + "\n"); - strBuilder.append("supplementalPageIdOff: " + supplementalPageIdOff + "\n"); return strBuilder.toString(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java index 2b7f12b..3cf0ba5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java @@ -22,7 +22,6 @@ package org.apache.hyracks.storage.am.btree.frames; import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public class BTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory { @@ -36,7 +35,7 @@ public class BTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory { @Override public IBTreeLeafFrame createFrame() { - return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter(), getLargePageHelper()); + return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter()); } @Override @@ -44,8 +43,4 @@ public class BTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory { return tupleWriterFactory; } - @Override - public ILargePageHelper getLargePageHelper() { - return BTreeLargeFrameHelper.INSTANCE; - } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java index 3043940..5995004 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java @@ -66,7 +66,6 @@ import org.apache.hyracks.storage.am.common.impls.TreeIndexDiskOrderScanCursor; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.BufferCache; -import org.apache.hyracks.storage.common.buffercache.CachedPage; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; @@ -103,8 +102,7 @@ public class BTree extends AbstractTreeIndex { RangePredicate diskOrderScanPred = new RangePredicate(null, null, true, true, ctx.cmp, ctx.cmp); int maxPageId = freePageManager.getMaxPage(ctx.metaFrame); int currentPageId = bulkloadLeafStart; - ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false, - largePageHelper); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false); page.acquireReadLatch(); try { cursor.setBufferCache(bufferCache); @@ -136,7 +134,7 @@ public class BTree extends AbstractTreeIndex { } private void validate(BTreeOpContext ctx, int pageId) throws HyracksDataException { - ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); ctx.interiorFrame.setPage(page); PageValidationInfo currentPvi = ctx.validationInfos.peekFirst(); @@ -220,8 +218,7 @@ public class BTree extends AbstractTreeIndex { ICachedPage originalPage = ctx.interiorFrame.getPage(); for (int i = 0; i < ctx.smPages.size(); i++) { int pageId = ctx.smPages.get(i); - ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, - largePageHelper); + ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); smPage.acquireWriteLatch(); try { ctx.interiorFrame.setPage(smPage); @@ -246,17 +243,17 @@ public class BTree extends AbstractTreeIndex { private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException { // Make sure the root is always in the same page. ICachedPage leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getLeftPage()), - false, largePageHelper); + false); leftNode.acquireWriteLatch(); try { int newLeftId = freePageManager.getFreePage(ctx.metaFrame); - ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true, - largePageHelper); + ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true); newLeftNode.acquireWriteLatch(); try { boolean largePage = false; if (leftNode.getBuffer().capacity() > newLeftNode.getBuffer().capacity()) { - bufferCache.resizePage(newLeftNode, leftNode.getBuffer().capacity() / bufferCache.getPageSize()); + bufferCache.resizePage(newLeftNode, leftNode.getBuffer().capacity() / bufferCache.getPageSize(), + ctx); largePage = true; } // Copy left child to new left child. @@ -268,7 +265,7 @@ public class BTree extends AbstractTreeIndex { long leftNodeLSN = ctx.interiorFrame.getPageLsn(); // Initialize new root (leftNode becomes new root). if (largePage) { - bufferCache.resizePage(leftNode, 1); + bufferCache.resizePage(leftNode, 1, ctx); ctx.interiorFrame.setPage(leftNode); ctx.interiorFrame.setLargeFlag(false); } else { @@ -372,7 +369,7 @@ public class BTree extends AbstractTreeIndex { switch (spaceStatus) { case EXPAND: { // TODO: avoid repeated calculation of tuple size - ctx.leafFrame.ensureCapacity(freePageManager, ctx.metaFrame, bufferCache, tuple); + ctx.leafFrame.ensureCapacity(bufferCache, tuple, ctx); } // fall-through case SUFFICIENT_CONTIGUOUS_SPACE: { @@ -432,8 +429,7 @@ public class BTree extends AbstractTreeIndex { } } int rightPageId = freePageManager.getFreePage(ctx.metaFrame); - ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true, - largePageHelper); + ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true); rightNode.acquireWriteLatch(); try { IBTreeLeafFrame rightFrame = ctx.createLeafFrame(); @@ -449,7 +445,7 @@ public class BTree extends AbstractTreeIndex { } else { ctx.modificationCallback.found(null, tuple); } - ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame, bufferCache); + ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, ctx, bufferCache); ctx.smPages.add(pageId); ctx.smPages.add(rightPageId); @@ -492,7 +488,7 @@ public class BTree extends AbstractTreeIndex { ctx.modificationCallback.found(beforeTuple, tuple); ctx.leafFrame.delete(tuple, oldTupleIndex); ctx.leafFrame.compact(); - ctx.leafFrame.ensureCapacity(freePageManager, ctx.metaFrame, bufferCache, tuple); + ctx.leafFrame.ensureCapacity(bufferCache, tuple, ctx); int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple); ctx.leafFrame.insert(tuple, targetTupleIndex); ctx.splitKey.reset(); @@ -550,8 +546,7 @@ public class BTree extends AbstractTreeIndex { switch (spaceStatus) { case INSUFFICIENT_SPACE: { int rightPageId = freePageManager.getFreePage(ctx.metaFrame); - ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true, - largePageHelper); + ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true); rightNode.acquireWriteLatch(); try { IBTreeFrame rightFrame = ctx.createInteriorFrame(); @@ -560,8 +555,7 @@ public class BTree extends AbstractTreeIndex { rightFrame.setMultiComparator(ctx.cmp); // instead of creating a new split key, use the existing // splitKey - ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey, freePageManager, - ctx.metaFrame, bufferCache); + ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey, ctx, bufferCache); ctx.smPages.add(pageId); ctx.smPages.add(rightPageId); ctx.interiorFrame.setSmFlag(true); @@ -631,7 +625,7 @@ public class BTree extends AbstractTreeIndex { } private ICachedPage isConsistent(int pageId, BTreeOpContext ctx) throws Exception { - ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); node.acquireReadLatch(); ctx.interiorFrame.setPage(node); boolean isConsistent = ctx.pageLsns.getLast() == ctx.interiorFrame.getPageLsn(); @@ -645,7 +639,7 @@ public class BTree extends AbstractTreeIndex { private void performOp(int pageId, ICachedPage parent, boolean parentIsReadLatched, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException { - ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); ctx.interiorFrame.setPage(node); // this check performs an unprotected read in the page // the following could happen: TODO fill out @@ -707,7 +701,7 @@ public class BTree extends AbstractTreeIndex { // Is there a propagated split key? if (ctx.splitKey.getBuffer() != null) { ICachedPage interiorNode = bufferCache.pin( - BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + BufferedFileHandle.getDiskPageId(fileId, pageId), false); interiorNode.acquireWriteLatch(); try { // Insert or update op. Both can cause split keys to propagate upwards. @@ -839,8 +833,7 @@ public class BTree extends AbstractTreeIndex { private BTreeOpContext createOpContext(IIndexAccessor accessor, IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback) { return new BTreeOpContext(accessor, leafFrameFactory, interiorFrameFactory, - freePageManager.getMetaDataFrameFactory().createFrame(), cmpFactories, modificationCallback, - searchCallback); + freePageManager, cmpFactories, modificationCallback, searchCallback); } @SuppressWarnings("rawtypes") @@ -857,7 +850,7 @@ public class BTree extends AbstractTreeIndex { public void printTree(int pageId, ICachedPage parent, boolean unpin, IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes, StringBuilder strBuilder, MultiComparator cmp) throws Exception { - ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false, largePageHelper); + ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); node.acquireReadLatch(); try { if (parent != null && unpin == true) { @@ -1077,21 +1070,18 @@ public class BTree extends AbstractTreeIndex { int headerSize = Math.max(leafFrame.getPageHeaderSize(), interiorFrame.getPageHeaderSize()); final int multiplier = (int) Math .ceil((double) tupleSize / (bufferCache.getPageSize() - headerSize)); - leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier); - ((CachedPage) leafFrontier.page).setLargePageHelper(largePageHelper); - leafFrame.setPage(leafFrontier.page); - leafFrame.initBuffer((byte) 0); if (multiplier > 1) { - int supplementalPages = multiplier - 1; - ((IBTreeLeafFrame) leafFrame).configureLargePage(supplementalPages, - freePageManager.getFreePageBlock(metaFrame, supplementalPages)); + leafFrontier.page = bufferCache.confiscateLargePage(dpid, multiplier, + freePageManager.getFreePageBlock(metaFrame, multiplier - 1)); } else { - ((IBTreeLeafFrame) leafFrame).setLargeFlag(true); + leafFrontier.page = bufferCache.confiscatePage(dpid); } + leafFrame.setPage(leafFrontier.page); + leafFrame.initBuffer((byte) 0); + ((IBTreeLeafFrame) leafFrame).setLargeFlag(true); } else { final long dpid = BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId); leafFrontier.page = bufferCache.confiscatePage(dpid); - ((CachedPage) leafFrontier.page).setLargePageHelper(largePageHelper); leafFrame.setPage(leafFrontier.page); leafFrame.initBuffer((byte) 0); } @@ -1161,7 +1151,6 @@ public class BTree extends AbstractTreeIndex { propagateBulk(level + 1, pagesToWrite); frontier.page = bufferCache.confiscatePage(BufferCache.INVALID_DPID); - ((CachedPage) frontier.page).setLargePageHelper(largePageHelper); interiorFrame.setPage(frontier.page); interiorFrame.initBuffer((byte) level); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java index 974860e..2297cc5 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeCountingSearchCursor.java @@ -25,7 +25,6 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; -import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper; import org.apache.hyracks.storage.am.common.api.ICursorInitialState; import org.apache.hyracks.storage.am.common.api.ISearchPredicate; import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; @@ -119,8 +118,7 @@ public class BTreeCountingSearchCursor implements ITreeIndexCursor { private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException { do { - ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false, - BTreeLargeFrameHelper.INSTANCE); + ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false); if (exclusiveLatchNodes) { nextLeaf.acquireWriteLatch(); page.releaseWriteLatch(isPageDirty); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java index 8513368..3649433 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeOpContext.java @@ -33,6 +33,7 @@ import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; import org.apache.hyracks.storage.am.btree.api.ITupleAcceptor; import org.apache.hyracks.storage.am.common.api.IIndexAccessor; import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; +import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback; import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor; @@ -43,8 +44,9 @@ import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; import org.apache.hyracks.storage.am.common.ophelpers.IntArrayList; import org.apache.hyracks.storage.am.common.ophelpers.LongArrayList; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; -public class BTreeOpContext implements IIndexOperationContext { +public class BTreeOpContext implements IIndexOperationContext, IExtraPageBlockHelper { private final int INIT_ARRAYLIST_SIZE = 6; public IIndexAccessor accessor; @@ -53,7 +55,8 @@ public class BTreeOpContext implements IIndexOperationContext { public ITreeIndexFrameFactory interiorFrameFactory; public IBTreeLeafFrame leafFrame; public IBTreeInteriorFrame interiorFrame; - public ITreeIndexMetaDataFrame metaFrame; + public final IMetaDataPageManager freePageManager; + public final ITreeIndexMetaDataFrame metaFrame; public IndexOperation op; public ITreeIndexCursor cursor; public BTreeCursorInitialState cursorInitialState; @@ -75,9 +78,9 @@ public class BTreeOpContext implements IIndexOperationContext { public final ITreeIndexTupleReference leafFrameTuple; public BTreeOpContext(IIndexAccessor accessor, ITreeIndexFrameFactory leafFrameFactory, - ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexMetaDataFrame metaFrame, - IBinaryComparatorFactory[] cmpFactories, IModificationOperationCallback modificationCallback, - ISearchOperationCallback searchCallback) { + ITreeIndexFrameFactory interiorFrameFactory, IMetaDataPageManager freePageManager, + IBinaryComparatorFactory[] cmpFactories, IModificationOperationCallback modificationCallback, + ISearchOperationCallback searchCallback) { this.accessor = accessor; if (cmpFactories[0] != null) { @@ -97,7 +100,8 @@ public class BTreeOpContext implements IIndexOperationContext { if (interiorFrame != null && this.cmp != null) { interiorFrame.setMultiComparator(cmp); } - this.metaFrame = metaFrame; + this.freePageManager = freePageManager; + this.metaFrame = freePageManager.getMetaDataFrameFactory().createFrame(); this.pageLsns = new LongArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE); this.smoCount = 0; this.modificationCallback = modificationCallback; @@ -233,4 +237,14 @@ public class BTreeOpContext implements IIndexOperationContext { this.modificationCallback = modificationCallback; this.searchCallback = searchCallback; } + + @Override + public int getFreeBlock(int size) throws HyracksDataException { + return freePageManager.getFreePageBlock(metaFrame, size); + } + + @Override + public void returnFreePageBlock(int blockPageId, int size) throws HyracksDataException { + freePageManager.addFreePageBlock(metaFrame, blockPageId, size); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java index 975bd9b..3301e37 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java @@ -25,7 +25,6 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.dataflow.common.util.TupleUtils; import org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame; -import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper; import org.apache.hyracks.storage.am.common.api.ICursorInitialState; import org.apache.hyracks.storage.am.common.api.IIndexAccessor; import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback; @@ -118,8 +117,7 @@ public class BTreeRangeSearchCursor implements ITreeIndexCursor { private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException { do { - ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false, - BTreeLargeFrameHelper.INSTANCE); + ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false); if (exclusiveLatchNodes) { nextLeaf.acquireWriteLatch(); page.releaseWriteLatch(isPageDirty); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java index 5b6cee9..7c2abb1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java @@ -38,7 +38,6 @@ import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; import org.apache.hyracks.storage.common.file.IFileMapProvider; public class BTreeUtils { @@ -49,7 +48,6 @@ public class BTreeUtils { ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory); ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory(); - ILargePageHelper largePageHelper = leafFrameFactory.getLargePageHelper(); IMetaDataPageManager freePageManager; freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory); BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, @@ -63,7 +61,6 @@ public class BTreeUtils { TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits); ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory); - ILargePageHelper largePageHelper = leafFrameFactory.getLargePageHelper(); BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, typeTraits.length, file); return btree; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java index 711db9e..037c183 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java @@ -27,7 +27,7 @@ import org.apache.hyracks.storage.am.common.frames.FrameOpSpaceStatus; import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; +import org.apache.hyracks.storage.common.buffercache.IExtraPageBlockHelper; public interface ITreeIndexFrame { @@ -73,7 +73,7 @@ public interface ITreeIndexFrame { public String printHeader(); public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey, - IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache) + IExtraPageBlockHelper extraPageBlockHelper, IBufferCache bufferCache) throws HyracksDataException, TreeIndexException; public ISlotManager getSlotManager(); @@ -104,6 +104,4 @@ public interface ITreeIndexFrame { public ITreeIndexTupleReference createTupleReference(); public void setMultiComparator(MultiComparator cmp); - - ILargePageHelper getLargePageHelper(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java index 248baf5..8a705dc 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java @@ -20,10 +20,7 @@ package org.apache.hyracks.storage.am.common.api; import java.io.Serializable; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; - public interface ITreeIndexFrameFactory extends Serializable { ITreeIndexFrame createFrame(); ITreeIndexTupleWriterFactory getTupleWriterFactory(); - ILargePageHelper getLargePageHelper(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/171ff2ff/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java index ea6bcac..72c6fe2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java @@ -31,7 +31,6 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference; import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter; import org.apache.hyracks.storage.am.common.ophelpers.SlotOffTupleOff; import org.apache.hyracks.storage.common.buffercache.ICachedPage; -import org.apache.hyracks.storage.common.buffercache.ILargePageHelper; public abstract class TreeIndexNSMFrame implements ITreeIndexFrame { @@ -51,14 +50,12 @@ public abstract class TreeIndexNSMFrame implements ITreeIndexFrame { protected ITreeIndexTupleWriter tupleWriter; protected ITreeIndexTupleReference frameTuple; - protected ILargePageHelper largePageHelper; - public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager, ILargePageHelper largePageHelper) { + public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager) { this.tupleWriter = tupleWriter; this.frameTuple = tupleWriter.createTupleReference(); this.slotManager = slotManager; this.slotManager.setFrame(this); - this.largePageHelper = largePageHelper; } @Override @@ -101,14 +98,10 @@ public abstract class TreeIndexNSMFrame implements ITreeIndexFrame { } } - public static boolean isLargePage(ByteBuffer buf) { + public boolean getLargeFlag() { return (buf.get(flagOff) & largeFlagBit) != 0; } - public boolean isLargePage() { - return isLargePage(buf); - } - @Override public boolean isInterior() { return buf.get(levelOff) > 0; @@ -342,8 +335,4 @@ public abstract class TreeIndexNSMFrame implements ITreeIndexFrame { return buf.capacity() - getFreeSpaceOff() - (getTupleCount() * slotManager.getSlotSize()); } - @Override - public ILargePageHelper getLargePageHelper() { - return largePageHelper; - } }