asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject [12/12] incubator-asterixdb git commit: ASTERIXDB-1436: Big Object Support For Storage
Date Fri, 13 May 2016 02:41:02 GMT
ASTERIXDB-1436: Big Object Support For Storage

Preliminary submit for big object storage support (values only).
Enable hyracks frames up to Integer.MAX_VALUE

Change-Id: Ie0a3cb855768dfd0cd1da4a5fc9f62aedebcbc5f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/840
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/1defc92a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/1defc92a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/1defc92a

Branch: refs/heads/master
Commit: 1defc92ae43cb36e1c06eb884c5359d5c1d6b0e5
Parents: a98d0de
Author: Michael Blow <mblow@apache.org>
Authored: Thu May 12 18:31:47 2016 -0700
Committer: Michael Blow <michael.blow@couchbase.com>
Committed: Thu May 12 19:40:40 2016 -0700

----------------------------------------------------------------------
 .../big_object_bulkload.1.ddl.aql               |  35 +++
 .../big_object_bulkload.2.update.aql            | 119 +++++++++
 .../big_object_bulkload.3.query.aql             |  23 ++
 .../big_object_insert.1.ddl.aql                 |  35 +++
 .../big_object_insert.2.update.aql              | 119 +++++++++
 .../big_object_insert.3.query.aql               |  23 ++
 .../big_object_bulkload.1.adm                   | 100 +++++++
 .../big_object_insert/big_object_insert.1.adm   | 100 +++++++
 .../feed-with-external-parser.1.adm             |   3 +-
 .../feed-with-multiple-indexes.1.adm            |   3 +-
 .../src/test/resources/runtimets/testsuite.xml  |  10 +
 .../asterix/common/transactions/ILogRecord.java |   9 +-
 .../asterix/common/transactions/LogRecord.java  |  45 ++--
 .../apache/asterix/test/aql/TestExecutor.java   |   3 +
 .../dataflow/FeedRecordDataFlowController.java  |   5 -
 .../external/dataflow/FeedTupleForwarder.java   |  16 --
 .../external/util/ExternalDataConstants.java    |   1 -
 .../apache/asterix/external/util/FeedUtils.java |   2 +-
 .../aql/statement/SubscribeFeedStatement.java   |   2 +-
 .../logging/ReplicationLogBuffer.java           |   8 +
 .../logging/ReplicationLogFlusher.java          |   5 +-
 .../management/ReplicationManager.java          |  18 +-
 .../management/service/logging/LogBuffer.java   |   4 +
 .../service/logging/LogBufferTailReader.java    |   6 +-
 .../management/service/logging/LogManager.java  |  22 +-
 .../logging/LogManagerWithReplication.java      |   6 +-
 .../management/service/logging/LogReader.java   |  93 ++++---
 .../service/logging/RemoteLogReader.java        |  57 ++--
 hyracks-fullstack/hyracks/hyracks-api/pom.xml   |   5 +
 .../apache/hyracks/api/comm/FrameConstants.java |  12 +-
 .../apache/hyracks/api/comm/FrameHelper.java    |  16 +-
 .../nc/resources/memory/FrameManager.java       |  18 +-
 .../common/comm/io/AbstractFrameAppender.java   |   2 +-
 .../common/comm/io/FrameFixedFieldAppender.java |   2 +-
 .../common/comm/io/FrameTupleAccessor.java      |   2 +-
 .../common/comm/io/FrameTupleAppender.java      |   2 +-
 .../comm/io/FrameTupleAppenderAccessor.java     |   2 +-
 .../common/io/MessagingFrameTupleAppender.java  |   2 +-
 .../dataflow/common/util/IntSerDeUtils.java     |  42 ---
 .../dataflow/common/util/ReflectionUtils.java   |  33 ---
 .../VariableFrameMemoryManager.java             |   2 +-
 .../std/sort/AbstractExternalSortRunMerger.java |   2 +-
 .../dataflow/std/sort/AbstractFrameSorter.java  |   2 +-
 .../sort/util/DeletableFrameTupleAppender.java  |   2 +-
 .../AbstractTupleMemoryManagerTest.java         |   2 +-
 .../VPartitionTupleBufferManagerTest.java       |   2 +-
 .../VariableTupleMemoryManagerTest.java         |   2 +-
 .../util/DeletableFrameTupleAppenderTest.java   |   2 +-
 .../storage/am/btree/api/IBTreeFrame.java       |   2 +
 .../storage/am/btree/api/IBTreeLeafFrame.java   |  14 +-
 .../frames/BTreeFieldPrefixNSMLeafFrame.java    |  37 ++-
 .../BTreeFieldPrefixNSMLeafFrameFactory.java    |   6 +
 .../am/btree/frames/BTreeLargeFrameHelper.java  |  44 ++++
 .../am/btree/frames/BTreeNSMInteriorFrame.java  |  50 ++--
 .../frames/BTreeNSMInteriorFrameFactory.java    |   6 +
 .../am/btree/frames/BTreeNSMLeafFrame.java      | 189 +++++++++++---
 .../btree/frames/BTreeNSMLeafFrameFactory.java  |   8 +-
 .../hyracks/storage/am/btree/impls/BTree.java   | 258 ++++++++++++-------
 .../btree/impls/BTreeCountingSearchCursor.java  |   5 +-
 .../am/btree/impls/BTreeRangeSearchCursor.java  |   4 +-
 .../storage/am/btree/util/BTreeUtils.java       |   3 +
 .../am/common/api/IMetaDataPageManager.java     |  15 +-
 .../storage/am/common/api/ITreeIndexFrame.java  |   8 +-
 .../am/common/api/ITreeIndexFrameFactory.java   |   7 +-
 .../am/common/frames/FrameOpSpaceStatus.java    |   7 +-
 .../am/common/frames/TreeIndexNSMFrame.java     |  58 ++++-
 .../freepage/LinkedMetaDataPageManager.java     |  15 ++
 .../am/common/impls/AbstractTreeIndex.java      |  34 ++-
 .../impls/TreeIndexDiskOrderScanCursor.java     |   3 +-
 .../am/common/util/TreeIndexStatsGatherer.java  |   2 +-
 .../freepage/VirtualMetaDataPageManager.java    |  12 +-
 .../impls/MultitenantVirtualBufferCache.java    |  16 ++
 .../am/lsm/common/impls/VirtualBufferCache.java |  60 ++++-
 .../storage/am/rtree/frames/RTreeNSMFrame.java  |  20 +-
 .../am/rtree/frames/RTreeNSMInteriorFrame.java  |   5 +-
 .../frames/RTreeNSMInteriorFrameFactory.java    |  10 +-
 .../am/rtree/frames/RTreeNSMLeafFrame.java      |   2 +-
 .../rtree/frames/RTreeNSMLeafFrameFactory.java  |  10 +-
 .../hyracks/storage/am/rtree/impls/RTree.java   |  14 +-
 .../buffercache/AsyncFIFOPageQueueManager.java  |  37 +--
 .../storage/common/buffercache/BufferCache.java | 182 +++++++++++--
 .../storage/common/buffercache/CachedPage.java  |  12 +-
 .../ClockPageReplacementStrategy.java           | 136 +++++++---
 .../common/buffercache/DebugBufferCache.java    |  17 ++
 .../buffercache/DirectBufferAllocator.java      |   4 +
 .../common/buffercache/FIFOLocalWriter.java     |   1 -
 .../common/buffercache/HeapBufferAllocator.java |   4 +
 .../common/buffercache/IBufferCache.java        |   6 +-
 .../buffercache/IBufferCacheInternal.java       |   2 +
 .../buffercache/ICacheMemoryAllocator.java      |  11 +
 .../common/buffercache/ILargePageHelper.java    |  25 ++
 .../buffercache/IPageReplacementStrategy.java   |   6 +
 .../ResourceHeapBufferAllocator.java            |   7 +-
 .../storage/common/file/BufferedFileHandle.java |   4 +-
 .../am/btree/OrderedIndexExamplesTest.java      |   8 +-
 .../storage/am/btree/FieldPrefixNSMTest.java    |   7 +-
 .../storage/am/btree/StorageManagerTest.java    |   4 +-
 .../org/apache/hyracks/util/IntSerDeUtils.java  |  42 +++
 .../apache/hyracks/util/ReflectionUtils.java    |  33 +++
 99 files changed, 1965 insertions(+), 533 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_bulkload/big_object_bulkload.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_bulkload/big_object_bulkload.1.ddl.aql
b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_bulkload/big_object_bulkload.1.ddl.aql
new file mode 100644
index 0000000..6670a05
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_bulkload/big_object_bulkload.1.ddl.aql
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : big_object_bulkload.aql
+ * Description     : bulkload insert of large objects
+ * Expected Result : Success
+ * Date            : 20th April 2016
+ */
+
+drop dataverse testdv2 if exists;
+create dataverse testdv2;
+use dataverse testdv2;
+
+create type testtype as open {
+  id: int64,
+  name: string
+}
+
+create dataset testds(testtype) primary key id;


Mime
View raw message