Return-Path: X-Original-To: apmail-asterixdb-commits-archive@minotaur.apache.org Delivered-To: apmail-asterixdb-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 846F318D01 for ; Thu, 18 Jun 2015 04:23:06 +0000 (UTC) Received: (qmail 60448 invoked by uid 500); 18 Jun 2015 04:23:05 -0000 Delivered-To: apmail-asterixdb-commits-archive@asterixdb.apache.org Received: (qmail 60416 invoked by uid 500); 18 Jun 2015 04:23:05 -0000 Mailing-List: contact commits-help@asterixdb.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.incubator.apache.org Delivered-To: mailing list commits@asterixdb.incubator.apache.org Received: (qmail 60407 invoked by uid 99); 18 Jun 2015 04:23:05 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 18 Jun 2015 04:23:05 +0000 X-ASF-Spam-Status: No, hits=-2000.4 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Thu, 18 Jun 2015 04:20:33 +0000 Received: (qmail 59518 invoked by uid 99); 18 Jun 2015 04:22:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 18 Jun 2015 04:22:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CDFD1E000B; Thu, 18 Jun 2015 04:22:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianfeng@apache.org To: commits@asterixdb.incubator.apache.org Date: Thu, 18 Jun 2015 04:22:18 -0000 Message-Id: <99751eb9db894c2eb7e65106e83134e0@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/14] incubator-asterixdb-hyracks git commit: VariableSizeFrame(VSizeFrame) support for Hyracks. X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-asterixdb-hyracks Updated Branches: refs/heads/master 0e5d5315e -> 0d87a57f7 http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java index 8598b70..231adbd 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java @@ -15,11 +15,10 @@ package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow; -import java.io.DataInputStream; import java.io.IOException; import java.nio.ByteBuffer; -import edu.uci.ics.hyracks.api.comm.FrameHelper; +import edu.uci.ics.hyracks.api.comm.VSizeFrame; import edu.uci.ics.hyracks.api.context.IHyracksTaskContext; import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor; import edu.uci.ics.hyracks.api.exceptions.HyracksDataException; @@ -27,9 +26,7 @@ import edu.uci.ics.hyracks.data.std.util.GrowableArray; import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor; import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender; -import edu.uci.ics.hyracks.dataflow.common.comm.util.ByteBufferInputStream; import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils; -import edu.uci.ics.hyracks.dataflow.common.util.IntSerDeUtils; import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable; import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer; import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.tokenizers.IToken; @@ -50,7 +47,6 @@ public class BinaryTokenizerOperatorNodePushable extends private ArrayTupleBuilder builder; private GrowableArray builderData; private FrameTupleAppender appender; - private ByteBuffer writeBuffer; public BinaryTokenizerOperatorNodePushable(IHyracksTaskContext ctx, RecordDescriptor inputRecDesc, RecordDescriptor outputRecDesc, @@ -68,12 +64,10 @@ public class BinaryTokenizerOperatorNodePushable extends @Override public void open() throws HyracksDataException { - accessor = new FrameTupleAccessor(ctx.getFrameSize(), inputRecDesc); - writeBuffer = ctx.allocateFrame(); + accessor = new FrameTupleAccessor(inputRecDesc); builder = new ArrayTupleBuilder(outputRecDesc.getFieldCount()); builderData = builder.getFieldData(); - appender = new FrameTupleAppender(ctx.getFrameSize()); - appender.reset(writeBuffer, true); + appender = new FrameTupleAppender(new VSizeFrame(ctx), true); writer.open(); } @@ -157,19 +151,8 @@ public class BinaryTokenizerOperatorNodePushable extends } - if (!appender.append(builder.getFieldEndOffsets(), - builder.getByteArray(), 0, builder.getSize())) { - FrameUtils.flushFrame(writeBuffer, writer); - appender.reset(writeBuffer, true); - - if (!appender.append(builder.getFieldEndOffsets(), - builder.getByteArray(), 0, builder.getSize())) { - throw new HyracksDataException("Record size (" - + builder.getSize() - + ") larger than frame size (" - + appender.getBuffer().capacity() + ")"); - } - } + FrameUtils.appendToWriter(writer, appender, builder.getFieldEndOffsets(), + builder.getByteArray(), 0, builder.getSize()); } @@ -179,9 +162,7 @@ public class BinaryTokenizerOperatorNodePushable extends @Override public void close() throws HyracksDataException { - if (appender.getTupleCount() > 0) { - FrameUtils.flushFrame(writeBuffer, writer); - } + appender.flush(writer, true); writer.close(); } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java index ddaeac0..95ec645 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java @@ -15,8 +15,10 @@ package edu.uci.ics.hyracks.storage.am.lsm.invertedindex.dataflow; +import edu.uci.ics.hyracks.api.comm.IFrame; import edu.uci.ics.hyracks.api.context.IHyracksTaskContext; import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider; +import edu.uci.ics.hyracks.api.exceptions.HyracksDataException; import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference; import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate; import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java index 59690c3..3c6c4cd 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/FixedSizeFrameTupleAccessor.java @@ -67,6 +67,11 @@ public class FixedSizeFrameTupleAccessor implements IFrameTupleAccessor { } @Override + public int getTupleLength(int tupleIndex) { + return getTupleEndOffset(tupleIndex) - getTupleStartOffset(tupleIndex); + } + + @Override public int getFieldSlotsLength() { return 0; } @@ -92,6 +97,11 @@ public class FixedSizeFrameTupleAccessor implements IFrameTupleAccessor { } @Override + public int getAbsoluteFieldStartOffset(int tupleIndex, int fIdx) { + return getTupleStartOffset(tupleIndex) + getFieldSlotsLength() + getFieldStartOffset(tupleIndex, fIdx); + } + + @Override public void reset(ByteBuffer buffer) { this.buffer = buffer; } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java index 1aded5f..a479815 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java @@ -549,7 +549,7 @@ public class OnDiskInvertedIndex implements IInvertedIndex { private final int FRAME_SIZE = 32768; @Override - public int getFrameSize() { + public int getInitialFrameSize() { return FRAME_SIZE; } @@ -563,8 +563,16 @@ public class OnDiskInvertedIndex implements IInvertedIndex { return ByteBuffer.allocate(FRAME_SIZE); } + @Override public ByteBuffer allocateFrame(int bytes) throws HyracksDataException { + return ByteBuffer.allocate(bytes); + } + + @Override public ByteBuffer reallocateFrame(ByteBuffer bytes, int newSizeInBytes, boolean copyOldData) throws HyracksDataException { + throw new HyracksDataException("TODO"); + } + @Override - public void deallocateFrames(int frameCount) { + public void deallocateFrames(int bytes) { // TODO Auto-generated method stub } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java index f3b019f..5b68c7b 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/AbstractTOccurrenceSearcher.java @@ -19,18 +19,18 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import edu.uci.ics.hyracks.api.comm.IFrame; import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor; +import edu.uci.ics.hyracks.api.comm.VSizeFrame; import edu.uci.ics.hyracks.api.context.IHyracksCommonContext; import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer; import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor; import edu.uci.ics.hyracks.api.exceptions.HyracksDataException; import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable; import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; -import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor; -import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender; +import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppenderAccessor; import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference; import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference; -import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer; import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer; import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator; import edu.uci.ics.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; @@ -59,9 +59,8 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc protected final MultiComparator invListCmp; protected final ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(QUERY_TOKEN_REC_DESC.getFieldCount()); - protected final ByteBuffer queryTokenFrame; - protected final FrameTupleAppender queryTokenAppender; - protected final FrameTupleAccessor queryTokenAccessor; + protected final IFrame queryTokenFrame; + protected final FrameTupleAppenderAccessor queryTokenAppender; protected final FrameTupleReference searchKey = new FrameTupleReference(); protected int occurrenceThreshold; @@ -78,10 +77,9 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc this.invListCursorFactory = new InvertedListCursorFactory(invIndex); this.invListCursorCache = new ObjectCache(invListCursorFactory, OBJECT_CACHE_INIT_SIZE, OBJECT_CACHE_EXPAND_SIZE); - this.queryTokenFrame = ctx.allocateFrame(); - this.queryTokenAppender = new FrameTupleAppender(ctx.getFrameSize()); - this.queryTokenAccessor = new FrameTupleAccessor(ctx.getFrameSize(), QUERY_TOKEN_REC_DESC); - this.queryTokenAccessor.reset(queryTokenFrame); + this.queryTokenFrame = new VSizeFrame(ctx); + this.queryTokenAppender = new FrameTupleAppenderAccessor(QUERY_TOKEN_REC_DESC); + this.queryTokenAppender.reset(queryTokenFrame, true); } public void reset() { @@ -116,7 +114,7 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc } public IFrameTupleAccessor createResultFrameTupleAccessor() { - return new FixedSizeFrameTupleAccessor(ctx.getFrameSize(), searchResult.getTypeTraits()); + return new FixedSizeFrameTupleAccessor(ctx.getInitialFrameSize(), searchResult.getTypeTraits()); } public ITupleReference createResultFrameTupleReference() { @@ -144,8 +142,10 @@ public abstract class AbstractTOccurrenceSearcher implements IInvertedIndexSearc ByteBuffer testBuf = buffer.get(i); resultFrameTupleAcc.reset(testBuf); for (int j = 0; j < resultFrameTupleAcc.getTupleCount(); j++) { - strBuffer.append(IntegerPointable.getInteger(resultFrameTupleAcc.getBuffer().array(), resultFrameTupleAcc.getFieldStartOffset(j, 0)) + ","); - strBuffer.append(IntegerPointable.getInteger(resultFrameTupleAcc.getBuffer().array(), resultFrameTupleAcc.getFieldStartOffset(j, 1)) + " "); + strBuffer.append(IntegerPointable.getInteger(resultFrameTupleAcc.getBuffer().array(), + resultFrameTupleAcc.getFieldStartOffset(j, 0)) + ","); + strBuffer.append(IntegerPointable.getInteger(resultFrameTupleAcc.getBuffer().array(), + resultFrameTupleAcc.getFieldStartOffset(j, 1)) + " "); } } System.out.println(strBuffer.toString()); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java index b7456e6..1e68099 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/PartitionedTOccurrenceSearcher.java @@ -24,7 +24,6 @@ import edu.uci.ics.hyracks.data.std.primitive.ShortPointable; import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference; import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference; -import edu.uci.ics.hyracks.dataflow.common.data.marshalling.ShortSerializerDeserializer; import edu.uci.ics.hyracks.storage.am.common.api.IIndexOperationContext; import edu.uci.ics.hyracks.storage.am.common.api.IndexException; import edu.uci.ics.hyracks.storage.am.common.tuples.ConcatenatingTupleReference; @@ -64,7 +63,7 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher lowerBoundTupleBuilder.addFieldEndOffset(); lowerBoundTuple.reset(lowerBoundTupleBuilder.getFieldEndOffsets(), lowerBoundTupleBuilder.getByteArray()); // Only needed for setting the number of fields in searchKey. - searchKey.reset(queryTokenAccessor, 0); + searchKey.reset(queryTokenAppender, 0); fullLowSearchKey.reset(); fullLowSearchKey.addTuple(searchKey); fullLowSearchKey.addTuple(lowerBoundTuple); @@ -75,7 +74,7 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher upperBoundTupleBuilder.addFieldEndOffset(); upperBoundTuple.reset(upperBoundTupleBuilder.getFieldEndOffsets(), upperBoundTupleBuilder.getByteArray()); // Only needed for setting the number of fields in searchKey. - searchKey.reset(queryTokenAccessor, 0); + searchKey.reset(queryTokenAppender, 0); fullHighSearchKey.reset(); fullHighSearchKey.addTuple(searchKey); fullHighSearchKey.addTuple(upperBoundTuple); @@ -93,7 +92,7 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher } tokenizeQuery(searchPred); - short numQueryTokens = (short) queryTokenAccessor.getTupleCount(); + short numQueryTokens = (short) queryTokenAppender.getTupleCount(); IInvertedIndexSearchModifier searchModifier = searchPred.getSearchModifier(); short numTokensLowerBound = searchModifier.getNumTokensLowerBound(numQueryTokens); @@ -109,7 +108,7 @@ public class PartitionedTOccurrenceSearcher extends AbstractTOccurrenceSearcher partitions.reset(numTokensLowerBound, numTokensUpperBound); cursorsOrderedByTokens.clear(); for (int i = 0; i < numQueryTokens; i++) { - searchKey.reset(queryTokenAccessor, i); + searchKey.reset(queryTokenAppender, i); if (!partInvIndex.openInvertedListPartitionCursors(this, ictx, numTokensLowerBound, numTokensUpperBound, partitions, cursorsOrderedByTokens)) { maxCountPossible--; http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java index c4056c5..4af9546 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/SearchResult.java @@ -54,8 +54,8 @@ public class SearchResult { // Integer for counting occurrences. typeTraits[invListFields.length] = IntegerPointable.TYPE_TRAITS; this.ctx = ctx; - appender = new FixedSizeFrameTupleAppender(ctx.getFrameSize(), typeTraits); - accessor = new FixedSizeFrameTupleAccessor(ctx.getFrameSize(), typeTraits); + appender = new FixedSizeFrameTupleAppender(ctx.getInitialFrameSize(), typeTraits); + accessor = new FixedSizeFrameTupleAccessor(ctx.getInitialFrameSize(), typeTraits); tuple = new FixedSizeTupleReference(typeTraits); buffers.add(ctx.allocateFrame()); } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java index 348ef75..c61a86e 100644 --- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java +++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/lsm/invertedindex/search/TOccurrenceSearcher.java @@ -38,12 +38,12 @@ public class TOccurrenceSearcher extends AbstractTOccurrenceSearcher { public void search(OnDiskInvertedIndexSearchCursor resultCursor, InvertedIndexSearchPredicate searchPred, IIndexOperationContext ictx) throws HyracksDataException, IndexException { tokenizeQuery(searchPred); - int numQueryTokens = queryTokenAccessor.getTupleCount(); + int numQueryTokens = queryTokenAppender.getTupleCount(); invListCursors.clear(); invListCursorCache.reset(); for (int i = 0; i < numQueryTokens; i++) { - searchKey.reset(queryTokenAccessor, i); + searchKey.reset(queryTokenAppender, i); IInvertedListCursor invListCursor = invListCursorCache.getNext(); invIndex.openInvertedListCursor(invListCursor, searchKey, ictx); invListCursors.add(invListCursor); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java index e1a196a..03ff58f 100644 --- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java +++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestJobletContext.java @@ -27,10 +27,12 @@ import edu.uci.ics.hyracks.api.job.profiling.counters.ICounterContext; import edu.uci.ics.hyracks.api.resources.IDeallocatable; import edu.uci.ics.hyracks.control.nc.io.IOManager; import edu.uci.ics.hyracks.control.nc.io.WorkspaceFileFactory; +import edu.uci.ics.hyracks.control.nc.resources.memory.FrameManager; public class TestJobletContext implements IHyracksJobletContext { private final int frameSize; private final INCApplicationContext appContext; + private final FrameManager frameManger; private JobId jobId; private WorkspaceFileFactory fileFactory; @@ -39,10 +41,23 @@ public class TestJobletContext implements IHyracksJobletContext { this.appContext = appContext; this.jobId = jobId; fileFactory = new WorkspaceFileFactory(this, (IOManager) getIOManager()); + this.frameManger = new FrameManager(frameSize); } - public ByteBuffer allocateFrame() { - return ByteBuffer.allocate(frameSize); + ByteBuffer allocateFrame() throws HyracksDataException { + return frameManger.allocateFrame(); + } + + public ByteBuffer allocateFrame(int bytes) throws HyracksDataException { + return frameManger.allocateFrame(bytes); + } + + ByteBuffer reallocateFrame(ByteBuffer tobeDeallocate, int newFrameSizeInBytes, boolean copyOldData) throws HyracksDataException { + return frameManger.reallocateFrame(tobeDeallocate, newFrameSizeInBytes, copyOldData); + } + + void deallocateFrames(int bytes) { + frameManger.deallocateFrames(bytes); } public int getFrameSize() { @@ -106,4 +121,5 @@ public class TestJobletContext implements IHyracksJobletContext { public ClassLoader getClassLoader() { return this.getClass().getClassLoader(); } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java index 2e4c812..9e395ab 100644 --- a/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java +++ b/hyracks/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java @@ -43,16 +43,29 @@ public class TestTaskContext implements IHyracksTaskContext { } @Override - public ByteBuffer allocateFrame() { + public ByteBuffer allocateFrame() throws HyracksDataException { return jobletContext.allocateFrame(); } - + + @Override + public ByteBuffer allocateFrame(int bytes) throws HyracksDataException { + return jobletContext.allocateFrame(bytes); + } + + @Override + public ByteBuffer reallocateFrame(ByteBuffer tobeDeallocate, int newSizeInBytes, boolean copyOldData) + throws HyracksDataException { + return jobletContext.reallocateFrame(tobeDeallocate,newSizeInBytes, copyOldData); + + } + @Override - public void deallocateFrames(int frameCount) { + public void deallocateFrames(int bytes) { + jobletContext.deallocateFrames(bytes); } @Override - public int getFrameSize() { + public int getInitialFrameSize() { return jobletContext.getFrameSize(); } http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java index 2c06c20..72e410a 100644 --- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java +++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java @@ -15,13 +15,14 @@ package edu.uci.ics.hyracks.storage.am.btree; import java.io.DataOutput; -import java.nio.ByteBuffer; import java.util.Random; import java.util.logging.Level; import org.junit.Test; +import edu.uci.ics.hyracks.api.comm.IFrame; import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor; +import edu.uci.ics.hyracks.api.comm.VSizeFrame; import edu.uci.ics.hyracks.api.context.IHyracksTaskContext; import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory; import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer; @@ -110,16 +111,16 @@ public class BTreeStatsTest extends AbstractBTreeTest { LOGGER.info("INSERTING INTO TREE"); } - ByteBuffer frame = ctx.allocateFrame(); - FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize()); + IFrame frame = new VSizeFrame(ctx); + FrameTupleAppender appender = new FrameTupleAppender(); ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount); DataOutput dos = tb.getDataOutput(); ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE }; RecordDescriptor recDesc = new RecordDescriptor(recDescSers); - IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc); - accessor.reset(frame); + IFrameTupleAccessor accessor = new FrameTupleAccessor(recDesc); + accessor.reset(frame.getBuffer()); FrameTupleReference tuple = new FrameTupleReference(); ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/0d87a57f/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java ---------------------------------------------------------------------- diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java index a86238a..49b9a8a 100644 --- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java +++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java @@ -16,14 +16,15 @@ package edu.uci.ics.hyracks.storage.am.btree; import java.io.DataOutput; -import java.nio.ByteBuffer; import java.util.Random; import java.util.logging.Level; import org.junit.Assert; import org.junit.Test; +import edu.uci.ics.hyracks.api.comm.IFrame; import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor; +import edu.uci.ics.hyracks.api.comm.VSizeFrame; import edu.uci.ics.hyracks.api.context.IHyracksTaskContext; import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator; import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer; @@ -69,8 +70,8 @@ public class FieldPrefixNSMTest extends AbstractBTreeTest { } } - ByteBuffer buf = ctx.allocateFrame(); - FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize()); + IFrame buf = new VSizeFrame(ctx); + FrameTupleAppender appender = new FrameTupleAppender(buf); ArrayTupleBuilder tb = new ArrayTupleBuilder(3); DataOutput dos = tb.getDataOutput(); @@ -78,8 +79,8 @@ public class FieldPrefixNSMTest extends AbstractBTreeTest { ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE }; RecordDescriptor recDesc = new RecordDescriptor(recDescSers); - IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc); - accessor.reset(buf); + IFrameTupleAccessor accessor = new FrameTupleAccessor(recDesc); + accessor.reset(buf.getBuffer()); FrameTupleReference tuple = new FrameTupleReference(); tb.reset();