asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [1/3] incubator-asterixdb-hyracks git commit: Support Change Feeds and Ingestion of Records with MetaData
Date Tue, 15 Mar 2016 23:34:54 GMT
Repository: incubator-asterixdb-hyracks
Updated Branches:
  refs/heads/master 3f2f53914 -> 7dabc19f7


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7dabc19f/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
index 142c64d..36b46a7 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexBulkLoadOperatorNodePushable.java
@@ -41,11 +41,11 @@ public class IndexBulkLoadOperatorNodePushable extends AbstractUnaryInputUnaryOu
     protected final long numElementsHint;
     protected final boolean checkIfEmptyIndex;
     protected final IIndexDataflowHelper indexHelper;
+    protected final IRecordDescriptorProvider recDescProvider;
+    protected final PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
     protected FrameTupleAccessor accessor;
     protected IIndex index;
     protected IIndexBulkLoader bulkLoader;
-    protected IRecordDescriptorProvider recDescProvider;
-    protected PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
 
     public IndexBulkLoadOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext
ctx, int partition,
             int[] fieldPermutation, float fillFactor, boolean verifyInput, long numElementsHint,
@@ -90,6 +90,7 @@ public class IndexBulkLoadOperatorNodePushable extends AbstractUnaryInputUnaryOu
                 throw new HyracksDataException(e);
             }
         }
+
         FrameUtils.flushFrame(buffer, writer);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7dabc19f/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java
b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java
index cf44305..9438632 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexInsertUpdateDeleteOperatorNodePushable.java
@@ -64,7 +64,7 @@ public class IndexInsertUpdateDeleteOperatorNodePushable extends AbstractUnaryIn
         this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc,
ctx, partition);
         this.recordDescProvider = recordDescProvider;
         this.op = op;
-        tuple.setFieldPermutation(fieldPermutation);
+        this.tuple.setFieldPermutation(fieldPermutation);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7dabc19f/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
index b315f8b..e3d00d5 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexCreateOperatorDescriptor.java
@@ -43,8 +43,8 @@ public class TreeIndexCreateOperatorDescriptor extends AbstractTreeIndexOperator
             ILocalResourceFactoryProvider localResourceFactoryProvider,
             IModificationOperationCallbackFactory modificationOpCallbackFactory) {
         super(spec, 0, 0, null, storageManager, lifecycleManagerProvider, fileSplitProvider,
typeTraits,
-                comparatorFactories, bloomFilterKeyFields, dataflowHelperFactory, null, false,
-                false, null, localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE,
modificationOpCallbackFactory);
+                comparatorFactories, bloomFilterKeyFields, dataflowHelperFactory, null, false,
false, null,
+                localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE, modificationOpCallbackFactory);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7dabc19f/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
index c44cb6b..1d55d4d 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
@@ -29,7 +29,7 @@ import org.apache.hyracks.util.encoding.VarLenIntEncoderDecoder;
 
 public class TypeAwareTupleWriter implements ITreeIndexTupleWriter {
 
-    protected ITypeTraits[] typeTraits;
+    protected final ITypeTraits[] typeTraits;
     protected VarLenIntEncoderDecoder.VarLenIntDecoder decoder = VarLenIntEncoderDecoder.createDecoder();
 
     public TypeAwareTupleWriter(ITypeTraits[] typeTraits) {
@@ -114,7 +114,7 @@ public class TypeAwareTupleWriter implements ITreeIndexTupleWriter {
     }
 
     protected int getNullFlagsBytes(ITupleReference tuple) {
-        return (int) Math.ceil((double) tuple.getFieldCount() / 8.0);
+        return (int) Math.ceil(tuple.getFieldCount() / 8.0);
     }
 
     protected int getFieldSlotsBytes(ITupleReference tuple) {
@@ -128,7 +128,7 @@ public class TypeAwareTupleWriter implements ITreeIndexTupleWriter {
     }
 
     protected int getNullFlagsBytes(int numFields) {
-        return (int) Math.ceil((double) numFields / 8.0);
+        return (int) Math.ceil(numFields / 8.0);
     }
 
     protected int getFieldSlotsBytes(ITupleReference tuple, int startField, int numFields)
{
@@ -145,10 +145,6 @@ public class TypeAwareTupleWriter implements ITreeIndexTupleWriter {
         return typeTraits;
     }
 
-    public void setTypeTraits(ITypeTraits[] typeTraits) {
-        this.typeTraits = typeTraits;
-    }
-
     @Override
     public int getCopySpaceRequired(ITupleReference tuple) {
         return bytesRequired(tuple);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7dabc19f/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index c175336..332ea78 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -168,7 +168,8 @@ public class LSMBTreeUtils {
             buddyBtreeCmpFactories[i] = cmpFactories[buddyBTreeFields[i]];
         }
 
-        TypeAwareTupleWriterFactory buddyBtreeTupleWriterFactory = new TypeAwareTupleWriterFactory(buddyBtreeTypeTraits);
+        TypeAwareTupleWriterFactory buddyBtreeTupleWriterFactory = new TypeAwareTupleWriterFactory(
+                buddyBtreeTypeTraits);
         ITreeIndexFrameFactory buddyBtreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
                 buddyBtreeTupleWriterFactory);
         ITreeIndexFrameFactory buddyBtreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(buddyBtreeTupleWriterFactory);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7dabc19f/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
index 7ee5802..4b8fc9b 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractMemoryLSMComponent.java
@@ -27,9 +27,9 @@ import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
 
 public abstract class AbstractMemoryLSMComponent extends AbstractLSMComponent {
 
-    private int writerCount;
     private final IVirtualBufferCache vbc;
     private final AtomicBoolean isModified;
+    private int writerCount;
     private boolean requestedToBeActive;
 
     public AbstractMemoryLSMComponent(IVirtualBufferCache vbc, boolean isActive, ILSMComponentFilter
filter) {
@@ -90,7 +90,7 @@ public abstract class AbstractMemoryLSMComponent extends AbstractLSMComponent
{
             case REPLICATE:
             case SEARCH:
                 if (state == ComponentState.READABLE_WRITABLE || state == ComponentState.READABLE_UNWRITABLE
-                || state == ComponentState.READABLE_UNWRITABLE_FLUSHING) {
+                        || state == ComponentState.READABLE_UNWRITABLE_FLUSHING) {
                     readerCount++;
                 } else {
                     return false;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7dabc19f/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
index efcd386..c4d2fcc 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMIndexSearchCursor.java
@@ -37,6 +37,8 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 
 public abstract class LSMIndexSearchCursor implements ITreeIndexCursor {
+    protected final ILSMIndexOperationContext opCtx;
+    protected final boolean returnDeletedTuples;
     protected PriorityQueueElement outputElement;
     protected IIndexCursor[] rangeCursors;
     protected PriorityQueueElement[] pqes;
@@ -46,8 +48,6 @@ public abstract class LSMIndexSearchCursor implements ITreeIndexCursor {
     protected boolean needPush;
     protected boolean includeMutableComponent;
     protected ILSMHarness lsmHarness;
-    protected final ILSMIndexOperationContext opCtx;
-    protected final boolean returnDeletedTuples;
 
     protected List<ILSMComponent> operationalComponents;
 
@@ -67,8 +67,10 @@ public abstract class LSMIndexSearchCursor implements ITreeIndexCursor
{
         if (outputPriorityQueue == null) {
             outputPriorityQueue = new PriorityQueue<PriorityQueueElement>(pqInitSize,
pqCmp);
             pqes = new PriorityQueueElement[pqInitSize];
-            for (int i = 0; i < rangeCursors.length; i++) {
+            for (int i = 0; i < pqInitSize; i++) {
                 pqes[i] = new PriorityQueueElement(i);
+            }
+            for (int i = 0; i < rangeCursors.length; i++) {
                 pushIntoPriorityQueue(pqes[i]);
             }
         } else {
@@ -183,7 +185,7 @@ public abstract class LSMIndexSearchCursor implements ITreeIndexCursor
{
     }
 
     protected void checkPriorityQueue() throws HyracksDataException, IndexException {
-        while (!outputPriorityQueue.isEmpty() || needPush == true) {
+        while (!outputPriorityQueue.isEmpty() || (needPush == true)) {
             if (!outputPriorityQueue.isEmpty()) {
                 PriorityQueueElement checkElement = outputPriorityQueue.peek();
                 // If there is no previous tuple or the previous tuple can be ignored
@@ -234,7 +236,7 @@ public abstract class LSMIndexSearchCursor implements ITreeIndexCursor
{
 
     public class PriorityQueueElement {
         private ITupleReference tuple;
-        private final int cursorIndex;
+        private int cursorIndex;
 
         public PriorityQueueElement(int cursorIndex) {
             tuple = null;
@@ -256,7 +258,7 @@ public abstract class LSMIndexSearchCursor implements ITreeIndexCursor
{
 
     public class PriorityQueueComparator implements Comparator<PriorityQueueElement>
{
 
-        protected final MultiComparator cmp;
+        protected MultiComparator cmp;
 
         public PriorityQueueComparator(MultiComparator cmp) {
             this.cmp = cmp;
@@ -287,7 +289,6 @@ public abstract class LSMIndexSearchCursor implements ITreeIndexCursor
{
     }
 
     protected void setPriorityQueueComparator() {
-        // is there a case where cmp != pqCmp ??
         if (pqCmp == null || cmp != pqCmp.getMultiComparator()) {
             pqCmp = new PriorityQueueComparator(cmp);
         }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/7dabc19f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fe44a12..cc25c34 100644
--- a/pom.xml
+++ b/pom.xml
@@ -156,7 +156,6 @@
            <exclude>**/src/test/resources/results/**</exclude>
            <exclude>**/src/test/resources/expected/**</exclude>
            <exclude>**/testcases/*.piglet</exclude>
-           <exclude>**/data/**/*.txt</exclude>
            <exclude>**/data/**/*.tbl</exclude>
            <exclude>**/data/**/*.ddl</exclude>
            <exclude>**/data/**/*.tsv</exclude>
@@ -174,6 +173,8 @@
            <exclude>**/target/**</exclude>
            <exclude>**/*.iml</exclude>
            <exclude>**/*.prefs</exclude>
+           <exclude>**/data/*.txt</exclude>
+           <exclude>**/data/**/*.txt</exclude>
            <exclude>**/.classpath</exclude>
            <exclude>**/.project</exclude>
       </excludes>


Mime
View raw message