asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject asterixdb git commit: Fixes for ASTERIXDB-1636
Date Fri, 30 Sep 2016 01:23:37 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 102ef590b -> 2685b60d9


Fixes for ASTERIXDB-1636

The index of the tuple field for filters from SecondaryIndexOperationsHelper and AqlMetadataProvider
differed. The one in AqlMetadataProvider was wrong, as it was attempting to take into account
the
presence of a partitioning field in the incoming tuple, which is not there in the case of
an
insert/upsert.

There was also an issue where on merge, for components with a filter page but no min/max,
the merge
would fail. I fixed this by skipping over null entries while getting the min/max of merging
components.

Finally, there was a very silly error in LSMComponentFilterManager which was causing the filter
page
to appear as blank, because the page was being pinned with the wrong argument. That is also
fixed.

Change-Id: Ib4bc413fcda9a5c98ae57f94e1c8a68fe9aacda3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1205
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Taewoo Kim <wangsaeu@yahoo.com>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/2685b60d
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/2685b60d
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/2685b60d

Branch: refs/heads/master
Commit: 2685b60d9e03a515fcc5260a3dd4399740f5ec40
Parents: 102ef59
Author: Ian Maxon <imaxon@apache.org>
Authored: Thu Sep 29 16:38:28 2016 -0700
Committer: Ian Maxon <imaxon@apache.org>
Committed: Thu Sep 29 18:15:53 2016 -0700

----------------------------------------------------------------------
 .../metadata/declared/AqlMetadataProvider.java  |  6 ++---
 .../common/impls/LSMComponentFilterManager.java | 25 +++++++++-----------
 .../invertedindex/impls/LSMInvertedIndex.java   | 10 ++++++--
 3 files changed, 22 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2685b60d/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 29087ba..e05aa25 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -1833,7 +1833,6 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId,
Strin
         // One token (+ optional partitioning field) + primary keys: [token,
         // number of token, PK]
         int numKeys = primaryKeys.size() + secondaryKeys.size();
-        int numTokenKeyPairFields = (!isPartitioned) ? 1 + primaryKeys.size() : 2 + primaryKeys.size();
         int numFilterFields = DatasetUtils.getFilterField(dataset) == null ? 0 : 1;
 
         // generate field permutations
@@ -1960,8 +1959,9 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId,
Strin
                 }
 
                 filterFieldsForNonBulkLoadOps = new int[numFilterFields];
-                filterFieldsForNonBulkLoadOps[0] = numTokenKeyPairFields;
-                invertedIndexFieldsForNonBulkLoadOps = new int[numTokenKeyPairFields];
+                //for non-bulk-loads, there is only <SK,PK,F> in the incoming tuples
+                filterFieldsForNonBulkLoadOps[0] = numKeys;
+                invertedIndexFieldsForNonBulkLoadOps = new int[numKeys];
                 for (int k = 0; k < invertedIndexFieldsForNonBulkLoadOps.length; k++)
{
                     invertedIndexFieldsForNonBulkLoadOps[k] = k;
                 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2685b60d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
index d0d7e69..93225cf 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMComponentFilterManager.java
@@ -56,36 +56,33 @@ public class LSMComponentFilterManager implements ILSMComponentFilterManager
{
     }
 
     @Override
-    public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex ) throws
HyracksDataException {
+    public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws
HyracksDataException {
         IMetaDataPageManager treeMetaManager = treeIndex.getMetaManager();
         ICachedPage filterPage = null;
         int componentFilterPageId = treeMetaManager.getFilterPageId();
         boolean appendOnly = false;
         int fileId = treeIndex.getFileId();
-        if(componentFilterPageId == LinkedMetaDataPageManager.NO_FILTER_IN_PLACE){//in-place
mode, no filter page yet
+        if (componentFilterPageId == LinkedMetaDataPageManager.NO_FILTER_IN_PLACE) { //in-place
mode, no filter page yet
             ITreeIndexMetaDataFrame metadataFrame = treeIndex.getMetaManager().getMetaDataFrameFactory().createFrame();
             int metaPageId = treeMetaManager.getFirstMetadataPage();
             ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId,
metaPageId), false);
             metadataPage.acquireWriteLatch();
-            try{
+            try {
                 metadataFrame.setPage(metadataPage);
                 componentFilterPageId = treeIndex.getMetaManager().getFreePage(metadataFrame);
                 metadataFrame.setLSMComponentFilterPageId(componentFilterPageId);
-            }
-            finally{
+            } finally {
                 metadataPage.releaseWriteLatch(true);
                 bufferCache.unpin(metadataPage);
             }
-        }
-        else if (componentFilterPageId <= LinkedMetaDataPageManager.NO_FILTER_APPEND_ONLY){
+        } else if (componentFilterPageId <= LinkedMetaDataPageManager.NO_FILTER_APPEND_ONLY)
{
             appendOnly = true;
             filterPage = treeMetaManager.getFilterPage();
-            if(filterPage == null){
+            if (filterPage == null) {
                 treeMetaManager.setFilterPage(bufferCache.confiscatePage(IBufferCache.INVALID_DPID));
                 filterPage = treeMetaManager.getFilterPage();
             }
-        }
-        else{// in place, not a new filter page
+        } else { // in place, not a new filter page
             filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, componentFilterPageId),
true);
         }
 
@@ -102,11 +99,10 @@ public class LSMComponentFilterManager implements ILSMComponentFilterManager
{
             }
 
         } finally {
-            if(!appendOnly){
+            if (!appendOnly) {
                 bufferCache.unpin(filterPage);
                 filterPage.releaseWriteLatch(true);
-            }
-            else{
+            } else {
                 filterPage.releaseWriteLatch(false);
             }
         }
@@ -122,7 +118,8 @@ public class LSMComponentFilterManager implements ILSMComponentFilterManager
{
         if (componentFilterPageId < 0)
             return false;
 
-        ICachedPage filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId,
componentFilterPageId), true);
+        ICachedPage filterPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId,
componentFilterPageId),
+                false);
 
         filterPage.acquireReadLatch();
         try {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/2685b60d/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 204e874..bf6e9b5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -645,8 +645,14 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
         if (component.getLSMComponentFilter() != null) {
             List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
             for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
-                filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
-                filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
+                ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
+                ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
+                if (min != null) {
+                    filterTuples.add(min);
+                }
+                if (max != null) {
+                    filterTuples.add(max);
+                }
             }
             filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
             filterManager.writeFilterInfo(component.getLSMComponentFilter(),


Mime
View raw message