trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sure...@apache.org
Subject [1/3] incubator-trafodion git commit: [TRAAFODION-2568] [TRAFODION-2574]
Date Fri, 14 Apr 2017 02:32:24 GMT
Repository: incubator-trafodion
Updated Branches:
  refs/heads/master 5b686a616 -> 0751af082


[TRAAFODION-2568] [TRAFODION-2574]

2568 - Predicates that are always false at compile time will cause their scan to have begin
and end keys reversed. No rows will be scanned. Idea due to Hans.
Files : ValueDesc.h/.cpp, SearchKey.h/.cpp and GenPreCode.cpp
2574 - Update on table with an index in which SET and WHERE clause contain an index key column
can now get a index driven scan. Thanknk you Qifan and Hans for your help.
Files : NormRelExpr.cpp, RelExpr.cpp and RelScan.h
Two other minor changes that will help get index plans. I am not able to get reproducible
test cases to open JIRAs on them.
Files : ScmCostMethod.cpp and Analyzer.cpp


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/8f92b924
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/8f92b924
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/8f92b924

Branch: refs/heads/master
Commit: 8f92b92466e2322a234cbc92355019307bb56cbd
Parents: 0c88f43
Author: Suresh Subbiah <sureshs@apache.org>
Authored: Fri Apr 7 04:15:00 2017 +0000
Committer: Suresh Subbiah <sureshs@apache.org>
Committed: Fri Apr 7 04:15:00 2017 +0000

----------------------------------------------------------------------
 core/sql/generator/GenPreCode.cpp    |  57 ++++++++-----
 core/sql/optimizer/Analyzer.cpp      |   6 +-
 core/sql/optimizer/NormRelExpr.cpp   |   8 ++
 core/sql/optimizer/RelExpr.cpp       | 136 +++++++++++++++++-------------
 core/sql/optimizer/RelScan.h         |   2 +-
 core/sql/optimizer/ScmCostMethod.cpp |   3 +-
 core/sql/optimizer/SearchKey.cpp     |   1 +
 core/sql/optimizer/SearchKey.h       |   7 ++
 core/sql/optimizer/ValueDesc.cpp     |  27 ++++++
 core/sql/optimizer/ValueDesc.h       |  10 +++
 10 files changed, 174 insertions(+), 83 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/generator/GenPreCode.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp
index f82bebc..d96378d 100644
--- a/core/sql/generator/GenPreCode.cpp
+++ b/core/sql/generator/GenPreCode.cpp
@@ -168,6 +168,10 @@ static NABoolean processConstHBaseKeys(Generator * generator,
       exePreds += relExpr->getSelectionPred();
       exePreds.subtractSet(keyPreds);
 
+      ValueId falseConst = NULL_VALUE_ID;
+      if (exePreds.containsFalseConstant(falseConst))
+        keyPreds += falseConst;
+
       HbaseSearchKey::makeHBaseSearchKeys(
            skey,
            skey->getIndexDesc()->getIndexKey(),
@@ -184,6 +188,15 @@ static NABoolean processConstHBaseKeys(Generator * generator,
       // picked up (to be used as the HBase search keys).
       exePreds += keyPreds;
 
+      if(falseConst != NULL_VALUE_ID)
+      {
+	for (CollIndex i = 0; i<mySearchKeys.entries(); i++ ) 
+        {
+          HbaseSearchKey* searchKey = mySearchKeys[i];
+	  searchKey->setIsFalsePred(TRUE);
+	}
+      }
+
       TableDesc *tdesc = NULL;
       if (mySearchKeys.entries()>0)
         {
@@ -3917,6 +3930,10 @@ RelExpr * FileScan::preCodeGen(Generator * generator,
           if (getSearchKey()) 
             exePreds += getSearchKey()->getExecutorPredicates();
 
+          ValueId falseConst = NULL_VALUE_ID;
+          if (exePreds.containsFalseConstant(falseConst))
+            replaceSearchKey = FALSE;
+
           // pick one search key and add the remaining
           // predicates (if any) to exePreds
           if (replaceSearchKey)
@@ -11224,27 +11241,27 @@ short HbaseAccess::processSQHbaseKeyPreds(Generator * generator,
 	  scanSpec.endKeyExclusive_ = FALSE;
 	  scanSpec.rowTS_ = -1;
 	  
-	  if ( !searchKey->areAllBeginKeysMissing() )
-	    {
-	      if ( (ct=searchKey->getCoveredLeadingKeys()) > 0 )
-		{
-		  ValueIdList beginKeyValues = searchKey->getBeginKeyValues();
-		  beginKeyValues.convertToTextKey(searchKey->getKeyColumns(), scanSpec.beginRowId_);
-		  scanSpec.beginKeyExclusive_ = searchKey->isBeginKeyExclusive();
-		} 
-	    } 
-	  
-	  if ( !searchKey->areAllEndKeysMissing() ) 
-	    {
-	      if ( (ct=searchKey->getCoveredLeadingKeys()) )
-		{
-		  ValueIdList endKeyValues = searchKey->getEndKeyValues();
-		  endKeyValues.convertToTextKey(searchKey->getKeyColumns(), scanSpec.endRowId_);
-		  scanSpec.endKeyExclusive_ = searchKey->isEndKeyExclusive();
-		} 
-	    } 
+          if ((( !searchKey->areAllBeginKeysMissing() ) && 
+	       ((ct=searchKey->getCoveredLeadingKeys()) > 0 )) ||
+	      searchKey->isFalsePred())
+	  {
+	    ValueIdList beginKeyValues = searchKey->getBeginKeyValues();
+	    beginKeyValues.convertToTextKey(searchKey->getKeyColumns(), 
+					    scanSpec.beginRowId_);
+	    scanSpec.beginKeyExclusive_ = 
+              searchKey->isBeginKeyExclusive();
+	  } 
 	  
-	  //	  scanSpec.addColumnNames(searchKey->getRequiredOutputColumns());
+	  if ((( !searchKey->areAllEndKeysMissing() ) && 
+	       (ct=searchKey->getCoveredLeadingKeys())) ||
+	      searchKey->isFalsePred())
+	  {
+	    ValueIdList endKeyValues = searchKey->getEndKeyValues();
+	    endKeyValues.convertToTextKey(searchKey->getKeyColumns(), 
+					  scanSpec.endRowId_);
+	    scanSpec.endKeyExclusive_ = searchKey->isEndKeyExclusive();
+	  }  
+	 
 	  listOfRangeRows.insertAt(listOfRangeRows.entries(), scanSpec);
 	}
     } // for

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/Analyzer.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/Analyzer.cpp b/core/sql/optimizer/Analyzer.cpp
index 1261f8c..1a2532e 100644
--- a/core/sql/optimizer/Analyzer.cpp
+++ b/core/sql/optimizer/Analyzer.cpp
@@ -820,12 +820,12 @@ TableAnalysis * QueryAnalysis::newTableAnalysis(RelExpr* tableExpr)
       return NULL;
   }
 
+  const IndexDesc *tableIndexDesc = tableDesc->getClusteringIndex();
   // Record the highest no of partitions for a table among all of the tables 
   // in a query. This is used in opt.cpp to adjust max parallelism.
   if (CmpCommon::getDefault(COMP_BOOL_24) == DF_ON)
   {
     // Get the number of partitions for the table
-    const IndexDesc *tableIndexDesc = tableDesc->getClusteringIndex();
     Lng32 tableNumOfPartns = tableIndexDesc->getNAFileSet()->getCountOfPartitions();
 
     // Check and save highest no of partitions
@@ -859,6 +859,10 @@ TableAnalysis * QueryAnalysis::newTableAnalysis(RelExpr* tableExpr)
   // append columns referenced in the scan required outputs
   tableExpr->getGroupAttr()->getCharacteristicOutputs()
     .findAllReferencedBaseCols(usedCols);
+ // add table's clustering key for index joins
+  ValueIdSet keyAsSet(tableIndexDesc->getIndexKey());
+  keyAsSet.findAllReferencedBaseCols(usedCols) ;
+
 
   const ValueIdSet tableCols(tableDesc->getColumnList());
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/NormRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NormRelExpr.cpp b/core/sql/optimizer/NormRelExpr.cpp
index 76bb055..b4b4269 100644
--- a/core/sql/optimizer/NormRelExpr.cpp
+++ b/core/sql/optimizer/NormRelExpr.cpp
@@ -6886,6 +6886,14 @@ RelExpr * GenericUpdate::normalizeNode(NormWA & normWARef)
      }
   }
 
+  /// YYY
+   if (getOperator().match(REL_ANY_UNARY_GEN_UPDATE))
+   {
+      Scan * scan = getLeftmostScanNode();
+      if (scan && scan->requiresHalloweenForUpdateUsingIndexScan())
+        setAvoidHalloween(TRUE);
+   }
+
   if (producedMergeIUDIndicator_ != NULL_VALUE_ID)
     {
       ValueId dummy;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/RelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp
index d2bd95f..d11ab29 100644
--- a/core/sql/optimizer/RelExpr.cpp
+++ b/core/sql/optimizer/RelExpr.cpp
@@ -8433,13 +8433,6 @@ void Scan::addIndexInfo()
     preds.clear();
     resultOld->convertToValueIdSet(preds, NULL, ITM_AND);
     doNotReplaceAnItemExpressionForLikePredicates(resultOld,preds,resultOld);
-
-//	 ValueIdSet resultSet;
-//	 revertBackToOldTreeUsingValueIdSet(preds, resultSet);
-//	 ItemExpr* resultOld =  resultSet.rebuildExprTree(ITM_AND,FALSE,FALSE);
-//	 preds.clear();
-//	 preds += resultSet;
-//	 doNotReplaceAnItemExpressionForLikePredicates(resultOld,preds,resultOld);
   }
 
 
@@ -9062,42 +9055,19 @@ void Scan::setTableAttributes(CANodeId nodeId)
   setBaseCardinality(MIN_ONE (tableDesc->getNATable()->getEstRowCount())) ;
 }
 
-NABoolean Scan::equalityPredOnCol(ItemExpr *col)
-{
-  //
-  // Returns TRUE if the column (col) has an equality predicate
-  // associated with it the scan's predicate list.
-  //
-  ValueIdSet pred = getSelectionPredicates();
-
-  for (ValueId vid=pred.init(); pred.next(vid); pred.advance(vid))
-  {
-    if ( vid.getItemExpr()->getOperatorType() != ITM_VEG_PREDICATE )
-      continue;
-
-    ItemExpr *expr = vid.getItemExpr();
-    ValueId id;
-    VEG *veg = ((VEGPredicate*)expr)->getVEG();
-
-    if (veg->getAllValues().referencesTheGivenValue(col->getValueId(), id))
-      return TRUE;
-  }
-
-  return FALSE;
-
-} // Scan::equalityPredOnCol
-
 NABoolean Scan::updateableIndex(IndexDesc *idx)
 {
   //
-  // Returns TRUE if the index (idx) can be used for a scan during an UPDATE.
-  // Otherwise, returns FALSE to prevent the "Halloween Update Problem".
+  // Returns TRUE if the index (idx) can be used for a scan during 
+  // an UPDATE. Halloween problem is protected with a sort using
+  // Scan::requiresHalloweenForUpdateUsingIndexScan(). 
+  // Returns FALSE only for certain embedded updates now.
   //
-  ValueIdSet
-    pred = getSelectionPredicates(),
-    dummySet;
 
+  if (!getGroupAttr()->isEmbeddedUpdate())
+    return TRUE ;
 
+  ValueIdSet pred = getSelectionPredicates(), dummySet;
   SearchKey searchKey(idx->getIndexKey(),
 	              idx->getOrderOfKeyValues(),
 	              getGroupAttr()->getCharacteristicInputs(),
@@ -9106,7 +9076,6 @@ NABoolean Scan::updateableIndex(IndexDesc *idx)
                       dummySet, // needed by the interface but not used here
                       idx
                       );
-
   // Unique index is OK to use.
   if (searchKey.isUnique())
     return TRUE;
@@ -9121,38 +9090,85 @@ NABoolean Scan::updateableIndex(IndexDesc *idx)
   {
     ItemExpr *updateCol = colUpdated[i].getItemExpr();
     CMPASSERT(updateCol->getOperatorType() == ITM_BASECOLUMN);
-
     for (CollIndex j = 0; j < indexKey.entries(); j++)
     {
       ItemExpr *keyCol = indexKey[j].getItemExpr();
-
-      ItemExpr *baseCol = ((IndexColumn*)keyCol)->getDefinition().getItemExpr();
+      ItemExpr *baseCol = 
+        ((IndexColumn*)keyCol)->getDefinition().getItemExpr();
       CMPASSERT(baseCol->getOperatorType() == ITM_BASECOLUMN);
-
-      // QSTUFF
-      if (getGroupAttr()->isEmbeddedUpdate()){
-        if (((BaseColumn*)updateCol)->getColNumber() ==
+      if (((BaseColumn*)updateCol)->getColNumber() ==
           ((BaseColumn*)baseCol)->getColNumber())
-          return FALSE;
-      }
-      // QSTUFF
+        return FALSE;
+    }
+  }
+  return TRUE;
+} // Scan::updateableIndex
+
+NABoolean Scan::requiresHalloweenForUpdateUsingIndexScan()
+{
+
+  // Returns TRUE if any non clustering index can be used for a scan 
+  // during an UPDATE and a key column of that index is in the SET
+  // clause of the update. If this method returns TRUE we will use a 
+  // sort node to prevent the "Halloween Update Problem".
+  //
 
-      if ((NOT(idx->isUniqueIndex() || idx->isClusteringIndex()))
-	     ||
-	  (CmpCommon::getDefault(UPDATE_CLUSTERING_OR_UNIQUE_INDEX_KEY) == DF_OFF))
+  // preds are in RangeSpec form
+  ValueIdSet preds = getSelectionPredicates();
+  const ValueIdList colUpdated = getTableDesc()->getColUpdated();
+  const LIST(IndexDesc *) & ixlist = getTableDesc()->getIndexes();
+
+  if ((colUpdated.entries() == 0) || (preds.entries() == 0) ||
+      (ixlist.entries() == 1) || // this is the clustering index
+      (CmpCommon::getDefault(UPDATE_CLUSTERING_OR_UNIQUE_INDEX_KEY) 
+       == DF_AGGRESSIVE)) // this setting means no Halloween protection
+    return FALSE;
+
+   if (CmpCommon::getDefault(RANGESPEC_TRANSFORMATION) == DF_ON)
+   {
+     ValueIdList selectionPredList(preds);
+     ItemExpr *inputItemExprTree = 
+       selectionPredList.rebuildExprTree(ITM_AND,FALSE,FALSE);
+     ItemExpr * resultOld = revertBackToOldTree(STMTHEAP, 
+                                                inputItemExprTree);
+     preds.clear();
+     resultOld->convertToValueIdSet(preds, NULL, ITM_AND);
+     doNotReplaceAnItemExpressionForLikePredicates(resultOld,preds,
+                                                   resultOld);
+   }
 
+  for (CollIndex indexNo = 0; indexNo < ixlist.entries(); indexNo++)
+  {
+    IndexDesc *idx = ixlist[indexNo];
+    if (idx->isClusteringIndex() || 
+        (idx->isUniqueIndex() && 
+        (CmpCommon::getDefault(UPDATE_CLUSTERING_OR_UNIQUE_INDEX_KEY) 
+         == DF_ON)))
+      continue ; // skip this idesc
+
+    const ValueIdList indexKey = idx->getIndexKey();
+    // Determine if the columns being updated are key columns. Each key
+    // column being updated must have an associated equality clause in
+    // the WHERE clause of the UPDATE for it to be used.
+    for (CollIndex i = 0; i < colUpdated.entries(); i++)
+    {
+      ItemExpr *updateCol = colUpdated[i].getItemExpr();
+      CMPASSERT(updateCol->getOperatorType() == ITM_BASECOLUMN);
+      for (CollIndex j = 0; j < indexKey.entries(); j++)
       {
-      if (((BaseColumn*)updateCol)->getColNumber() ==
-          ((BaseColumn*)baseCol)->getColNumber() AND
-	  NOT equalityPredOnCol(baseCol))
-	return FALSE;
+        ItemExpr *keyCol = indexKey[j].getItemExpr();
+        ItemExpr *baseCol = 
+          ((IndexColumn*)keyCol)->getDefinition().getItemExpr();
+        CMPASSERT(baseCol->getOperatorType() == ITM_BASECOLUMN);
+        if (((BaseColumn*)updateCol)->getColNumber() ==
+            ((BaseColumn*)baseCol)->getColNumber() AND
+            NOT preds.containsAsEquiLocalPred(baseCol->getValueId()))
+          return TRUE;
+      }
     }
   }
-  }
-
-  return TRUE;
-
-} // Scan::updateableIndex
+  return FALSE;
+}
 
 // how many index descriptors can be used with this scan node?
 CollIndex Scan::numUsableIndexes()

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/RelScan.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelScan.h b/core/sql/optimizer/RelScan.h
index 4f24dc0..0977d80 100644
--- a/core/sql/optimizer/RelScan.h
+++ b/core/sql/optimizer/RelScan.h
@@ -431,8 +431,8 @@ public:
 
   // find out which indexes on the base table are usable for this scan
   void addIndexInfo();
-  NABoolean equalityPredOnCol(ItemExpr*);
   NABoolean updateableIndex(IndexDesc*);
+  NABoolean requiresHalloweenForUpdateUsingIndexScan();
 
   // iterate over the usable indexes (index-only and index joins)
   CollIndex numUsableIndexes();

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/ScmCostMethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ScmCostMethod.cpp b/core/sql/optimizer/ScmCostMethod.cpp
index 16a0d87..71abb11 100644
--- a/core/sql/optimizer/ScmCostMethod.cpp
+++ b/core/sql/optimizer/ScmCostMethod.cpp
@@ -3783,9 +3783,10 @@ void CostMethodHbaseUpdateOrDelete::computeIOCostsForCursorOperation(
   ) const
 {
   const CostScalar & kbPerBlock = CIDesc->getBlockSizeInKb();
+  // if rowsize is bigger than blocksize, rowsPerBlock will be 1.
   const CostScalar rowsPerBlock =
     ((kbPerBlock * csOneKiloBytes) /
-     CIDesc->getNAFileSet()->getRecordLength()).getFloor();
+     CIDesc->getNAFileSet()->getRecordLength()).getCeiling();
   CostScalar totalIndexBlocks(csZero);
 
   if (probesInOrder)

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/SearchKey.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/SearchKey.cpp b/core/sql/optimizer/SearchKey.cpp
index 07482bf..424770b 100644
--- a/core/sql/optimizer/SearchKey.cpp
+++ b/core/sql/optimizer/SearchKey.cpp
@@ -2656,6 +2656,7 @@ HbaseSearchKey::HbaseSearchKey(const ValueIdList & keyColumns,
 
    requiredOutputColumns_ = keyColumns_;
    requiredOutputColumns_.insert(nonKeyColumns_);
+   isFalsePred_ = FALSE;
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/SearchKey.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/SearchKey.h b/core/sql/optimizer/SearchKey.h
index 1a684d1..7b7ebca 100644
--- a/core/sql/optimizer/SearchKey.h
+++ b/core/sql/optimizer/SearchKey.h
@@ -1027,6 +1027,9 @@ public:
   //
   NABoolean isEqualTo(const HbaseSearchKey* other) const;
 
+  NABoolean isFalsePred() {return isFalsePred_ ;}
+  void setIsFalsePred(NABoolean val) {isFalsePred_ = val ;}
+
   // -----------------------------------------------------------------------
 
 protected:
@@ -1042,6 +1045,10 @@ protected:
    ValueIdSet requiredOutputColumns_;
 
    ValueIdSet nonKeyPreds_;
+
+   // is a constant folded pred that returns FALSE always. SK has 
+   // min/max for begin/end reversed.
+   NABoolean isFalsePred_;
 };
 
 #endif /* SEARCHKEY_H */

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/ValueDesc.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ValueDesc.cpp b/core/sql/optimizer/ValueDesc.cpp
index c308854..b65abdc 100644
--- a/core/sql/optimizer/ValueDesc.cpp
+++ b/core/sql/optimizer/ValueDesc.cpp
@@ -4480,6 +4480,33 @@ NABoolean ValueIdSet::containsAnyTrue(ValueId &refAnyTrue ) const
    return FALSE;
 } // ValueIdSet::containsAnyTrue
 
+NABoolean ValueIdSet::containsFalseConstant(ValueId &falseConstant ) const
+{
+
+   for ( ValueId vid = init(); next(vid); advance(vid))
+   {
+     OperatorTypeEnum OpType = vid.getItemExpr()->getOperatorType();
+     
+     if( OpType == ITM_RETURN_FALSE )
+     {
+       falseConstant = vid;
+       return TRUE;
+     }
+     else if ( OpType == ITM_CONSTANT )
+     {
+       NABoolean negate;
+       ConstValue *cv = vid.getItemExpr()->castToConstValue(negate);
+       if( cv && cv->isAFalseConstant())
+       {
+         falseConstant = vid;
+         return TRUE;
+       }
+     }
+   }
+   return FALSE;
+
+} // ValueIdSet::containsFalseConstant
+
 // -----------------------------------------------------------------------
 // Methods used for debugging and display.
 // -----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8f92b924/core/sql/optimizer/ValueDesc.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ValueDesc.h b/core/sql/optimizer/ValueDesc.h
index d69d988..27d3be2 100644
--- a/core/sql/optimizer/ValueDesc.h
+++ b/core/sql/optimizer/ValueDesc.h
@@ -1481,6 +1481,16 @@ public:
   // ---------------------------------------------------------------------
   NABoolean containsAnyTrue(ValueId &refAnyTrue ) const;
 
+  // ---------------------------------------------------------------------
+  // Returns a boolean to indicate if this vid set has a vid that 
+  // corresponds to a FlaseConstant. Such a constant is generated 
+  // during constant folding. For example a user given predicate such as 
+  // 1 = 0 will cause constant folding to generate such a Constant. 
+  // The reference to a ValueId given is initialized to the ValueID 
+  // of the FalseConstant.
+  // ---------------------------------------------------------------------
+  NABoolean containsFalseConstant(ValueId &falseConstant ) const;
+
   // for each OLAP LEAD function cotnained in this, add the equivalent
   // OLAP LEAD function for each element (as the child of LEAD) in input, 
   // and save the new function in result


Mime
View raw message