trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hzel...@apache.org
Subject [1/2] incubator-trafodion git commit: [TRAFODION-1753] Wrong HASH2 partitioning for salted tables
Date Sat, 16 Jan 2016 23:35:43 GMT
Repository: incubator-trafodion
Updated Branches:
  refs/heads/master 9409c8ea0 -> b6f49b677


[TRAFODION-1753] Wrong HASH2 partitioning for salted tables

Fix to create the HASH2 partitioning function for salted tables
with the number of salt buckets as the number of partitions and
with only the salt columns as the partitioning key.


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/2f896f58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/2f896f58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/2f896f58

Branch: refs/heads/master
Commit: 2f896f58f485b89dfc01bdc0ee00c776a26e2562
Parents: 9359d88
Author: Hans Zeller <hzeller@apache.org>
Authored: Thu Jan 14 17:40:48 2016 +0000
Committer: Hans Zeller <hzeller@apache.org>
Committed: Thu Jan 14 17:40:48 2016 +0000

----------------------------------------------------------------------
 core/sql/optimizer/NATable.cpp          | 73 +++++++++++++++-------------
 core/sql/sqlcat/desc.h                  |  1 -
 core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 18 ++-----
 3 files changed, 43 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2f896f58/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index d6d24f8..0b01dc2 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -2193,22 +2193,31 @@ static PartitioningFunction * createHash2PartitioningFunction
 
 
 static 
-NodeMap* createNodeMapForHbase(desc_struct* desc, const NATable* table, NAMemory* heap)
+NodeMap* createNodeMapForHbase(desc_struct* desc, const NATable* table,
+                               int numSaltBuckets, NAMemory* heap)
 {
    Int32 partns = 0;
+   Int32 numRegions = 0;
    desc_struct* hrk = desc;
  
    while ( hrk ) {
-      partns++;
-      hrk=hrk->header.next;
+     numRegions++;
+     hrk=hrk->header.next;
    }
 
+   if (numSaltBuckets <= 1)
+     partns = numRegions;
+   else
+     partns = numSaltBuckets;
+
    NodeMap* nodeMap = new (heap) 
        NodeMap(heap, partns, NodeMapEntry::ACTIVE, NodeMap::HBASE);
 
    // get nodeNames of region servers by making a JNI call
    // do it only for multiple partition table
-   if (partns > 1 && (CmpCommon::getDefault(TRAF_ALLOW_ESP_COLOCATION) == DF_ON))
{
+   // TBD: co-location for tables where # of salt buckets and # regions don't match
+   if (partns > 1 && (CmpCommon::getDefault(TRAF_ALLOW_ESP_COLOCATION) == DF_ON)
&&
+       (numSaltBuckets <= 1 || numSaltBuckets == numRegions)) {
      ARRAY(const char *) nodeNames(heap, partns);
      if (table->getRegionsNodeName(partns, nodeNames)) {
        for (Int32 p=0; p < partns; p++) {
@@ -2229,13 +2238,15 @@ NodeMap* createNodeMapForHbase(desc_struct* desc, const NATable* table,
NAMemory
 
 static 
 PartitioningFunction*
-createHash2PartitioningFunctionForHBase(desc_struct* desc, const NATable * table, 
+createHash2PartitioningFunctionForHBase(desc_struct* desc,
+                                        const NATable * table,
+                                        int numSaltBuckets,
                                         NAMemory* heap)
 {
 
    desc_struct* hrk = desc;
  
-   NodeMap* nodeMap = createNodeMapForHbase(desc, table, heap);
+   NodeMap* nodeMap = createNodeMapForHbase(desc, table, numSaltBuckets, heap);
 
    Int32 partns = nodeMap->getNumEntries();
 
@@ -2634,7 +2645,7 @@ createRangePartitioningFunctionForMultiRegionHBase(Int32 partns,
 			                const NAColumnArray & partKeyColArray,
                                         NAMemory* heap)
 {
-   NodeMap* nodeMap = createNodeMapForHbase(desc, table, heap);
+   NodeMap* nodeMap = createNodeMapForHbase(desc, table, -1, heap);
 
    struct desc_struct* 
       partns_desc = ( table->isHbaseCellTable() || table->isHbaseRowTable()) ?
@@ -3842,7 +3853,7 @@ NABoolean createNAFileSets(desc_struct * table_desc       /*IN*/,
                                        order == ASCENDING);
           
           if ( table->isHbaseTable() && 
-               indexColumn->isComputedColumnAlways() ) 
+               indexColumn->isSaltColumn() ) 
             {
               
               // examples of the saltClause string:
@@ -3855,27 +3866,24 @@ NABoolean createNAFileSets(desc_struct * table_desc       /*IN*/,
                                                           strlen(saltClause), 
                                                           CharInfo::ISO88591);
               
-              if ( saltExpr &&
-                   saltExpr->getOperatorType() == ITM_HASH2_DISTRIB) {
-                
-                // get the # of salted partitions from saltClause
-                ItemExprList csList(CmpCommon::statementHeap());
-                saltExpr->findAll(ITM_CONSTANT, csList, FALSE, FALSE);
+              CMPASSERT(saltExpr &&
+                        saltExpr->getOperatorType() == ITM_HASH2_DISTRIB);
                 
-                // get #salted partitions from last ConstValue in the list
-                if ( csList.entries() > 0 ) {
-                  ConstValue* ct = (ConstValue*)csList[csList.entries()-1];
-                  
-                  if ( ct->canGetExactNumericValue() )  {
-                    numOfSaltedPartitions = ct->getExactNumericValue();
-                  }
+              // get the # of salted partitions from saltClause
+              ItemExprList csList(CmpCommon::statementHeap());
+              saltExpr->findAll(ITM_CONSTANT, csList, FALSE, FALSE);
+
+              // get #salted partitions from last ConstValue in the list
+              if ( csList.entries() > 0 ) {
+                ConstValue* ct = (ConstValue*)csList[csList.entries()-1];
+
+                if ( ct->canGetExactNumericValue() )  {
+                  numOfSaltedPartitions = ct->getExactNumericValue();
                 }
               }
-              
+
               // collect all ColReference objects into hbaseSaltColumnList.
-              CMPASSERT(saltExpr != NULL);
               saltExpr->findAll(ITM_REFERENCE, hbaseSaltColumnList, FALSE, FALSE);
-              
             }
           
 	  if (isTheClusteringKey)
@@ -4121,22 +4129,21 @@ NABoolean createNAFileSets(desc_struct * table_desc       /*IN*/,
               Int32 splits = findDescEntries(hbd);
 
               // Do Hash2 only if the table is salted orignally 
-              // and the current number of partitions is greater than 1.
+              // and the current number of HBase regions is greater than 1.
               if ( doHash2 )
                  doHash2 = (numOfSaltedPartitions > 0 && splits > 1);
 
-              if ( doHash2 && hbd && hbd->header.nodetype == DESC_HBASE_HASH2_REGION_TYPE
) {
+              if ( hbd )
+                if ( doHash2 ) {
 	           partFunc = createHash2PartitioningFunctionForHBase(
 	    	      ((table_desc_struct*)table_desc)->hbase_regionkey_desc,
                       table,
-		      heap);
+                      numOfSaltedPartitions,
+                      heap);
 
                    partitioningKeyColumns = hbaseSaltOnColumns;
-
-              } else 
-                if ((!doHash2 && (hbd && hbd->header.nodetype == DESC_HBASE_HASH2_REGION_TYPE))
-                      ||
-                (hbd && hbd->header.nodetype == DESC_HBASE_RANGE_REGION_TYPE))
+                }
+                else
 	           partFunc = createRangePartitioningFunctionForHBase(
 	    	      ((table_desc_struct*)table_desc)->hbase_regionkey_desc,
                       table,
@@ -4144,7 +4151,7 @@ NABoolean createNAFileSets(desc_struct * table_desc       /*IN*/,
 		      heap);
               else {
 
-	        // Range partitioned or single partition table
+	        // no region descriptor, range partitioned or single partition table
 	        partFunc = createRangePartitioningFunction(
 	    	   files_desc->body.files_desc.partns_desc,
 	    	   partitioningKeyColumns,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2f896f58/core/sql/sqlcat/desc.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcat/desc.h b/core/sql/sqlcat/desc.h
index 3e98d6c..c4f61dc 100644
--- a/core/sql/sqlcat/desc.h
+++ b/core/sql/sqlcat/desc.h
@@ -61,7 +61,6 @@ enum desc_nodetype {
   DESC_VIEW_TYPE,
   DESC_SCHEMA_LABEL_TYPE,
   DESC_SEQUENCE_GENERATOR_TYPE,
-  DESC_HBASE_HASH2_REGION_TYPE,
   DESC_HBASE_RANGE_REGION_TYPE,
   DESC_ROUTINE_TYPE,
   DESC_LIBRARY_TYPE

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/2f896f58/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
index eb4efbd..0456728 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
@@ -8619,12 +8619,6 @@ void populateRegionDescForEndKey(char* buf, Int32 len, struct desc_struct*
targe
    target->body.hbase_region_desc.endKeyLen = len;
 }
 
-void populateRegionDescAsHASH2(char* buf, Int32 len, struct desc_struct* target, NAMemory*)
-{
-   target->header.nodetype = DESC_HBASE_HASH2_REGION_TYPE;
-   populateRegionDescForEndKey(buf, len, target);
-}
-
 void populateRegionDescAsRANGE(char* buf, Int32 len, struct desc_struct* target, NAMemory*)
 {
    target->header.nodetype = DESC_HBASE_RANGE_REGION_TYPE;
@@ -9402,15 +9396,9 @@ desc_struct * CmpSeabaseDDL::getSeabaseUserTableDesc(const NAString
&catName,
       ExpHbaseInterface* ehi =CmpSeabaseDDL::allocEHI();
       ByteArrayList* bal = ehi->getRegionEndKeys(extNameForHbase);
 
-      // Set the header.nodetype to either HASH2 or RANGE based on whether
-      // the table is salted or not.  
-      if (tableIsSalted && CmpCommon::getDefault(HBASE_HASH2_PARTITIONING) == DF_ON)

-        ((table_desc_struct*)tableDesc)->hbase_regionkey_desc = 
-          assembleDescs(bal, populateRegionDescAsHASH2, STMTHEAP);
-      else
-       if ( CmpCommon::getDefault(HBASE_RANGE_PARTITIONING) == DF_ON ) 
-         ((table_desc_struct*)tableDesc)->hbase_regionkey_desc = 
-            assembleDescs(bal, populateRegionDescAsRANGE, STMTHEAP);
+      // create a list of region descriptors
+      ((table_desc_struct*)tableDesc)->hbase_regionkey_desc = 
+        assembleDescs(bal, populateRegionDescAsRANGE, STMTHEAP);
       delete bal;
 
       // if this is base table or index and hbase object doesn't exist, then this object


Mime
View raw message