trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dbirds...@apache.org
Subject [1/3] incubator-trafodion git commit: resolve JIRA-1495
Date Mon, 28 Sep 2015 20:10:24 GMT
Repository: incubator-trafodion
Updated Branches:
  refs/heads/master 0a8da2987 -> 91cd23464


resolve JIRA-1495


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/d315ba78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/d315ba78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/d315ba78

Branch: refs/heads/master
Commit: d315ba78e7d70be8c24229b2161c9344efedcb46
Parents: 85f1168
Author: Qifan Chen <qfc@dev02.trafodion.org>
Authored: Mon Sep 28 14:17:28 2015 +0000
Committer: Qifan Chen <qfc@dev02.trafodion.org>
Committed: Mon Sep 28 14:17:28 2015 +0000

----------------------------------------------------------------------
 core/sql/optimizer/NodeMap.cpp      |  2 +-
 core/sql/regress/hive/EXPECTED009   | 32 +++++++++-----------------------
 core/sql/regress/tools/runallsb     |  1 +
 core/sql/sqlcomp/DefaultConstants.h |  2 ++
 core/sql/sqlcomp/nadefaults.cpp     |  4 +++-
 core/sql/ustat/hs_globals.cpp       | 30 +++++++++++++++++++-----------
 core/sql/ustat/hs_globals.h         |  6 ++++--
 7 files changed, 39 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d315ba78/core/sql/optimizer/NodeMap.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NodeMap.cpp b/core/sql/optimizer/NodeMap.cpp
index f020d28..f11426a 100644
--- a/core/sql/optimizer/NodeMap.cpp
+++ b/core/sql/optimizer/NodeMap.cpp
@@ -1381,7 +1381,7 @@ NABoolean NodeMap::smooth(Lng32 numNodes)
     } 
   }
 
-  // Find how many entries wth the highest frequency, and compute the number of entris with
+  // Find how many entries wth the highest frequency, and compute the number of entries with
   // normal frequency (normEntries) and the number of nodes that appear with normal frequency

   // (normalNodesCt).
   Lng32 count = 0;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d315ba78/core/sql/regress/hive/EXPECTED009
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED009 b/core/sql/regress/hive/EXPECTED009
index 9706107..6b7a304 100644
--- a/core/sql/regress/hive/EXPECTED009
+++ b/core/sql/regress/hive/EXPECTED009
@@ -22,20 +22,10 @@
 >>-- make sure external schemas exist in Trafodion
 >>execute get_hv_schemas;
 
-SCHEMA_NAME
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-
-_HV_SCH_T009_                                                                           
                                                                                         
                                                                             
-
---- 1 row(s) selected.
+--- 0 row(s) selected.
 >>execute get_hb_schemas;
 
-SCHEMA_NAME
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-
-_HBASESTATS_                                                                            
                                                                                         
                                                                             
-
---- 1 row(s) selected.
+--- 0 row(s) selected.
 >>
 >>obey TEST009(test_hive1);
 >>-- create external tables for precreated tables
@@ -55,10 +45,9 @@ _HBASESTATS_
 SCHEMA_NAME
 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-_HV_SCH_T009_                                                                           
                                                                                         
                                                                             
 _HV_HIVE_                                                                               
                                                                                         
                                                                             
 
---- 2 row(s) selected.
+--- 1 row(s) selected.
 >>
 >>-- Verify tables were created
 >>showddl trafodion."_HV_HIVE_".customer;
@@ -182,9 +171,7 @@ CREATE EXTERNAL TABLE TRAFODION."_HV_HIVE_".PROMOTION
 >>-- create external tables from hive schema sch_t009
 >>create external table t009t1 for hive.sch_t009.t009t1;
 
-*** ERROR[1390] Object TRAFODION."_HV_SCH_T009_".T009T1 already exists in Trafodion.
-
---- SQL operation failed with errors.
+--- SQL operation complete.
 >>create external table t009t2 for hive.sch_t009.t009t2;
 
 --- SQL operation complete.
@@ -194,8 +181,8 @@ CREATE EXTERNAL TABLE TRAFODION."_HV_HIVE_".PROMOTION
 SCHEMA_NAME
 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-_HV_SCH_T009_                                                                           
                                                                                         
                                                                             
 _HV_HIVE_                                                                               
                                                                                         
                                                                             
+_HV_SCH_T009_                                                                           
                                                                                         
                                                                             
 
 --- 2 row(s) selected.
 >>showddl trafodion."_HV_SCH_T009_".t009t1;
@@ -459,9 +446,9 @@ ROW_ID      CATALOG_NAME
          2  TRAFODION                                                                   
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             _HV_HIVE_   
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            PROMOTION                                    
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
          3  HIVE                                                                        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             HIVE        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            PROMOTION                                    
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
          4  TRAFODION                                                                   
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             _HV_SCH_T009_
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
       
                                            T009T2                                       
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
-         5  HIVE                                                                        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             SCH_T009    
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            T009T2                                       
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
-         6  HIVE                                                                        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             HIVE        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            CUSTOMER                                     
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
-         7  HIVE                                                                        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             HIVE        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            ITEM                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
+         5  HIVE                                                                        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             HIVE        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            ITEM                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
+         6  HIVE                                                                        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             SCH_T009    
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            T009T2                                       
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
+         7  HIVE                                                                        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             HIVE        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            CUSTOMER                                     
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
          8  HIVE                                                                        
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                             SCH_T009    
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
        
                                            T009T1                                       
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                                                                                         
                
 
 --- 8 row(s) selected.
@@ -518,11 +505,10 @@ ROW_ID      CATALOG_NAME
 SCHEMA_NAME
 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
 
-_HBASESTATS_                                                                            
                                                                                         
                                                                             
 _HB__ROW__                                                                              
                                                                                         
                                                                             
 _HB__CELL__                                                                             
                                                                                         
                                                                             
 
---- 3 row(s) selected.
+--- 2 row(s) selected.
 >>
 >>-- Verify tables were created
 >>showddl trafodion."_HB__ROW__"."baseball";

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d315ba78/core/sql/regress/tools/runallsb
----------------------------------------------------------------------
diff --git a/core/sql/regress/tools/runallsb b/core/sql/regress/tools/runallsb
index d8a5585..78a5925 100755
--- a/core/sql/regress/tools/runallsb
+++ b/core/sql/regress/tools/runallsb
@@ -42,6 +42,7 @@ export SQLMX_REGRESS=1
 export SEABASE_REGRESS=2
 
 TEST_SUBDIRS="core compGeneral executor seabase fullstack2 charsets qat catman1 udr hive"

+TEST_SUBDIRS="catman1 udr hive" 
 
 #
 # Make sure we're running from the regress subdir

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d315ba78/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h
index bb5c4f4..4c7f229 100644
--- a/core/sql/sqlcomp/DefaultConstants.h
+++ b/core/sql/sqlcomp/DefaultConstants.h
@@ -3758,6 +3758,8 @@ enum DefaultConstants
   HBASE_RANGE_PARTITIONING_PARTIAL_COLS,
   MERGE_WITH_UNIQUE_INDEX,
 
+  USTAT_MIN_CHAR_DATASIZE_FOR_IS,
+
   // This enum constant must be the LAST one in the list; it's a count,
   // not an Attribute (it's not IN DefaultDefaults; it's the SIZE of it)!
   __NUM_DEFAULT_ATTRIBUTES

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d315ba78/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 4e7a765..1e775a3 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -3522,8 +3522,10 @@ XDDkwd__(SUBQUERY_UNNESTING,			"ON"),
  XDDui___(USTAT_MAX_READ_AGE_IN_MIN,            "5760"),
   DDui___(USTAT_MAX_SAMPLE_AGE,                 "365"),  // For R2.5 set to a year so user
created samples won't be removed.
 
+  DDflt0_(USTAT_MIN_CHAR_DATASIZE_FOR_IS,      "1000"),  // max data size in MB for char
type to use 
+                                                         // internal sort without checking
UEC.
   DDflt0_(USTAT_MIN_CHAR_UEC_FOR_IS,            "0.2"),  // minimum UEC for char type to
use internal sort
-  DDflt0_(USTAT_MIN_DEC_BIN_UEC_FOR_IS,         "0.03"), // minimum UEC for binary types
to use internal sort
+  DDflt0_(USTAT_MIN_DEC_BIN_UEC_FOR_IS,         "0.0"),  // minimum UEC for binary types
to use internal sort
 
   DDflt0_(USTAT_MIN_ESTIMATE_FOR_ROWCOUNT,      "10000000"),
   DDui1__(USTAT_MIN_ROWCOUNT_FOR_CTS_SAMPLE,    "10000"),

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d315ba78/core/sql/ustat/hs_globals.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.cpp b/core/sql/ustat/hs_globals.cpp
index 1471cd1..7addcf2 100644
--- a/core/sql/ustat/hs_globals.cpp
+++ b/core/sql/ustat/hs_globals.cpp
@@ -5401,7 +5401,7 @@ Lng32 HSGlobalsClass::CollectStatistics()
         if (trySampleTableBypassForIS && multiGroup ) {
 
               if (CmpCommon::getDefault(USTAT_USE_INTERNAL_SORT_FOR_MC) == DF_ON &&
-                  allGroupsFitInMemory()) 
+                  allGroupsFitInMemory(maxRowsToRead)) 
               {
                 // if both single and MC groups can fit in memory, turn on
                 // performing MC in memory flag.
@@ -10257,7 +10257,7 @@ bool isInternalSortType(HSColumnStruct &col)
 // from the column's existing histogram). If there is no existing histogram for
 // the column, the values used default to 0, and internal sort will not be used.
 //
-NABoolean isInternalSortEfficient(HSColGroupStruct *group)
+NABoolean isInternalSortEfficient(Int64 rows, HSColGroupStruct *group)
 {
   HSLogMan *LM = HSLogMan::Instance();
   Lng32 dataType = group->ISdatatype;
@@ -10295,10 +10295,16 @@ NABoolean isInternalSortEfficient(HSColGroupStruct *group)
     }
   else if (DFS2REC::isAnyCharacter(dataType))
     {
-      // For char types, number of distinct values must be at least 
+      // For char types, if the total amount of data (rows * length) is less than 
+      // USTAT_MIN_CHAR_DATASIZE_FOR_IS (default to 1000 MB), use IS. Otherwise
+      // the number of distinct values must be at least 
       // USTAT_MIN_CHAR_UEC_FOR_IS of total (default 20%).
-      uecRateMinForIS = CmpCommon::getDefaultNumeric(USTAT_MIN_CHAR_UEC_FOR_IS);
-      returnVal = (uecRate >= uecRateMinForIS);
+      if ( rows * group->ISlength < 1024*1024*CmpCommon::getDefaultNumeric(USTAT_MIN_CHAR_DATASIZE_FOR_IS)
)
+        returnVal = TRUE;
+      else {
+        uecRateMinForIS = CmpCommon::getDefaultNumeric(USTAT_MIN_CHAR_UEC_FOR_IS);
+        returnVal = (uecRate >= uecRateMinForIS);
+      }
     }
   else
     returnVal = TRUE;  // No threshold established yet for other types; use IS
@@ -10327,7 +10333,7 @@ NABoolean isInternalSortEfficient(HSColGroupStruct *group)
   return returnVal;
 }
 
-NABoolean HSGlobalsClass::allGroupsFitInMemory()
+NABoolean HSGlobalsClass::allGroupsFitInMemory(Int64 rows)
 {
   Int64 memLeft = getMaxMemory();
 
@@ -10346,7 +10352,7 @@ NABoolean HSGlobalsClass::allGroupsFitInMemory()
       if (group->memNeeded > 0 &&        // was set to 0 if exceeds address
space
           group->memNeeded < memLeft &&
           isInternalSortType(group->colSet[0]) &&
-          isInternalSortEfficient(group))  
+          isInternalSortEfficient(rows, group))  
         {
           count++;
           memLeft -= group->memNeeded;
@@ -10393,7 +10399,7 @@ Int32 HSGlobalsClass::getColsToProcess(Int64 rows,
 
   do
     {
-      numColsSelected = selectSortBatch(internalSortWhenBetter,
+      numColsSelected = selectSortBatch(rows, internalSortWhenBetter,
                                         trySampleTableBypass);
       if (numColsSelected > 0)
         numColsToProcess = allocateMemoryForInternalSortColumns(rows);
@@ -10431,8 +10437,9 @@ Int32 HSGlobalsClass::getColsToProcess(Int64 rows,
 // select all columns if they will all fit in memory at once, and only consider
 // expected individual column performance if this can't be done.
 //
-Int32 HSGlobalsClass::selectSortBatch(NABoolean ISonlyWhenBetter,
-                                    NABoolean trySampleInMemory)
+Int32 HSGlobalsClass::selectSortBatch(Int64 rows, 
+                                      NABoolean ISonlyWhenBetter,
+                                      NABoolean trySampleInMemory)
 {
   HSLogMan *LM = HSLogMan::Instance();
 
@@ -10469,7 +10476,8 @@ Int32 HSGlobalsClass::selectSortBatch(NABoolean ISonlyWhenBetter,
           group->memNeeded > 0 &&        // was set to 0 if exceeds address
space
           group->memNeeded < memLeft &&
           isInternalSortType(group->colSet[0]) &&
-          (trySampleInMemory || !ISonlyWhenBetter || isInternalSortEfficient(group)))  //@ZXuec
+          (trySampleInMemory || !ISonlyWhenBetter || 
+           isInternalSortEfficient(rows, group)))  //@ZXuec
         {
           group->state = PENDING;
           count++;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d315ba78/core/sql/ustat/hs_globals.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.h b/core/sql/ustat/hs_globals.h
index e6a6592..76ee429 100644
--- a/core/sql/ustat/hs_globals.h
+++ b/core/sql/ustat/hs_globals.h
@@ -160,6 +160,7 @@ class ISFixedChar
     // NABasicObject does not define array versions of those operators. Even if
     // it did, we wouldn't subclass it because it would make the objects bigger
     // (NABasicObject has a heap ptr member variable).
+    /*
     void* operator new[](size_t size)
     {
       return STMTHEAP->allocateMemory(size, FALSE);
@@ -169,6 +170,7 @@ class ISFixedChar
     {
       STMTHEAP->deallocateMemory(addr);
     }
+*/
 
     // Note that we forego the usual convention of having operator= return a
     // reference to the assigned-to object. This is just to make this operation
@@ -1701,7 +1703,7 @@ private:
 
     // Select a set of columns that will fit in available memory so they can
     // be sorted internally.
-    Int32 selectSortBatch(NABoolean ISonlyWhenBetter,
+    Int32 selectSortBatch(Int64 rows, NABoolean ISonlyWhenBetter,
                         NABoolean trySampleInMemory);
 
     // Select a set of columns that can be IUS updated in memory in one batch.
@@ -1714,7 +1716,7 @@ private:
 
     // Determine if all groups (both single and MC) can fit in memory for internal sort.
     // No space is actually allocated and no state is set for each group.
-    NABoolean allGroupsFitInMemory();
+    NABoolean allGroupsFitInMemory(Int64 rows);
 
     // Determine the next batch of columns to be processed with internal sort
     // by calling selectSortBatch() and ensuring that adequate memory can be


Mime
View raw message