trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From se...@apache.org
Subject [5/8] trafodion git commit: [TRAFODION-3234] Refactor hive meta calls to be less resource intensive to support hive partitions if needed
Date Wed, 05 Dec 2018 21:54:48 GMT
[TRAFODION-3234] Refactor hive meta calls to be less resource intensive to support hive partitions
if needed

Fixes for hive/TEST005 failure.
This also made some of the earlier failed test cases in this test suite to return correct
results.

Data modification check was unnecessarily done in the scan operator. This has been removed.


Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/7be2672d
Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/7be2672d
Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/7be2672d

Branch: refs/heads/master
Commit: 7be2672d6fcad8e48d7508ff8a04b51bbde48441
Parents: 7633327
Author: selvaganesang <selva.govindarajan@esgyn.com>
Authored: Tue Dec 4 15:09:25 2018 +0000
Committer: selvaganesang <selva.govindarajan@esgyn.com>
Committed: Tue Dec 4 15:09:25 2018 +0000

----------------------------------------------------------------------
 core/sql/executor/ExHdfsScan.cpp  |  99 +++----------------------
 core/sql/executor/hiveHook.cpp    |  10 ++-
 core/sql/optimizer/HDFSHook.cpp   |  67 +++++++----------
 core/sql/regress/hive/EXPECTED005 | 128 +++++++++++++++------------------
 core/sql/regress/hive/TEST005     |   4 --
 5 files changed, 98 insertions(+), 210 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index c49a6a0..c4318d6 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -446,11 +446,12 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
                 step_ = ASSIGN_RANGES_AT_RUNTIME;
                 break;
               }
-	    else if (getHdfsFileInfoListAsArray().isEmpty())
-	      {
-                step_ = CHECK_FOR_DATA_MOD_AND_DONE;
-		break;
-	      }
+	    else {
+               if (useLibhdfsScan_)
+                   step_ = INIT_HDFS_CURSOR;
+               else
+                   step_ = SETUP_HDFS_SCAN; 
+	    }
 
 	    beginRangeNum_ =  
 	      *(Lng32*)hdfsScanTdb().getHdfsFileRangeBeginList()->get(myInstNum_);
@@ -459,11 +460,8 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
 	      *(Lng32*)hdfsScanTdb().getHdfsFileRangeNumList()->get(myInstNum_);
 
 	    currRangeNum_ = beginRangeNum_;
-
-	    if (numRanges_ > 0)
-              step_ = CHECK_FOR_DATA_MOD;
-            else
-              step_ = CHECK_FOR_DATA_MOD_AND_DONE;
+            if (numRanges_ <= 0)
+               step_ = DONE;
 	  }          
 	  break;
 
@@ -479,85 +477,6 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
           else
             step_ = DONE;
           break;
-
-        case CHECK_FOR_DATA_MOD:
-        case CHECK_FOR_DATA_MOD_AND_DONE:
-          {
-            char * dirPath = hdfsScanTdb().hdfsRootDir_;
-            Int64 modTS = hdfsScanTdb().modTSforDir_;
-            if ((dirPath == NULL) || (modTS == -1))
-              dataModCheckDone_ = TRUE;
-
-            if (NOT dataModCheckDone_)
-              {
-                dataModCheckDone_ = TRUE;
-
-                Lng32 numOfPartLevels = hdfsScanTdb().numOfPartCols_;
-
-                if (hdfsScanTdb().hdfsDirsToCheck())
-                  {
-                    // TBD
-                  }
-             
-                Int64 failedModTS = -1;
-                Lng32 failedLocBufLen = 1000;
-                char failedLocBuf[failedLocBufLen];
-                retcode = ExpLOBinterfaceDataModCheck
-                  (lobGlob_,
-                   dirPath,
-                   hdfsScanTdb().hostName_,
-                   hdfsScanTdb().port_,
-                   modTS,
-                   numOfPartLevels,
-                   failedModTS,
-                   failedLocBuf, failedLocBufLen);
-              
-                if (retcode < 0)
-                  {
-                    Lng32 cliError = 0;
-		    
-                    Lng32 intParam1 = -retcode;
-                    ComDiagsArea * diagsArea = NULL;
-                    ExRaiseSqlError(getHeap(), &diagsArea, 
-                                    (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
-                                    NULL, &intParam1, 
-                                    &cliError, 
-                                    NULL, 
-                                    "HDFS",
-                                    (char*)"ExpLOBInterfaceDataModCheck",
-                                    getLobErrStr(intParam1));
-                    pentry_down->setDiagsArea(diagsArea);
-                    step_ = HANDLE_ERROR_AND_DONE;
-                    break;
-                  }  
-
-                if (retcode == 1) // check failed
-                  {
-                    char errStr[200];
-                    str_sprintf(errStr, "genModTS = %ld, failedModTS = %ld", 
-                                modTS, failedModTS);
-
-                    ComDiagsArea * diagsArea = NULL;
-                    ExRaiseSqlError(getHeap(), &diagsArea, 
-                                    (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR), NULL,
-                                    NULL, NULL, NULL,
-                                    errStr);
-                    pentry_down->setDiagsArea(diagsArea);
-                    step_ = HANDLE_ERROR_AND_DONE;
-                    break;
-                  }
-              }
-
-            if (step_ == CHECK_FOR_DATA_MOD_AND_DONE)
-              step_ = DONE;
-            else {
-              if (useLibhdfsScan_)
-                 step_ = INIT_HDFS_CURSOR;
-              else
-                 step_ = SETUP_HDFS_SCAN;
-            }
-          }        
-          break;
         case SETUP_HDFS_SCAN:
           {   
              if (hdfsScan_ != NULL)
@@ -810,7 +729,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
                       }
                     else
                       ExRaiseSqlError(getHeap(), &diagsArea, 
-                                      (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
+                                      (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE));
                     pentry_down->setDiagsArea(diagsArea);
                     step_ = HANDLE_ERROR_AND_DONE;
                     break;

http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/executor/hiveHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/hiveHook.cpp b/core/sql/executor/hiveHook.cpp
index 284ba5c..234249a 100644
--- a/core/sql/executor/hiveHook.cpp
+++ b/core/sql/executor/hiveHook.cpp
@@ -307,8 +307,8 @@ struct hive_tbl_desc* HiveMetaData::getTableDesc( const char* schemaName,
 
     while (ptr) {
 
-      if ( !(strcmp(ptr->tblName_, tblName)
-             ||strcmp(ptr->schName_, schemaName))) {
+      if ( !(stricmp(ptr->tblName_, tblName)
+             ||stricmp(ptr->schName_, schemaName))) {
         if ((NOT rereadFromMD) && (validate(ptr)))
            return ptr;
         else {
@@ -332,6 +332,9 @@ struct hive_tbl_desc* HiveMetaData::getTableDesc( const char* schemaName,
 
       ptr = ptr->next_;
    }
+   
+   if (validateOnly)
+     return NULL;
 
    HVC_RetCode hvcRetcode;
    HiveClient_JNI *hiveClient = HiveClient_JNI::newInstance(heap_, hvcRetcode);
@@ -569,7 +572,8 @@ Int32 hive_tbl_desc::getSortColNum(const char* name)
 
 Int64 hive_tbl_desc::redeftime()
 {
-  Int64 result = creationTS_;
+  // creationTS_ is in seconds 
+  Int64 result = creationTS_ * 1000;
   if (redefineTS_ !=  -1)
       result = redefineTS_;
   return result;

http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/optimizer/HDFSHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp
index ed9b2fd..cc2bab5 100644
--- a/core/sql/optimizer/HDFSHook.cpp
+++ b/core/sql/optimizer/HDFSHook.cpp
@@ -854,50 +854,40 @@ NABoolean HHDFSTableStats::populate(struct hive_tbl_desc *htd)
   Int32 hdfsPort = -1;
   NAString tableDir;
 
-  if (hsd)
-    {
-      if (hsd->isTextFile())
+  if (hsd) {
+     if (hsd->isTextFile())
         type_ = TEXT_;
-      else if (hsd->isSequenceFile())
+     else if (hsd->isSequenceFile())
         type_ = SEQUENCE_;
-      else if (hsd->isOrcFile())
+     else if (hsd->isOrcFile())
         type_ = ORC_;
-      else
+     else
         type_ = UNKNOWN_;
-    }
-
-  while (hsd && diags_.isSuccess())
-    {
-      // split table URL into host, port and filename
-      if (! splitLocation(hsd->location_,
-                          hdfsHost,
-                          hdfsPort,
-                          tableDir,
-                          diags_,
-                          hdfsPortOverride_)) {
-        return FALSE;
-      }
-
-      if (! connectHDFS(hdfsHost, hdfsPort)) {
-        return FALSE; // diags_ is set
-      }
-
-      // put back fully qualified URI
-      tableDir = hsd->location_;
-
-      // get the fine-resolution timestamp before visiting
-      // the tree, to avoid losing any updates while this
-      // method is executing
-      computeModificationTSmsec();
-
-      if (diags_.isSuccess())
+  }
+  // split table URL into host, port and filename
+  if (! splitLocation(hsd->location_,
+                hdfsHost,
+                hdfsPort,
+                tableDir,
+                diags_,
+                hdfsPortOverride_)) 
+      return FALSE;
+  if (! connectHDFS(hdfsHost, hdfsPort)) 
+     return FALSE; // diags_ is set
+  // put back fully qualified URI
+  tableDir = hsd->location_;
+  computeModificationTSmsec();
+  if (diags_.isSuccess()) {
+     htd->setRedeftime(modificationTSInMillisec_);
+     while (hsd && diags_.isSuccess()) {
         // visit the directory
-      processDirectory(tableDir, hsd->buckets_, 
+        processDirectory(hsd->location_, hsd->buckets_, 
                        hsd->isTrulyText(), 
                        hsd->getRecordTerminator());
 
-      hsd = hsd->next_;
-    }
+        hsd = hsd->next_;
+     }
+  }
 
   disconnectHDFS();
   validationJTimestamp_ = JULIANTIMESTAMP();
@@ -1147,8 +1137,6 @@ void HHDFSTableStats::disconnectHDFS()
 
 void HHDFSTableStats::computeModificationTSmsec()
 {
-  if (modificationTSInMillisec_ <= 0)
-    {
       HDFS_Client_RetCode rc;
 
       // get a millisecond-resolution timestamp via JNI
@@ -1167,9 +1155,6 @@ void HHDFSTableStats::computeModificationTSmsec()
           diags_.recordError(errMsg, "HHDFSTableStats::computeModificationTSmsec");
           modificationTSInMillisec_ = -1;
         }
-    }
-
-  return;
 }
 
 OsimHHDFSStatsBase* HHDFSTableStats::osimSnapShot(NAMemory * heap)

http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/regress/hive/EXPECTED005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005
index c3f6a7b..d9475d8 100644
--- a/core/sql/regress/hive/EXPECTED005
+++ b/core/sql/regress/hive/EXPECTED005
@@ -72,7 +72,7 @@
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542230997205, failedModTS = 1542231173977, failedLoc
= hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900115861, failedModTS = 1543900283891, failedLoc
= hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
 
 C_PREFERRED_CUST_FLAG      (EXPR)              
 -------------------------  --------------------
@@ -108,7 +108,7 @@ Y                                          9525
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542230997205, failedModTS = 1542231173977, failedLoc
= hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900115861, failedModTS = 1543900283891, failedLoc
= hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
 
 C_PREFERRED_CUST_FLAG      (EXPR)              
 -------------------------  --------------------
@@ -118,8 +118,6 @@ N                                          9789
 Y                                          9525
 
 --- 3 row(s) selected.
->>-- because we don't invalidate in the executor,
->>-- this should still return 0 rows
 >>
 >>execute s2;
 
@@ -151,7 +149,7 @@ Y                                          9525
 (EXPR)              
 --------------------
 
-                   0
+                9789
 
 --- 1 row(s) selected.
 >>-- although this should get an NATable cache
@@ -177,7 +175,7 @@ Y                                          9525
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231178414, failedModTS = 1542231182305, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/newtable
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900288480, failedModTS = 1543900294265, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/newtable
 
 A                        
 -------------------------
@@ -227,7 +225,7 @@ xyz
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231173977, failedModTS = 1542231203852, failedLoc
= hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900283891, failedModTS = 1543900317574, failedLoc
= hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
 
 C_PREFERRED_CUST_FLAG      (EXPR)              
 -------------------------  --------------------
@@ -237,12 +235,11 @@ N                                         19631
 Y                                         18984
 
 --- 3 row(s) selected.
->>-- s1 should still return 0 rows - for now
 >>execute s2;
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231173977, failedModTS = 1542231203852, failedLoc
= hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900283891, failedModTS = 1543900317574, failedLoc
= hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
 
 C_PREFERRED_CUST_FLAG      (EXPR)              
 -------------------------  --------------------
@@ -268,13 +265,12 @@ Y                                         18984
                    0
 
 --- 1 row(s) selected.
->>-- s1 should still return 0 rows - for now
 >>execute s2part;
 
 (EXPR)              
 --------------------
 
-                   0
+                9789
 
 --- 1 row(s) selected.
 >>execute s3part;
@@ -282,7 +278,7 @@ Y                                         18984
 (EXPR)              
 --------------------
 
-                   0
+               19314
 
 --- 1 row(s) selected.
 >>
@@ -298,7 +294,7 @@ Y                                         18984
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231200755, failedModTS = 1542231206302, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/newtable
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900313695, failedModTS = 1543900320754, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/newtable
 
 A            B                        
 -----------  -------------------------
@@ -354,7 +350,7 @@ Y                                         18984
 (EXPR)              
 --------------------
 
-                   0
+                9789
 
 --- 1 row(s) selected.
 >>execute s4part;
@@ -362,7 +358,7 @@ Y                                         18984
 (EXPR)              
 --------------------
 
-                   0
+               19999
 
 --- 1 row(s) selected.
 >>select count(*) from tbl_utf8;
@@ -631,18 +627,18 @@ C1           C2                    C3                         C4   
           C
 --- 1 row(s) selected.
 >>load with continue on error into trafodion.seabase.traf_tbl_bad select * from tbl_bad;
 Task:  LOAD            Status: Started    Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task:  CLEANUP         Status: Started    Time: 2018-11-14 21:34:43.60760
-Task:  CLEANUP         Status: Ended      Time: 2018-11-14 21:34:43.76350
-Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.016
-Task:  LOADING DATA    Status: Started    Time: 2018-11-14 21:34:43.76422
+Task:  CLEANUP         Status: Started    Time: 2018-12-04 05:13:10.600264
+Task:  CLEANUP         Status: Ended      Time: 2018-12-04 05:13:10.612952
+Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.013
+Task:  LOADING DATA    Status: Started    Time: 2018-12-04 05:13:10.613035
        Rows Processed: 8 
        Error Rows:     5 
-Task:  LOADING DATA    Status: Ended      Time: 2018-11-14 21:34:43.275300
-Task:  LOADING DATA    Status: Ended      Elapsed Time:    00:00:00.199
-Task:  COMPLETION      Status: Started    Time: 2018-11-14 21:34:43.275441
+Task:  LOADING DATA    Status: Ended      Time: 2018-12-04 05:13:10.766853
+Task:  LOADING DATA    Status: Ended      Elapsed Time:    00:00:00.154
+Task:  COMPLETION      Status: Started    Time: 2018-12-04 05:13:10.766892
        Rows Loaded:    3 
-Task:  COMPLETION      Status: Ended      Time: 2018-11-14 21:34:44.92332
-Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.817
+Task:  COMPLETION      Status: Ended      Time: 2018-12-04 05:13:11.231800
+Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.465
 
 --- 3 row(s) loaded.
 >>select count(*) from trafodion.seabase.traf_tbl_bad;
@@ -658,19 +654,19 @@ Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.817
 --- 3 row(s) deleted.
 >>load with log error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad;
 Task:  LOAD            Status: Started    Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task:  CLEANUP         Status: Started    Time: 2018-11-14 21:34:45.188714
-Task:  CLEANUP         Status: Ended      Time: 2018-11-14 21:34:45.202159
-Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.013
-       Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181114_213445
-Task:  LOADING DATA    Status: Started    Time: 2018-11-14 21:34:45.202236
+Task:  CLEANUP         Status: Started    Time: 2018-12-04 05:13:12.257338
+Task:  CLEANUP         Status: Ended      Time: 2018-12-04 05:13:12.272165
+Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.015
+       Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181204_051312
+Task:  LOADING DATA    Status: Started    Time: 2018-12-04 05:13:12.272230
        Rows Processed: 8 
        Error Rows:     5 
-Task:  LOADING DATA    Status: Ended      Time: 2018-11-14 21:34:45.474416
-Task:  LOADING DATA    Status: Ended      Elapsed Time:    00:00:00.272
-Task:  COMPLETION      Status: Started    Time: 2018-11-14 21:34:45.474451
+Task:  LOADING DATA    Status: Ended      Time: 2018-12-04 05:13:12.508271
+Task:  LOADING DATA    Status: Ended      Elapsed Time:    00:00:00.236
+Task:  COMPLETION      Status: Started    Time: 2018-12-04 05:13:12.508318
        Rows Loaded:    3 
-Task:  COMPLETION      Status: Ended      Time: 2018-11-14 21:34:46.101985
-Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.628
+Task:  COMPLETION      Status: Ended      Time: 2018-12-04 05:13:13.152197
+Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.644
 
 --- 3 row(s) loaded.
 >>select count(*) from trafodion.seabase.traf_tbl_bad;
@@ -683,19 +679,19 @@ Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.628
 --- 1 row(s) selected.
 >>load with log error rows to '/user/trafodion/bulkload/logs/TEST005' into trafodion.seabase.traf_tbl_bad
select * from tbl_bad;
 Task:  LOAD            Status: Started    Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task:  CLEANUP         Status: Started    Time: 2018-11-14 21:34:47.154307
-Task:  CLEANUP         Status: Ended      Time: 2018-11-14 21:34:47.162209
-Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.008
-       Logging Location: /user/trafodion/bulkload/logs/TEST005/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181114_213447
-Task:  LOADING DATA    Status: Started    Time: 2018-11-14 21:34:47.162253
+Task:  CLEANUP         Status: Started    Time: 2018-12-04 05:13:14.138305
+Task:  CLEANUP         Status: Ended      Time: 2018-12-04 05:13:14.142425
+Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.004
+       Logging Location: /user/trafodion/bulkload/logs/TEST005/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181204_051314
+Task:  LOADING DATA    Status: Started    Time: 2018-12-04 05:13:14.142474
        Rows Processed: 8 
        Error Rows:     5 
-Task:  LOADING DATA    Status: Ended      Time: 2018-11-14 21:34:47.473145
-Task:  LOADING DATA    Status: Ended      Elapsed Time:    00:00:00.311
-Task:  COMPLETION      Status: Started    Time: 2018-11-14 21:34:47.473192
+Task:  LOADING DATA    Status: Ended      Time: 2018-12-04 05:13:14.379124
+Task:  LOADING DATA    Status: Ended      Elapsed Time:    00:00:00.237
+Task:  COMPLETION      Status: Started    Time: 2018-12-04 05:13:14.379186
        Rows Loaded:    3 
-Task:  COMPLETION      Status: Ended      Time: 2018-11-14 21:34:48.262413
-Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.789
+Task:  COMPLETION      Status: Ended      Time: 2018-12-04 05:13:14.991405
+Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.612
 
 --- 3 row(s) loaded.
 >>select count(*) from trafodion.seabase.traf_tbl_bad;
@@ -711,10 +707,10 @@ Task:  COMPLETION      Status: Ended      Elapsed Time:    00:00:00.789
 --- 6 row(s) deleted.
 >>load with stop after 3 error rows into trafodion.seabase.traf_tbl_bad select * from
tbl_bad;
 Task:  LOAD            Status: Started    Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task:  CLEANUP         Status: Started    Time: 2018-11-14 21:34:49.353872
-Task:  CLEANUP         Status: Ended      Time: 2018-11-14 21:34:49.366515
-Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.013
-Task:  LOADING DATA    Status: Started    Time: 2018-11-14 21:34:49.366571
+Task:  CLEANUP         Status: Started    Time: 2018-12-04 05:13:16.65887
+Task:  CLEANUP         Status: Ended      Time: 2018-12-04 05:13:16.72797
+Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.007
+Task:  LOADING DATA    Status: Started    Time: 2018-12-04 05:13:16.72836
 
 *** ERROR[8113] The maximum number of error rows is exceeded.
 
@@ -729,11 +725,11 @@ Task:  LOADING DATA    Status: Started    Time: 2018-11-14 21:34:49.366571
 --- 1 row(s) selected.
 >>load with log error rows, stop after 3 error rows into trafodion.seabase.traf_tbl_bad
select * from tbl_bad;
 Task:  LOAD            Status: Started    Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task:  CLEANUP         Status: Started    Time: 2018-11-14 21:34:50.532300
-Task:  CLEANUP         Status: Ended      Time: 2018-11-14 21:34:50.542271
-Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.010
-       Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181114_213450
-Task:  LOADING DATA    Status: Started    Time: 2018-11-14 21:34:50.542323
+Task:  CLEANUP         Status: Started    Time: 2018-12-04 05:13:17.218725
+Task:  CLEANUP         Status: Ended      Time: 2018-12-04 05:13:17.234042
+Task:  CLEANUP         Status: Ended      Elapsed Time:    00:00:00.015
+       Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181204_051317
+Task:  LOADING DATA    Status: Started    Time: 2018-12-04 05:13:17.234092
 
 *** ERROR[8113] The maximum number of error rows is exceeded.
 
@@ -823,7 +819,7 @@ Task:  LOADING DATA    Status: Started    Time: 2018-11-14 21:34:50.542323
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231316268, failedModTS = 1542231318359, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900418287, failedModTS = 1543900423606, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive
 
 A          
 -----------
@@ -882,7 +878,7 @@ A            B
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231324672, failedModTS = 1542231325768, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900429674, failedModTS = 1543900430804, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive
 
 --- 0 row(s) selected.
 >>insert into hive.hive.thive values (10, 20);
@@ -906,7 +902,7 @@ A            B
 
 *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
 
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231326748, failedModTS = 1542231328569, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1543900431824, failedModTS = 1543900433636, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive
 
 --- 0 row(s) selected.
 >>
@@ -995,7 +991,7 @@ t005part.a	t005part.b	t005part.c
 >>invoke hive.hive.thive_insert_smallint;
 
 -- Definition of hive table HIVE.HIVE.THIVE_INSERT_SMALLINT
--- Definition current  Wed Nov 14 21:39:06 2018
+-- Definition current  Tue Dec  4 05:17:20 2018
 
   (
     A                                SMALLINT
@@ -1016,7 +1012,7 @@ CREATE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT
 /* Trafodion DDL */
 
 REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT;
-/* ObjectUID = 4263396855838728432 */
+/* ObjectUID = 4414832062427972126 */
 
 --- SQL operation complete.
 >>
@@ -1029,10 +1025,6 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT;
 >>insert into hive.hive.thive_insert_smallint select * from 
 +> (values (10), (11111111), (21), (22222222));
 
-*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
-
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231120366, failedModTS = 1542231546673, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive_insert_smallint
-
 --- 4 row(s) inserted.
 >>select * from hive.hive.thive_insert_smallint;
 
@@ -1070,10 +1062,6 @@ A
 >>insert into hive.hive.thive_insert_smallint select * from 
 +> (values (10), (11111111), (21), (22222222));
 
-*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
-
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231548908, failedModTS = 1542231550707, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive_insert_smallint
-
 --- 2 row(s) inserted.
 >>select * from hive.hive.thive_insert_smallint;
 
@@ -1113,7 +1101,7 @@ A
 >>invoke hive.hive.thive_insert_varchar;
 
 -- Definition of hive table HIVE.HIVE.THIVE_INSERT_VARCHAR
--- Definition current  Wed Nov 14 21:39:15 2018
+-- Definition current  Tue Dec  4 05:17:29 2018
 
   (
     A                                VARCHAR(1 CHAR) CHARACTER SET UTF8 COLLATE
@@ -1138,7 +1126,7 @@ CREATE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR
 /* Trafodion DDL */
 
 REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR;
-/* ObjectUID = 4263396855838729348 */
+/* ObjectUID = 4414832062427972695 */
 
 --- SQL operation complete.
 >>cqd hive_insert_error_mode '1';
@@ -1149,10 +1137,6 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR;
 --- SQL operation complete.
 >>insert into hive.hive.thive_insert_varchar values ('abcddcba','efghijkl');
 
-*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
-
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: compiledModTS = 1542231129568, failedModTS = 1542231555639, failedLoc
= hdfs://localhost:9000/user/hive/warehouse/thive_insert_varchar
-
 --- 1 row(s) inserted.
 >>
 >>cqd hive_max_string_length_in_bytes '20';

http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/regress/hive/TEST005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST005 b/core/sql/regress/hive/TEST005
index 6521786..bf87a69 100644
--- a/core/sql/regress/hive/TEST005
+++ b/core/sql/regress/hive/TEST005
@@ -122,8 +122,6 @@ prepare s2part from
   --order by 1
   ;
 execute s1;
--- because we don't invalidate in the executor,
--- this should still return 0 rows
 
 execute s2;
 -- should get an NATable cache
@@ -173,11 +171,9 @@ prepare s3part from
   --order by 1
   ;
 execute s1;
--- s1 should still return 0 rows - for now
 execute s2;
 execute s3;
 execute s1part;
--- s1 should still return 0 rows - for now
 execute s2part;
 execute s3part;
 


Mime
View raw message