trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ansha...@apache.org
Subject [7/8] incubator-trafodion git commit: hive data modification detection: commit #4
Date Tue, 31 May 2016 22:53:42 GMT
hive data modification detection: commit #4


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/51a9c73e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/51a9c73e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/51a9c73e

Branch: refs/heads/master
Commit: 51a9c73ed37c3711178483c4a200f17a92d1c08d
Parents: b1a8f02
Author: Anoop Sharma <anoop.sharma@esgyn.com>
Authored: Tue May 31 18:03:40 2016 +0000
Committer: Anoop Sharma <anoop.sharma@esgyn.com>
Committed: Tue May 31 18:03:40 2016 +0000

----------------------------------------------------------------------
 core/sql/cli/SessionDefaults.cpp        |   2 +
 core/sql/comexe/ComTdbExeUtil.cpp       |   6 +-
 core/sql/comexe/ComTdbExeUtil.h         |  12 ++-
 core/sql/comexe/ComTdbHdfsScan.h        |   4 +-
 core/sql/executor/ExExeUtil.h           |   9 +-
 core/sql/executor/ExExeUtilMisc.cpp     |  95 ++++++++++++++-----
 core/sql/executor/ExFastTransport.cpp   |   2 +-
 core/sql/executor/ExHdfsScan.cpp        |   2 +-
 core/sql/exp/ExpErrorEnums.h            |   1 +
 core/sql/exp/ExpLOBaccess.cpp           | 133 ++++-----------------------
 core/sql/exp/ExpLOBaccess.h             |   5 +-
 core/sql/exp/ExpLOBinterface.h          |   5 +-
 core/sql/generator/GenFastTransport.cpp |   2 +-
 core/sql/generator/GenRelExeUtil.cpp    |   5 +-
 core/sql/optimizer/BindRelExpr.cpp      |   3 +-
 core/sql/optimizer/HDFSHook.cpp         |  50 ++++------
 core/sql/optimizer/NATable.cpp          |   3 +
 core/sql/optimizer/RelExeUtil.h         |   9 +-
 core/sql/optimizer/RelFastTransport.cpp |   1 +
 core/sql/regress/hive/EXPECTED003       |  37 ++++++--
 core/sql/regress/hive/EXPECTED005       |  36 +++++---
 core/sql/regress/hive/EXPECTED015       |  59 ++++++------
 core/sql/regress/hive/TEST003           |   9 +-
 core/sql/regress/hive/TEST005           |  11 +--
 core/sql/regress/hive/TEST015           |   3 +-
 25 files changed, 240 insertions(+), 264 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/cli/SessionDefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/SessionDefaults.cpp b/core/sql/cli/SessionDefaults.cpp
index 024754b..5e79138 100644
--- a/core/sql/cli/SessionDefaults.cpp
+++ b/core/sql/cli/SessionDefaults.cpp
@@ -782,6 +782,8 @@ static const AQRInfo::AQRErrorMap aqrErrorMap[] =
   // parallel purgedata failed
   AQREntry(   8022,      0,      3,    60,      0,   0, "",    0,     1),
 
+  // hive data modification timestamp mismatch.
+  // query will be AQR'd and hive metadata will be reloaded.
   AQREntry(   8436,      0,      1,     0,      0,   2, "04:05",  0,     0),
 
   // FS memory errors

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.cpp b/core/sql/comexe/ComTdbExeUtil.cpp
index 3442714..e3dd630 100644
--- a/core/sql/comexe/ComTdbExeUtil.cpp
+++ b/core/sql/comexe/ComTdbExeUtil.cpp
@@ -1189,7 +1189,8 @@ ComTdbExeUtilFastDelete::ComTdbExeUtilFastDelete(
      NABoolean isHiveTruncate,
      char * hiveTableLocation,
      char * hiveHostName,
-     Lng32 hivePortNum)
+     Lng32 hivePortNum,
+     Int64 hiveModTS)
      : ComTdbExeUtil(ComTdbExeUtil::FAST_DELETE_,
 		     NULL, 0, (Int16)SQLCHARSETCODE_UNKNOWN,
 		     tableName, tableNameLen,
@@ -1211,7 +1212,8 @@ ComTdbExeUtilFastDelete::ComTdbExeUtilFastDelete(
        lobNumArray_(lobNumArray),
        hiveTableLocation_(hiveTableLocation),
        hiveHdfsHost_(hiveHostName),
-       hiveHdfsPort_(hivePortNum)
+       hiveHdfsPort_(hivePortNum),
+       hiveModTS_(hiveModTS)
 {
   setIsHiveTruncate(isHiveTruncate);
   setNodeType(ComTdb::ex_FAST_DELETE);

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h
index ea3ef69..eb70eac 100644
--- a/core/sql/comexe/ComTdbExeUtil.h
+++ b/core/sql/comexe/ComTdbExeUtil.h
@@ -1518,7 +1518,8 @@ public:
 			  NABoolean ishiveTruncate = FALSE,
 			  char * hiveTableLocation = NULL,
                           char * hiveHostName = NULL,
-                          Lng32 hivePortNum = 0
+                          Lng32 hivePortNum = 0,
+                          Int64 hiveModTS = -1
 			  );
 
   Long pack(void *);
@@ -1569,6 +1570,11 @@ public:
     return hiveHdfsPort_;
   }
 
+  Lng32 getHiveModTS() const
+  {
+    return hiveModTS_;
+  }
+
   // ---------------------------------------------------------------------
   // Used by the internal SHOWPLAN command to get attributes of a TDB.
   // ---------------------------------------------------------------------
@@ -1645,7 +1651,9 @@ private:
   NABasicPtr  hiveTableLocation_;                    // 56-63
   NABasicPtr hiveHdfsHost_;                          // 64-71
   Int32 hiveHdfsPort_;                               // 72-75
-  char fillersComTdbExeUtilFastDelete_[52];          // 76-127
+  char fillers1_[4];                                 // 76-79
+  Int64 hiveModTS_;                                  // 80-87
+  char fillersComTdbExeUtilFastDelete_[40];          // 88-127
 };
 
 class ComTdbExeUtilGetStatistics : public ComTdbExeUtil

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbHdfsScan.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbHdfsScan.h b/core/sql/comexe/ComTdbHdfsScan.h
index ac83311..392f7cf 100755
--- a/core/sql/comexe/ComTdbHdfsScan.h
+++ b/core/sql/comexe/ComTdbHdfsScan.h
@@ -135,7 +135,7 @@ class ComTdbHdfsScan : public ComTdb
   char fillersComTdbHdfsScan1_[2];                            // 190 - 191
   NABasicPtr nullFormat_;                                     // 192 - 199
 
-  // next 3 params used to check if data under hdfsFileDir
+  // next 4 params are used to check if data under hdfsFileDir
   // was modified after query was compiled.
   NABasicPtr hdfsRootDir_;                                     // 200 - 207
   Int64  modTSforDir_;                                         // 208 - 215
@@ -196,7 +196,7 @@ public:
                  char * loggingLocation = NULL,
                  char * errCountId = NULL,
 
-                 // next 3 params used to check if data under hdfsFileDir
+                 // next 4 params are used to check if data under hdfsFileDir
                  // was modified after query was compiled.
                  char * hdfsRootDir  = NULL,
                  Int64  modTSforDir   = -1,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index 6ed29c1..062362b 100755
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -3299,6 +3299,7 @@ class ExExeUtilHiveTruncateTcb : public ExExeUtilTcb
     {
       INITIAL_,
       ERROR_,
+      DATA_MOD_CHECK_,
       EMPTY_DIRECTORY_,
       DONE_
     };
@@ -3306,18 +3307,10 @@ class ExExeUtilHiveTruncateTcb : public ExExeUtilTcb
   ExExeUtilFastDeleteTdb & fdTdb() const
     {return (ExExeUtilFastDeleteTdb &) tdb;};
 
-
-//  short doHiveTruncate(char * objectName,
-//                     NABoolean isIndex,
-//                     NABoolean fastDelUsingResetEOF);
-
   short injectError(const char * val);
 
   Step step_;
 
-  char  hdfsHost_[500];
-  int  hdfsPort_;
-  char  hiveTableLocation_[513];
   int   numExistingFiles_;
   void * lobGlob_;
 };

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExExeUtilMisc.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilMisc.cpp b/core/sql/executor/ExExeUtilMisc.cpp
index 1aea6c1..267d22e 100644
--- a/core/sql/executor/ExExeUtilMisc.cpp
+++ b/core/sql/executor/ExExeUtilMisc.cpp
@@ -2240,13 +2240,7 @@ ExExeUtilHiveTruncateTcb::ExExeUtilHiveTruncateTcb(
   qparent_.down->allocatePstate(this);
 
   numExistingFiles_ = 0;
-  memset (hdfsHost_, '\0', sizeof(hdfsHost_));
-  memset (hiveTableLocation_, '\0', sizeof(hiveTableLocation_));
 
-  strncpy(hdfsHost_, fdTdb().getHiveHdfsHost(), sizeof(hdfsHost_));
-  hdfsPort_ = fdTdb().getHiveHdfsPort();
-  char * outputPath =  fdTdb().getHiveTableLocation();
-  strncpy(hiveTableLocation_, outputPath, 512);
   step_ = INITIAL_;
 }
 
@@ -2290,26 +2284,84 @@ short ExExeUtilHiveTruncateTcb::work()
       case INITIAL_:
       {
 
-        //nothing for now --
-        // more stuff later
+        if (fdTdb().getHiveModTS() > 0)
+          step_ = DATA_MOD_CHECK_;
+        else
+          step_ = EMPTY_DIRECTORY_;
+      }
+      break;
+
+      case DATA_MOD_CHECK_:
+      {
+        cliRC = ExpLOBinterfaceDataModCheck
+          (lobGlob_,
+           fdTdb().getHiveTableLocation(),
+           fdTdb().getHiveHdfsHost(),
+           fdTdb().getHiveHdfsPort(),
+           fdTdb().getHiveModTS(),
+           0);
+
+        if (cliRC < 0)
+        {
+          Lng32 cliError = 0;
+          
+          Lng32 intParam1 = -cliRC;
+          ComDiagsArea * diagsArea = NULL;
+          ExRaiseSqlError(getHeap(), &diagsArea, 
+                          (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
+                          NULL, &intParam1, 
+                          &cliError, 
+                          NULL, 
+                          "HDFS",
+                          (char*)"ExpLOBInterfaceEmptyDirectory",
+                          getLobErrStr(intParam1));
+          pentry_down->setDiagsArea(diagsArea);
+          step_ = ERROR_;
+          break;
+        }
+
+        if (cliRC == 1) // data mod check failed
+        {
+          ComDiagsArea * diagsArea = NULL;
+          ExRaiseSqlError(getHeap(), &diagsArea, 
+                          (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
+          pentry_down->setDiagsArea(diagsArea);
+          
+          step_ = ERROR_;
+          break;
+        }
+   
         step_ = EMPTY_DIRECTORY_;
       }
-        break;
+      break;
 
       case EMPTY_DIRECTORY_:
       {
-        Lng32 retCode= ExpLOBinterfaceEmptyDirectory(
-                                    lobGlob_,
-                                    (char*)"",                  //name is empty
-                                    hiveTableLocation_,
-                                    Lob_HDFS_File,
-                                    hdfsHost_,
-                                    hdfsPort_,
-                                    0 ,
-                                    1 ,
-                                    0);
-        if (retCode != 0)
+        cliRC = ExpLOBinterfaceEmptyDirectory(
+             lobGlob_,
+             (char*)"",                  //name is empty
+             fdTdb().getHiveTableLocation(),
+             Lob_HDFS_File,
+             fdTdb().getHiveHdfsHost(),
+             fdTdb().getHiveHdfsPort(),
+             0 ,
+             1 ,
+             0);
+        if (cliRC != 0)
         {
+          Lng32 cliError = 0;
+          
+          Lng32 intParam1 = -cliRC;
+          ComDiagsArea * diagsArea = NULL;
+          ExRaiseSqlError(getHeap(), &diagsArea, 
+                          (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
+                          NULL, &intParam1, 
+                          &cliError, 
+                          NULL, 
+                          "HDFS",
+                          (char*)"ExpLOBInterfaceEmptyDirectory",
+                          getLobErrStr(intParam1));
+          pentry_down->setDiagsArea(diagsArea);
           step_ = ERROR_;
         }
         else
@@ -2318,6 +2370,7 @@ short ExExeUtilHiveTruncateTcb::work()
         }
       }
       break;
+
       case ERROR_:
       {
         if (qparent_.up->isFull())
@@ -2350,7 +2403,7 @@ short ExExeUtilHiveTruncateTcb::work()
 
         step_ = DONE_;
       }
-        break;
+      break;
 
       case DONE_:
       {

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExFastTransport.cpp b/core/sql/executor/ExFastTransport.cpp
index 45c3959..bb8d4dc 100644
--- a/core/sql/executor/ExFastTransport.cpp
+++ b/core/sql/executor/ExFastTransport.cpp
@@ -738,7 +738,7 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
       {
         ComDiagsArea * diagsArea = NULL;
         ExRaiseSqlError(getHeap(), &diagsArea, 
-                        (ExeErrorCode)(8436));
+                        (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
         pentry_down->setDiagsArea(diagsArea);
         pstate.step_ = EXTRACT_ERROR;
         break;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index b204fb2..82e01b3 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -468,7 +468,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
                   {
                     ComDiagsArea * diagsArea = NULL;
                     ExRaiseSqlError(getHeap(), &diagsArea, 
-                                    (ExeErrorCode)(8436));
+                                    (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
                     pentry_down->setDiagsArea(diagsArea);
                     step_ = HANDLE_ERROR_AND_DONE;
                     break;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpErrorEnums.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpErrorEnums.h b/core/sql/exp/ExpErrorEnums.h
index de4d00f..987fe7e 100644
--- a/core/sql/exp/ExpErrorEnums.h
+++ b/core/sql/exp/ExpErrorEnums.h
@@ -153,6 +153,7 @@ enum ExeErrorCode
   EXE_IS_BITWISE_AND_ERROR		= 8431,
   EXE_UNSIGNED_OVERFLOW                 = 8432,
   EXE_INVALID_CHARACTER                 = 8433,
+  EXE_HIVE_DATA_MOD_CHECK_ERROR         = 8436,
   EXE_HISTORY_BUFFER_TOO_SMALL		= 8440,
   EXE_OLAP_OVERFLOW_NOT_SUPPORTED       = 8441,
   EXE_ERROR_FROM_LOB_INTERFACE          = 8442,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 7db4a40..ad127ea 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -110,104 +110,6 @@ ExLob::~ExLob()
    
 }
 
-#ifdef __ignore
-Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, 
-                               char *dir, 
-			       LobsStorage storage,
-                               char *hdfsServer, Int64 hdfsPort,
-                               char *lobLocation,
-                               int bufferSize , short replication ,
-                               int blockSize, Int64 lobMaxSize, 
-                               ExLobGlobals *lobGlobals)
-{
-  int openFlags;
-  mode_t filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
-  struct timespec startTime;
-  struct timespec endTime;
-  Int64 secs, nsecs, totalnsecs;
- 
-  if (dir) 
-    {
-      if (dir_.empty()) 
-	{
-	  dir_ = string(dir);
-	}
-
-      if (lobFile)
-        snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s", dir_.c_str(), lobFile);
-      
-    } 
-  else if (lobFile)
-    { 
-      snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s", lobFile);
-      
-    }
-
-  hdfsServer_ = hdfsServer;
-  hdfsPort_ = hdfsPort;
-
-  if (fs_ == NULL)
-    {
-      fs_ = hdfsConnect(hdfsServer_, hdfsPort_);
-      if (fs_ == NULL)
-        return LOB_HDFS_CONNECT_ERROR;
-    }
-
-  if (lobGlobals)
-    lobGlobals->setHdfsFs(fs_);
-  
-  if (storage_ != Lob_Invalid_Storage)
-    {
-      return LOB_INIT_ERROR;
-    } 
-  else 
-    {
-      storage_ = storage;
-    }
-
-  stats_.init(); 
-
-  if (lobLocation)
-    lobLocation_ = lobLocation;
-  clock_gettime(CLOCK_MONOTONIC, &startTime);
-
-  clock_gettime(CLOCK_MONOTONIC, &endTime);
-
-  secs = endTime.tv_sec - startTime.tv_sec;
-  nsecs = endTime.tv_nsec - startTime.tv_nsec;
-  if (nsecs < 0) 
-    {
-      secs--;
-      nsecs += NUM_NSECS_IN_SEC;
-    }
-  totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
-  stats_.hdfsConnectionTime += totalnsecs;
-    
-  if (mode == EX_LOB_CREATE) 
-    { 
-      // check if file is already created
-      hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
-      if (fInfo != NULL) 
-	{
-	  hdfsFreeFileInfo(fInfo, 1);
-	  return LOB_DATA_FILE_CREATE_ERROR;
-	} 
-      openFlags = O_WRONLY | O_CREAT;   
-      fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags, bufferSize, replication, blockSize);
-      if (!fdData_) 
-	{
-          return LOB_DATA_FILE_CREATE_ERROR;
-	}
-      hdfsCloseFile(fs_, fdData_);
-      fdData_ = NULL;
-     
-    }
-  lobGlobalHeap_ = lobGlobals->getHeap();    
-  return LOB_OPER_OK;
-    
-}
-#endif
-
 Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode, 
                                char *dir, 
 			       LobsStorage storage,
@@ -573,34 +475,34 @@ Ex_Lob_Error ExLob::emptyDirectory()
     int numExistingFiles=0;
     hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, lobDataFile_);
     if (fileInfos == NULL)
-      {
-        return LOB_DATA_FILE_NOT_FOUND_ERROR; //here a directory
-      }
+    {
+      return LOB_DIR_NAME_ERROR;
+    }
 
     fileInfos = hdfsListDirectory(fs_, lobDataFile_, &numExistingFiles);
-    if (fileInfos == NULL)
-      {
-        return LOB_OPER_OK;
-      }
-    
-    for (int i = 0; i < numExistingFiles; i++) 
+    if (fileInfos == NULL) // empty directory
     {
-#ifdef USE_HADOOP_1
-      int retCode = hdfsDelete(fs_, fileInfos[i].mName);
-#else
-      int retCode = hdfsDelete(fs_, fileInfos[i].mName, 0);
-#endif
+      return LOB_OPER_OK;
+    }
+
+    NABoolean error = FALSE;
+    for (int i = 0; ((NOT error) && (i < numExistingFiles)); i++) 
+    {
+      // if dir, recursively delete it and everything under it
+      int retCode = hdfsDelete(fs_, fileInfos[i].mName, 1);
       if (retCode !=0)
       {
-        //ex_assert(retCode == 0, "delete returned error");
-        return LOB_DATA_FILE_DELETE_ERROR;
+        error = TRUE;
       }
     }
+
     if (fileInfos)
     {
       hdfsFreeFileInfo(fileInfos, numExistingFiles);
     }
-    
+
+    if (error)
+      return LOB_DATA_FILE_DELETE_ERROR;
 
     return LOB_OPER_OK;
 }
@@ -2274,6 +2176,7 @@ Ex_Lob_Error ExLobsOper (
   if (globPtr == NULL)
     {
       if ((operation == Lob_Init) ||
+          (operation == Lob_Empty_Directory) ||
           (operation == Lob_Data_Mod_Check))
 	{
 	  globPtr = (void *) new ExLobGlobals();

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index 416529d..2cce09e 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -481,10 +481,7 @@ class ExLob
 
   // dirPath: path to needed directory (includes directory name)
   // modTS is the latest timestamp on any file/dir under dirPath.
-  // numFilesInDir is the total number of files under dirPath.
-  // This method validates that current modTS is not greater then input modTS
-  // and current number of files in dirPath are the same as input numFilesInDir.
-  // If either condition is not true, then check fails.
+  // This method validates that current modTS is not greater then input modTS.
   // Return: LOB_OPER_OK, if passes. LOB_DATA_MOD_CHECK_ERROR, if fails.
   Ex_Lob_Error dataModCheck(
        char * dirPath, 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBinterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h
index 689e422..859417e 100644
--- a/core/sql/exp/ExpLOBinterface.h
+++ b/core/sql/exp/ExpLOBinterface.h
@@ -293,10 +293,7 @@ Lng32 ExpLOBinterfacePurgeBackupLobDataFile(void *& lobGlob,  char
*hdfsServer,
 
 // dirPath: path to needed directory (includes directory name)
 // modTS is the latest timestamp on any file/dir under dirPath.
-// numFilesInDir is the total number of files under dirPath.
-// This method validates that current modTS is not greater then input modTS
-// and current number of files in dirPath are the same as input numFilesInDir.
-// If either condition is not true, then check fails.
+// This method validates that current modTS is not greater then input modTS.
 // Return: 1, if check fails. 0, if passes. -1, if error.
 Lng32 ExpLOBinterfaceDataModCheck(void * lobGlob,
                                   char * dirPath,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/generator/GenFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenFastTransport.cpp b/core/sql/generator/GenFastTransport.cpp
index 7e1ee3e..6a4b4e7 100644
--- a/core/sql/generator/GenFastTransport.cpp
+++ b/core/sql/generator/GenFastTransport.cpp
@@ -624,7 +624,7 @@ PhysicalFastExtract::codeGen(Generator *generator)
   {
     newTdb->setIsHiveInsert(1);
     newTdb->setIncludeHeader(0);
-    setOverwriteHiveTable( getOverwriteHiveTable());
+    newTdb->setOverwriteHiveTable( getOverwriteHiveTable());
   }
   else
   {

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/generator/GenRelExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp
index 55eba37..7629657 100644
--- a/core/sql/generator/GenRelExeUtil.cpp
+++ b/core/sql/generator/GenRelExeUtil.cpp
@@ -3237,12 +3237,11 @@ short ExeUtilFastDelete::codeGen(Generator * generator)
 			    (ex_cri_desc *)(generator->getCriDesc(Generator::DOWN)),
 			    (queue_index)getDefault(GEN_DDL_SIZE_DOWN),
 			    (queue_index)getDefault(GEN_DDL_SIZE_UP),
-#pragma nowarn(1506)   // warning elimination 
 			    getDefault(GEN_DDL_NUM_BUFFERS),
 			    getDefault(GEN_DDL_BUFFER_SIZE),
 			    isHiveTable(),
-			    hiveTableLocation, hiveHdfsHost, hiveHdfsPort);
-#pragma warn(1506)  // warning elimination 
+			    hiveTableLocation, hiveHdfsHost, hiveHdfsPort,
+                            hiveModTS_);
 
   if (doPurgedataCat_)
     exe_util_tdb->setDoPurgedataCat(TRUE);

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp
index b81fdcf..9c3f0c2 100644
--- a/core/sql/optimizer/BindRelExpr.cpp
+++ b/core/sql/optimizer/BindRelExpr.cpp
@@ -9195,7 +9195,8 @@ RelExpr *Insert::bindNode(BindWA *bindWA)
                           TRUE,
                           new (bindWA->wHeap()) NAString(tableDir),
                           new (bindWA->wHeap()) NAString(hostName),
-                          hdfsPort);
+                          hdfsPort,
+                          hTabStats->getModificationTS());
 
       //new root to prevent  error 4056 when binding
       newRelExpr = new (bindWA->wHeap()) RelRoot(newRelExpr);

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/HDFSHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp
index fda6611..0d6fd34 100644
--- a/core/sql/optimizer/HDFSHook.cpp
+++ b/core/sql/optimizer/HDFSHook.cpp
@@ -1070,59 +1070,41 @@ void HHDFSTableStats::print(FILE *ofd)
   fprintf(ofd,"====================================================================\n");
 }
 
-//extern __thread hdfsFS *globalFS;
-hdfsFS *globalFS;
-
 NABoolean HHDFSTableStats::connectHDFS(const NAString &host, Int32 port)
 {
   NABoolean result = TRUE;
 
   // establish connection to HDFS if needed
-  if (globalFS == NULL ||
-      *globalFS == NULL ||
+  if (fs_ == NULL ||
       currHdfsHost_ != host ||
       currHdfsPort_ != port)
     {
-      if (globalFS && *globalFS)
-        disconnectHDFS();
-
-      if (globalFS == NULL)
+      if (fs_)
         {
-          globalFS = new hdfsFS;
-          *globalFS = NULL;
+          hdfsDisconnect(fs_);
+          fs_ = NULL;
         }
-
-      if (*globalFS == NULL)
+      fs_ = hdfsConnect(host, port);
+      
+      if (fs_ == NULL)
         {
-          *globalFS = hdfsConnect(host, port);
+          NAString errMsg("hdfsConnect to ");
+          errMsg += host;
+          errMsg += ":";
+          errMsg += port;
+          errMsg += " failed";
+          diags_.recordError(errMsg, "HHDFSTableStats::connectHDFS");
+          result = FALSE;
         }
-
       currHdfsHost_ = host;
       currHdfsPort_ = port;
     }
-
-  fs_ = *globalFS;
-  if (fs_ == NULL)
-    {
-      NAString errMsg("hdfsConnect to ");
-      errMsg += host;
-      errMsg += ":";
-      errMsg += port;
-      errMsg += " failed";
-      diags_.recordError(errMsg, "HHDFSTableStats::connectHDFS");
-      result = FALSE;
-    }
-  
   return result;
 }
 
 void HHDFSTableStats::disconnectHDFS()
 {
-  if (globalFS && *globalFS)
-    {
-      hdfsDisconnect(*globalFS);
-      *globalFS = NULL;
-    }
-
+  if (fs_)
+    hdfsDisconnect(fs_);
   fs_ = NULL;
 }

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index 81eb5b0..66d18b6 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -7308,6 +7308,9 @@ NATable * NATableDB::get(const ExtendedQualName* key, BindWA* bindWA,
NABoolean
      }
   }
 
+  // the reload cqd will be set during aqr after compiletime and runtime
+  // timestamp mismatch is detected.
+  // If set, reload hive metadata.
   if ((cachedNATable->isHiveTable()) &&
       (CmpCommon::getDefault(HIVE_DATA_MOD_CHECK) == DF_ON) &&
       (CmpCommon::getDefault(TRAF_RELOAD_NATABLE_CACHE) == DF_ON))

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/RelExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelExeUtil.h b/core/sql/optimizer/RelExeUtil.h
index 5d0150b..5c782e9 100644
--- a/core/sql/optimizer/RelExeUtil.h
+++ b/core/sql/optimizer/RelExeUtil.h
@@ -1034,7 +1034,8 @@ public:
 		    NABoolean isHiveTable = FALSE,
 		    NAString * hiveTableLocation = NULL,
                     NAString * hiveHostName = NULL,
-                    Int32 hiveHdfsPort = 0)
+                    Int32 hiveHdfsPort = 0,
+                    Int64 hiveModTS = -1)
        : ExeUtilExpr(FAST_DELETE_, name, exprNode, NULL, stmtText, stmtTextCharSet, oHeap),
          doPurgedataCat_(doPurgedataCat),
          noLog_(noLog), ignoreTrigger_(ignoreTrigger),
@@ -1044,7 +1045,8 @@ public:
          offlineTable_(FALSE),
          doLabelPurgedata_(FALSE),
          numLOBs_(0),
-         isHiveTable_(isHiveTable)
+         isHiveTable_(isHiveTable),
+         hiveModTS_(hiveModTS)
   {
     if (isHiveTable )
       {
@@ -1123,6 +1125,9 @@ private:
   NAString  hiveTableLocation_;
   NAString hiveHostName_;
   Int32 hiveHdfsPort_;
+
+  // timestamp of hiveTableLocation. 
+  Int64 hiveModTS_;
 };
 
 class ExeUtilMaintainObject : public ExeUtilExpr

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/RelFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelFastTransport.cpp b/core/sql/optimizer/RelFastTransport.cpp
index 45d4168..4734d2a 100644
--- a/core/sql/optimizer/RelFastTransport.cpp
+++ b/core/sql/optimizer/RelFastTransport.cpp
@@ -98,6 +98,7 @@ RelExpr * FastExtract::copyTopNode(RelExpr *derivedNode,
   result->recordSeparator_ = recordSeparator_ ;
   result->selectList_ = selectList_;
   result->isSequenceFile_ = isSequenceFile_;
+  result->overwriteHiveTable_ = overwriteHiveTable_;
 
   return RelExpr::copyTopNode(result, outHeap);
 }

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED003 b/core/sql/regress/hive/EXPECTED003
index 79cdb5a..0d0c3d0 100644
--- a/core/sql/regress/hive/EXPECTED003
+++ b/core/sql/regress/hive/EXPECTED003
@@ -2,21 +2,18 @@
 >>set schema hive.hive;
 
 --- SQL operation complete.
->>cqd attempt_esp_parallelism 'off';
-
---- SQL operation complete.
->>cqd hive_max_esps  '1';
-
---- SQL operation complete.
->>cqd PARALLEL_NUM_ESPS '1';
-
---- SQL operation complete.
+>>--cqd attempt_esp_parallelism 'off';
+>>--cqd hive_max_esps  '1';
+>>--cqd PARALLEL_NUM_ESPS '1';
 >>cqd HIVE_MAX_STRING_LENGTH '25' ;
 
 --- SQL operation complete.
 >>cqd mode_seahive 'ON';
 
 --- SQL operation complete.
+>>cqd auto_query_retry_warnings 'ON';
+
+--- SQL operation complete.
 >>
 >>prepare s from insert into hive.ins_customer select * from hive.customer;
 
@@ -135,6 +132,10 @@ P_PROMO_SK   P_PROMO_ID                 P_START_DATE_SK  P_END_DATE_SK
 P_ITEM_S
 
 *** ERROR[8822] The statement was not prepared.
 
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[4023] The degree of each row value constructor (20) must equal the degree of
the target table column list (19).
+
 >>-- number of columns doesn't match
 >>
 >>prepare s from
@@ -144,6 +145,10 @@ P_PROMO_SK   P_PROMO_ID                 P_START_DATE_SK  P_END_DATE_SK
 P_ITEM_S
 
 *** ERROR[8822] The statement was not prepared.
 
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[4039] Column T_TIME_SK is of type INTEGER, incompatible with the value's type,
CHAR(1).
+
 >>-- wrong data types
 >>
 >>
@@ -172,6 +177,10 @@ P_PROMO_SK   P_PROMO_ID                 P_START_DATE_SK  P_END_DATE_SK
 P_ITEM_S
 >>
 >>insert OVERWRITE TABLE hive.ins_customer_address select * from hive.customer_address;
 
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
 --- 50000 row(s) inserted.
 >>
 >>select count(*) from hive.customer_address;
@@ -193,6 +202,10 @@ P_PROMO_SK   P_PROMO_ID                 P_START_DATE_SK  P_END_DATE_SK
 P_ITEM_S
 >>--execute  again
 >>insert OVERWRITE TABLE hive.ins_customer_address select * from hive.customer_address;
 
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
 --- 50000 row(s) inserted.
 >>
 >>select count(*) from hive.customer_address;
@@ -595,7 +608,7 @@ test lp bug # 1355477
 > SS_SOLD_DATE_SK  SS_STORE_SK  SS_QUANTITY
 > ---------------  -----------  -----------
 >>--execute again --overwrite should get rid og existing data from previous run
->>control query shape union(cut,esp_exchange(cut));
+>>control query shape union(cut, esp_exchange(cut));
 
 --- SQL operation complete.
 >>prepare s from insert overwrite table ins_store_sales_summary select ss_sold_date_sk,ss_store_sk,
sum (ss_quantity) from store_sales group by  ss_sold_date_sk ,ss_store_sk;
@@ -624,6 +637,10 @@ LC   RC   OP   OPERATOR              OPT       DESCRIPTION          
CARD
 --- SQL operation complete.
 >>execute s;
 
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
 --- 12768 row(s) inserted.
 >>control query shape cut;
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005
index 8e26184..eaa414e 100644
--- a/core/sql/regress/hive/EXPECTED005
+++ b/core/sql/regress/hive/EXPECTED005
@@ -161,11 +161,12 @@ Y                                          9525
 >>insert into newtable values ('abc');
 
 --- 1 row(s) inserted.
->>cqd query_cache '0';
-
---- SQL operation complete.
 >>select * from newtable;
 
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
 A                        
 -------------------------
 
@@ -173,9 +174,6 @@ abc
 
 --- 1 row(s) selected.
 >>-- expect to see the row, but only because query cache is off
->>cqd query_cache reset;
-
---- SQL operation complete.
 >>
 >>insert into hiveregr5.newtable2 values ('xyz');
 
@@ -289,9 +287,6 @@ A            B
 >>-- overwrite the table with auto-generated partitions
 >>sh regrhive.ksh -v -f $REGRTSTDIR/TEST005_d.hive.sql;
 >>
->>cqd query_cache '0';
-
---- SQL operation complete.
 >>prepare s4 from 
 +>  select c_preferred_cust_flag,
 +>         count(*) 
@@ -652,14 +647,18 @@ C1           C2                    C3                         C4   
           C
 >>sh echo "create table thive(a int);" > TEST005_junk;
 >>sh regrhive.ksh -f TEST005_junk;
 >>
->>select * from hive.hive.thive;
+>>select a from hive.hive.thive;
 
 --- 0 row(s) selected.
 >>
 >>sh echo "insert into thive values (1);" > TEST005_junk;
 >>sh regrhive.ksh -f TEST005_junk;
 >>
->>select * from hive.hive.thive;
+>>select a from hive.hive.thive;
+
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
 
 A          
 -----------
@@ -670,7 +669,7 @@ A
 >>insert into hive.hive.thive values (2);
 
 --- 1 row(s) inserted.
->>select * from hive.hive.thive;
+>>select a from hive.hive.thive;
 
 A          
 -----------
@@ -689,6 +688,19 @@ A
 >>sh echo "insert into thive values (1,2);" > TEST005_junk;
 >>sh regrhive.ksh -f TEST005_junk;
 >>
+>>select a from hive.hive.thive;
+
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry. 
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
+A          
+-----------
+
+          1
+
+--- 1 row(s) selected.
+>>
 >>select * from hive.hive.thive;
 
 A            B          

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED015
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED015 b/core/sql/regress/hive/EXPECTED015
index f1d6067..162a3bc 100644
--- a/core/sql/regress/hive/EXPECTED015
+++ b/core/sql/regress/hive/EXPECTED015
@@ -115,10 +115,11 @@
 >>cqd COMPRESSED_INTERNAL_FORMAT_DEFRAG_RATIO '100';
 
 --- SQL operation complete.
->>cqd query_cache '0';
+>>cqd HIVE_NUM_ESPS_PER_DATANODE '3';
 
 --- SQL operation complete.
->>cqd HIVE_NUM_ESPS_PER_DATANODE '3';
+>>
+>>cqd auto_query_retry_warnings 'ON';
 
 --- SQL operation complete.
 >>
@@ -209,9 +210,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.T015T2
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.T015T2
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.T015T2
        Rows Processed: 5 
-Task:  PREPARATION     Status: Ended      ET: 00:00:00.184
+Task:  PREPARATION     Status: Ended      ET: 00:00:00.051
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.T015T2
-Task:  COMPLETION      Status: Ended      ET: 00:00:00.359
+Task:  COMPLETION      Status: Ended      ET: 00:00:00.781
 
 --- 5 row(s) loaded.
 >>
@@ -233,9 +234,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.T015T2
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.T015T2
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.T015T2
        Rows Processed: 5 
-Task:  PREPARATION     Status: Ended      ET: 00:00:00.176
+Task:  PREPARATION     Status: Ended      ET: 00:00:00.162
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.T015T2
-Task:  COMPLETION      Status: Ended      ET: 00:00:00.203
+Task:  COMPLETION      Status: Ended      ET: 00:00:00.183
 
 --- 5 row(s) loaded.
 >>
@@ -258,9 +259,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.T015T2
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.T015T2
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.T015T2
        Rows Processed: 5 
-Task:  PREPARATION     Status: Ended      ET: 00:00:00.195
+Task:  PREPARATION     Status: Ended      ET: 00:00:00.064
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.T015T2
-Task:  COMPLETION      Status: Ended      ET: 00:00:00.318
+Task:  COMPLETION      Status: Ended      ET: 00:00:00.327
 
 --- 5 row(s) loaded.
 >>
@@ -288,9 +289,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.T015T2
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.T015T2
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.T015T2
        Rows Processed: 5 
-Task:  PREPARATION     Status: Ended      ET: 00:00:00.179
+Task:  PREPARATION     Status: Ended      ET: 00:00:00.164
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.T015T2
-Task:  COMPLETION      Status: Ended      ET: 00:00:00.380
+Task:  COMPLETION      Status: Ended      ET: 00:00:00.268
 
 --- 5 row(s) loaded.
 >>
@@ -461,9 +462,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.CUSTOMER_ADDRE
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
        Rows Processed: 5000 
-Task:  PREPARATION     Status: Ended      ET: 00:00:08.673
+Task:  PREPARATION     Status: Ended      ET: 00:00:08.885
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task:  COMPLETION      Status: Ended      ET: 00:00:00.402
+Task:  COMPLETION      Status: Ended      ET: 00:00:01.379
 
 --- 5000 row(s) loaded.
 >>
@@ -543,9 +544,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.CUSTOMER_ADDRE
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
        Rows Processed: 5000 
-Task:  PREPARATION     Status: Ended      ET: 00:00:01.062
+Task:  PREPARATION     Status: Ended      ET: 00:00:01.054
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
-Task:  COMPLETION      Status: Ended      ET: 00:00:00.294
+Task:  COMPLETION      Status: Ended      ET: 00:00:00.820
 
 --- 5000 row(s) loaded.
 >>
@@ -644,9 +645,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOG
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
        Rows Processed: 5000 
-Task:  PREPARATION     Status: Ended      ET: 00:00:04.130
+Task:  PREPARATION     Status: Ended      ET: 00:00:03.594
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task:  COMPLETION      Status: Ended      ET: 00:00:00.631
+Task:  COMPLETION      Status: Ended      ET: 00:00:00.323
 
 --- 5000 row(s) loaded.
 >>
@@ -691,8 +692,6 @@ CD_DEMO_SK   CD_GENDER  CD_MARITAL_STATUS  CD_EDUCATION_STATUS   CD_PURCHASE_EST
 >>---------------------
 >>select count(*) from hive.hive.customer_demographics where cd_demo_sk <= 5000;
 
-*** WARNING[6008] Statistics for column (CD_DEMO_SK) from table HIVE.HIVE.CUSTOMER_DEMOGRAPHICS
were not available. As a result, the access path chosen might not be the best possible.
-
 (EXPR)              
 --------------------
 
@@ -750,9 +749,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOG
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
        Rows Processed: 5000 
-Task:  PREPARATION     Status: Ended      ET: 00:00:06.200
+Task:  PREPARATION     Status: Ended      ET: 00:00:05.596
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task:  COMPLETION      Status: Ended      ET: 00:00:01.735
+Task:  COMPLETION      Status: Ended      ET: 00:00:01.203
 
 --- 5000 row(s) loaded.
 >>
@@ -1262,9 +1261,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOG
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
        Rows Processed: 5000 
-Task:  PREPARATION     Status: Ended      ET: 00:00:06.517
+Task:  PREPARATION     Status: Ended      ET: 00:00:04.891
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task:  COMPLETION      Status: Ended      ET: 00:00:02.242
+Task:  COMPLETION      Status: Ended      ET: 00:00:02.090
 
 --- 5000 row(s) loaded.
 >>
@@ -1305,11 +1304,11 @@ Task:  DISABLE INDEXE  Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOG
 Task:  DISABLE INDEXE  Status: Ended      Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
        Rows Processed: 1000 
-Task:  PREPARATION     Status: Ended      ET: 00:00:04.908
+Task:  PREPARATION     Status: Ended      ET: 00:00:04.588
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task:  COMPLETION      Status: Ended      ET: 00:00:01.357
+Task:  COMPLETION      Status: Ended      ET: 00:00:01.075
 Task:  POPULATE INDEX  Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task:  POPULATE INDEX  Status: Ended      ET: 00:00:12.289
+Task:  POPULATE INDEX  Status: Ended      ET: 00:00:11.865
 
 --- 1000 row(s) loaded.
 >>
@@ -1354,11 +1353,11 @@ Task:  DISABLE INDEXE  Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOG
 Task:  DISABLE INDEXE  Status: Ended      Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
        Rows Processed: 5000 
-Task:  PREPARATION     Status: Ended      ET: 00:00:05.592
+Task:  PREPARATION     Status: Ended      ET: 00:00:05.812
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task:  COMPLETION      Status: Ended      ET: 00:00:02.018
+Task:  COMPLETION      Status: Ended      ET: 00:00:02.106
 Task:  POPULATE INDEX  Status: Started    Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task:  POPULATE INDEX  Status: Ended      ET: 00:00:14.563
+Task:  POPULATE INDEX  Status: Ended      ET: 00:00:12.813
 
 --- 5000 row(s) loaded.
 >>
@@ -1532,9 +1531,9 @@ Task:  CLEANUP         Status: Started    Object: TRAFODION.HBASE."customer_addr
 Task:  CLEANUP         Status: Ended      Object: TRAFODION.HBASE."customer_address_delim"
 Task:  PREPARATION     Status: Started    Object: TRAFODION.HBASE."customer_address_delim"
        Rows Processed: 5000 
-Task:  PREPARATION     Status: Ended      ET: 00:00:04.154
+Task:  PREPARATION     Status: Ended      ET: 00:00:03.585
 Task:  COMPLETION      Status: Started    Object: TRAFODION.HBASE."customer_address_delim"
-Task:  COMPLETION      Status: Ended      ET: 00:00:01.729
+Task:  COMPLETION      Status: Ended      ET: 00:00:01.575
 
 --- 5000 row(s) loaded.
 >>

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST003 b/core/sql/regress/hive/TEST003
index 7e3641b..3761906 100644
--- a/core/sql/regress/hive/TEST003
+++ b/core/sql/regress/hive/TEST003
@@ -66,11 +66,12 @@ order by s desc;
 log LOG003 clear;
 
 set schema hive.hive;
-cqd attempt_esp_parallelism 'off';
-cqd hive_max_esps  '1';
-cqd PARALLEL_NUM_ESPS '1';
+--cqd attempt_esp_parallelism 'off';
+--cqd hive_max_esps  '1';
+--cqd PARALLEL_NUM_ESPS '1';
 cqd HIVE_MAX_STRING_LENGTH '25' ;
 cqd mode_seahive 'ON';
+cqd auto_query_retry_warnings 'ON';
 
 prepare s from insert into hive.ins_customer select * from hive.customer;
 execute s;
@@ -187,7 +188,7 @@ sh diff -b LOG003_ORIG_STORE_SALES_SUMMARY.DAT LOG003_INS_STORE_SALES_SUMMARY.DA
 
 log LOG003;
 --execute again --overwrite should get rid og existing data from previous run
-control query shape union(cut,esp_exchange(cut));
+control query shape union(cut, esp_exchange(cut));
 prepare s from insert overwrite table ins_store_sales_summary select ss_sold_date_sk,ss_store_sk,
sum (ss_quantity) from store_sales group by  ss_sold_date_sk ,ss_store_sk; 
 explain options 'f' s;
 execute s;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST005 b/core/sql/regress/hive/TEST005
index e44b633..cd3fa56 100644
--- a/core/sql/regress/hive/TEST005
+++ b/core/sql/regress/hive/TEST005
@@ -144,10 +144,8 @@ where c_customer_sk between 20000 and 39999;
 select * from newtable;
 -- no rows, but should know the new table
 insert into newtable values ('abc');
-cqd query_cache '0';
 select * from newtable;
 -- expect to see the row, but only because query cache is off
-cqd query_cache reset;
 
 insert into hiveregr5.newtable2 values ('xyz');
 select * from hiveregr5.newtable2;
@@ -190,7 +188,6 @@ select a,b from newtable;
 -- overwrite the table with auto-generated partitions
 sh regrhive.ksh -v -f $REGRTSTDIR/TEST005_d.hive.sql;
 
-cqd query_cache '0';
 prepare s4 from 
   select c_preferred_cust_flag,
          count(*) 
@@ -292,14 +289,14 @@ sh regrhive.ksh -f TEST005_junk;
 sh echo "create table thive(a int);" > TEST005_junk;
 sh regrhive.ksh -f TEST005_junk;
 
-select * from hive.hive.thive;
+select a from hive.hive.thive;
 
 sh echo "insert into thive values (1);" > TEST005_junk;
 sh regrhive.ksh -f TEST005_junk;
 
-select * from hive.hive.thive;
+select a from hive.hive.thive;
 insert into hive.hive.thive values (2);
-select * from hive.hive.thive;
+select a from hive.hive.thive;
 
 sh echo "drop table thive;" > TEST005_junk;
 sh regrhive.ksh -f TEST005_junk;
@@ -310,6 +307,8 @@ sh regrhive.ksh -f TEST005_junk;
 sh echo "insert into thive values (1,2);" > TEST005_junk;
 sh regrhive.ksh -f TEST005_junk;
 
+select a from hive.hive.thive;
+
 select * from hive.hive.thive;
 
 log;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST015
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST015 b/core/sql/regress/hive/TEST015
index ed0dd12..dfac48f 100644
--- a/core/sql/regress/hive/TEST015
+++ b/core/sql/regress/hive/TEST015
@@ -39,9 +39,10 @@ obey TEST015(setup);
 cqd COMPRESSED_INTERNAL_FORMAT 'ON';
 cqd COMPRESSED_INTERNAL_FORMAT_BMO 'ON';
 cqd COMPRESSED_INTERNAL_FORMAT_DEFRAG_RATIO '100';
-cqd query_cache '0';
 cqd HIVE_NUM_ESPS_PER_DATANODE '3';
 
+cqd auto_query_retry_warnings 'ON';
+
 obey TEST015(test_bulk_load_simple);
 
 log;


Mime
View raw message