trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sand...@apache.org
Subject [1/5] incubator-trafodion git commit: Fix for AQR for missing hive tables and misc lob related fixes TRAFODION-2770
Date Fri, 20 Oct 2017 06:15:25 GMT
Repository: incubator-trafodion
Updated Branches:
  refs/heads/master 5071a2088 -> c7199d5aa


Fix for AQR for missing hive tables and misc lob related fixes TRAFODION-2770


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/17ba6b8a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/17ba6b8a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/17ba6b8a

Branch: refs/heads/master
Commit: 17ba6b8acb69a3feb3cfd3c67ae6abc37d30b242
Parents: f49b86f
Author: Sandhya Sundaresan <sandhya@apache.org>
Authored: Mon Oct 16 17:54:18 2017 +0000
Committer: Sandhya Sundaresan <sandhya@apache.org>
Committed: Mon Oct 16 17:54:18 2017 +0000

----------------------------------------------------------------------
 core/sql/cli/SessionDefaults.cpp    |  2 +-
 core/sql/cli/Statement.cpp          | 19 ++++++++---
 core/sql/executor/ExExeUtilLoad.cpp |  3 +-
 core/sql/executor/ExHdfsScan.cpp    | 10 ++++--
 core/sql/exp/ExpLOBaccess.cpp       | 21 ++++++++++--
 core/sql/optimizer/ItemFunc.h       |  2 +-
 core/sql/regress/hive/DIFF003.KNOWN | 56 +++++++++++++++++++++++++++-----
 core/sql/regress/hive/FILTER005     |  2 ++
 8 files changed, 96 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/cli/SessionDefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/SessionDefaults.cpp b/core/sql/cli/SessionDefaults.cpp
index 25dd67c..966ce77 100644
--- a/core/sql/cli/SessionDefaults.cpp
+++ b/core/sql/cli/SessionDefaults.cpp
@@ -824,7 +824,7 @@ static const AQRInfo::AQRErrorMap aqrErrorMap[] =
   AQREntry(   8575,      0,      1,     0,      1,   0, "",    0,     0), 
 
   // table not found
-  AQREntry(   8577,      0,      1,     0,      1,   0, "",    0,     0), 
+  AQREntry(   8577,      0,      1,     0,      1,   2, "04:05",    0,  0), 
 
   // sim check failure
   AQREntry(   8579,      0,      1,     0,      1,   0, "",    0,     0),

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/cli/Statement.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Statement.cpp b/core/sql/cli/Statement.cpp
index 2169373..52f7e5a 100644
--- a/core/sql/cli/Statement.cpp
+++ b/core/sql/cli/Statement.cpp
@@ -2051,11 +2051,22 @@ RETCODE Statement::doHiveTableSimCheck(TrafSimilarityTableInfo *si,
                 << DgString2(getLobErrStr(intParam1))
                 << DgInt0(intParam1)
                 << DgInt1(0);
-
-      if (intParam1 == LOB_DATA_FILE_NOT_FOUND_ERROR)
+      if (intParam1 == LOB_DATA_READ_ERROR)
         {
-          diagsArea << DgSqlCode(-EXE_TABLE_NOT_FOUND)
-                    << DgString0(si->tableName());
+          if ((failedLocBufLen > 0) && (strlen(failedLocBuf) > 0))
+            {
+              char errBuf[strlen(si->tableName()) + 100 + failedLocBufLen];
+              str_sprintf(errBuf, "%s (fileLoc: %s)",
+                          si->tableName(), failedLocBuf);
+              diagsArea << DgSqlCode(-EXE_TABLE_NOT_FOUND)
+                        << DgString0(errBuf);              
+            }
+          else
+            {
+              diagsArea << DgSqlCode(-EXE_TABLE_NOT_FOUND)
+                        << DgString0(si->tableName());
+            }
+          simCheckFailed = TRUE;
         }
 
       return ERROR;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index 57345bf..a945556 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -2758,7 +2758,8 @@ void ExExeUtilLobExtractTcb::freeResources()
     getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()->
     getStatement()->getContext();
   //close any open cursors.
-   retcode = ExpLOBInterfaceSelectCursor
+  if (lobHandle_ and lobName_)
+    retcode = ExpLOBInterfaceSelectCursor
 	      (lobGlobs,
 	       lobName_, 
 	       lobLoc_,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index 3b9baa5..00a673a 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -619,10 +619,16 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
                   {
                     ComDiagsArea * diagsArea = NULL;
                     if (hdfsErrorDetail == ENOENT)
-                      ExRaiseSqlError(getHeap(), &diagsArea, 
+                      {
+                        char errBuf[strlen(hdfsScanTdb().tableName()) + 
+                                    strlen(hdfsFileName_) + 100];
+                        str_sprintf(errBuf, "%s (fileLoc: %s)",
+                                    hdfsScanTdb().tableName(), hdfsFileName_);
+                        ExRaiseSqlError(getHeap(), &diagsArea, 
                                       (ExeErrorCode)(EXE_TABLE_NOT_FOUND), NULL,
                                       NULL, NULL, NULL,
-                                      hdfsScanTdb().tableName());
+                                      errBuf);
+                      }
                     else
                       ExRaiseSqlError(getHeap(), &diagsArea, 
                                       (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 455fec1..6eced07 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -451,8 +451,25 @@ Ex_Lob_Error ExLob::dataModCheck(
   // find mod time of root dir
   hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, dirPath);
   if (fileInfos == NULL)
-    {
-      return LOB_DATA_FILE_NOT_FOUND_ERROR;
+    {       
+      Lng32 failedFileLen = strlen(dirPath);
+      Lng32 copyLen = (failedFileLen > (*failedLocBufLen-1) 
+                       ? (*failedLocBufLen-1) : failedFileLen);
+      Int32 hdfserror = errno;
+      char hdfsErrStr[20];
+      sprintf(hdfsErrStr,"(errno %d)",errno);
+      str_cpy_and_null(failedLocBuf, dirPath, copyLen,
+                       '\0', ' ', TRUE);
+      str_cat_c(failedLocBuf,hdfsErrStr);
+      *failedLocBufLen = copyLen;
+      if (errno)
+        {
+          // Allow for hdfs error. AQR will find the new hive mapped files
+          // if the hive table has been remapped to new data files
+          return LOB_DATA_FILE_NOT_FOUND_ERROR;
+        }
+      else
+        return LOB_DATA_READ_ERROR;
     }
     
   Int64 currModTS = fileInfos[0].mLastMod;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/optimizer/ItemFunc.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h
index e4bdf07..c60447d 100644
--- a/core/sql/optimizer/ItemFunc.h
+++ b/core/sql/optimizer/ItemFunc.h
@@ -2921,7 +2921,7 @@ public:
   short lobNum_;
   LobsStorage lobStorageType_;
   NAString lobStorageLocation_;
-  Int32 lobMaxSize_; // In MB units
+  Int64 lobMaxSize_; // In byte units
   Int32 lobMaxChunkMemSize_; //In MB Units
   Int32 lobGCLimit_ ;//In MB Units
   Int32 hdfsPort_;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/regress/hive/DIFF003.KNOWN
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/DIFF003.KNOWN b/core/sql/regress/hive/DIFF003.KNOWN
index 3480a0d..f8803ea 100644
--- a/core/sql/regress/hive/DIFF003.KNOWN
+++ b/core/sql/regress/hive/DIFF003.KNOWN
@@ -1,10 +1,50 @@
-192,195d191
-< *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry.
+543,546d542
+< *** WARNING[6008] Statistics for column (SS_SOLD_DATE_SK) from table HIVE.HIVE.STORE_SALES
were not available. As a result, the access path chosen might not be the best possible.
 < 
-< *** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: removed
+< *** WARNING[6008] Statistics for column (SS_STORE_SK) from table HIVE.HIVE.STORE_SALES
were not available. As a result, the access path chosen might not be the best possible.
 < 
-216a213,216
-> *** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry
was 0 seconds. See next entry for the error that caused this retry.
-> 
-> *** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
DataModMismatchDetails: removed
-> 
+555,560c551,557
+< 6 . 7 root 2.92E+006
+< 5 . 6 esp_exchange 1:2(hash2) 2.92E+006
+< 3 . 5 hive_insert INS_STORE_SALES_SUMM 2.92E+006
+< 2 . 3 hash_groupby 2.92E+006
+< 1 . 2 esp_exchange 2(hash2):2(hash2) 2.92E+006
+< . . 1 hive_scan STORE_SALES 2.92E+006
+---
+> 7 . 8 root 1.09E+004
+> 6 . 7 esp_exchange 1:2(hash2) 1.09E+004
+> 4 . 6 hive_insert INS_STORE_SALES_SUMM 1.09E+004
+> 3 . 4 hash_partial_groupby 1.09E+004
+> 2 . 3 esp_exchange 2(hash2):2(hash2) 1.09E+004
+> 1 . 2 hash_partial_groupby 1.09E+004
+> . . 1 hive_scan STORE_SALES 2.75E+006
+597,601c594
+< 3,10c3,4
+< < *** WARNING[6008] Statistics for column (SS_SOLD_DATE_SK) from table HIVE.HIVE.STORE_SALES
were not available. As a result, the access path chosen might not be the best possible.
+< <
+< < *** WARNING[6008] Statistics for column (SS_STORE_SK) from table HIVE.HIVE.STORE_SALES
were not available. As a result, the access path chosen might not be the best possible.
+< <
+---
+> 3,6c3,4
+615,618d607
+< *** WARNING[6008] Statistics for column (SS_SOLD_DATE_SK) from table HIVE.HIVE.STORE_SALES
were not available. As a result, the access path chosen might not be the best possible.
+< 
+< *** WARNING[6008] Statistics for column (SS_STORE_SK) from table HIVE.HIVE.STORE_SALES
were not available. As a result, the access path chosen might not be the best possible.
+< 
+627,633c616,623
+< 8 . 9 root 2.92E+006
+< 1 7 8 blocked_union 2.92E+006
+< 6 . 7 esp_exchange 1:2(hash2) 2.92E+006
+< 4 . 6 hive_insert INS_STORE_SALES_SUMM 2.92E+006
+< 3 . 4 hash_groupby 2.92E+006
+< 2 . 3 esp_exchange 2(hash2):2(hash2) 2.92E+006
+< . . 2 hive_scan STORE_SALES 2.92E+006
+---
+> 9 . 10 root 1.09E+004
+> 1 8 9 blocked_union 1.09E+004
+> 7 . 8 esp_exchange 1:2(hash2) 1.09E+004
+> 5 . 7 hive_insert INS_STORE_SALES_SUMM 1.09E+004
+> 4 . 5 hash_partial_groupby 1.09E+004
+> 3 . 4 esp_exchange 2(hash2):2(hash2) 1.09E+004
+> 2 . 3 hash_partial_groupby 1.09E+004
+> . . 2 hive_scan STORE_SALES 2.75E+006

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/17ba6b8a/core/sql/regress/hive/FILTER005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/FILTER005 b/core/sql/regress/hive/FILTER005
index b128157..08be9a6 100755
--- a/core/sql/regress/hive/FILTER005
+++ b/core/sql/regress/hive/FILTER005
@@ -31,4 +31,6 @@ sed "
 s/Logging Location: \/user\/trafodion\/bulkload\/logs\/ERR.*/Logging Location:@loggingLocation@/
 s/Logging Location: \/user\/trafodion\/bulkload\/logs\/TEST005\/ERR.*/Logging Location:@TEST005LoggingLocation@/
 s/Time:.*/Time:@time@/
+s/^\*\*\* WARNING\[8436\].*/Warning 8436 or 8577 removed/g
+s/^\*\*\* WARNING\[8577\].*/Warning 8436 or 8577 removed/g
 " $fil


Mime
View raw message