trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hzel...@apache.org
Subject [1/2] incubator-trafodion git commit: TRAFODION-2002 check columns for Hive inserts
Date Thu, 19 May 2016 21:32:47 GMT
Repository: incubator-trafodion
Updated Branches:
  refs/heads/master c41b39bab -> 444cd9cc8


TRAFODION-2002 check columns for Hive inserts

Also TRAFODION-1904 column list in insert/select into Hive table is ignored

Added checks for matching number and data types, so that the
data to be inserted matches the column layout of the Hive target
table.

Using a column list for insert into a Hive table now will raise
SQL error 4223 instead of being ignored (so it is still not supported).


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3205c6a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3205c6a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3205c6a2

Branch: refs/heads/master
Commit: 3205c6a2470146297ea5215a3e82eab6b74c1541
Parents: 855759f
Author: Hans Zeller <hzeller@apache.org>
Authored: Wed May 18 22:25:21 2016 +0000
Committer: Hans Zeller <hzeller@apache.org>
Committed: Wed May 18 22:25:21 2016 +0000

----------------------------------------------------------------------
 core/sql/optimizer/BindRelExpr.cpp      | 44 +++++++++++++++++++++++++++-
 core/sql/optimizer/RelFastTransport.cpp |  4 +--
 core/sql/optimizer/RelFastTransport.h   | 14 ++++-----
 core/sql/regress/hive/EXPECTED003       | 26 ++++++++++++++++
 core/sql/regress/hive/TEST003           | 11 +++++++
 5 files changed, 89 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp
index 0fa2288..59d0b9b 100644
--- a/core/sql/optimizer/BindRelExpr.cpp
+++ b/core/sql/optimizer/BindRelExpr.cpp
@@ -9150,6 +9150,14 @@ RelExpr *Insert::bindNode(BindWA *bindWA)
         bindWA->setErrStatus();
         return this;
     }
+
+    // specifying a list of column names to insert to is not yet supported
+    if (insertColTree_) {
+      *CmpCommon::diags() << DgSqlCode(-4223)
+                          << DgString0("Target column list for insert into Hive table");
+      bindWA->setErrStatus();
+      return this;
+    }
      
     //    NABoolean isSequenceFile = (*hTabStats)[0]->isSequenceFile();
     const NABoolean isSequenceFile = hTabStats->isSequenceFile();
@@ -9160,7 +9168,7 @@ RelExpr *Insert::bindNode(BindWA *bindWA)
                                  new (bindWA->wHeap()) NAString(hiveTablePath),
                                  new (bindWA->wHeap()) NAString(hostName),
                                  hdfsPort,
-                                 TRUE,
+                                 getTableDesc(),
                                  new (bindWA->wHeap()) NAString(getTableName().getQualifiedNameObj().getObjectName()),
                                  FastExtract::FILE,
                                  bindWA->wHeap());
@@ -16676,6 +16684,40 @@ RelExpr * FastExtract::bindNode(BindWA *bindWA)
   // can also get from child Root compExpr
   ValueIdList vidList;
   childRETDesc->getValueIdList(vidList, USER_COLUMN);
+
+  if (isHiveInsert())
+    {
+      // validate number of columns and column types of the select list
+      ValueIdList tgtCols;
+
+      hiveTableDesc_->getUserColumnList(tgtCols);
+
+      if (vidList.entries() != tgtCols.entries())
+        {
+          // 4023 degree of row value constructor must equal that of target table
+          *CmpCommon::diags() << DgSqlCode(-4023)
+                              << DgInt0(vidList.entries())
+                              << DgInt1(tgtCols.entries());
+          bindWA->setErrStatus();
+          return NULL;
+        }
+
+      // Check that the source and target types are compatible.
+      for (CollIndex j=0; j<vidList.entries(); j++)
+        {
+          Assign *tmpAssign = new(bindWA->wHeap())
+            Assign(tgtCols[j].getItemExpr(), vidList[j].getItemExpr());
+
+          if ( CmpCommon::getDefault(ALLOW_IMPLICIT_CHAR_CASTING) == DF_ON )
+            tmpAssign->tryToDoImplicitCasting(bindWA);
+          const NAType *targetType = tmpAssign->synthesizeType();
+          if (!targetType) {
+            bindWA->setErrStatus();
+            return NULL;
+          }
+        }
+    }
+
   setSelectList(vidList);
 
   if (includeHeader())

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/optimizer/RelFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelFastTransport.cpp b/core/sql/optimizer/RelFastTransport.cpp
index 5049d76..8daa6c2 100644
--- a/core/sql/optimizer/RelFastTransport.cpp
+++ b/core/sql/optimizer/RelFastTransport.cpp
@@ -49,7 +49,7 @@ FastExtract::FastExtract(const FastExtract & other)
   targetName_ = other.targetName_;
   hdfsHostName_ = other.hdfsHostName_;
   hdfsPort_ = other.hdfsPort_;
-  isHiveInsert_ = other.isHiveInsert_;
+  hiveTableDesc_ = other.hiveTableDesc_;
   hiveTableName_ = other.hiveTableName_;
   delimiter_ = other.delimiter_;
   isAppend_ = other.isAppend_;
@@ -87,7 +87,7 @@ RelExpr * FastExtract::copyTopNode(RelExpr *derivedNode,
   result->targetName_ = targetName_;
   result->hdfsHostName_ = hdfsHostName_;
   result->hdfsPort_ = hdfsPort_;
-  result->isHiveInsert_= isHiveInsert_;
+  result->hiveTableDesc_= hiveTableDesc_;
   result->hiveTableName_ = hiveTableName_;
   result->delimiter_ = delimiter_;
   result->isAppend_ = isAppend_;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/optimizer/RelFastTransport.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelFastTransport.h b/core/sql/optimizer/RelFastTransport.h
index d576f6e..f0088e1 100644
--- a/core/sql/optimizer/RelFastTransport.h
+++ b/core/sql/optimizer/RelFastTransport.h
@@ -105,7 +105,7 @@ public :
     targetName_(*targName, oHeap),
     hdfsHostName_(oHeap),
     hdfsPort_(0),
-    isHiveInsert_(FALSE),
+    hiveTableDesc_(NULL),
     delimiter_(*delim, oHeap),
     isAppend_(isAppend),
     includeHeader_(needsHeader),
@@ -126,7 +126,7 @@ public :
     targetName_(*targName, oHeap),
     hdfsHostName_(oHeap),
     hdfsPort_(0),
-    isHiveInsert_(FALSE),
+    hiveTableDesc_(NULL),
     delimiter_(oHeap),
     isAppend_(FALSE),
     includeHeader_(FALSE),
@@ -146,7 +146,7 @@ public :
     targetName_(oHeap),
     hdfsHostName_(oHeap),
     hdfsPort_(0),
-    isHiveInsert_(FALSE),
+    hiveTableDesc_(NULL),
     delimiter_(oHeap),
     isAppend_(FALSE),
     includeHeader_(FALSE),
@@ -162,7 +162,7 @@ public :
       NAString* targName,
       NAString* hostName,
       Int32 portNum,
-      NABoolean isHiveInsert,
+      TableDesc *hiveTableDesc,
       NAString* hiveTableName,
       ExtractDest targType,
       CollHeap *oHeap = CmpCommon::statementHeap())
@@ -171,7 +171,7 @@ public :
     targetName_(*targName, oHeap),
     hdfsHostName_(*hostName, oHeap),
     hdfsPort_(portNum),
-    isHiveInsert_(isHiveInsert),
+    hiveTableDesc_(hiveTableDesc),
     hiveTableName_(*hiveTableName, oHeap),
     delimiter_(oHeap),
     isAppend_(FALSE),
@@ -269,7 +269,7 @@ public :
   const NAString& getHdfsHostName() const {return hdfsHostName_;}
   Int32 getHdfsPort() const {return hdfsPort_;}
   const NAString& getHiveTableName() const {return hiveTableName_;}
-  NABoolean isHiveInsert() const {return isHiveInsert_;}
+  NABoolean isHiveInsert() const {return (hiveTableDesc_ != NULL);}
   const NAString& getDelimiter() const {return delimiter_;}
   NABoolean isAppend() const {return isAppend_;}
   NABoolean includeHeader() const {return includeHeader_ ;}
@@ -321,7 +321,7 @@ private:
   NAString nullString_;
   NAString recordSeparator_;
   NABoolean isAppend_;
-  NABoolean isHiveInsert_;
+  TableDesc *hiveTableDesc_;
   NAString hiveTableName_;
   NABoolean overwriteHiveTable_;
   NABoolean isSequenceFile_;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/regress/hive/EXPECTED003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED003 b/core/sql/regress/hive/EXPECTED003
index 8c8006a..5648797 100644
--- a/core/sql/regress/hive/EXPECTED003
+++ b/core/sql/regress/hive/EXPECTED003
@@ -123,6 +123,32 @@ P_PROMO_SK   P_PROMO_ID                 P_START_DATE_SK  P_END_DATE_SK
 P_ITEM_S
 
 --- 20 row(s) selected.
 >>
+>>-- some negative tests
+>>insert into hive.ins_promotion (p_promo_sk, p_item_sk) select * from hive.promotion;
+
+*** ERROR[4223] Target column list for insert into Hive table not supported in this software
version.
+
+*** ERROR[8822] The statement was not prepared.
+
+>>-- target column list is not supported
+>>
+>>insert into hive.ins_promotion select *, p_promo_sk from hive.promotion;
+
+*** ERROR[4023] The degree of each row value constructor (20) must equal the degree of the
target table column list (19).
+
+*** ERROR[8822] The statement was not prepared.
+
+>>-- number of columns doesn't match
+>>
+>>prepare s from
++>insert into hive.ins_time_dim values ('a', 2, 3, 4, 5, 6, 'c', 'd', 'e', 'f');
+
+*** ERROR[4039] Column T_TIME_SK is of type INTEGER, incompatible with the value's type,
CHAR(1).
+
+*** ERROR[8822] The statement was not prepared.
+
+>>-- wrong data types
+>>
 >>
 >>--try new HIVE SYNTAX
 >>--------------

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/regress/hive/TEST003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST003 b/core/sql/regress/hive/TEST003
index f442420..ada9c87 100644
--- a/core/sql/regress/hive/TEST003
+++ b/core/sql/regress/hive/TEST003
@@ -87,6 +87,17 @@ select count(*) from hive.ins_promotion;
 --select from hive table where row delimiter was explicettely specified
 select [first 20] * from hive.ins_promotion order by p_promo_sk ;
 
+-- some negative tests
+insert into hive.ins_promotion (p_promo_sk, p_item_sk) select * from hive.promotion;
+-- target column list is not supported
+
+insert into hive.ins_promotion select *, p_promo_sk from hive.promotion;
+-- number of columns doesn't match
+
+prepare s from
+insert into hive.ins_time_dim values ('a', 2, 3, 4, 5, 6, 'c', 'd', 'e', 'f');
+-- wrong data types
+
 
 --try new HIVE SYNTAX
 --------------


Mime
View raw message