trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dbirds...@apache.org
Subject [01/14] incubator-trafodion git commit: jira-1443 final.
Date Mon, 21 Sep 2015 23:12:16 GMT
Repository: incubator-trafodion
Updated Branches:
  refs/heads/master a982a5c4a -> e1093020f


jira-1443 final.


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3d212c68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3d212c68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3d212c68

Branch: refs/heads/master
Commit: 3d212c68f2a4bca5388178068117da5f89119b80
Parents: 0434c60
Author: Qifan Chen <qfc@dev02.trafodion.org>
Authored: Thu Sep 17 03:54:24 2015 +0000
Committer: Qifan Chen <qfc@dev02.trafodion.org>
Committed: Thu Sep 17 03:54:24 2015 +0000

----------------------------------------------------------------------
 .../sql/scripts/install_hadoop_regr_test_env    |  15 ++-
 core/sql/common/ComSmallDefs.h                  |   5 +-
 core/sql/optimizer/BindRelExpr.cpp              |   3 +-
 core/sql/optimizer/BindWA.cpp                   |   4 +-
 core/sql/optimizer/BindWA.h                     |   4 +-
 core/sql/optimizer/NATable.cpp                  |  97 ++++++++--------
 core/sql/optimizer/NATable.h                    |  22 +++-
 core/sql/optimizer/ObjectNames.cpp              |  12 ++
 core/sql/optimizer/ObjectNames.h                |   3 +
 core/sql/sqlcomp/CmpSeabaseDDLtable.cpp         |   8 --
 core/sql/ustat/hs_cli.cpp                       |  12 ++
 core/sql/ustat/hs_globals.cpp                   |  24 +++-
 core/sql/ustat/hs_globals.h                     |  15 ++-
 core/sql/ustat/hs_la.cpp                        | 112 ++++++++++++++++++-
 core/sql/ustat/hs_la.h                          |  13 ++-
 core/sql/ustat/hs_parser.cpp                    |  82 +++++++-------
 core/sql/ustat/hs_util.cpp                      |   3 +
 17 files changed, 319 insertions(+), 115 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sqf/sql/scripts/install_hadoop_regr_test_env
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/install_hadoop_regr_test_env b/core/sqf/sql/scripts/install_hadoop_regr_test_env
index 292d2bd..ee1bac9 100755
--- a/core/sqf/sql/scripts/install_hadoop_regr_test_env
+++ b/core/sqf/sql/scripts/install_hadoop_regr_test_env
@@ -199,7 +199,6 @@ fi
   ./dsdgen -force $FORCE -dir $MY_TPCDS_DATA_DIR -scale $SCALE -table store       >>${MY_LOG_FILE}
2>&1
   ./dsdgen -force $FORCE -dir $MY_TPCDS_DATA_DIR -scale $SCALE -table promotion   >>${MY_LOG_FILE}
2>&1
 
-  echo "Copying generated data to HDFS..."
   $MY_HDFS_CMD dfs -mkdir /hive/tpcds                        >>${MY_LOG_FILE} 2>&1
   $MY_HDFS_CMD dfs -mkdir /hive/tpcds/date_dim               >>${MY_LOG_FILE} 2>&1
   $MY_HDFS_CMD dfs -mkdir /hive/tpcds/time_dim               >>${MY_LOG_FILE} 2>&1
@@ -214,6 +213,20 @@ fi
 
   cd $MY_TPCDS_DATA_DIR
 
+  which iconv >>${MY_LOG_FILE} 2>&1
+  if (( $? != 0 ))
+  then
+    echo "iconv utility not available. The data will be in ISO-8859-1 format."
+  else
+    echo "Converting the data into UTF-8 format ..."
+    for t in date_dim time_dim item customer customer_demographics household_demographics
customer_address store promotion store_sales
+      do
+        iconv -f ISO-8859-1 -t UTF-8 -o ${t}.utf8.dat ${t}.dat >>${MY_LOG_FILE} 2>&1
+        mv ${t}.utf8.dat ${t}.dat
+      done
+  fi
+
+  echo "Copying generated data to HDFS..."
   for t in date_dim time_dim item customer customer_demographics household_demographics customer_address
store promotion store_sales
     do
       $MY_HDFS_CMD dfs -put ${t}.dat /hive/tpcds/${t}        >>${MY_LOG_FILE} 2>&1

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/common/ComSmallDefs.h
----------------------------------------------------------------------
diff --git a/core/sql/common/ComSmallDefs.h b/core/sql/common/ComSmallDefs.h
index 028ca1b..6e893c5 100644
--- a/core/sql/common/ComSmallDefs.h
+++ b/core/sql/common/ComSmallDefs.h
@@ -116,7 +116,7 @@ typedef NABoolean               ComBoolean;
 #define HIVE_SYSTEM_CATALOG          "HIVE"
 #define HIVE_SYSTEM_SCHEMA           "HIVE"
 #define HIVE_STATS_CATALOG           "TRAFODION"
-#define HIVE_STATS_SCHEMA            "HIVESTATS"
+#define HIVE_STATS_SCHEMA            "\"_HIVESTATS_\""
 #define HIVE_EXT_SCHEMA_PREFIX       "_HV_"
 
 #define HBASE_SYSTEM_CATALOG          "HBASE"
@@ -127,6 +127,9 @@ typedef NABoolean               ComBoolean;
 #define HBASE_HISTINT_PK               "SB_HISTOGRAM_INTERVALS_PK"
 #define HBASE_EXT_SCHEMA_PREFIX       "_HB_"
 
+#define HBASE_STATS_CATALOG          "TRAFODION"
+#define HBASE_STATS_SCHEMA           "\"_HBASESTATS_\""
+
 #define TRAFODION_SYSCAT_LIT              "TRAFODION"
 #define SEABASE_SYSTEM_SCHEMA           "SEABASE"
 #define SEABASE_OLD_PRIVMGR_SCHEMA         "PRIVMGR_MD"

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp
index 2d400bb..74fa8f3 100644
--- a/core/sql/optimizer/BindRelExpr.cpp
+++ b/core/sql/optimizer/BindRelExpr.cpp
@@ -1573,7 +1573,8 @@ NATable *BindWA::getNATable(CorrName& corrName,
   // native HIVE or HBASE objects unless the allowExternalTables flag is set.  
   // allowExternalTables is set for drop table and SHOWDDL statements.  
   // TDB - may want to merge the Trafodion version with the native version.
-  if ((table) && (table->isExternalTable() && (! bindWA->allowExternalTables())))
+  if ((table) && (table->isExternalTable() && (! bindWA->allowExternalTables())
&&
+                  !table->isHistogramTable()))
     {
       *CmpCommon::diags() << DgSqlCode(-4258)
                           << DgTableName(table->getTableName().getQualifiedNameAsAnsiString());

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/BindWA.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindWA.cpp b/core/sql/optimizer/BindWA.cpp
index ce83134..b80edff 100644
--- a/core/sql/optimizer/BindWA.cpp
+++ b/core/sql/optimizer/BindWA.cpp
@@ -114,7 +114,7 @@ void BindScope::mergeOuterRefs(const ValueIdSet& other)
 // ***********************************************************************
 // BindWA()
 // ***********************************************************************
-BindWA::BindWA(SchemaDB *schemaDB, CmpContext* cmpContext, NABoolean inDDL)
+BindWA::BindWA(SchemaDB *schemaDB, CmpContext* cmpContext, NABoolean inDDL, NABoolean allowExtTables)
      : schemaDB_(schemaDB)
      , currentCmpContext_(cmpContext)
      , inputVars_(cmpContext ? cmpContext->statementHeap() : NULL)
@@ -131,7 +131,7 @@ BindWA::BindWA(SchemaDB *schemaDB, CmpContext* cmpContext, NABoolean inDDL)
   //     , inRIMaint_(FALSE)
      , inViewWithCheckOption_(NULL)
      , viewCount_(0)
-     , allowExternalTables_(FALSE)
+     , allowExternalTables_(allowExtTables)
      , errFlag_(FALSE)
      , uniqueNum_(0)
      , uniqueIudNum_(0) //++Triggers,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/BindWA.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindWA.h b/core/sql/optimizer/BindWA.h
index 11b1fdd..92406c5 100644
--- a/core/sql/optimizer/BindWA.h
+++ b/core/sql/optimizer/BindWA.h
@@ -1178,7 +1178,8 @@ public:
   // --------------------------------------------------------------------
   BindWA(SchemaDB *schemaDB,
   	 CmpContext *currentCmpContext = NULL,
-	 NABoolean inDDL = FALSE);
+	 NABoolean inDDL = FALSE,
+	 NABoolean allowExternalTables = FALSE);
 
   // copy ctor
   BindWA (const BindWA & orig, CollHeap * h=0) ; // not written
@@ -1797,7 +1798,6 @@ private:
   // --------------------------------------------------------------------
   NABoolean inDDL_;
 
-
   //---------------------------------------------------------------------
   // does the query tree contain any update nodes?
   //---------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index 5a68aa8..dbf8aed 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -4890,6 +4890,34 @@ Int64 lookupObjectUid( const QualifiedName& qualName
   return objectUID;
 }
 
+NABoolean NATable::fetchObjectUIDForNativeTable(const CorrName& corrName)
+{
+   NAString adjustedName = ComConvertNativeNameToTrafName
+         (corrName.getQualifiedNameObj().getCatalogName(),
+          corrName.getQualifiedNameObj().getUnqualifiedSchemaNameAsAnsiString(),
+          corrName.getQualifiedNameObj().getUnqualifiedObjectNameAsAnsiString());
+   QualifiedName extObjName (adjustedName, 3, STMTHEAP);
+
+   Lng32 diagsMark = CmpCommon::diags()->mark();
+   objectUID_ = ::lookupObjectUid(extObjName, COM_BASE_TABLE_OBJECT);
+
+   // If the objectUID is not found, then the table is not externally defined
+   // in Trafodion, set the objectUID to 0
+   // If an unexpected error occurs, then return with the error
+   if (objectUID_ <= 0)
+     {
+       if (CmpCommon::diags()->contains(-1389))
+         {
+           CmpCommon::diags()->rewind(diagsMark);
+           objectUID_ = 0;
+         }
+       else
+         return FALSE;
+     }
+
+   return TRUE;
+}
+
 // -----------------------------------------------------------------------
 // NATable::NATable() constructor
 // -----------------------------------------------------------------------
@@ -5110,6 +5138,12 @@ NATable::NATable(BindWA *bindWA,
     setIsExternalTable(TRUE);
   }
  
+  if (qualifiedName_.getQualifiedNameObj().isHistograms() || 
+      qualifiedName_.getQualifiedNameObj().isHistogramIntervals())
+  {
+    setIsHistogramTable(TRUE);
+  }
+ 
   insertMode_ = table_desc->body.table_desc.insertMode;
 
   setRecordLength(table_desc->body.table_desc.record_length);
@@ -5128,29 +5162,10 @@ NATable::NATable(BindWA *bindWA,
   // been defined in Trafodion use this value, otherwise, set to 0
   if (isHbaseCell_ || isHbaseRow_)
     {
-      NAString adjustedName = ComConvertNativeNameToTrafName
-         (corrName.getQualifiedNameObj().getCatalogName(),
-          corrName.getQualifiedNameObj().getUnqualifiedSchemaNameAsAnsiString(),
-          corrName.getQualifiedNameObj().getUnqualifiedObjectNameAsAnsiString());
-      QualifiedName extObjName (adjustedName, 3, STMTHEAP);
-
-      Lng32 diagsMark = CmpCommon::diags()->mark();
-      objectUID_ = ::lookupObjectUid(extObjName, COM_BASE_TABLE_OBJECT);
+      if ( !fetchObjectUIDForNativeTable(corrName) )
+        return;
 
-      // If the objectUID is not found, then the table is not externally defined
-      // in Trafodion, set the objectUID to 0
-      // If an unexpected error occurs, then return with the error
-      if (objectUID_ <= 0)
-        {
-          if (CmpCommon::diags()->contains(-1389))
-            {
-              CmpCommon::diags()->rewind(diagsMark);
-              objectUID_ = 0;
-            }
-          else
-            return;
-        }
-      else
+      if (objectUID_ > 0 )
         setHasExternalTable(TRUE);
     }
 
@@ -5781,31 +5796,12 @@ NATable::NATable(BindWA *bindWA,
   // If the HIVE table has been registered in Trafodion, get the objectUID
   // from Trafodion, otherwise, set it to 0.
   // TBD - does getQualifiedNameObj handle delimited names correctly?
-  NAString adjustedName = ComConvertNativeNameToTrafName 
-                            ( corrName.getQualifiedNameObj().getCatalogName(),
-                              corrName.getQualifiedNameObj().getSchemaName(),
-                              corrName.getQualifiedNameObj().getObjectName()); 
-  QualifiedName extObjName (adjustedName, 3, STMTHEAP);
-
-  Lng32 diagsMark = CmpCommon::diags()->mark();
-  objectUID_ = ::lookupObjectUid(extObjName, COM_BASE_TABLE_OBJECT);
-
-  // If the objectUID is not found, then the table is not externally defined
-  // in Trafodion, set the objectUID to 0
-  // If an unexpected error occurs, then return with the error
-  if (objectUID_ <= 0)
-    {
-      if (CmpCommon::diags()->contains(-1389))
-        {
-          CmpCommon::diags()->rewind(diagsMark);
-          objectUID_ = 0;
-        }
-      else
-        return;
-    }
-  else
+  if ( !fetchObjectUIDForNativeTable(corrName) )
+     return;
+
+  if ( objectUID_ > 0 )
     setHasExternalTable(TRUE);
- 
+
   // for HIVE objects, the schema owner and table owner is HIVE_ROLE_ID
   if (CmpCommon::context()->isAuthorizationEnabled())
   {
@@ -6883,10 +6879,19 @@ void NATable::setupPrivInfo()
 // Query the metadata to find the object uid of the table. This is used when
 // the uid for a metadata table is requested, since 0 is usually stored for
 // these tables.
+// 
+// On return, the "Object Not Found" error (1389) is filtered out from 
+// CmpCommon::diags().
 Int64 NATable::lookupObjectUid()
 {
+    Lng32 diagsMark = CmpCommon::diags()->mark();
+
     QualifiedName qualName = getExtendedQualName().getQualifiedNameObj();
     objectUID_ = ::lookupObjectUid(qualName, objectType_);
+
+    if (CmpCommon::diags()->contains(-1389))
+      CmpCommon::diags()->rewind(diagsMark);
+
     return objectUID_.get_value();
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/NATable.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.h b/core/sql/optimizer/NATable.h
index 6a22781..ec19d14 100644
--- a/core/sql/optimizer/NATable.h
+++ b/core/sql/optimizer/NATable.h
@@ -509,6 +509,7 @@ public:
 
   const ComUID &getCatalogUid() const           { return catalogUID_; }
   const ComUID &getSchemaUid() const            { return schemaUID_; }
+
   const ComUID &objectUid() const
   {
     if (objectUID_.get_value() == 0)
@@ -516,7 +517,17 @@ public:
     return objectUID_;
   }
 
-  Int64 lookupObjectUid();  // Used to look up uid on demand for metadata tables
+  // fetch the object UID that is associated with the external 
+  // table object (if any) for a native table.
+  // Set objectUID_ to 0 if no such external table exists;
+  // set objectUID_ to -1 if there is error during the fetch operation;
+  NABoolean fetchObjectUIDForNativeTable(const CorrName& corrName);
+
+
+  Int64 lookupObjectUid();  // Used to look up uid on demand for metadata tables.
+                            // On return, the "Object Not Found" error (-1389) 
+                            // is filtered out from CmpCommon::diags().
+
   bool isEnabledForDDLQI() const;
 
   const ComObjectType &getObjectType() const { return objectType_; }
@@ -710,6 +721,12 @@ public:
   NABoolean hasExternalTable() const
   {  return (flags_ & HAS_EXTERNAL_TABLE) != 0; }
 
+  void setIsHistogramTable( NABoolean value )
+  {  value ? flags_ |= IS_HISTOGRAM_TABLE : flags_ &= ~IS_HISTOGRAM_TABLE; }
+
+  NABoolean isHistogramTable() const
+  {  return (flags_ & IS_HISTOGRAM_TABLE) != 0; }
+
   const CheckConstraintList &getCheckConstraints() const
                                                 { return checkConstraints_; }
   const AbstractRIConstraintList &getUniqueConstraints() const
@@ -924,7 +941,8 @@ private:
     SERIALIZED_ENCODED_COLUMN = 0x00020000,
     SERIALIZED_COLUMN         = 0x00040000,
     IS_EXTERNAL_TABLE         = 0x00080000,
-    HAS_EXTERNAL_TABLE        = 0x00100000
+    HAS_EXTERNAL_TABLE        = 0x00100000,
+    IS_HISTOGRAM_TABLE        = 0x00200000
   };
     
   UInt32 flags_;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/ObjectNames.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ObjectNames.cpp b/core/sql/optimizer/ObjectNames.cpp
index f2fe28b..75a9462 100644
--- a/core/sql/optimizer/ObjectNames.cpp
+++ b/core/sql/optimizer/ObjectNames.cpp
@@ -670,6 +670,7 @@ NABoolean QualifiedName::isSeabasePrivMgrMD() const
     (getCatalogName(), getSchemaName());
 }
 
+
 // -----------------------------------------------------------------------
 // Methods for class CorrName
 // -----------------------------------------------------------------------
@@ -1385,6 +1386,17 @@ void QualifiedName::print(FILE* ofd, const char* indent, const char*
title) cons
 #endif
 }
 
+NABoolean QualifiedName::isHistograms() const
+{
+   return (getObjectName() == "SB_HISTOGRAMS");
+}
+
+NABoolean QualifiedName::isHistogramIntervals() const
+{
+   return (getObjectName() == "SB_HISTOGRAM_INTERVALS");
+}
+
+
 void ExtendedQualName::print(FILE* ofd, const char* indent, const char* title) const
 {
 #ifndef NDEBUG

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/ObjectNames.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ObjectNames.h b/core/sql/optimizer/ObjectNames.h
index d7f15cf..981025f 100644
--- a/core/sql/optimizer/ObjectNames.h
+++ b/core/sql/optimizer/ObjectNames.h
@@ -322,6 +322,9 @@ public:
   NABoolean isSeabaseMD() const;
   NABoolean isSeabasePrivMgrMD() const;
 
+  NABoolean isHistograms() const;
+  NABoolean isHistogramIntervals() const;
+
   void setObjectName(const NAString &objName)   { objectName_ = objName; }
   void setObjectNameSpace(ComAnsiNameSpace objNameSpace)   { objectNameSpace_ = objNameSpace;
}
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
index 36663f5..e7f7b29 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
@@ -249,14 +249,6 @@ void CmpSeabaseDDL::createSeabaseTableLike(
   ComObjectName srcTableName(createTableNode->getLikeSourceTableName(), COM_TABLE_NAME);
   srcTableName.applyDefaults(currCatAnsiName, currSchAnsiName);
   
-  // If source table is external, return an error.  
-  // TBD - allow create table like a native (external) HIVE or HBASE table
-  if (ComIsTrafodionExternalSchemaName(srcTableName.getSchemaNamePart().getInternalName()))
-    {
-      *SqlParser_Diags << DgSqlCode(-CAT_UNSUPPORTED_COMMAND_ERROR);
-      return;
-    }
-  
   CorrName cn(srcTableName.getObjectNamePart().getInternalName(),
               STMTHEAP,
               srcTableName.getSchemaNamePart().getInternalName(),

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_cli.cpp b/core/sql/ustat/hs_cli.cpp
index 9d1da06..94fce0f 100644
--- a/core/sql/ustat/hs_cli.cpp
+++ b/core/sql/ustat/hs_cli.cpp
@@ -540,6 +540,17 @@ Lng32 HSSample::create(NAString& tblName, NABoolean unpartitioned,
NABoolean isP
 
     NAString tempTabName = tblName;
     NAString userTabName = objDef->getObjectFullName();
+
+    // If the table is a native one, convert the fully qualified user table name NT
+    // to a fully qualified external table name ET. The sample table will be coreated
+    // like ET.
+    if ( HSGlobalsClass::isNativeCat(objDef->getCatName(HSTableDef::EXTERNAL_FORMAT)))
{
+      userTabName = ComConvertNativeNameToTrafName(
+                      objDef->getCatName(HSTableDef::EXTERNAL_FORMAT),
+                      objDef->getSchemaName(HSTableDef::EXTERNAL_FORMAT),
+                      objDef->getObjectName(HSTableDef::EXTERNAL_FORMAT));
+    }
+
     NAString userLocation;
     ComObjectName *sampleName;
     NAString tableOptions;
@@ -681,6 +692,7 @@ Lng32 HSSample::create(NAString& tblName, NABoolean unpartitioned,
NABoolean isP
                             - UERR_UNABLE_TO_CREATE_OBJECT,
                             NULL,
                             tempTabName, objDef);
+
     HSHandleError(retcode);
 
     //***********************************************************

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_globals.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.cpp b/core/sql/ustat/hs_globals.cpp
index 6e0e9c0..d5dba38 100644
--- a/core/sql/ustat/hs_globals.cpp
+++ b/core/sql/ustat/hs_globals.cpp
@@ -2745,6 +2745,7 @@ THREAD_P NABoolean HSGlobalsClass::performISForMC_ = FALSE;
 HSGlobalsClass::HSGlobalsClass(ComDiagsArea &diags)
   : catSch(new(STMTHEAP) NAString(STMTHEAP)),
     isHbaseTable(FALSE),
+    isHiveTable(FALSE),
     user_table(new(STMTHEAP) NAString(STMTHEAP)),
     numPartitions(0),
     hstogram_table(new(STMTHEAP) NAString(STMTHEAP)),
@@ -2862,6 +2863,7 @@ Lng32 HSGlobalsClass::Initialize()
        defaultHiveCatName = new (GetCliGlobals()->exCollHeap()) NAString("");
     else
       (*defaultHiveCatName) = "";
+
     CmpCommon::getDefault(HIVE_CATALOG, (*defaultHiveCatName), FALSE);
     (*defaultHiveCatName).toUpper();
 
@@ -2869,7 +2871,8 @@ Lng32 HSGlobalsClass::Initialize()
        defaultHbaseCatName = new (GetCliGlobals()->exCollHeap()) NAString("");
     else
       (*defaultHbaseCatName) = "";
-    CmpCommon::getDefault(SEABASE_CATALOG, (*defaultHbaseCatName), FALSE);
+
+    CmpCommon::getDefault(HBASE_CATALOG, (*defaultHbaseCatName), FALSE);
     (*defaultHbaseCatName).toUpper();
 
                                               /*==============================*/
@@ -2887,6 +2890,22 @@ Lng32 HSGlobalsClass::Initialize()
         HSHandleError(retcode);
         TM->Commit(); // Must commit this transaction (even if schema didn't get created).
       }
+                                              /*=====================================*/
+                                              /*   CREATE HBASE STATS SCHEMA         */
+                                              /*   typically as trafodion.hbasestats */
+                                              /*=====================================*/
+    if (isNativeHbaseCat(objDef->getCatName()))
+      {
+        HSTranMan *TM = HSTranMan::Instance(); // Must have transaction around this.
+        TM->Begin("Create schema for native hbase stats.");
+        NAString ddl = "CREATE SCHEMA IF NOT EXISTS ";
+        ddl.append(HBASE_STATS_CATALOG).append('.').append(HBASE_STATS_SCHEMA);
+        retcode = HSFuncExecQuery(ddl, -UERR_INTERNAL_ERROR, NULL,
+                                  "Creating schema for native HBase statistics", NULL,
+                                  NULL);
+        HSHandleError(retcode);
+        TM->Commit(); // Must commit this transaction (even if schema didn't get created).
+      }
                                               /*==============================*/
                                               /*    CREATE HISTOGRM TABLES    */
                                               /*==============================*/
@@ -3938,7 +3957,7 @@ Lng32 HSSample::make(NABoolean rowCountIsEstimate, // input
     // On SQ, alter the sample table to audit afterwards. There are performance 
     // issues with non-audited tables on SQ. For Trafodion, however, this alter
     // is not supported, so skip it.
-     if (!hs_globals->isHbaseTable)
+     if (!hs_globals->isHbaseTable && !hs_globals->isHiveTable)
        {
          LM->StartTimer("Set audit attribute on sample table");
          SQL_EXEC_SetParserFlagsForExSqlComp_Internal(hsALLOW_SPECIALTABLETYPE);
@@ -15510,3 +15529,4 @@ Lng32 HSGlobalsClass::CollectStatisticsWithFastStats()
 
   return retcode;
 }
+

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_globals.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.h b/core/sql/ustat/hs_globals.h
index a44899a..e55f27a 100644
--- a/core/sql/ustat/hs_globals.h
+++ b/core/sql/ustat/hs_globals.h
@@ -1374,11 +1374,21 @@ public:
     // See if catName is the name of an HBase catalog.
     static NABoolean isHbaseCat(const NAString& catName)
     {
+      return ((catName == TRAFODION_SYSCAT_LIT) || isNativeHbaseCat(catName));
+    }
+
+    static NABoolean isNativeHbaseCat(const NAString& catName)
+    {
       return (((defaultHbaseCatName != NULL) && (catName == (*defaultHbaseCatName)))
||
-              ((catName == TRAFODION_SYSCAT_LIT) ||
-	       (catName == HBASE_SYSTEM_CATALOG)));
+              (catName == HBASE_SYSTEM_CATALOG));
     }
 
+    static NABoolean isNativeCat(const NAString& catName)
+    {
+      return (isNativeHbaseCat(catName) || isHiveCat(catName));
+    }
+
+
     static NABoolean isHBaseMeta(const NAString& schName)
     { return (schName == "_MD_" || schName == "_PRIVMGR_MD_"); }
 
@@ -1560,6 +1570,7 @@ public:
     NAString      *catSch;                         /* catalog+schema name     */
     NAString      *user_table;                     /* object name             */
     NABoolean     isHbaseTable;                    /* ustat on HBase table    */
+    NABoolean     isHiveTable;                     /* ustat on Hive table     */
     ComAnsiNameSpace nameSpace;                    /* object namespace    ++MV*/
     Int64          numPartitions;                  /* # of partns in object   */
     NAString      *hstogram_table;                 /* HISTOGRM table          */

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_la.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_la.cpp b/core/sql/ustat/hs_la.cpp
index 9ba464f..77a5ed1 100644
--- a/core/sql/ustat/hs_la.cpp
+++ b/core/sql/ustat/hs_la.cpp
@@ -121,7 +121,10 @@ NABoolean HSSqTableDef::objExists(labelDetail detail)
 
 void HSTableDef::setNATable()
   {
-    BindWA bindWA(ActiveSchemaDB(), CmpCommon::context());
+    //BindWA bindWA(ActiveSchemaDB(), CmpCommon::context(), FALSE, getIsNativeHbaseOrHive());
+    BindWA bindWA(ActiveSchemaDB(), CmpCommon::context(), FALSE, 
+                  HSGlobalsClass::isNativeCat(*catalog_));
+
     CorrName corrName(*object_, STMTHEAP, *schema_, *catalog_);
     if (isVolatile())
       corrName.setIsVolatile(TRUE);
@@ -848,6 +851,9 @@ NABoolean HSHiveTableDef::objExists(labelDetail unused)
   if (!naTbl_)
     return FALSE;
 
+  if (!setObjectUID())
+    return FALSE;
+
   tableStats_ = naTbl_->getClusteringIndex()->getHHDFSTableStats();
 
   *ansiName_ = *catalog_;
@@ -922,7 +928,6 @@ NABoolean HSHiveTableDef::objExists(labelDetail unused)
     return FALSE;
   }
 
-  objectUID_ = hiveTblDesc_->tblID_;
   objActualFormat_ = SQLMX;
 
   return TRUE;
@@ -994,13 +999,107 @@ Lng32 HSHiveTableDef::DescribeColumnNames()
 }
 
 //=====================================================
+//
+NAString HSHbaseTableDef::getHistLoc(formatType format) const
+{
+  return HBASE_STATS_CATALOG "." HBASE_STATS_SCHEMA;
+}
+
+static 
+Lng32 CreateExternalTable(const NAString& catName, const NAString& schName, const
NAString& nativeTableName)
+{
+   HSLogMan *LM = HSLogMan::Instance();
+   if (LM->LogNeeded())
+      {
+        snprintf(LM->msg, sizeof(LM->msg), "Creating external table table for %s on
demand.",
+                          nativeTableName.data());
+        LM->Log(LM->msg);
+      }
+
+   // do not have to worry about the catalog and schema for the new external table 
+   // here. These names will be determined by the processing logic. 
+   NAString ddl = "CREATE EXTERNAL TABLE ";
+   ddl.append(nativeTableName);
+   ddl.append(" FOR ");
+   ddl.append(catName);
+   ddl.append(".");
+   ddl.append(schName);
+   ddl.append(".");
+   ddl.append(nativeTableName);
+   ddl.append(" ATTRIBUTE ");
+
+   HSTranMan *TM = HSTranMan::Instance();
+   if (TM->InTransaction())
+      ddl.append(" AUDIT");
+   else
+      ddl.append("NO AUDIT");
+
+   Lng32 retcode = HSFuncExecDDL(ddl.data(), - UERR_INTERNAL_ERROR, NULL,
+                            "Create external table", NULL);
+
+   if (retcode < 0 && LM->LogNeeded())
+      {
+        snprintf(LM->msg, sizeof(LM->msg), "Creation of the external table failed.");
+        LM->Log(LM->msg);
+      }
+
+   return retcode;
+}
+
+NABoolean HSTableDef::setObjectUID()
+{
+  objectUID_ = naTbl_->objectUid().get_value();
+
+  if (objectUID_ <= 0 && HSGlobalsClass::isNativeCat(getCatName(EXTERNAL_FORMAT))
) {
+
+    // If objectUID is not set, it means there is no corresponding
+    // external table created for it. Need to create one here.
+    NAString catName = getCatName(EXTERNAL_FORMAT);
+    NAString schName = getSchemaName(EXTERNAL_FORMAT);
+    NAString objName = getObjectName(EXTERNAL_FORMAT);
+    Lng32 retcode = CreateExternalTable(catName, schName, objName);
+
+    if (retcode != 0)
+      return FALSE;
+
+    setNATable();
+    if (!naTbl_)
+      return FALSE;
+
+    CorrName corrName(getObjectName(), STMTHEAP, getSchemaName(), getCatName());
+
+    if ( !naTbl_->fetchObjectUIDForNativeTable(corrName) )
+      return FALSE;
+
+    objectUID_ = naTbl_->objectUid().get_value();
+
+    HSLogMan *LM = HSLogMan::Instance();
+    if (LM->LogNeeded()) {
+       sprintf(LM->msg, "NATable::fetchObjectUIDForNativeTable() returns %ld\n", objectUID_);
+       LM->Log(LM->msg);
+    }
+  }
+
+  return (objectUID_ > 0);
+}
 
 NABoolean HSHbaseTableDef::objExists(labelDetail unused)
 {
+  HSLogMan *LM = HSLogMan::Instance();
+
+  if (LM->LogNeeded()) {
+     sprintf(LM->msg, "call HSHbaseTableDef::objExists\n");
+     LM->Log(LM->msg);
+  }
+
   setNATable();
   if (!naTbl_)
     return FALSE;
 
+  if (!setObjectUID())
+    return FALSE;
+
+
   *ansiName_ = getCatName(EXTERNAL_FORMAT);
   ansiName_->append(".");
   ansiName_->append(getSchemaName(EXTERNAL_FORMAT));
@@ -1018,7 +1117,14 @@ NABoolean HSHbaseTableDef::objExists(labelDetail unused)
   Lng32 retcode_ = getColumnNames();
   HSHandleError(retcode_);
 
-  objectUID_ = naTbl_->objectUid().get_value();
+  if (LM->LogNeeded()) {
+     sprintf(LM->msg, "objectUID_ set to %ld\n", objectUID_);
+     sprintf(LM->msg, "naTbl_->objectUid() is %ld\n", naTbl_->objectUid().get_value());
+     LM->Log(LM->msg);
+           
+  }
+
+
   objActualFormat_ = SQLMX;
 
   return TRUE;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_la.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_la.h b/core/sql/ustat/hs_la.h
index 43c0aab..97db4ac 100644
--- a/core/sql/ustat/hs_la.h
+++ b/core/sql/ustat/hs_la.h
@@ -58,6 +58,7 @@ class HSTableDef : public NABasicObject
     HSTableDef(const ComObjectName &tableName,
                const hs_table_type tableType,
                const ComAnsiNameSpace nameSpace);
+
     ~HSTableDef();
     virtual NABoolean objExists(labelDetail detail = MIN_INFO) = 0;
     virtual NABoolean publicSchemaExists() = 0;
@@ -68,6 +69,7 @@ class HSTableDef : public NABasicObject
     NAString getObjectFullName() const;
     NATable* getNATable() const {return naTbl_;}
     void setNATable();
+
     virtual NAString getCatalogLoc(formatType format = INTERNAL_FORMAT) const = 0;
     NAString getPrimaryLoc(formatType format = INTERNAL_FORMAT) const;
     virtual NAString getHistLoc(formatType format = INTERNAL_FORMAT) const = 0;
@@ -113,6 +115,9 @@ class HSTableDef : public NABasicObject
     virtual tblOrigin getTblOrigin() const = 0;
 
   protected:
+    NABoolean setObjectUID();
+
+  protected:
     NAString           *tableName_;
     NAString           *ansiName_;
     NAString           *guardianName_;
@@ -350,12 +355,8 @@ class HSHbaseTableDef : public HSTableDef
         HS_ASSERT(FALSE);  // MP only
         return "";
       }
-    // Unlike Hive, for HBase we must put the hist and hist_ints tables in
-    // HBase, so we use same cat/sch as source table.
-    NAString getHistLoc(formatType format = INTERNAL_FORMAT) const
-      {
-        return getPrimaryLoc(format);  // inherited from HSTableDef
-      }
+    NAString getHistLoc(formatType format = INTERNAL_FORMAT) const;
+
     Lng32 getFileType()  const
       {
         return -1;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_parser.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_parser.cpp b/core/sql/ustat/hs_parser.cpp
index f444245..82158dc 100644
--- a/core/sql/ustat/hs_parser.cpp
+++ b/core/sql/ustat/hs_parser.cpp
@@ -148,6 +148,7 @@ Lng32 AddTableName( const hs_table_type type
                  )
   {
     HSGlobalsClass *hs_globals = GetHSContext();
+    
     NAString catName, schName, objName;
     NAString extName;
     NAString defaultCat, defaultSch;
@@ -225,7 +226,7 @@ Lng32 AddTableName( const hs_table_type type
               catName = ActiveSchemaDB()->getDefaultSchema().getCatalogName();
           }
   
-        if (schema)
+        if (schema) 
           schName = schema;
         else
           {
@@ -299,6 +300,7 @@ Lng32 AddTableName( const hs_table_type type
                                                 volIntName,
                                                 hs_globals->tableType,
                                                 hs_globals->nameSpace);
+
       	if (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO))
           {
             // now look into the regular schema
@@ -340,45 +342,47 @@ Lng32 AddTableName( const hs_table_type type
 	  }
 	
 
-  hs_globals->objDef = HSTableDef::create(STMTHEAP,
+       hs_globals->objDef = HSTableDef::create(STMTHEAP,
                                           intName,
                                           hs_globals->tableType,
                                           hs_globals->nameSpace);
 
-  // try public schema if an object is not qualified and not found
-  if ((NOT schema) && 
-      (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO)))
-	  {
-      NAString pubSch = ActiveSchemaDB()->getDefaults().getValue(PUBLIC_SCHEMA_NAME);
-      ComSchemaName pubSchema(pubSch);
-      if (NOT pubSchema.getSchemaNamePart().isEmpty())
-      {
-        NAString pubSchName = pubSchema.getSchemaNamePart().getInternalName();
-        NAString pubCatName = (pubSchema.getCatalogNamePart().isEmpty() ? 
-          catName:pubSchema.getCatalogNamePart().getInternalName());
-      	ComObjectName pubIntName(pubCatName, pubSchName, objName,
-			                           COM_UNKNOWN_NAME, ComAnsiNamePart::INTERNAL_FORMAT);
-	      if (pubIntName.isValid())
-	      {
-          HSTableDef *pubObjDef = HSTableDef::create(STMTHEAP,
-                                                     pubIntName, 
-                                                     hs_globals->tableType,
-                                                     hs_globals->nameSpace);
-          if (pubObjDef->objExists(HSTableDef::MAX_INFO))
+       // try public schema if an object is not qualified and not found
+       if ((NOT schema) && 
+           (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO)))
+       {
+          NAString pubSch = ActiveSchemaDB()->getDefaults().getValue(PUBLIC_SCHEMA_NAME);
+          ComSchemaName pubSchema(pubSch);
+          if (NOT pubSchema.getSchemaNamePart().isEmpty())
           {
-            hs_globals->objDef = pubObjDef;
+            NAString pubSchName = pubSchema.getSchemaNamePart().getInternalName();
+            NAString pubCatName = (pubSchema.getCatalogNamePart().isEmpty() ? 
+              catName:pubSchema.getCatalogNamePart().getInternalName());
+      	    ComObjectName pubIntName(pubCatName, pubSchName, objName,
+                                     COM_UNKNOWN_NAME, ComAnsiNamePart::INTERNAL_FORMAT);
+	      
+            if (pubIntName.isValid())
+	     {
+                HSTableDef *pubObjDef = HSTableDef::create(STMTHEAP,
+                                                           pubIntName, 
+                                                           hs_globals->tableType,
+                                                           hs_globals->nameSpace);
+
+                if (pubObjDef->objExists(HSTableDef::MAX_INFO))
+                {
+                  hs_globals->objDef = pubObjDef;
+                }
+	     }
           }
-	      }
-      }
-	  }
+       }
 
-	if (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO))
-	  {
-	    HSFuncMergeDiags(-UERR_OBJECT_INACCESSIBLE, extName);
-	    retcode = -1;
-	    HSHandleError(retcode);
-	  }
+      if (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO))
+      {
+         HSFuncMergeDiags(-UERR_OBJECT_INACCESSIBLE, extName);
+         retcode = -1;
+         HSHandleError(retcode);
       }
+    }
 
     //10-040123-2660 We only support tables. We do not allow views.
     // Tables can be metadata tables.
@@ -398,6 +402,7 @@ Lng32 AddTableName( const hs_table_type type
     *hs_globals->user_table = hs_globals->objDef->getObjectFullName();
     hs_globals->tableFormat = hs_globals->objDef->getObjectFormat();
     hs_globals->isHbaseTable = HSGlobalsClass::isHbaseCat(catName);
+    hs_globals->isHiveTable = HSGlobalsClass::isHiveCat(catName);
 
     if (hs_globals->tableFormat == SQLMX)
       {
@@ -428,21 +433,20 @@ Lng32 AddTableName( const hs_table_type type
             LM->Log(LM->msg);
          }
 
-        const char* period = strchr(hs_globals->catSch->data(), '.');
-        NAString catName(hs_globals->catSch->data(),
-                         period - hs_globals->catSch->data());
+        NAString catName(hs_globals->objDef->getCatName());
+
+        *hs_globals->hstogram_table = getHistogramsTableLocation(hs_globals->catSch->data(),
FALSE);
+
+        *hs_globals->hsintval_table = getHistogramsTableLocation(hs_globals->catSch->data(),
FALSE);
+
         NABoolean isHbaseOrHive = HSGlobalsClass::isHbaseCat(catName) ||
                                   HSGlobalsClass::isHiveCat(catName);
 
-        *hs_globals->hstogram_table = 
-            getHistogramsTableLocation(hs_globals->catSch->data(), FALSE);
         if (isHbaseOrHive)
           hs_globals->hstogram_table->append(".").append(HBASE_HIST_NAME);
         else
           hs_globals->hstogram_table->append(".HISTOGRAMS");
 
-        *hs_globals->hsintval_table = 
-            getHistogramsTableLocation(hs_globals->catSch->data(), FALSE);
         if (isHbaseOrHive)
           hs_globals->hsintval_table->append(".").append(HBASE_HISTINT_NAME);
         else

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_util.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_util.cpp b/core/sql/ustat/hs_util.cpp
index 672b19c..f676bf5 100644
--- a/core/sql/ustat/hs_util.cpp
+++ b/core/sql/ustat/hs_util.cpp
@@ -1147,6 +1147,9 @@ NAString getHistogramsTableLocation(
     {
       NAString catName = regularLocation;
       catName.remove(catName.first('.'));
+      if ( HSGlobalsClass::isNativeHbaseCat(catName) )
+        histLoc = HBASE_STATS_CATALOG "." HBASE_STATS_SCHEMA;
+      else
       if (HSGlobalsClass::isHiveCat(catName))
         histLoc = HIVE_STATS_CATALOG "." HIVE_STATS_SCHEMA;
       else



Mime
View raw message