Return-Path: X-Original-To: apmail-trafodion-commits-archive@www.apache.org Delivered-To: apmail-trafodion-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7718018212 for ; Mon, 21 Sep 2015 23:12:33 +0000 (UTC) Received: (qmail 19393 invoked by uid 500); 21 Sep 2015 23:12:27 -0000 Delivered-To: apmail-trafodion-commits-archive@trafodion.apache.org Received: (qmail 19364 invoked by uid 500); 21 Sep 2015 23:12:27 -0000 Mailing-List: contact commits-help@trafodion.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@trafodion.apache.org Delivered-To: mailing list commits@trafodion.apache.org Received: (qmail 19353 invoked by uid 99); 21 Sep 2015 23:12:27 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Sep 2015 23:12:27 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 965E6F529E for ; Mon, 21 Sep 2015 23:12:26 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.802 X-Spam-Level: * X-Spam-Status: No, score=1.802 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RP_MATCHES_RCVD=0.001, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-eu-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id GVMUAwsAAE2Q for ; Mon, 21 Sep 2015 23:12:19 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-eu-west.apache.org (ASF Mail Server at mx1-eu-west.apache.org) with SMTP id 5578F24B3C for ; Mon, 21 Sep 2015 23:12:17 +0000 (UTC) Received: (qmail 19294 invoked by uid 99); 21 Sep 2015 23:12:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Sep 2015 23:12:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 59557E0440; Mon, 21 Sep 2015 23:12:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dbirdsall@apache.org To: commits@trafodion.incubator.apache.org Date: Mon, 21 Sep 2015 23:12:16 -0000 Message-Id: <1cd1807c7e694f65a9b349f2bc748380@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/14] incubator-trafodion git commit: jira-1443 final. Repository: incubator-trafodion Updated Branches: refs/heads/master a982a5c4a -> e1093020f jira-1443 final. Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3d212c68 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3d212c68 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3d212c68 Branch: refs/heads/master Commit: 3d212c68f2a4bca5388178068117da5f89119b80 Parents: 0434c60 Author: Qifan Chen Authored: Thu Sep 17 03:54:24 2015 +0000 Committer: Qifan Chen Committed: Thu Sep 17 03:54:24 2015 +0000 ---------------------------------------------------------------------- .../sql/scripts/install_hadoop_regr_test_env | 15 ++- core/sql/common/ComSmallDefs.h | 5 +- core/sql/optimizer/BindRelExpr.cpp | 3 +- core/sql/optimizer/BindWA.cpp | 4 +- core/sql/optimizer/BindWA.h | 4 +- core/sql/optimizer/NATable.cpp | 97 ++++++++-------- core/sql/optimizer/NATable.h | 22 +++- core/sql/optimizer/ObjectNames.cpp | 12 ++ core/sql/optimizer/ObjectNames.h | 3 + core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 8 -- core/sql/ustat/hs_cli.cpp | 12 ++ core/sql/ustat/hs_globals.cpp | 24 +++- core/sql/ustat/hs_globals.h | 15 ++- core/sql/ustat/hs_la.cpp | 112 ++++++++++++++++++- core/sql/ustat/hs_la.h | 13 ++- core/sql/ustat/hs_parser.cpp | 82 +++++++------- core/sql/ustat/hs_util.cpp | 3 + 17 files changed, 319 insertions(+), 115 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sqf/sql/scripts/install_hadoop_regr_test_env ---------------------------------------------------------------------- diff --git a/core/sqf/sql/scripts/install_hadoop_regr_test_env b/core/sqf/sql/scripts/install_hadoop_regr_test_env index 292d2bd..ee1bac9 100755 --- a/core/sqf/sql/scripts/install_hadoop_regr_test_env +++ b/core/sqf/sql/scripts/install_hadoop_regr_test_env @@ -199,7 +199,6 @@ fi ./dsdgen -force $FORCE -dir $MY_TPCDS_DATA_DIR -scale $SCALE -table store >>${MY_LOG_FILE} 2>&1 ./dsdgen -force $FORCE -dir $MY_TPCDS_DATA_DIR -scale $SCALE -table promotion >>${MY_LOG_FILE} 2>&1 - echo "Copying generated data to HDFS..." $MY_HDFS_CMD dfs -mkdir /hive/tpcds >>${MY_LOG_FILE} 2>&1 $MY_HDFS_CMD dfs -mkdir /hive/tpcds/date_dim >>${MY_LOG_FILE} 2>&1 $MY_HDFS_CMD dfs -mkdir /hive/tpcds/time_dim >>${MY_LOG_FILE} 2>&1 @@ -214,6 +213,20 @@ fi cd $MY_TPCDS_DATA_DIR + which iconv >>${MY_LOG_FILE} 2>&1 + if (( $? != 0 )) + then + echo "iconv utility not available. The data will be in ISO-8859-1 format." + else + echo "Converting the data into UTF-8 format ..." + for t in date_dim time_dim item customer customer_demographics household_demographics customer_address store promotion store_sales + do + iconv -f ISO-8859-1 -t UTF-8 -o ${t}.utf8.dat ${t}.dat >>${MY_LOG_FILE} 2>&1 + mv ${t}.utf8.dat ${t}.dat + done + fi + + echo "Copying generated data to HDFS..." for t in date_dim time_dim item customer customer_demographics household_demographics customer_address store promotion store_sales do $MY_HDFS_CMD dfs -put ${t}.dat /hive/tpcds/${t} >>${MY_LOG_FILE} 2>&1 http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/common/ComSmallDefs.h ---------------------------------------------------------------------- diff --git a/core/sql/common/ComSmallDefs.h b/core/sql/common/ComSmallDefs.h index 028ca1b..6e893c5 100644 --- a/core/sql/common/ComSmallDefs.h +++ b/core/sql/common/ComSmallDefs.h @@ -116,7 +116,7 @@ typedef NABoolean ComBoolean; #define HIVE_SYSTEM_CATALOG "HIVE" #define HIVE_SYSTEM_SCHEMA "HIVE" #define HIVE_STATS_CATALOG "TRAFODION" -#define HIVE_STATS_SCHEMA "HIVESTATS" +#define HIVE_STATS_SCHEMA "\"_HIVESTATS_\"" #define HIVE_EXT_SCHEMA_PREFIX "_HV_" #define HBASE_SYSTEM_CATALOG "HBASE" @@ -127,6 +127,9 @@ typedef NABoolean ComBoolean; #define HBASE_HISTINT_PK "SB_HISTOGRAM_INTERVALS_PK" #define HBASE_EXT_SCHEMA_PREFIX "_HB_" +#define HBASE_STATS_CATALOG "TRAFODION" +#define HBASE_STATS_SCHEMA "\"_HBASESTATS_\"" + #define TRAFODION_SYSCAT_LIT "TRAFODION" #define SEABASE_SYSTEM_SCHEMA "SEABASE" #define SEABASE_OLD_PRIVMGR_SCHEMA "PRIVMGR_MD" http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/BindRelExpr.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp index 2d400bb..74fa8f3 100644 --- a/core/sql/optimizer/BindRelExpr.cpp +++ b/core/sql/optimizer/BindRelExpr.cpp @@ -1573,7 +1573,8 @@ NATable *BindWA::getNATable(CorrName& corrName, // native HIVE or HBASE objects unless the allowExternalTables flag is set. // allowExternalTables is set for drop table and SHOWDDL statements. // TDB - may want to merge the Trafodion version with the native version. - if ((table) && (table->isExternalTable() && (! bindWA->allowExternalTables()))) + if ((table) && (table->isExternalTable() && (! bindWA->allowExternalTables()) && + !table->isHistogramTable())) { *CmpCommon::diags() << DgSqlCode(-4258) << DgTableName(table->getTableName().getQualifiedNameAsAnsiString()); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/BindWA.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindWA.cpp b/core/sql/optimizer/BindWA.cpp index ce83134..b80edff 100644 --- a/core/sql/optimizer/BindWA.cpp +++ b/core/sql/optimizer/BindWA.cpp @@ -114,7 +114,7 @@ void BindScope::mergeOuterRefs(const ValueIdSet& other) // *********************************************************************** // BindWA() // *********************************************************************** -BindWA::BindWA(SchemaDB *schemaDB, CmpContext* cmpContext, NABoolean inDDL) +BindWA::BindWA(SchemaDB *schemaDB, CmpContext* cmpContext, NABoolean inDDL, NABoolean allowExtTables) : schemaDB_(schemaDB) , currentCmpContext_(cmpContext) , inputVars_(cmpContext ? cmpContext->statementHeap() : NULL) @@ -131,7 +131,7 @@ BindWA::BindWA(SchemaDB *schemaDB, CmpContext* cmpContext, NABoolean inDDL) // , inRIMaint_(FALSE) , inViewWithCheckOption_(NULL) , viewCount_(0) - , allowExternalTables_(FALSE) + , allowExternalTables_(allowExtTables) , errFlag_(FALSE) , uniqueNum_(0) , uniqueIudNum_(0) //++Triggers, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/BindWA.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/BindWA.h b/core/sql/optimizer/BindWA.h index 11b1fdd..92406c5 100644 --- a/core/sql/optimizer/BindWA.h +++ b/core/sql/optimizer/BindWA.h @@ -1178,7 +1178,8 @@ public: // -------------------------------------------------------------------- BindWA(SchemaDB *schemaDB, CmpContext *currentCmpContext = NULL, - NABoolean inDDL = FALSE); + NABoolean inDDL = FALSE, + NABoolean allowExternalTables = FALSE); // copy ctor BindWA (const BindWA & orig, CollHeap * h=0) ; // not written @@ -1797,7 +1798,6 @@ private: // -------------------------------------------------------------------- NABoolean inDDL_; - //--------------------------------------------------------------------- // does the query tree contain any update nodes? //--------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/NATable.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp index 5a68aa8..dbf8aed 100644 --- a/core/sql/optimizer/NATable.cpp +++ b/core/sql/optimizer/NATable.cpp @@ -4890,6 +4890,34 @@ Int64 lookupObjectUid( const QualifiedName& qualName return objectUID; } +NABoolean NATable::fetchObjectUIDForNativeTable(const CorrName& corrName) +{ + NAString adjustedName = ComConvertNativeNameToTrafName + (corrName.getQualifiedNameObj().getCatalogName(), + corrName.getQualifiedNameObj().getUnqualifiedSchemaNameAsAnsiString(), + corrName.getQualifiedNameObj().getUnqualifiedObjectNameAsAnsiString()); + QualifiedName extObjName (adjustedName, 3, STMTHEAP); + + Lng32 diagsMark = CmpCommon::diags()->mark(); + objectUID_ = ::lookupObjectUid(extObjName, COM_BASE_TABLE_OBJECT); + + // If the objectUID is not found, then the table is not externally defined + // in Trafodion, set the objectUID to 0 + // If an unexpected error occurs, then return with the error + if (objectUID_ <= 0) + { + if (CmpCommon::diags()->contains(-1389)) + { + CmpCommon::diags()->rewind(diagsMark); + objectUID_ = 0; + } + else + return FALSE; + } + + return TRUE; +} + // ----------------------------------------------------------------------- // NATable::NATable() constructor // ----------------------------------------------------------------------- @@ -5110,6 +5138,12 @@ NATable::NATable(BindWA *bindWA, setIsExternalTable(TRUE); } + if (qualifiedName_.getQualifiedNameObj().isHistograms() || + qualifiedName_.getQualifiedNameObj().isHistogramIntervals()) + { + setIsHistogramTable(TRUE); + } + insertMode_ = table_desc->body.table_desc.insertMode; setRecordLength(table_desc->body.table_desc.record_length); @@ -5128,29 +5162,10 @@ NATable::NATable(BindWA *bindWA, // been defined in Trafodion use this value, otherwise, set to 0 if (isHbaseCell_ || isHbaseRow_) { - NAString adjustedName = ComConvertNativeNameToTrafName - (corrName.getQualifiedNameObj().getCatalogName(), - corrName.getQualifiedNameObj().getUnqualifiedSchemaNameAsAnsiString(), - corrName.getQualifiedNameObj().getUnqualifiedObjectNameAsAnsiString()); - QualifiedName extObjName (adjustedName, 3, STMTHEAP); - - Lng32 diagsMark = CmpCommon::diags()->mark(); - objectUID_ = ::lookupObjectUid(extObjName, COM_BASE_TABLE_OBJECT); + if ( !fetchObjectUIDForNativeTable(corrName) ) + return; - // If the objectUID is not found, then the table is not externally defined - // in Trafodion, set the objectUID to 0 - // If an unexpected error occurs, then return with the error - if (objectUID_ <= 0) - { - if (CmpCommon::diags()->contains(-1389)) - { - CmpCommon::diags()->rewind(diagsMark); - objectUID_ = 0; - } - else - return; - } - else + if (objectUID_ > 0 ) setHasExternalTable(TRUE); } @@ -5781,31 +5796,12 @@ NATable::NATable(BindWA *bindWA, // If the HIVE table has been registered in Trafodion, get the objectUID // from Trafodion, otherwise, set it to 0. // TBD - does getQualifiedNameObj handle delimited names correctly? - NAString adjustedName = ComConvertNativeNameToTrafName - ( corrName.getQualifiedNameObj().getCatalogName(), - corrName.getQualifiedNameObj().getSchemaName(), - corrName.getQualifiedNameObj().getObjectName()); - QualifiedName extObjName (adjustedName, 3, STMTHEAP); - - Lng32 diagsMark = CmpCommon::diags()->mark(); - objectUID_ = ::lookupObjectUid(extObjName, COM_BASE_TABLE_OBJECT); - - // If the objectUID is not found, then the table is not externally defined - // in Trafodion, set the objectUID to 0 - // If an unexpected error occurs, then return with the error - if (objectUID_ <= 0) - { - if (CmpCommon::diags()->contains(-1389)) - { - CmpCommon::diags()->rewind(diagsMark); - objectUID_ = 0; - } - else - return; - } - else + if ( !fetchObjectUIDForNativeTable(corrName) ) + return; + + if ( objectUID_ > 0 ) setHasExternalTable(TRUE); - + // for HIVE objects, the schema owner and table owner is HIVE_ROLE_ID if (CmpCommon::context()->isAuthorizationEnabled()) { @@ -6883,10 +6879,19 @@ void NATable::setupPrivInfo() // Query the metadata to find the object uid of the table. This is used when // the uid for a metadata table is requested, since 0 is usually stored for // these tables. +// +// On return, the "Object Not Found" error (1389) is filtered out from +// CmpCommon::diags(). Int64 NATable::lookupObjectUid() { + Lng32 diagsMark = CmpCommon::diags()->mark(); + QualifiedName qualName = getExtendedQualName().getQualifiedNameObj(); objectUID_ = ::lookupObjectUid(qualName, objectType_); + + if (CmpCommon::diags()->contains(-1389)) + CmpCommon::diags()->rewind(diagsMark); + return objectUID_.get_value(); } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/NATable.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/NATable.h b/core/sql/optimizer/NATable.h index 6a22781..ec19d14 100644 --- a/core/sql/optimizer/NATable.h +++ b/core/sql/optimizer/NATable.h @@ -509,6 +509,7 @@ public: const ComUID &getCatalogUid() const { return catalogUID_; } const ComUID &getSchemaUid() const { return schemaUID_; } + const ComUID &objectUid() const { if (objectUID_.get_value() == 0) @@ -516,7 +517,17 @@ public: return objectUID_; } - Int64 lookupObjectUid(); // Used to look up uid on demand for metadata tables + // fetch the object UID that is associated with the external + // table object (if any) for a native table. + // Set objectUID_ to 0 if no such external table exists; + // set objectUID_ to -1 if there is error during the fetch operation; + NABoolean fetchObjectUIDForNativeTable(const CorrName& corrName); + + + Int64 lookupObjectUid(); // Used to look up uid on demand for metadata tables. + // On return, the "Object Not Found" error (-1389) + // is filtered out from CmpCommon::diags(). + bool isEnabledForDDLQI() const; const ComObjectType &getObjectType() const { return objectType_; } @@ -710,6 +721,12 @@ public: NABoolean hasExternalTable() const { return (flags_ & HAS_EXTERNAL_TABLE) != 0; } + void setIsHistogramTable( NABoolean value ) + { value ? flags_ |= IS_HISTOGRAM_TABLE : flags_ &= ~IS_HISTOGRAM_TABLE; } + + NABoolean isHistogramTable() const + { return (flags_ & IS_HISTOGRAM_TABLE) != 0; } + const CheckConstraintList &getCheckConstraints() const { return checkConstraints_; } const AbstractRIConstraintList &getUniqueConstraints() const @@ -924,7 +941,8 @@ private: SERIALIZED_ENCODED_COLUMN = 0x00020000, SERIALIZED_COLUMN = 0x00040000, IS_EXTERNAL_TABLE = 0x00080000, - HAS_EXTERNAL_TABLE = 0x00100000 + HAS_EXTERNAL_TABLE = 0x00100000, + IS_HISTOGRAM_TABLE = 0x00200000 }; UInt32 flags_; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/ObjectNames.cpp ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/ObjectNames.cpp b/core/sql/optimizer/ObjectNames.cpp index f2fe28b..75a9462 100644 --- a/core/sql/optimizer/ObjectNames.cpp +++ b/core/sql/optimizer/ObjectNames.cpp @@ -670,6 +670,7 @@ NABoolean QualifiedName::isSeabasePrivMgrMD() const (getCatalogName(), getSchemaName()); } + // ----------------------------------------------------------------------- // Methods for class CorrName // ----------------------------------------------------------------------- @@ -1385,6 +1386,17 @@ void QualifiedName::print(FILE* ofd, const char* indent, const char* title) cons #endif } +NABoolean QualifiedName::isHistograms() const +{ + return (getObjectName() == "SB_HISTOGRAMS"); +} + +NABoolean QualifiedName::isHistogramIntervals() const +{ + return (getObjectName() == "SB_HISTOGRAM_INTERVALS"); +} + + void ExtendedQualName::print(FILE* ofd, const char* indent, const char* title) const { #ifndef NDEBUG http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/optimizer/ObjectNames.h ---------------------------------------------------------------------- diff --git a/core/sql/optimizer/ObjectNames.h b/core/sql/optimizer/ObjectNames.h index d7f15cf..981025f 100644 --- a/core/sql/optimizer/ObjectNames.h +++ b/core/sql/optimizer/ObjectNames.h @@ -322,6 +322,9 @@ public: NABoolean isSeabaseMD() const; NABoolean isSeabasePrivMgrMD() const; + NABoolean isHistograms() const; + NABoolean isHistogramIntervals() const; + void setObjectName(const NAString &objName) { objectName_ = objName; } void setObjectNameSpace(ComAnsiNameSpace objNameSpace) { objectNameSpace_ = objNameSpace; } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp index 36663f5..e7f7b29 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp @@ -249,14 +249,6 @@ void CmpSeabaseDDL::createSeabaseTableLike( ComObjectName srcTableName(createTableNode->getLikeSourceTableName(), COM_TABLE_NAME); srcTableName.applyDefaults(currCatAnsiName, currSchAnsiName); - // If source table is external, return an error. - // TBD - allow create table like a native (external) HIVE or HBASE table - if (ComIsTrafodionExternalSchemaName(srcTableName.getSchemaNamePart().getInternalName())) - { - *SqlParser_Diags << DgSqlCode(-CAT_UNSUPPORTED_COMMAND_ERROR); - return; - } - CorrName cn(srcTableName.getObjectNamePart().getInternalName(), STMTHEAP, srcTableName.getSchemaNamePart().getInternalName(), http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_cli.cpp ---------------------------------------------------------------------- diff --git a/core/sql/ustat/hs_cli.cpp b/core/sql/ustat/hs_cli.cpp index 9d1da06..94fce0f 100644 --- a/core/sql/ustat/hs_cli.cpp +++ b/core/sql/ustat/hs_cli.cpp @@ -540,6 +540,17 @@ Lng32 HSSample::create(NAString& tblName, NABoolean unpartitioned, NABoolean isP NAString tempTabName = tblName; NAString userTabName = objDef->getObjectFullName(); + + // If the table is a native one, convert the fully qualified user table name NT + // to a fully qualified external table name ET. The sample table will be coreated + // like ET. + if ( HSGlobalsClass::isNativeCat(objDef->getCatName(HSTableDef::EXTERNAL_FORMAT))) { + userTabName = ComConvertNativeNameToTrafName( + objDef->getCatName(HSTableDef::EXTERNAL_FORMAT), + objDef->getSchemaName(HSTableDef::EXTERNAL_FORMAT), + objDef->getObjectName(HSTableDef::EXTERNAL_FORMAT)); + } + NAString userLocation; ComObjectName *sampleName; NAString tableOptions; @@ -681,6 +692,7 @@ Lng32 HSSample::create(NAString& tblName, NABoolean unpartitioned, NABoolean isP - UERR_UNABLE_TO_CREATE_OBJECT, NULL, tempTabName, objDef); + HSHandleError(retcode); //*********************************************************** http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_globals.cpp ---------------------------------------------------------------------- diff --git a/core/sql/ustat/hs_globals.cpp b/core/sql/ustat/hs_globals.cpp index 6e0e9c0..d5dba38 100644 --- a/core/sql/ustat/hs_globals.cpp +++ b/core/sql/ustat/hs_globals.cpp @@ -2745,6 +2745,7 @@ THREAD_P NABoolean HSGlobalsClass::performISForMC_ = FALSE; HSGlobalsClass::HSGlobalsClass(ComDiagsArea &diags) : catSch(new(STMTHEAP) NAString(STMTHEAP)), isHbaseTable(FALSE), + isHiveTable(FALSE), user_table(new(STMTHEAP) NAString(STMTHEAP)), numPartitions(0), hstogram_table(new(STMTHEAP) NAString(STMTHEAP)), @@ -2862,6 +2863,7 @@ Lng32 HSGlobalsClass::Initialize() defaultHiveCatName = new (GetCliGlobals()->exCollHeap()) NAString(""); else (*defaultHiveCatName) = ""; + CmpCommon::getDefault(HIVE_CATALOG, (*defaultHiveCatName), FALSE); (*defaultHiveCatName).toUpper(); @@ -2869,7 +2871,8 @@ Lng32 HSGlobalsClass::Initialize() defaultHbaseCatName = new (GetCliGlobals()->exCollHeap()) NAString(""); else (*defaultHbaseCatName) = ""; - CmpCommon::getDefault(SEABASE_CATALOG, (*defaultHbaseCatName), FALSE); + + CmpCommon::getDefault(HBASE_CATALOG, (*defaultHbaseCatName), FALSE); (*defaultHbaseCatName).toUpper(); /*==============================*/ @@ -2887,6 +2890,22 @@ Lng32 HSGlobalsClass::Initialize() HSHandleError(retcode); TM->Commit(); // Must commit this transaction (even if schema didn't get created). } + /*=====================================*/ + /* CREATE HBASE STATS SCHEMA */ + /* typically as trafodion.hbasestats */ + /*=====================================*/ + if (isNativeHbaseCat(objDef->getCatName())) + { + HSTranMan *TM = HSTranMan::Instance(); // Must have transaction around this. + TM->Begin("Create schema for native hbase stats."); + NAString ddl = "CREATE SCHEMA IF NOT EXISTS "; + ddl.append(HBASE_STATS_CATALOG).append('.').append(HBASE_STATS_SCHEMA); + retcode = HSFuncExecQuery(ddl, -UERR_INTERNAL_ERROR, NULL, + "Creating schema for native HBase statistics", NULL, + NULL); + HSHandleError(retcode); + TM->Commit(); // Must commit this transaction (even if schema didn't get created). + } /*==============================*/ /* CREATE HISTOGRM TABLES */ /*==============================*/ @@ -3938,7 +3957,7 @@ Lng32 HSSample::make(NABoolean rowCountIsEstimate, // input // On SQ, alter the sample table to audit afterwards. There are performance // issues with non-audited tables on SQ. For Trafodion, however, this alter // is not supported, so skip it. - if (!hs_globals->isHbaseTable) + if (!hs_globals->isHbaseTable && !hs_globals->isHiveTable) { LM->StartTimer("Set audit attribute on sample table"); SQL_EXEC_SetParserFlagsForExSqlComp_Internal(hsALLOW_SPECIALTABLETYPE); @@ -15510,3 +15529,4 @@ Lng32 HSGlobalsClass::CollectStatisticsWithFastStats() return retcode; } + http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_globals.h ---------------------------------------------------------------------- diff --git a/core/sql/ustat/hs_globals.h b/core/sql/ustat/hs_globals.h index a44899a..e55f27a 100644 --- a/core/sql/ustat/hs_globals.h +++ b/core/sql/ustat/hs_globals.h @@ -1374,11 +1374,21 @@ public: // See if catName is the name of an HBase catalog. static NABoolean isHbaseCat(const NAString& catName) { + return ((catName == TRAFODION_SYSCAT_LIT) || isNativeHbaseCat(catName)); + } + + static NABoolean isNativeHbaseCat(const NAString& catName) + { return (((defaultHbaseCatName != NULL) && (catName == (*defaultHbaseCatName))) || - ((catName == TRAFODION_SYSCAT_LIT) || - (catName == HBASE_SYSTEM_CATALOG))); + (catName == HBASE_SYSTEM_CATALOG)); } + static NABoolean isNativeCat(const NAString& catName) + { + return (isNativeHbaseCat(catName) || isHiveCat(catName)); + } + + static NABoolean isHBaseMeta(const NAString& schName) { return (schName == "_MD_" || schName == "_PRIVMGR_MD_"); } @@ -1560,6 +1570,7 @@ public: NAString *catSch; /* catalog+schema name */ NAString *user_table; /* object name */ NABoolean isHbaseTable; /* ustat on HBase table */ + NABoolean isHiveTable; /* ustat on Hive table */ ComAnsiNameSpace nameSpace; /* object namespace ++MV*/ Int64 numPartitions; /* # of partns in object */ NAString *hstogram_table; /* HISTOGRM table */ http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_la.cpp ---------------------------------------------------------------------- diff --git a/core/sql/ustat/hs_la.cpp b/core/sql/ustat/hs_la.cpp index 9ba464f..77a5ed1 100644 --- a/core/sql/ustat/hs_la.cpp +++ b/core/sql/ustat/hs_la.cpp @@ -121,7 +121,10 @@ NABoolean HSSqTableDef::objExists(labelDetail detail) void HSTableDef::setNATable() { - BindWA bindWA(ActiveSchemaDB(), CmpCommon::context()); + //BindWA bindWA(ActiveSchemaDB(), CmpCommon::context(), FALSE, getIsNativeHbaseOrHive()); + BindWA bindWA(ActiveSchemaDB(), CmpCommon::context(), FALSE, + HSGlobalsClass::isNativeCat(*catalog_)); + CorrName corrName(*object_, STMTHEAP, *schema_, *catalog_); if (isVolatile()) corrName.setIsVolatile(TRUE); @@ -848,6 +851,9 @@ NABoolean HSHiveTableDef::objExists(labelDetail unused) if (!naTbl_) return FALSE; + if (!setObjectUID()) + return FALSE; + tableStats_ = naTbl_->getClusteringIndex()->getHHDFSTableStats(); *ansiName_ = *catalog_; @@ -922,7 +928,6 @@ NABoolean HSHiveTableDef::objExists(labelDetail unused) return FALSE; } - objectUID_ = hiveTblDesc_->tblID_; objActualFormat_ = SQLMX; return TRUE; @@ -994,13 +999,107 @@ Lng32 HSHiveTableDef::DescribeColumnNames() } //===================================================== +// +NAString HSHbaseTableDef::getHistLoc(formatType format) const +{ + return HBASE_STATS_CATALOG "." HBASE_STATS_SCHEMA; +} + +static +Lng32 CreateExternalTable(const NAString& catName, const NAString& schName, const NAString& nativeTableName) +{ + HSLogMan *LM = HSLogMan::Instance(); + if (LM->LogNeeded()) + { + snprintf(LM->msg, sizeof(LM->msg), "Creating external table table for %s on demand.", + nativeTableName.data()); + LM->Log(LM->msg); + } + + // do not have to worry about the catalog and schema for the new external table + // here. These names will be determined by the processing logic. + NAString ddl = "CREATE EXTERNAL TABLE "; + ddl.append(nativeTableName); + ddl.append(" FOR "); + ddl.append(catName); + ddl.append("."); + ddl.append(schName); + ddl.append("."); + ddl.append(nativeTableName); + ddl.append(" ATTRIBUTE "); + + HSTranMan *TM = HSTranMan::Instance(); + if (TM->InTransaction()) + ddl.append(" AUDIT"); + else + ddl.append("NO AUDIT"); + + Lng32 retcode = HSFuncExecDDL(ddl.data(), - UERR_INTERNAL_ERROR, NULL, + "Create external table", NULL); + + if (retcode < 0 && LM->LogNeeded()) + { + snprintf(LM->msg, sizeof(LM->msg), "Creation of the external table failed."); + LM->Log(LM->msg); + } + + return retcode; +} + +NABoolean HSTableDef::setObjectUID() +{ + objectUID_ = naTbl_->objectUid().get_value(); + + if (objectUID_ <= 0 && HSGlobalsClass::isNativeCat(getCatName(EXTERNAL_FORMAT)) ) { + + // If objectUID is not set, it means there is no corresponding + // external table created for it. Need to create one here. + NAString catName = getCatName(EXTERNAL_FORMAT); + NAString schName = getSchemaName(EXTERNAL_FORMAT); + NAString objName = getObjectName(EXTERNAL_FORMAT); + Lng32 retcode = CreateExternalTable(catName, schName, objName); + + if (retcode != 0) + return FALSE; + + setNATable(); + if (!naTbl_) + return FALSE; + + CorrName corrName(getObjectName(), STMTHEAP, getSchemaName(), getCatName()); + + if ( !naTbl_->fetchObjectUIDForNativeTable(corrName) ) + return FALSE; + + objectUID_ = naTbl_->objectUid().get_value(); + + HSLogMan *LM = HSLogMan::Instance(); + if (LM->LogNeeded()) { + sprintf(LM->msg, "NATable::fetchObjectUIDForNativeTable() returns %ld\n", objectUID_); + LM->Log(LM->msg); + } + } + + return (objectUID_ > 0); +} NABoolean HSHbaseTableDef::objExists(labelDetail unused) { + HSLogMan *LM = HSLogMan::Instance(); + + if (LM->LogNeeded()) { + sprintf(LM->msg, "call HSHbaseTableDef::objExists\n"); + LM->Log(LM->msg); + } + setNATable(); if (!naTbl_) return FALSE; + if (!setObjectUID()) + return FALSE; + + *ansiName_ = getCatName(EXTERNAL_FORMAT); ansiName_->append("."); ansiName_->append(getSchemaName(EXTERNAL_FORMAT)); @@ -1018,7 +1117,14 @@ NABoolean HSHbaseTableDef::objExists(labelDetail unused) Lng32 retcode_ = getColumnNames(); HSHandleError(retcode_); - objectUID_ = naTbl_->objectUid().get_value(); + if (LM->LogNeeded()) { + sprintf(LM->msg, "objectUID_ set to %ld\n", objectUID_); + sprintf(LM->msg, "naTbl_->objectUid() is %ld\n", naTbl_->objectUid().get_value()); + LM->Log(LM->msg); + + } + + objActualFormat_ = SQLMX; return TRUE; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_la.h ---------------------------------------------------------------------- diff --git a/core/sql/ustat/hs_la.h b/core/sql/ustat/hs_la.h index 43c0aab..97db4ac 100644 --- a/core/sql/ustat/hs_la.h +++ b/core/sql/ustat/hs_la.h @@ -58,6 +58,7 @@ class HSTableDef : public NABasicObject HSTableDef(const ComObjectName &tableName, const hs_table_type tableType, const ComAnsiNameSpace nameSpace); + ~HSTableDef(); virtual NABoolean objExists(labelDetail detail = MIN_INFO) = 0; virtual NABoolean publicSchemaExists() = 0; @@ -68,6 +69,7 @@ class HSTableDef : public NABasicObject NAString getObjectFullName() const; NATable* getNATable() const {return naTbl_;} void setNATable(); + virtual NAString getCatalogLoc(formatType format = INTERNAL_FORMAT) const = 0; NAString getPrimaryLoc(formatType format = INTERNAL_FORMAT) const; virtual NAString getHistLoc(formatType format = INTERNAL_FORMAT) const = 0; @@ -113,6 +115,9 @@ class HSTableDef : public NABasicObject virtual tblOrigin getTblOrigin() const = 0; protected: + NABoolean setObjectUID(); + + protected: NAString *tableName_; NAString *ansiName_; NAString *guardianName_; @@ -350,12 +355,8 @@ class HSHbaseTableDef : public HSTableDef HS_ASSERT(FALSE); // MP only return ""; } - // Unlike Hive, for HBase we must put the hist and hist_ints tables in - // HBase, so we use same cat/sch as source table. - NAString getHistLoc(formatType format = INTERNAL_FORMAT) const - { - return getPrimaryLoc(format); // inherited from HSTableDef - } + NAString getHistLoc(formatType format = INTERNAL_FORMAT) const; + Lng32 getFileType() const { return -1; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_parser.cpp ---------------------------------------------------------------------- diff --git a/core/sql/ustat/hs_parser.cpp b/core/sql/ustat/hs_parser.cpp index f444245..82158dc 100644 --- a/core/sql/ustat/hs_parser.cpp +++ b/core/sql/ustat/hs_parser.cpp @@ -148,6 +148,7 @@ Lng32 AddTableName( const hs_table_type type ) { HSGlobalsClass *hs_globals = GetHSContext(); + NAString catName, schName, objName; NAString extName; NAString defaultCat, defaultSch; @@ -225,7 +226,7 @@ Lng32 AddTableName( const hs_table_type type catName = ActiveSchemaDB()->getDefaultSchema().getCatalogName(); } - if (schema) + if (schema) schName = schema; else { @@ -299,6 +300,7 @@ Lng32 AddTableName( const hs_table_type type volIntName, hs_globals->tableType, hs_globals->nameSpace); + if (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO)) { // now look into the regular schema @@ -340,45 +342,47 @@ Lng32 AddTableName( const hs_table_type type } - hs_globals->objDef = HSTableDef::create(STMTHEAP, + hs_globals->objDef = HSTableDef::create(STMTHEAP, intName, hs_globals->tableType, hs_globals->nameSpace); - // try public schema if an object is not qualified and not found - if ((NOT schema) && - (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO))) - { - NAString pubSch = ActiveSchemaDB()->getDefaults().getValue(PUBLIC_SCHEMA_NAME); - ComSchemaName pubSchema(pubSch); - if (NOT pubSchema.getSchemaNamePart().isEmpty()) - { - NAString pubSchName = pubSchema.getSchemaNamePart().getInternalName(); - NAString pubCatName = (pubSchema.getCatalogNamePart().isEmpty() ? - catName:pubSchema.getCatalogNamePart().getInternalName()); - ComObjectName pubIntName(pubCatName, pubSchName, objName, - COM_UNKNOWN_NAME, ComAnsiNamePart::INTERNAL_FORMAT); - if (pubIntName.isValid()) - { - HSTableDef *pubObjDef = HSTableDef::create(STMTHEAP, - pubIntName, - hs_globals->tableType, - hs_globals->nameSpace); - if (pubObjDef->objExists(HSTableDef::MAX_INFO)) + // try public schema if an object is not qualified and not found + if ((NOT schema) && + (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO))) + { + NAString pubSch = ActiveSchemaDB()->getDefaults().getValue(PUBLIC_SCHEMA_NAME); + ComSchemaName pubSchema(pubSch); + if (NOT pubSchema.getSchemaNamePart().isEmpty()) { - hs_globals->objDef = pubObjDef; + NAString pubSchName = pubSchema.getSchemaNamePart().getInternalName(); + NAString pubCatName = (pubSchema.getCatalogNamePart().isEmpty() ? + catName:pubSchema.getCatalogNamePart().getInternalName()); + ComObjectName pubIntName(pubCatName, pubSchName, objName, + COM_UNKNOWN_NAME, ComAnsiNamePart::INTERNAL_FORMAT); + + if (pubIntName.isValid()) + { + HSTableDef *pubObjDef = HSTableDef::create(STMTHEAP, + pubIntName, + hs_globals->tableType, + hs_globals->nameSpace); + + if (pubObjDef->objExists(HSTableDef::MAX_INFO)) + { + hs_globals->objDef = pubObjDef; + } + } } - } - } - } + } - if (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO)) - { - HSFuncMergeDiags(-UERR_OBJECT_INACCESSIBLE, extName); - retcode = -1; - HSHandleError(retcode); - } + if (NOT hs_globals->objDef->objExists(HSTableDef::MAX_INFO)) + { + HSFuncMergeDiags(-UERR_OBJECT_INACCESSIBLE, extName); + retcode = -1; + HSHandleError(retcode); } + } //10-040123-2660 We only support tables. We do not allow views. // Tables can be metadata tables. @@ -398,6 +402,7 @@ Lng32 AddTableName( const hs_table_type type *hs_globals->user_table = hs_globals->objDef->getObjectFullName(); hs_globals->tableFormat = hs_globals->objDef->getObjectFormat(); hs_globals->isHbaseTable = HSGlobalsClass::isHbaseCat(catName); + hs_globals->isHiveTable = HSGlobalsClass::isHiveCat(catName); if (hs_globals->tableFormat == SQLMX) { @@ -428,21 +433,20 @@ Lng32 AddTableName( const hs_table_type type LM->Log(LM->msg); } - const char* period = strchr(hs_globals->catSch->data(), '.'); - NAString catName(hs_globals->catSch->data(), - period - hs_globals->catSch->data()); + NAString catName(hs_globals->objDef->getCatName()); + + *hs_globals->hstogram_table = getHistogramsTableLocation(hs_globals->catSch->data(), FALSE); + + *hs_globals->hsintval_table = getHistogramsTableLocation(hs_globals->catSch->data(), FALSE); + NABoolean isHbaseOrHive = HSGlobalsClass::isHbaseCat(catName) || HSGlobalsClass::isHiveCat(catName); - *hs_globals->hstogram_table = - getHistogramsTableLocation(hs_globals->catSch->data(), FALSE); if (isHbaseOrHive) hs_globals->hstogram_table->append(".").append(HBASE_HIST_NAME); else hs_globals->hstogram_table->append(".HISTOGRAMS"); - *hs_globals->hsintval_table = - getHistogramsTableLocation(hs_globals->catSch->data(), FALSE); if (isHbaseOrHive) hs_globals->hsintval_table->append(".").append(HBASE_HISTINT_NAME); else http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3d212c68/core/sql/ustat/hs_util.cpp ---------------------------------------------------------------------- diff --git a/core/sql/ustat/hs_util.cpp b/core/sql/ustat/hs_util.cpp index 672b19c..f676bf5 100644 --- a/core/sql/ustat/hs_util.cpp +++ b/core/sql/ustat/hs_util.cpp @@ -1147,6 +1147,9 @@ NAString getHistogramsTableLocation( { NAString catName = regularLocation; catName.remove(catName.first('.')); + if ( HSGlobalsClass::isNativeHbaseCat(catName) ) + histLoc = HBASE_STATS_CATALOG "." HBASE_STATS_SCHEMA; + else if (HSGlobalsClass::isHiveCat(catName)) histLoc = HIVE_STATS_CATALOG "." HIVE_STATS_SCHEMA; else