Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 09054200D19 for ; Fri, 22 Sep 2017 00:24:52 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 0766D1609E4; Thu, 21 Sep 2017 22:24:52 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id F1D1D1609E1 for ; Fri, 22 Sep 2017 00:24:50 +0200 (CEST) Received: (qmail 50751 invoked by uid 500); 21 Sep 2017 22:24:50 -0000 Mailing-List: contact commits-help@trafodion.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@trafodion.apache.org Delivered-To: mailing list commits@trafodion.apache.org Received: (qmail 50742 invoked by uid 99); 21 Sep 2017 22:24:50 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 21 Sep 2017 22:24:50 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 950A118FD5E for ; Thu, 21 Sep 2017 22:24:49 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.222 X-Spam-Level: X-Spam-Status: No, score=-4.222 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001, SPF_PASS=-0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id fSFPdg3F1WRZ for ; Thu, 21 Sep 2017 22:24:47 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 4A0305F3FE for ; Thu, 21 Sep 2017 22:24:46 +0000 (UTC) Received: (qmail 50555 invoked by uid 99); 21 Sep 2017 22:24:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 21 Sep 2017 22:24:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id C335DF5834; Thu, 21 Sep 2017 22:24:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sureshs@apache.org To: commits@trafodion.incubator.apache.org Date: Thu, 21 Sep 2017 22:24:46 -0000 Message-Id: <16143d6c8cd04894bba6022b40d84e62@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [3/5] incubator-trafodion git commit: [TRAFODION-2730] process loop when infer_charset cqd is ON and query invalidation is done archived-at: Thu, 21 Sep 2017 22:24:52 -0000 [TRAFODION-2730] process loop when infer_charset cqd is ON and query invalidation is done This PR also includes fixes for [TRAFODION-2747] and [TRAFODION-2748]. The changes are small and the problems not severe. Please see JIRA for an explanation Add aditional change has been made to explain output. cache_blocks ON is now added to explain for HBase scans. Previously explain would only show cache_blocks OFF, if blockcache was disabled for a HBase scan. Files 2730 - QCache.cpp 2747 - ExSequence.cpp, ExAtp.h 2748 - GenPreCode.cpp cache_block - GenExplain.cpp, 4 regression EXPECTED files Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/e68aa43a Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/e68aa43a Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/e68aa43a Branch: refs/heads/master Commit: e68aa43ae2f99e571095763b6e7acd51114295da Parents: 574452e Author: Suresh Subbiah Authored: Mon Sep 18 16:27:49 2017 +0000 Committer: Suresh Subbiah Committed: Mon Sep 18 16:27:49 2017 +0000 ---------------------------------------------------------------------- core/sql/executor/ExSequence.cpp | 15 +- core/sql/exp/ExpAtp.h | 17 ++ core/sql/generator/GenExplain.cpp | 7 +- core/sql/generator/GenPreCode.cpp | 6 + core/sql/regress/executor/EXPECTED131 | 4 + core/sql/regress/executor/EXPECTED140 | 29 +- core/sql/regress/seabase/EXPECTED010 | 448 +++++++++++++++++------------ core/sql/regress/seabase/EXPECTED011 | 13 +- core/sql/sqlcomp/QCache.cpp | 9 +- 9 files changed, 334 insertions(+), 214 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e68aa43a/core/sql/executor/ExSequence.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExSequence.cpp b/core/sql/executor/ExSequence.cpp index 6377b6e..1ed811b 100644 --- a/core/sql/executor/ExSequence.cpp +++ b/core/sql/executor/ExSequence.cpp @@ -350,6 +350,8 @@ ExSequenceTcb::ExSequenceTcb (const ExSequenceTdb & myTdb, ((void*)this, GetHistoryRow, GetHistoryRowOLAP); checkPartitionChangeExpr()->fixup(0, getExpressionMode(), this, space, heap_, FALSE, glob); } + + workAtp_->getTupp(myTdb.tuppIndex_) = new(space) tupp_descriptor; } // Destructor @@ -404,6 +406,8 @@ void ExSequenceTcb::freeResources() } firstOLAPBuffer_ = NULL; lastOLAPBuffer_ = NULL; + + workAtp_->getTupp(myTdb().tuppIndex_).release(); } // work - doit... @@ -461,6 +465,10 @@ short ExSequenceTcb::work() pentry_down = qparent_.down->getHeadEntry(); pstate = (ExSequencePrivateState*) pentry_down->pstate; request = pentry_down->downState.request; + workAtp_->copyPartialAtp(pentry_down->getAtp(),0, + MINOF(myTdb().tuppIndex_, + pentry_down->numTuples())-1); + // copy temp and input tupps. Last tupp history row // Take any child replies and process them. Return the processed // rows as long the parent queue has room. @@ -610,10 +618,6 @@ short ExSequenceTcb::work() // case ex_queue::Q_OK_MMORE: { - tupp_descriptor histTupp; - workAtp_->copyAtp(pentry_down->getAtp()); - workAtp_->getTupp(myTdb().tuppIndex_) = &histTupp; - if ( checkPartitionChangeExpr() && currentHistRowPtr_) { @@ -1117,7 +1121,8 @@ short ExSequenceTcb::work() = pentry_down->downState.parentIndex; pentry_up->upState.downIndex = qparent_.down->getHeadIndex(); pentry_up->upState.setMatchNo(pstate->matchCount_); - + workAtp_->releasePartialAtp(0,MINOF(myTdb().tuppIndex_, + pentry_down->numTuples())-1); qparent_.down->removeHead(); qparent_.up->insert(); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e68aa43a/core/sql/exp/ExpAtp.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpAtp.h b/core/sql/exp/ExpAtp.h index 24ac649..1a3ea18 100644 --- a/core/sql/exp/ExpAtp.h +++ b/core/sql/exp/ExpAtp.h @@ -91,6 +91,8 @@ class atp_struct short last_tupp); inline unsigned short numTuples() const; inline void release(); // de-initialize + inline void releasePartialAtp(short from_first_tupp, + short last_tupp) ; // // Methods for manipulating diagnostics area. @@ -311,4 +313,19 @@ inline void atp_struct::release() setDiagsArea(0); } +inline void atp_struct::releasePartialAtp(short from_first_tupp, + short last_tupp) +{ + // Release each tupp. + // + for(Int32 i=from_first_tupp; i<=last_tupp; i++) + { + tuppArray_[i].release(); + } + + // null-out diags area pointer; setDiagsArea releases existing + // reference to diags area if it is non-null + setDiagsArea(0); +} + #endif http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e68aa43a/core/sql/generator/GenExplain.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenExplain.cpp b/core/sql/generator/GenExplain.cpp index 4cc0a0d..77d9c9f 100644 --- a/core/sql/generator/GenExplain.cpp +++ b/core/sql/generator/GenExplain.cpp @@ -949,10 +949,11 @@ HbaseAccess::addSpecificExplainInfo(ExplainTupleMaster *explainTuple, description += cacheBuf ; description += " " ; - if (!(((ComTdbHbaseAccess *)tdb)->getHbasePerfAttributes()->cacheBlocks())) { - description += "cache_blocks: " ; + description += "cache_blocks: " ; + if (!(((ComTdbHbaseAccess *)tdb)->getHbasePerfAttributes()->cacheBlocks())) description += "OFF " ; - } + else + description += "ON " ; if ((((ComTdbHbaseAccess *)tdb)->getHbasePerfAttributes()->useSmallScanner())) { description += "small_scanner: " ; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e68aa43a/core/sql/generator/GenPreCode.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenPreCode.cpp b/core/sql/generator/GenPreCode.cpp index 84edc29..e23a2bf 100644 --- a/core/sql/generator/GenPreCode.cpp +++ b/core/sql/generator/GenPreCode.cpp @@ -4190,6 +4190,12 @@ RelExpr * FileScan::preCodeGen(Generator * generator, // assign individual files and blocks to each ESPs ((NodeMap *) getPartFunc()->getNodeMap())->assignScanInfos(hiveSearchKey_); generator->setProcessLOB(TRUE); + + // flag set for HBase scan in HbaseAccess::preCodeGen + // unique scan unlikely for hive scans except + // with predicate on virtual cols. + if (!(searchKey() && searchKey()->isUnique())) + generator->oltOptInfo()->setMultipleRowsReturned(TRUE); } } http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e68aa43a/core/sql/regress/executor/EXPECTED131 ---------------------------------------------------------------------- diff --git a/core/sql/regress/executor/EXPECTED131 b/core/sql/regress/executor/EXPECTED131 index f5309b9..f56fced 100644 --- a/core/sql/regress/executor/EXPECTED131 +++ b/core/sql/regress/executor/EXPECTED131 @@ -103,6 +103,7 @@ DESCRIPTION columns ................ all begin_keys(excl) ....... 2 end_keys(excl) ......... 5 + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:1 key_columns ............ UNIQ @@ -146,6 +147,7 @@ DESCRIPTION columns ................ all begin_keys(excl) ....... 2 end_keys(excl) ........ 99 + cache_blocks ........... ON column_retrieved ....... #1:1 key_columns ............ UNIQ executor_predicates .... (UNIQ > 2) and (UNIQ < 99) @@ -301,6 +303,7 @@ DESCRIPTION columns ................ all begin_keys(excl) ....... 2 end_keys(excl) ........ 30 + cache_blocks ........... ON column_retrieved ....... #1:1 key_columns ............ UNIQ executor_predicates .... (UNIQ > 2) and (UNIQ < 30) @@ -344,6 +347,7 @@ DESCRIPTION columns ................ all begin_keys(excl) ....... 2 end_keys(excl) ........ 30 + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:1 key_columns ............ UNIQ http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e68aa43a/core/sql/regress/executor/EXPECTED140 ---------------------------------------------------------------------- diff --git a/core/sql/regress/executor/EXPECTED140 b/core/sql/regress/executor/EXPECTED140 index 1f2086f..d7ef8ae 100644 --- a/core/sql/regress/executor/EXPECTED140 +++ b/core/sql/regress/executor/EXPECTED140 @@ -124,18 +124,18 @@ +> , SS_NET_PROFIT +>from hive.hive.store_sales where ss_sold_date_sk is not null; Task: LOAD Status: Started Object: TRAFODION.SCH.T140C -Task: CLEANUP Status: Started Time: 2017-03-15 05:17:16.759 -Task: CLEANUP Status: Ended Time: 2017-03-15 05:17:16.781 -Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.022 -Task: LOADING DATA Status: Started Time: 2017-03-15 05:17:16.781 +Task: CLEANUP Status: Started Time: 2017-09-18 15:49:46.789 +Task: CLEANUP Status: Ended Time: 2017-09-18 15:49:46.816 +Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.027 +Task: LOADING DATA Status: Started Time: 2017-09-18 15:49:46.816 Rows Processed: 2750311 Error Rows: 0 -Task: LOADING DATA Status: Ended Time: 2017-03-15 05:18:33.540 -Task: LOADING DATA Status: Ended Elapsed Time: 00:01:16.759 -Task: COMPLETION Status: Started Time: 2017-03-15 05:18:33.540 +Task: LOADING DATA Status: Ended Time: 2017-09-18 15:50:33.561 +Task: LOADING DATA Status: Ended Elapsed Time: 00:00:46.744 +Task: COMPLETION Status: Started Time: 2017-09-18 15:50:33.561 Rows Loaded: 2750311 -Task: COMPLETION Status: Ended Time: 2017-03-15 05:18:34.657 -Task: COMPLETION Status: Ended Elapsed Time: 00:00:01.117 +Task: COMPLETION Status: Ended Time: 2017-09-18 15:50:35.404 +Task: COMPLETION Status: Ended Elapsed Time: 00:00:01.843 --- 2750311 row(s) loaded. >>update statistics for table t140c on every column sample; @@ -193,6 +193,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:3 pushed_down_rpn ........ (#1:4>?) @@ -255,6 +256,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:2,#1:6 pushed_down_rpn ........ (#1:4<=?) @@ -304,6 +306,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:2,#1:6 pushed_down_rpn ........ (#1:4<=?) @@ -365,6 +368,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:6 pushed_down_rpn ........ (#1:4=?)(#1:6 is_not_null.)AND @@ -422,6 +426,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:3,#1:6 pushed_down_rpn ........ (#1:4?)OR @@ -517,6 +522,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:6 pushed_down_rpn ........ (#1:7=.?)(#1:6 is_not_null.)AND @@ -573,6 +579,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:3,#1:6 pushed_down_rpn ........ (#1:7<.?)(#1:7>.?)OR @@ -682,6 +689,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON small_scanner .......... ON column_retrieved ....... #1:6 pushed_down_rpn ........ (#1:6>=.?)(#1:6<=.?)AND @@ -775,6 +783,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON parallel_scanner ....... 2 column_retrieved ....... #1:3 key_columns ............ _SALT_, UNIQ, UNIQ2 @@ -846,6 +855,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON parallel_scanner ....... 1 column_retrieved ....... #1:3 key_columns ............ _SALT_, UNIQ, UNIQ2 @@ -912,6 +922,7 @@ DESCRIPTION columns ................ all begin_keys(incl) end_keys(incl) + cache_blocks ........... ON parallel_scanner ....... 1 column_retrieved ....... #1:3 key_columns ............ _SALT_, UNIQ, UNIQ2