Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id AEA62200CFD for ; Wed, 6 Sep 2017 18:04:33 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id AD3FD1609C2; Wed, 6 Sep 2017 16:04:33 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2FCC71609BA for ; Wed, 6 Sep 2017 18:04:32 +0200 (CEST) Received: (qmail 51803 invoked by uid 500); 6 Sep 2017 16:04:30 -0000 Mailing-List: contact commits-help@trafodion.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: commits@trafodion.apache.org Delivered-To: mailing list commits@trafodion.apache.org Received: (qmail 51794 invoked by uid 99); 6 Sep 2017 16:04:30 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Sep 2017 16:04:30 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 2AD7018B066 for ; Wed, 6 Sep 2017 16:04:30 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -3.722 X-Spam-Level: X-Spam-Status: No, score=-3.722 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_NUMSUBJECT=0.5, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001, SPF_PASS=-0.001] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id mBRZ4ydELlP3 for ; Wed, 6 Sep 2017 16:04:25 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id 0D1715F6BE for ; Wed, 6 Sep 2017 16:04:24 +0000 (UTC) Received: (qmail 51543 invoked by uid 99); 6 Sep 2017 16:04:24 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 06 Sep 2017 16:04:24 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1B248F56A7; Wed, 6 Sep 2017 16:04:24 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sandhya@apache.org To: commits@trafodion.incubator.apache.org Date: Wed, 06 Sep 2017 16:04:29 -0000 Message-Id: <7f64d45ab0b14589b07d08f4b274fe09@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [7/8] incubator-trafodion git commit: Merge remote branch 'origin/pr/1216/head' into merge_1216 archived-at: Wed, 06 Sep 2017 16:04:33 -0000 Merge remote branch 'origin/pr/1216/head' into merge_1216 Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/f94c9ea3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/f94c9ea3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/f94c9ea3 Branch: refs/heads/master Commit: f94c9ea3310dc179c544a083d0aff99e6d9b0742 Parents: 47cd6a9 Author: Sandhya Sundaresan Authored: Mon Sep 4 06:58:05 2017 +0000 Committer: Sandhya Sundaresan Committed: Tue Sep 5 20:39:12 2017 +0000 ---------------------------------------------------------------------- core/sql/cli/Cli.cpp | 5 +- core/sql/clitest/blobtest.cpp | 12 +- core/sql/executor/ExExeUtil.h | 7 +- core/sql/executor/ExExeUtilLoad.cpp | 255 ++++++++----- core/sql/exp/ExpLOBaccess.cpp | 135 +++++-- core/sql/exp/ExpLOBaccess.h | 4 +- core/sql/exp/ExpLOBenums.h | 4 +- core/sql/exp/ExpLOBinterface.cpp | 57 ++- core/sql/exp/ExpLOBinterface.h | 13 + core/sql/generator/GenRelExeUtil.cpp | 2 +- core/sql/regress/executor/EXPECTED130 | 78 ++-- core/sql/regress/hive/EXPECTED018 | 585 +++++++++++++++-------------- 12 files changed, 688 insertions(+), 469 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/cli/Cli.cpp ---------------------------------------------------------------------- diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp index 1a6eaf5..a25a31b 100644 --- a/core/sql/cli/Cli.cpp +++ b/core/sql/cli/Cli.cpp @@ -10261,7 +10261,10 @@ Lng32 SQLCLI_LOBcliInterface NADELETEBASIC(query, currContext.exHeap()); if (NOT (inCliInterface && (*inCliInterface))) - delete cliInterface; + { + delete cliInterface; + cliInterface = NULL; + } if (cliRC < 0) { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/clitest/blobtest.cpp ---------------------------------------------------------------------- diff --git a/core/sql/clitest/blobtest.cpp b/core/sql/clitest/blobtest.cpp index a9a457a..492bbd6 100644 --- a/core/sql/clitest/blobtest.cpp +++ b/core/sql/clitest/blobtest.cpp @@ -92,7 +92,7 @@ Int32 extractLobToBuffer(CliGlobals *cliglob, char * lobHandle, Int64 &lengthOfL while ((retcode != 100) && !(retcode<0)) { retcode = cliInterface.clearExecFetchClose(NULL,NULL,statusBuf, &statusBufLen); - if (!retcode) + if (retcode>= 0) { memcpy((char*)&(lobFinalBuf[i]),(char *)lobDataBuf,lobExtractLen); i += lobExtractLen; @@ -107,6 +107,9 @@ Int32 extractLobToBuffer(CliGlobals *cliglob, char * lobHandle, Int64 &lengthOfL fclose(lobFileId); } + str_sprintf(query,"extract lobtobuffer(lob '%s', LOCATION %Ld, SIZE 0) ", lobHandle, (Int64)lobDataBuf); + + cliInterface.clearExecFetchClose(NULL,NULL,statusBuf, &statusBufLen); delete lobFinalBuf; delete query; delete lobDataBuf; @@ -142,13 +145,16 @@ Int32 extractLobToFileInChunks(CliGlobals *cliglob, char * lobHandle, char *fil while ((retcode != 100) && !(retcode<0)) { retcode = cliInterface.clearExecFetchClose(NULL,NULL,statusBuf, &statusBufLen); - if (!retcode) + if (retcode>= 0) { byteCount=fwrite(lobDataBuf,sizeof(char),*inputOutputAddr, lobFileId); cout << "Wrote " << byteCount << " bytes to file : " << filename << endl; } } - + lobExtractLen = 0; + str_sprintf(query,"extract lobtobuffer(lob '%s', LOCATION %Ld, SIZE %Ld) ", lobHandle, (Int64)lobDataBuf, inputOutputAddr); + retcode = cliInterface.executeImmediatePrepare(query); + cliInterface.clearExecFetchClose(NULL,NULL,statusBuf, &statusBufLen); fclose(lobFileId); http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/executor/ExExeUtil.h ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h index 51695c6..07d3b22 100644 --- a/core/sql/executor/ExExeUtil.h +++ b/core/sql/executor/ExExeUtil.h @@ -2931,6 +2931,8 @@ public: ex_globals * glob = 0); virtual short work(); + virtual ~ExExeUtilLobExtractTcb(); + virtual void freeResources(); ExExeUtilLobExtractTdb & lobTdb() const { @@ -2978,7 +2980,7 @@ public: char * lobData2_; Int64 lobDataSpecifiedExtractLen_; Int64 lobDataLen_; - Lng32 remainingBytes_; + Int64 remainingBytes_; Lng32 currPos_; Lng32 numChildRows_; Int64 requestTag_; @@ -3036,7 +3038,8 @@ public: ex_globals * glob = 0); virtual short work(); - + virtual ~ExExeUtilLobUpdateTcb(); + virtual void freeResources(); ExExeUtilLobUpdateTdb & lobTdb() const { return (ExExeUtilLobUpdateTdb &) tdb; http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/executor/ExExeUtilLoad.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp index dfbbac4..614adde 100644 --- a/core/sql/executor/ExExeUtilLoad.cpp +++ b/core/sql/executor/ExExeUtilLoad.cpp @@ -2749,6 +2749,45 @@ ExExeUtilLobExtractTcb::ExExeUtilLobExtractTcb } +void ExExeUtilLobExtractTcb::freeResources() +{ + Lng32 cliRC = 0; + Lng32 retcode = 0; + void * lobGlobs = getLobGlobals()->lobAccessGlobals(); + ContextCli *currContext = + getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()-> + getStatement()->getContext(); + //close any open cursors. + retcode = ExpLOBInterfaceSelectCursor + (lobGlobs, + lobName_, + lobLoc_, + lobType_, + lobTdb().getLobHdfsServer(), + lobTdb().getLobHdfsPort(), + + lobHandleLen_, lobHandle_, + 0, //cursor bytes + NULL, //cursor id + requestTag_, + Lob_Buffer, + 0, // not check status + 1, // waited op + + 0, lobDataSpecifiedExtractLen_, + lobDataLen_, lobData_, + 3, // close + 0); // open type not applicable + + NADELETE(lobGlobals_,LOBglobals,currContext->exHeap()); + lobGlobals_ = NULL; +} + +ExExeUtilLobExtractTcb::~ExExeUtilLobExtractTcb() +{ + freeResources(); +} + short ExExeUtilLobExtractTcb::work() { Lng32 cliRC = 0; @@ -2996,7 +3035,49 @@ short ExExeUtilLobExtractTcb::work() } } - + Int16 flags; + Lng32 lobNum; + Int64 uid, inDescSyskey, descPartnKey; + short schNameLen; + char schName[1024]; + ExpLOBoper::extractFromLOBhandle(&flags, &lobType_, &lobNum, &uid, + &inDescSyskey, &descPartnKey, + &schNameLen, (char *)schName, + (char *)lobHandle_, (Lng32)lobHandleLen_); + + //Retrieve the lobLocation for this lobNum which will be used + //in the other steps_ which open and read lob data file. + short *lobNumList = new (getHeap()) short[1]; + short *lobTypList = new (getHeap()) short[1]; + char **lobLocList = new (getHeap()) char*[1]; + char **lobColNameList = new (getHeap()) char*[1]; + lobLocList[0] = new (getHeap()) char[1024]; + lobColNameList[0] = new (getHeap()) char[256]; + + Lng32 numLobs = lobNum; + Lng32 cliRC = SQL_EXEC_LOBddlInterface + ( + schName, + schNameLen, + uid, + numLobs, + LOB_CLI_SELECT_UNIQUE, + lobNumList, + lobTypList, + lobLocList,lobColNameList,lobTdb().getLobHdfsServer(), + lobTdb().getLobHdfsPort(),0,FALSE); + if (cliRC < 0) + { + getDiagsArea()->mergeAfter(diags); + + step_ = HANDLE_ERROR_; + break; + } + + strcpy(lobLoc_, lobLocList[0]); + NADELETEBASIC(lobColNameList[0],getHeap()); + NADELETEBASIC(lobNumList,getHeap()); + NADELETEBASIC(lobTypList,getHeap()); if (lobTdb().getToType() == ComTdbExeUtilLobExtract::TO_BUFFER_) step_ = EXTRACT_LOB_DATA_; else @@ -3029,28 +3110,31 @@ short ExExeUtilLobExtractTcb::work() &schNameLen, (char *)schName, (char *)lobHandle_, (Lng32)lobHandleLen_); - - //Retrieve the total length of this lob using the handle info and return to the caller - Int64 dummy = 0; - cliRC = SQL_EXEC_LOBcliInterface(lobHandle_, lobHandleLen_,NULL,NULL,NULL,NULL,LOB_CLI_SELECT_LOBLENGTH,LOB_CLI_ExecImmed, 0,&lobDataLen_, &dummy, &dummy,0,0,FALSE); - if (cliRC < 0) - { - getDiagsArea()->mergeAfter(diags); - - step_ = HANDLE_ERROR_; - break; - } - if (lobTdb().retrieveLength()) - { - if ((lobTdb().getBufAddr() != -1) && (lobTdb().getBufAddr() != 0)) - str_cpy_all((char *)lobTdb().getBufAddr(), (char *)&lobDataLen_,sizeof(Int64)); - str_sprintf(statusString_," LOB Length : %d", lobDataLen_); - step_ = RETURN_STATUS_; - break; - } - else - step_ = EXTRACT_LOB_DATA_; - break; + + lobName_ = ExpLOBoper::ExpGetLOBname(uid, lobNum, lobNameBuf_, 1000); + + //Retrieve the total length of this lob using the handle info and return to the caller + + retcode = ExpLOBInterfaceGetLobLength( lobGlobs, + lobName_, + lobLoc_, + lobType_, + lobTdb().getLobHdfsServer(), + lobTdb().getLobHdfsPort(), + lobHandleLen_, lobHandle_, + lobDataLen_); + + if (lobTdb().retrieveLength()) + { + if ((lobTdb().getBufAddr() != -1) && (lobTdb().getBufAddr() != 0)) + str_cpy_all((char *)lobTdb().getBufAddr(), (char *)&lobDataLen_,sizeof(Int64)); + str_sprintf(statusString_," LOB Length : %d", lobDataLen_); + step_ = RETURN_STATUS_; + break; + } + else + step_ = EXTRACT_LOB_DATA_; + break; } case EXTRACT_LOB_DATA_ : @@ -3069,38 +3153,16 @@ short ExExeUtilLobExtractTcb::work() lobDataSpecifiedExtractLen_ = lobTdb().totalBufSize_; - + if (lobDataSpecifiedExtractLen_ == 0) + { + // Passed in length is 0 indicates the caller is done with + // this lobhandle and wants to close this cursor + step_ = CLOSE_CURSOR_; + break; + } - short *lobNumList = new (getHeap()) short[1]; - short *lobTypList = new (getHeap()) short[1]; - char **lobLocList = new (getHeap()) char*[1]; - char **lobColNameList = new (getHeap()) char*[1]; - lobLocList[0] = new (getHeap()) char[1024]; - lobColNameList[0] = new (getHeap()) char[256]; - - Lng32 numLobs = lobNum; - Lng32 cliRC = SQL_EXEC_LOBddlInterface - ( - schName, - schNameLen, - uid, - numLobs, - LOB_CLI_SELECT_UNIQUE, - lobNumList, - lobTypList, - lobLocList,lobColNameList,lobTdb().getLobHdfsServer(), - lobTdb().getLobHdfsPort(),0,FALSE); - if (cliRC < 0) - { - getDiagsArea()->mergeAfter(diags); - - step_ = HANDLE_ERROR_; - break; - } - - strcpy(lobLoc_, lobLocList[0]); - + // Read the lob contents into target file @@ -3204,6 +3266,15 @@ short ExExeUtilLobExtractTcb::work() if (lobTdb().getToType() == ComTdbExeUtilLobExtract::TO_BUFFER_) so = Lob_Buffer; lobDataSpecifiedExtractLen_ = *((Int64 *)(lobTdb().dataExtractSizeIOAddr())); + + if (lobDataSpecifiedExtractLen_ == 0) + { + // Passed in length is 0 indicates the caller is done with + // this lobhandle and wants to close this cursor + step_ = CLOSE_CURSOR_; + break; + } + retcode = ExpLOBInterfaceSelectCursor (lobGlobs, lobName_, @@ -3246,7 +3317,7 @@ short ExExeUtilLobExtractTcb::work() break; } - remainingBytes_ = (Lng32)lobDataOutputLen; + remainingBytes_ = lobDataOutputLen; currPos_ = 0; @@ -3309,33 +3380,11 @@ short ExExeUtilLobExtractTcb::work() getLobErrStr(intParam1)); step_ = HANDLE_ERROR_; break; - } + } step_ = DONE_; } break; - - case RETURN_STRING_: - { - if (qparent_.up->isFull()) - return WORK_OK; - - Lng32 size = MINOF((Lng32)lobTdb().dataExtractSizeIOAddr(), (Lng32)remainingBytes_); - - moveRowToUpQueue(&lobData_[currPos_], size); - - remainingBytes_ -= size; - currPos_ += size; - - if (remainingBytes_ <= 0) - { - step_ = READ_CURSOR_; - qparent_.down->removeHead(); - } - - return WORK_RESCHEDULE_AND_RETURN; - } - break; case RETURN_STATUS_: { @@ -3442,6 +3491,18 @@ ExExeUtilLobUpdateTcb::ExExeUtilLobUpdateTcb currContext->exHeap(),currContext,lobTdb().getLobHdfsServer(), lobTdb().getLobHdfsPort()); } +ExExeUtilLobUpdateTcb::~ExExeUtilLobUpdateTcb() +{ + freeResources(); +} + +void ExExeUtilLobUpdateTcb::freeResources() +{ + ContextCli *currContext = + getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()-> + getStatement()->getContext(); + NADELETE(lobGlobals_,LOBglobals,currContext->exHeap()); +} short ExExeUtilLobUpdateTcb::work() { @@ -3560,7 +3621,7 @@ short ExExeUtilLobUpdateTcb::work() char **lobColNameList = new (getHeap()) char*[1]; lobLocList[0] = new (getHeap()) char[1024]; lobColNameList[0] = new (getHeap()) char[256]; - + Lng32 numLobs = lobNum; Lng32 cliRC = SQL_EXEC_LOBddlInterface ( @@ -3582,6 +3643,10 @@ short ExExeUtilLobUpdateTcb::work() } strcpy(lobLoc_, lobLocList[0]); + NADELETEBASIC(lobLocList[0],getHeap()); + NADELETEBASIC(lobColNameList[0],getHeap()); + NADELETEBASIC(lobNumList,getHeap()); + NADELETEBASIC(lobTypList,getHeap()); char outLobHandle[LOB_HANDLE_LEN]; Int32 outHandleLen; Int64 requestTag = 0; @@ -3674,6 +3739,10 @@ short ExExeUtilLobUpdateTcb::work() } strcpy(lobLoc_, lobLocList[0]); + NADELETEBASIC(lobLocList[0],getHeap()); + NADELETEBASIC(lobColNameList[0],getHeap()); + NADELETEBASIC(lobNumList,getHeap()); + NADELETEBASIC(lobTypList,getHeap()); char outLobHandle[LOB_HANDLE_LEN]; Int32 outHandleLen; Int64 requestTag = 0; @@ -3766,33 +3835,15 @@ short ExExeUtilLobUpdateTcb::work() } strcpy(lobLoc_, lobLocList[0]); + NADELETEBASIC(lobLocList[0],getHeap()); + NADELETEBASIC(lobColNameList[0],getHeap()); + NADELETEBASIC(lobNumList,getHeap()); + NADELETEBASIC(lobTypList,getHeap()); + char outLobHandle[LOB_HANDLE_LEN]; Int32 outHandleLen; Int64 requestTag = 0; - /* retcode = ExpLOBInterfaceDelete(lobGlobs, - lobTdb().getLobHdfsServer(), - lobTdb().getLobHdfsPort(), - lobName_, - lobLoc_, - lobHandleLen_, - lobHandle_, - requestTag_, - getLobGlobals()->xnId(), - inDescSyskey, - 0,1); - if (retcode < 0) - { - Lng32 cliError = 0; - - Lng32 intParam1 = -retcode; - ComDiagsArea * diagsArea = getDiagsArea(); - ExRaiseSqlError(getHeap(), &diagsArea, - (ExeErrorCode)(8442), NULL, &intParam1, - &cliError, NULL, (char*)"ExpLOBInterfaceUpdate", - getLobErrStr(intParam1)); - step_ = HANDLE_ERROR_; - break; - } */ + retcode = ExpLOBInterfaceUpdate(lobGlobs, lobTdb().getLobHdfsServer(), lobTdb().getLobHdfsPort(), http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/exp/ExpLOBaccess.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp index 3764752..af2846c 100644 --- a/core/sql/exp/ExpLOBaccess.cpp +++ b/core/sql/exp/ExpLOBaccess.cpp @@ -230,21 +230,6 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff if (cliErr == 100 ) { isEOD= TRUE; - cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleLenIn, - NULL, NULL, - (char *)&dummyParam, (Lng32 *)&dummyParam, - LOB_CLI_SELECT_CLOSE, LOB_CLI_ExecImmed, - &dummyParam, &dummyParam, - &dummyParam, &dummyParam, - &cliInterface, - transId,lobTrace_); - if (cliErr <0 ) - { - str_sprintf(logBuf, "LOB_CLI_SELECT_CLOSE Returned cli error %d",cliErr); - lobDebugInfo(logBuf,0,__LINE__,lobTrace_); - err = LOB_DESC_READ_ERROR; - return err; - } } else @@ -256,9 +241,17 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff char temp[blackBoxLen+1]; str_cpy_and_null(temp, blackBox, blackBoxLen, '\0', '0', TRUE); lobDataFile_ = temp; + outOffset = offset; + err=statSourceFile(temp,outSize); + if (err != LOB_OPER_OK) + return err; } - outOffset = offset; - outSize = size; + else + { + outOffset = offset; + outSize = size; + } + } str_sprintf(logBuf, " Returned after ::fetchCursor %Ld,%Ld",outOffset,outSize); @@ -804,6 +797,62 @@ Ex_Lob_Error ExLob::readExternalSourceFile(char *srcfile, char *&fileData, Int32 return LOB_OPER_OK; } +Ex_Lob_Error ExLob::getLength(char *handleIn, Int32 handleInLen,Int64 &outLobLen,LobsSubOper so, Int64 transId) +{ + char logBuf[4096]; + Int32 cliErr = 0; + Ex_Lob_Error err=LOB_OPER_OK; + char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6]; + Int32 blackBoxLen = 0; + Int64 dummy = 0; + Int32 dummy2 = 0; + if (so != Lob_External_File) + { + + cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen,NULL,NULL,NULL,NULL,LOB_CLI_SELECT_LOBLENGTH,LOB_CLI_ExecImmed, 0,&outLobLen, 0, 0,0,transId,lobTrace_); + + if (cliErr < 0 ) { + str_sprintf(logBuf,"CLI SELECT_LOBLENGTH returned error %d",cliErr); + lobDebugInfo(logBuf, 0,__LINE__,lobTrace_); + + return LOB_DESC_READ_ERROR; + } + } + else + { + //Get the lob external filename from the descriptor file and get the length of the file + cliErr = SQL_EXEC_LOBcliInterface(handleIn, + handleInLen, + blackBox, &blackBoxLen, + NULL, 0, + LOB_CLI_SELECT_UNIQUE, LOB_CLI_ExecImmed, + &dummy, &dummy, + &dummy, &dummy, + 0, + transId,lobTrace_); + if (cliErr < 0 ) { + str_sprintf(logBuf,"CLI SELECT_LOBLENGTH returned error %d",cliErr); + lobDebugInfo(logBuf, 0,__LINE__,lobTrace_); + + return LOB_DESC_READ_ERROR; + } + if (blackBox && blackBoxLen >0 ) + { + // we have received the external data file name from the + // descriptor table + + char temp[blackBoxLen+1]; + str_cpy_and_null(temp, blackBox, blackBoxLen, '\0', '0', TRUE); + + + err=statSourceFile(temp,outLobLen); + if (err != LOB_OPER_OK) + return err; + } + + } + return err; +} Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOper, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemSize,Int64 lobGCLimit, char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char *handleOut, Int32 &handleOutLen, Int64 xnId, void *lobGlobals) { Ex_Lob_Error err=LOB_OPER_OK; @@ -1566,7 +1615,7 @@ Ex_Lob_Error ExLob::readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 h if (cursor.eod_) { // remove cursor from the map. // server has already closed the cursor. - closeCursor(handleIn, handleInLen); + closeCursor(handleIn, handleInLen,transId); // indicate EOD to SQL operLen = 0; return LOB_OPER_OK; @@ -1585,17 +1634,48 @@ Ex_Lob_Error ExLob::readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 h -Ex_Lob_Error ExLob::closeCursor(char *handleIn, Int32 handleInLen) +Ex_Lob_Error ExLob::closeCursor(char *handleIn, Int32 handleInLen, Int64 transId) { char logBuf[4096]; + Int64 dummyParam = 0; + Int32 cliErr = 0; + Ex_Lob_Error err = LOB_OPER_OK; lobCursors_it it = lobCursors_.find(string(handleIn, handleInLen)); + if (it == lobCursors_.end()) + { + // cursor already closed + return LOB_OPER_OK; + } + + void *cliInterface = it->second.cliInterface_; + if (cliInterface) + { + cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen, + NULL, NULL, + (char *)&dummyParam, (Lng32 *)&dummyParam, + LOB_CLI_SELECT_CLOSE, LOB_CLI_ExecImmed, + &dummyParam, &dummyParam, + &dummyParam, &dummyParam, + &cliInterface, + transId,lobTrace_); + if (cliErr <0 ) + { + str_sprintf(logBuf, "LOB_CLI_SELECT_CLOSE Returned cli error %d",cliErr); + lobDebugInfo(logBuf,0,__LINE__,lobTrace_); + err = LOB_DESC_READ_ERROR; + return err; + } + } if (it != lobCursors_.end()) { str_sprintf(logBuf,"closing cursor for handle"); lobDebugInfo(logBuf,0,__LINE__,lobTrace_); lobCursors_.erase(it); } - return LOB_OPER_OK; + + + + return LOB_OPER_OK; } Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, Int64 lobMaxSize, Int64 lobMaxChunkMemLen, char *handleIn, Int32 handleInLen, Int64 lobGCLimit, void *lobGlobals) @@ -2021,6 +2101,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I operLen += bytesRead; tgt += bytesRead; } + hdfsCloseFile(fs_, fdData_); fdData_ = NULL; return LOB_OPER_OK; @@ -2110,7 +2191,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr, if (err==LOB_OPER_OK) closeCursor(handleIn, - handleLenIn); + handleLenIn,transId); else return err; } @@ -2225,7 +2306,7 @@ Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 siz tgtOffset += chunkSize; } closeCursor(handleIn, - handleInLen); + handleInLen,transId); } close(fdDestFile); return LOB_OPER_OK; @@ -2352,7 +2433,7 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s } closeCursor(handleIn, - handleInLen); + handleInLen,transId); } hdfsCloseFile(fs_, fdTgtFile); fdTgtFile=NULL; @@ -2637,7 +2718,11 @@ Ex_Lob_Error ExLobsOper ( if (blackBox) (lobGlobals->getHeap())->deallocateMemory((char*) blackBox); break; - + case Lob_GetLength: + { + err = lobPtr->getLength(handleIn, handleInLen,retOperLen,subOperation,transId); + } + break; case Lob_ReadDesc: // read desc only. Needed for pass thru. err = lobPtr->getDesc(desc,handleIn,handleInLen,(char *)blackBox, &blackBoxLen,handleOut,handleOutLen,transId); retOperLen = 0; @@ -2704,7 +2789,7 @@ Ex_Lob_Error ExLobsOper ( break; case Lob_CloseCursor: - err = lobPtr->closeCursor(handleIn, handleInLen); + err = lobPtr->closeCursor(handleIn, handleInLen,transId); break; case Lob_CloseDataCursorSimple: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/exp/ExpLOBaccess.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h index acb63fc..3a5a8ca 100644 --- a/core/sql/exp/ExpLOBaccess.h +++ b/core/sql/exp/ExpLOBaccess.h @@ -436,7 +436,7 @@ class ExLob : public NABasicObject Ex_Lob_Error purgeLob(); Ex_Lob_Error closeFile(); LobInputOutputFileType fileType(char *ioFileName); - Ex_Lob_Error closeCursor(char *handleIn, Int32 handleInLen); + Ex_Lob_Error closeCursor(char *handleIn, Int32 handleInLen,Int64 transId); Ex_Lob_Error closeDataCursorSimple(const char *fileName, ExLobGlobals *lobGlobals); Ex_Lob_Error doSanityChecks(char *dir, LobsStorage storage, @@ -496,7 +496,7 @@ class ExLob : public NABasicObject ExLobStats *getStats() { return &stats_; } NAHeap *getLobGlobalHeap() { return lobGlobalHeap_;} - + Ex_Lob_Error getLength(char *handleIn, Int32 handleInLen,Int64 &outLobLen,LobsSubOper so, Int64 transId); // ExLobRequest *getRequest() { return &request_; } //The next 2 functions are not active at this point. They serve as an example http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/exp/ExpLOBenums.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBenums.h b/core/sql/exp/ExpLOBenums.h index 37af1d0..d04b080 100644 --- a/core/sql/exp/ExpLOBenums.h +++ b/core/sql/exp/ExpLOBenums.h @@ -259,7 +259,9 @@ typedef enum { Lob_Cleanup, // destroy everything under globals Lob_PerformGC, Lob_RestoreLobDataFile, - Lob_PurgeBackupLobDataFile + Lob_PurgeBackupLobDataFile, + Lob_GetLength + } LobsOper; typedef enum { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/exp/ExpLOBinterface.cpp ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBinterface.cpp b/core/sql/exp/ExpLOBinterface.cpp index 5016a01..9b58344 100644 --- a/core/sql/exp/ExpLOBinterface.cpp +++ b/core/sql/exp/ExpLOBinterface.cpp @@ -982,8 +982,9 @@ Lng32 ExpLOBInterfaceSelectCursor(void * exLobGlob, waitedOp, exLobGlob, 0, - hdfsDetailError, 0,0,0,0,0,0,0, + hdfsDetailError,0,0,0,0,0,0,0, openType + ); if (err != LOB_OPER_OK) @@ -994,6 +995,60 @@ Lng32 ExpLOBInterfaceSelectCursor(void * exLobGlob, return LOB_ACCESS_SUCCESS; } + +Lng32 ExpLOBInterfaceGetLobLength(void * exLobGlob, + char * lobName, + char * lobLoc, + Lng32 lobType, + char * lobHdfsServer, + Lng32 lobHdfsPort, + Int32 handleLen, + char * lobHandle, + Int64 &outLobLen + + ) +{ + Ex_Lob_Error err; + + Int64 dummyParam = 0; + Int32 dummyParam2 = 0; + Ex_Lob_Error status; + Int64 cliError=0; + + LobsOper lo; + LobsSubOper so; + LobsStorage ls = (LobsStorage)lobType; + if (ls == Lob_External_HDFS_File) + so = Lob_External_File; + else + so = Lob_Buffer; + err = ExLobsOper(lobName, + lobHandle, handleLen, + lobHdfsServer, lobHdfsPort, + NULL, dummyParam2, + dummyParam, dummyParam, + outLobLen, + dummyParam, dummyParam, + status, cliError, + lobLoc, ls, //Lob_HDFS_File, + NULL, 0, + dummyParam,NULL, + Lob_GetLength, + so, + TRUE, + exLobGlob, + 0, + 0, 0,0,0,0,0,0,0, + 0 + ); + + if (err != LOB_OPER_OK) + { + return -err; + } + + return LOB_ACCESS_SUCCESS; +} Lng32 ExpLOBinterfaceStats( void * exLobGlob, ExLobStats * lobStats, http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/exp/ExpLOBinterface.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h index a8245c8..88b6fe9 100644 --- a/core/sql/exp/ExpLOBinterface.h +++ b/core/sql/exp/ExpLOBinterface.h @@ -331,6 +331,19 @@ Lng32 ExpLOBinterfaceEmptyDirectory(void * lobGlob, short replication =0, int blocksize=0); +Lng32 ExpLOBInterfaceGetLobLength(void * exLobGlob, + char * lobName, + char * lobLoc, + Lng32 lobType, + char * lobHdfsServer, + Lng32 lobHdfsPort, + Int32 handleLen, + char * lobHandle, + Int64 &outLobLen + + ); + + /* class HdfsFileInfo { http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f94c9ea3/core/sql/generator/GenRelExeUtil.cpp ---------------------------------------------------------------------- diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp index bd0461f..9d0df8c 100644 --- a/core/sql/generator/GenRelExeUtil.cpp +++ b/core/sql/generator/GenRelExeUtil.cpp @@ -4298,7 +4298,7 @@ if (handleInStringFormat_) { exe_util_tdb->setRetrieveLength(TRUE); } - exe_util_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)); + exe_util_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024); generator->setCriDesc(givenDesc, Generator::DOWN); generator->setCriDesc(returnedDesc, Generator::UP);