trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sure...@apache.org
Subject [03/13] incubator-trafodion git commit: Fixes for extract to files and dealing with issues ith 2 or more lob columns.
Date Fri, 25 Sep 2015 21:13:16 GMT
Fixes for extract to files and dealing with issues ith 2 or more lob columns.


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/b3e9a310
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/b3e9a310
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/b3e9a310

Branch: refs/heads/master
Commit: b3e9a310421f4df7da13b079c6b4c918a7182285
Parents: d8b4f82
Author: Sandhya Sundaresan <sandhya@apache.org>
Authored: Wed Aug 19 17:49:59 2015 +0000
Committer: Sandhya Sundaresan <sandhya@apache.org>
Committed: Wed Aug 19 17:49:59 2015 +0000

----------------------------------------------------------------------
 core/sql/cli/Cli.cpp                      |    5 +-
 core/sql/cli/Cli.h                        |    3 +-
 core/sql/cli/CliExtern.cpp                |    5 +-
 core/sql/comexe/ComTdbExeUtil.h           |    8 +-
 core/sql/executor/ExExeUtil.h             |    2 +-
 core/sql/executor/ExExeUtilLoad.cpp       |   21 +-
 core/sql/exp/ExpLOB.cpp                   |   37 +-
 core/sql/exp/ExpLOB.h                     |    2 +-
 core/sql/exp/ExpLOBaccess.cpp             | 1018 +++++++++++++-----------
 core/sql/exp/ExpLOBaccess.h               |   24 +-
 core/sql/exp/ExpLOBenums.h                |    4 +-
 core/sql/exp/ExpLOBexternal.h             |    3 +-
 core/sql/exp/ExpLOBinterface.cpp          |   25 +-
 core/sql/exp/ExpLOBinterface.h            |   11 +-
 core/sql/optimizer/BindItemExpr.cpp       |    2 +
 core/sql/optimizer/ItemFunc.h             |    4 +-
 core/sql/optimizer/NATable.cpp            |    2 +-
 core/sql/optimizer/RelExeUtil.h           |    2 +-
 core/sql/parser/sqlparser.y               |   22 +-
 core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp |    2 +-
 core/sql/sqlcomp/CmpSeabaseDDLtable.cpp   |    7 +-
 21 files changed, 683 insertions(+), 526 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index 46dce39..1a20d62 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -10289,7 +10289,8 @@ Lng32 SQLCLI_LOBddlInterface
  /*IN*/     LOBcliQueryType qType,
  /*IN*/     short *lobNumList,
  /*IN*/     short *lobTypList,
- /*IN*/     char* *lobLocList
+ /*IN*/     char* *lobLocList,
+ /*IN*/    Int64 lobMaxSize
  )
 {
   Lng32 cliRC = 0;
@@ -10373,7 +10374,7 @@ Lng32 SQLCLI_LOBddlInterface
 	    Lng32 rc = ExpLOBoper::createLOB
 	      (NULL, currContext.exHeap(),
 	       lobLocList[i],
-	       objectUID, lobNumList[i]);
+	       objectUID, lobNumList[i], lobMaxSize);
 	    
 	    if (rc)
 	      {

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/cli/Cli.h
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.h b/core/sql/cli/Cli.h
index cb0be3b..75ed6ae 100644
--- a/core/sql/cli/Cli.h
+++ b/core/sql/cli/Cli.h
@@ -917,7 +917,8 @@ Lng32 SQLCLI_LOBddlInterface
  /*IN*/     LOBcliQueryType qType,
  /*IN*/     short *lobNumList,
  /*IN*/     short *lobTypList,
- /*IN*/     char* *lobLocList
+ /*IN*/     char* *lobLocList,
+ /*IN*/     Int64 lobMaxSize
  );
 
 Lng32 SQLCLI_LOBloader2sqlInterface

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/cli/CliExtern.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/CliExtern.cpp b/core/sql/cli/CliExtern.cpp
index dc22672..13e4e8c 100644
--- a/core/sql/cli/CliExtern.cpp
+++ b/core/sql/cli/CliExtern.cpp
@@ -7422,7 +7422,8 @@ Lng32 SQL_EXEC_LOBddlInterface
  /*IN*/     LOBcliQueryType qType,
  /*IN*/     short *lobNumList,
  /*IN*/     short *lobTypList,
- /*IN*/     char* *lobLocList
+ /*IN*/     char* *lobLocList,
+ /*IN */    Int64 lobMaxSize
  )
 {
   Lng32 retcode;
@@ -7442,7 +7443,7 @@ Lng32 SQL_EXEC_LOBddlInterface
 				      qType,
 				      lobNumList,
 				      lobTypList,
-				      lobLocList);
+				       lobLocList, lobMaxSize);
     }
   catch(...)
     {

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/comexe/ComTdbExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h
index 1681e93..98add96 100644
--- a/core/sql/comexe/ComTdbExeUtil.h
+++ b/core/sql/comexe/ComTdbExeUtil.h
@@ -2735,6 +2735,10 @@ public:
   void setErrorIfNotExists(NABoolean v)
   {(v ? flags_ |= ERROR_IF_NOT_EXISTS : flags_ &= ~ERROR_IF_NOT_EXISTS); };
   NABoolean errorIfNotExists() { return (flags_ & ERROR_IF_NOT_EXISTS) != 0; };
+
+  void setErrorIfExists(NABoolean v)
+  {(v ? flags_ |= ERROR_IF_EXISTS : flags_ &= ~ERROR_IF_EXISTS); };
+  NABoolean errorIfExists() { return (flags_ & ERROR_IF_EXISTS) != 0; };
   
   void setTruncateExisting(NABoolean v)
   {(v ? flags_ |= TRUNCATE_EXISTING : flags_ &= ~TRUNCATE_EXISTING); };
@@ -2752,7 +2756,9 @@ private:
     WITH_CREATE        = 0x0004,
     RETRIEVE_LENGTH    =0x0008,
     ERROR_IF_NOT_EXISTS =0x0010,
-    TRUNCATE_EXISTING = 0x0020
+    ERROR_IF_EXISTS     =0x0020,
+    TRUNCATE_EXISTING = 0x0040
+    
   };
 
   NABasicPtr handle_;                                      // 00-07

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index f44757c..6ab075a 100755
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -2911,7 +2911,7 @@ public:
     EMPTY_,
     GET_NO_CHILD_HANDLE_,
     GET_LOB_HANDLE_,
-    RETRIEVE_LOB_HANDLE_LENGTH_,
+    RETRIEVE_LOB_LENGTH_,
     EXTRACT_LOB_DATA_,
     RETURN_STATUS_,
     SEND_REQ_TO_CHILD_,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index eb3a663..e86e057 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -2886,13 +2886,11 @@ short ExExeUtilLobExtractTcb::work()
 		  }
 
 	      }
-	    if (lobTdb().retrieveLength())
-	      step_ = RETRIEVE_LOB_HANDLE_LENGTH_;
-	    else
-	      step_ = EXTRACT_LOB_DATA_;
+	    
+	    step_ = RETRIEVE_LOB_LENGTH_;	    
 	    break;
 	  }
-	case RETRIEVE_LOB_HANDLE_LENGTH_ : 
+	case RETRIEVE_LOB_LENGTH_ : 
 	  {
 	    Int16 flags;
 	    Lng32  lobNum;
@@ -2915,12 +2913,15 @@ short ExExeUtilLobExtractTcb::work()
 		     step_ = HANDLE_ERROR_;
 		     break;
 		   }
-		else
+		if  (lobTdb().retrieveLength())
 		  {
 		    str_sprintf(statusString_," LOB Length : %d", lobDataLen);
 		    step_ = RETURN_STATUS_;
 		    break;	
-		  }	  
+		  }
+		else
+		  step_ = EXTRACT_LOB_DATA_;
+		break;
 	      
 	  }
 	case EXTRACT_LOB_DATA_ :
@@ -2956,7 +2957,7 @@ short ExExeUtilLobExtractTcb::work()
 	       LOB_CLI_SELECT_UNIQUE,
 	       lobNumList,
 	       lobTypList,
-	       lobLocList);
+	       lobLocList,0);
 	    if (cliRC < 0)
 	      {
 		getDiagsArea()->mergeAfter(diags);
@@ -2972,7 +2973,7 @@ short ExExeUtilLobExtractTcb::work()
 
 	    if (lobTdb().getToType() == ComTdbExeUtilLobExtract::TO_FILE_)
 	      {
-		LobTgtFileFlags tgtFlags = Lob_Append_Or_Create;
+		LobTgtFileFlags tgtFlags = Lob_Error_Or_Create;
 		if (lobTdb().errorIfNotExists() && !lobTdb().truncateExisting())
 		  tgtFlags = Lob_Append_Or_Error;
 		if (lobTdb().truncateExisting() && !lobTdb().errorIfNotExists())
@@ -3004,7 +3005,7 @@ short ExExeUtilLobExtractTcb::work()
 		    ComDiagsArea * diagsArea = getDiagsArea();
 		    ExRaiseSqlError(getHeap(), &diagsArea, 
 				    (ExeErrorCode)(8442), NULL, &intParam1, 
-				    &cliError, NULL, (char*)"ExpLOBInterfaceSelectCursor",
+				    &cliError, NULL, (char*)"ExpLOBInterfaceSelect",
 				    getLobErrStr(intParam1));
 		    step_ = HANDLE_ERROR_;
 		    break;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/exp/ExpLOB.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.cpp b/core/sql/exp/ExpLOB.cpp
index 65072c5..2a96c14 100644
--- a/core/sql/exp/ExpLOB.cpp
+++ b/core/sql/exp/ExpLOB.cpp
@@ -163,7 +163,7 @@ char * ExpLOBoper::ExpGetLOBMDName(Lng32 schNameLen, char * schName,
 }
 Lng32 ExpLOBoper::createLOB(void * lobGlob, void * lobHeap, 
 			    char * lobLoc,
-			    Int64 uid, Lng32 num)
+			    Int64 uid, Lng32 num, Int64 lobMaxSize)
 {
   char buf[100];
   
@@ -183,7 +183,7 @@ Lng32 ExpLOBoper::createLOB(void * lobGlob, void * lobHeap,
   else
     lobGlobL = lobGlob;
 
-  rc = ExpLOBinterfaceCreate(lobGlobL, lobName, lobLoc);
+  rc = ExpLOBinterfaceCreate(lobGlobL, lobName, lobLoc, Lob_HDFS_File,(char *)"default",lobMaxSize);
 
   return rc;
 }
@@ -681,9 +681,14 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[],
 
   Lng32 cliError = 0;
   char * lobData = NULL;
+  lobData= new(h) char[lobLen];
   //send lobData only if it's a lob_file operation
   if (so == Lob_File)
-    lobData = op_data[1];
+    {
+      str_cpy_and_null(lobData,op_data[1],lobLen,'\0',' ',TRUE);
+      
+    }
+ 
   LobsOper lo ;
  
   if (lobOperStatus == CHECK_STATUS_)
@@ -789,8 +794,16 @@ ex_expr::exp_return_type ExpLOBiud::insertData(Lng32 handleLen,
     return ex_expr::EXPR_ERROR;
 
   lobLen = getOperand(1)->getLength();
-  char * lobData = op_data[1];
-LobsOper lo ;
+  
+  char * lobData = NULL;
+  if(fromFile())
+    {
+      lobData = new (h) char[lobLen];  
+      str_cpy_and_null(lobData,op_data[1],lobLen,'\0',' ',TRUE);
+    }
+    else
+      lobData = op_data[1];
+  LobsOper lo ;
  
   if (lobOperStatus == CHECK_STATUS_)
     lo = Lob_Check_Status;
@@ -800,8 +813,8 @@ LobsOper lo ;
     lo = Lob_InsertData;
 
   LobsSubOper so = Lob_None;
-  if (fromFile())
-    so = Lob_File;
+  if (fromFile())    
+    so = Lob_File;       
   else if (fromString() || fromLoad())
     so = Lob_Memory;
   else if (fromLob())
@@ -1195,7 +1208,8 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
 	 lobLen, 
 	 data,
 	 fromLobName, fromSchNameLen, fromSchName,
-	 fromDescKey, fromDescTS);
+	 fromDescKey, fromDescTS,
+	 getLobMaxSize(), getLobMaxChunkMemSize());
     }
   else
     {
@@ -1217,7 +1231,8 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
 	 lobLen, 
 	 data,
 	 fromLobName, fromSchNameLen, fromSchName,
-	 fromDescKey, fromDescTS);
+	 fromDescKey, fromDescTS,
+	 getLobMaxSize(), getLobMaxChunkMemSize());
     }
 
   if (rc < 0)
@@ -1382,7 +1397,7 @@ ex_expr::exp_return_type ExpLOBconvert::eval(char *op_data[],
 				 (lobOperStatus == CHECK_STATUS_ ? 1 : 0),
  				 waitedOp,
 
-				 descKey, lobLen, lobLen, tgtFileName,getLobMaxChunkMemSize());
+				 0, lobLen, lobLen, tgtFileName,getLobMaxChunkMemSize());
     }
   else if (toString())
     {
@@ -1407,7 +1422,7 @@ ex_expr::exp_return_type ExpLOBconvert::eval(char *op_data[],
 				 (lobOperStatus == CHECK_STATUS_ ? 1 : 0),
  				 waitedOp,
 
-				 descKey, lobLen, lobLen, lobData,getLobMaxChunkMemSize());
+				 0, lobLen, lobLen, lobData,getLobMaxChunkMemSize());
 
       if (rc == LOB_ACCESS_PREEMPT)
 	{

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/exp/ExpLOB.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.h b/core/sql/exp/ExpLOB.h
index 50ee631..a043f4e 100644
--- a/core/sql/exp/ExpLOB.h
+++ b/core/sql/exp/ExpLOB.h
@@ -234,7 +234,7 @@ public:
 
   static Lng32 createLOB(void * lobGlob, void * lobHeap,
 			 char * lobLoc,
-			 Int64 uid, Lng32 lobNum);
+			 Int64 uid, Lng32 lobNum, Int64 lobMAxSize);
 
   static Lng32 dropLOB(void * lobGlob, void * lobHeap, 
 		       char * lobLoc,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 5fc555b..669777d 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -116,7 +116,7 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
 			       LobsStorage storage,
                                char *hdfsServer, Int64 hdfsPort,
                                int bufferSize , short replication ,
-                               int blockSize, ExLobGlobals *lobGlobals)
+                               int blockSize, Int64 lobMaxSize, ExLobGlobals *lobGlobals)
 {
   int openFlags;
   mode_t filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
@@ -207,7 +207,7 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
 	      return LOB_DESC_FILE_CREATE_ERROR;
 	    }
 	  //write empty header info into it. 
-	  ExLobDescHeader header(LOB_DATA_FILE_SIZE_MAX);
+	  ExLobDescHeader header(lobMaxSize);
       
 	  Int64 numWritten = 0;
 	  numWritten = hdfsWrite(fs_, fdDesc_, (void *)&header, sizeof(ExLobDescHeader));
@@ -319,15 +319,14 @@ Ex_Lob_Error ExLob::writeData(Int64 offset, char *data, Int32 size, Int64 &operL
        return LOB_DATA_FILE_OPEN_ERROR;
      }
 
-     if (hdfsWrite(fs_, fdData_, data, size) == -1) {
+     if ((operLen = hdfsWrite(fs_, fdData_, data, size)) == -1) {
        return LOB_DATA_WRITE_ERROR;
      }
      if (hdfsFlush(fs_, fdData_)) {
        return LOB_DATA_FLUSH_ERROR;
      }
 
-    operLen = size;
-
+    
     return LOB_OPER_OK;
 }
 
@@ -559,7 +558,7 @@ Ex_Lob_Error ExLob::readHdfsSourceFile(char *srcfile, char *&fileData, Int32 &si
        return LOB_SOURCE_FILE_OPEN_ERROR;
      }
 
-     //TBD We need to be able to read a very large file in a loop as well. 
+     
      fileData = (char *) (getLobGlobalHeap())->allocateMemory(size);
      if (fileData == (char *)-1) {
        return LOB_SOURCE_DATA_ALLOC_ERROR;
@@ -657,7 +656,7 @@ Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOpe
       {
 	return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
       }
-    err = allocateDesc((unsigned int)sourceLen, descNumOut, dataOffset);
+    err = allocateDesc((unsigned int)sourceLen, descNumOut, dataOffset, lobMaxSize);
 
     operLen = 0; 
     if (err != LOB_OPER_OK)
@@ -675,34 +674,15 @@ Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOpe
     return err;
 }
 
-Ex_Lob_Error ExLob::writeLobData(char *source, Int64 sourceLen, LobsSubOper subOperation, 
-				 Int64 descNumIn, Int64 &operLen, Int64 lobMaxChunkMemSize)
+Ex_Lob_Error ExLob::writeLobData(char *source, Int64 sourceLen, LobsSubOper subOperation, Int64 tgtOffset,Int64 &operLen, Int64 lobMaxChunkMemSize)
 {
-    Int64 dataOffset;
     Ex_Lob_Error err; 
-    ExLobDesc desc;
-
-    operLen = 0;
-
-    // get offset from desc row
-    err = getDesc(desc);
-    if (err != LOB_OPER_OK)
-       return err;
-
-    int cliErr = request_.getCliError();
-    if (cliErr < 0 || cliErr == 100) { // some error or EOD.
-       return LOB_DESC_READ_ERROR;
-    }
-    
-    if ((source == NULL)) { 
-       return LOB_SOURCE_DATA_ERROR;
-    }
-
+   
     char *inputAddr = source;
-    Int64 inputSize = desc.getSize();
-    Int64 writeOffset = desc.getOffset();
     Int64 readOffset = 0;
     Int32 allocMemSize = 0;
+    Int64 inputSize = sourceLen;
+    Int64 writeOffset = tgtOffset;
     while(inputSize > 0)
       {
         allocMemSize = MINOF(lobMaxChunkMemSize, inputSize);
@@ -742,7 +722,7 @@ Ex_Lob_Error ExLob::writeLobData(char *source, Int64 sourceLen, LobsSubOper subO
     return err;
 }
 
-Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 offset, Int64 &operLen)
+Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size,  Int64 &operLen)
 {
    Ex_Lob_Error err = LOB_OPER_OK; 
 
@@ -750,29 +730,25 @@ Ex_Lob_Error ExLob::readToMem(char *memAddr, Int64 size, Int64 offset, Int64 &op
    int cliErr;
 
    operLen = 0;
-
-   /* sss
-   if (fdData_ || (openFlags_ != O_RDONLY)) 
-   {
-      hdfsCloseFile(fs_, fdData_);
-      fdData_=NULL;
-   }
-   openFlags_ = O_RDONLY;
-   fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
-   if (!fdData_) {
-     return LOB_DATA_FILE_OPEN_ERROR;
-     }*/
+   ExLobDesc desc;
+   Int64 sizeToRead = 0;
+   
+   
+   err = getDesc(desc);
+   sizeToRead = MINOF(size,desc.getSize());
    
-   err = readDataToMem(memAddr, offset, size, operLen);
+   if (getRequest()->getBlackBoxLen() == -1)
+     sizeToRead = size;
+   err = readDataToMem(memAddr, desc.getOffset(),sizeToRead, operLen);
 
    return err;
 }
 LobInputOutputFileType ExLob::fileType(char *ioFileName)
 {
   std::string fileTgt(ioFileName);
-  std:string hdfsDirStr("hdfs:///");
+  std:string hdfsDirStr("hdfs://");
   std::string httpStr("http://");
-  std:: string fileDirStr("file:///");
+  std:: string fileDirStr("file://");
   short found = 0;
   LobInputOutputFileType  filetype;
   bool isHdfs = FALSE;
@@ -795,26 +771,44 @@ LobInputOutputFileType ExLob::fileType(char *ioFileName)
     else
       return LOCAL_FILE;
 }
-Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 offset, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags)
+Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 tgtLength, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags)
 {
-  Ex_Lob_Error err; 
- 
+  Ex_Lob_Error err = LOB_OPER_OK; 
+  Int64 srcOffset = 0;
+  Int64 srcLength = 0;
   LobInputOutputFileType tgtType = fileType(tgtFileName);
+  ExLobDesc desc;
+  err = getDesc(desc);
+  if (err != LOB_OPER_OK)
+    return err;
+  if (getRequest()->getBlackBoxLen() == -1)  // mxlobsrvr returned -1 indicating multiple chunks for this particular lob handle
+    {
+      //the data retrieval in chunks is handled in readDataToMem.
+    }
+  else if (tgtLength <=0 )
+    {
+      return LOB_SOURCE_FILE_READ_ERROR;
+    }
+  else
+    {
+      srcOffset = desc.getOffset();
+      
+    }
   if (tgtType == HDFS_FILE)
     {
-      err = readDataToHdfsFile(tgtFileName,  0 /*offset*/, operLen, lobMaxChunkMemLen, fileflags);
+      err = readDataToHdfsFile(tgtFileName,  srcOffset , tgtLength,operLen, lobMaxChunkMemLen, fileflags);
       if (err != LOB_OPER_OK)
 	return err;
     }
   else if(tgtType == CURL_FILE)
     {
-      err = readDataToExternalFile(tgtFileName, 0, operLen, lobMaxChunkMemLen, fileflags);
+      err = readDataToExternalFile(tgtFileName, srcOffset, tgtLength, operLen, lobMaxChunkMemLen, fileflags);
       if (err != LOB_OPER_OK)
 	return err;
     }
   else if (tgtType == LOCAL_FILE)
-    {
-      err = readDataToLocalFile(tgtFileName,0 /*offset */,operLen, lobMaxChunkMemLen, fileflags);
+    { 
+      err = readDataToLocalFile(tgtFileName,srcOffset, tgtLength,operLen, lobMaxChunkMemLen, fileflags);
       if (err != LOB_OPER_OK)
 	return err;
     }
@@ -824,16 +818,27 @@ Ex_Lob_Error ExLob::readToFile(char *tgtFileName, Int64 offset, Int64 &operLen,
   return LOB_OPER_OK;
 }
 
-Ex_Lob_Error ExLob::append(char *data, Int64 size, Int64 headDescNum, Int64 &operLen)
+Ex_Lob_Error ExLob::append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemSize)
 {
     Ex_Lob_Error err = LOB_OPER_OK;
     Int64 dummyParam;
     Int64 dataOffset=0;
-    err = allocateDesc((unsigned int)size, dummyParam, dataOffset);
+    Int64 sourceLen = size;
+    if (so == Lob_File)
+      {
+	err = statSourceFile(data, sourceLen); 
+	if (err != LOB_OPER_OK)
+	  return err;
+      }
+    if (sourceLen <= 0 || sourceLen > lobMaxSize)
+      {
+	return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
+      }
+    err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize);
     if (err != LOB_OPER_OK)
       return err;
     request_.setType(Lob_Req_Append);
-    request_.getDesc().setSize(size);
+    request_.getDesc().setSize(sourceLen);
     request_.setDataOffset(dataOffset);
     request_.send();
 
@@ -848,34 +853,64 @@ Ex_Lob_Error ExLob::append(char *data, Int64 size, Int64 headDescNum, Int64 &ope
        return LOB_DESC_APPEND_ERROR;
     }
 
-    
-    // write new lob desc
-    if (size != -1)
-    { 
-      err = writeData(NULL, data, size, operLen);
-      hdfsCloseFile(fs_, fdData_);
-      fdData_ = NULL;
+   
+    err = writeLobData(data, sourceLen,so,dataOffset,operLen,lobMaxChunkMemSize);
+    if (err != LOB_OPER_OK)
+      return err;
+    return LOB_OPER_OK;
+}
+Ex_Lob_Error ExLob::insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize)
+{
+   Ex_Lob_Error err; 
+   ExLobDesc desc;
 
-      if (err != LOB_OPER_OK)
-        return err;
+   operLen = 0;
+
+   // get offset and input size from desc (the one that was just                           inserted into the descriptor handle table)
+   err = getDesc(desc);
+   if (err != LOB_OPER_OK)
+     return err;
+
+    int cliErr = request_.getCliError();
+    if (cliErr < 0 || cliErr == 100) { // some error or EOD.
+       return LOB_DESC_READ_ERROR;
+    }
+    
+    if ((data == NULL)) { 
+       return LOB_SOURCE_DATA_ERROR;
     }
 
-    operLen = size;
+    char *inputAddr = data;
+    Int64 inputSize = desc.getSize();
+    Int64 tgtOffset = desc.getOffset();
+    err = writeLobData(inputAddr, inputSize,so, tgtOffset, 
+		       operLen,lobMaxChunkMemSize);
+    if (err != LOB_OPER_OK)
+      return err;
     return LOB_OPER_OK;
 }
-
-Ex_Lob_Error ExLob::update(char *data, Int64 size, Int64 headDescNum, Int64 &operLen)
+Ex_Lob_Error ExLob::update(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize)
 {
     Ex_Lob_Error err = LOB_OPER_OK;
     Int64 dummyParam;
     Int64 dataOffset = 0;
-
-    err = allocateDesc((unsigned int)size, dummyParam, dataOffset);
+    Int64 sourceLen = size;
+    if (so == Lob_File)
+      {
+	err = statSourceFile(data, sourceLen); 
+	if (err != LOB_OPER_OK)
+	  return err;
+      }
+    if (sourceLen <= 0 || sourceLen > lobMaxSize)
+      {
+	return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
+      }
+    err = allocateDesc((unsigned int)sourceLen, dummyParam, dataOffset, lobMaxSize);
     if (err != LOB_OPER_OK)
       return err;
     // send a message to mxlobsrvr to do an update into descriptor tables
     request_.setType(Lob_Req_Update);
-    request_.getDesc().setSize(size);
+    request_.getDesc().setSize(sourceLen);
     request_.setDataOffset(dataOffset);
 
     request_.send();
@@ -892,22 +927,9 @@ Ex_Lob_Error ExLob::update(char *data, Int64 size, Int64 headDescNum, Int64 &ope
     }
 
    
-
-    // write new lob desc
-    if (size != -1)
-    {
-
-     
-      err = writeData(NULL, data, size, operLen);
-      hdfsCloseFile(fs_, fdData_);
-      fdData_ = NULL;
-
-      if (err != LOB_OPER_OK)
-        return err;
-    }
-
-    operLen = size;
-
+    err = writeLobData(data, sourceLen,so,dataOffset,operLen,lobMaxChunkMemSize);
+    if (err != LOB_OPER_OK)
+      return err;
     return LOB_OPER_OK;
 }
 
@@ -1335,7 +1357,7 @@ Ex_Lob_Error ExLob::doSanityChecks(char *dir, LobsStorage storage,
 
 
 
-Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset)
+Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, Int64 lobMaxSize)
 {
     Ex_Lob_Error err = LOB_OPER_OK;
     Lng32 retval = 0;
@@ -1351,7 +1373,7 @@ Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset)
 	fdDesc_ = NULL;
 	return LOB_DESC_FILE_OPEN_ERROR;
       }
-    ExLobDescHeader header;
+      ExLobDescHeader header(lobMaxSize);
     numRead = hdfsPread(fs_,fdDesc_, 0, (void *)&header, sizeof(ExLobDescHeader) );
     if (numRead <=0)
       {
@@ -1473,7 +1495,6 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
       offset = cursor.descOffset_ + cursor.bytesRead_;
       // #endif
 
-#ifdef SQ_USE_HDFS
       if (!fdData_ || (openFlags_ != O_RDONLY)) 
       {
          hdfsCloseFile(fs_, fdData_);
@@ -1507,18 +1528,13 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
          cursor.eod_ = true; 
          continue;
       }
-#else
-      bytesRead = pread(fdData_, tgt, bytesToCopy, cursor.descOffset_ + cursor.bytesRead_);
-      if (bytesRead == -1) {
-         return LOB_DATA_READ_ERROR;
-      }
-#endif
+
 
       cursor.bytesRead_ += bytesRead;
       operLen += bytesRead;
       tgt += bytesRead;
    }
-
+   hdfsCloseFile(fs_, fdData_);
    return LOB_OPER_OK;
 }
 
@@ -1552,7 +1568,7 @@ Ex_Lob_Error ExLob::readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cur
       offset = cursor.descOffset_ + cursor.bytesRead_;
 
       // gets chunks of 64KB. Uses readDirect internally.
-      //bytesRead = hdfsPread(fs_, fdData_, offset, tgt, bytesToCopy);
+      // bytesRead = hdfsPread(fs_, fdData_, offset, tgt, bytesToCopy);
       bytesRead = hdfsRead(fs_, fdData_, tgt, bytesToCopy);
 
       stats_.numHdfsReqs++;
@@ -1587,16 +1603,12 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
                                   Int64 offset, Int64 size, Int64 &operLen)
 
 { 
-  Ex_Lob_Error err;
+  Ex_Lob_Error err = LOB_OPER_OK;
   operLen = 0;
-  NABoolean multipleChunks = FALSE;
-  ExLobDesc *arrayOfDescs=NULL;
-  ExLobDesc desc;
-  Int64 sizeToRead = 0;
   Int64 bytesRead = 0;
-  err = getDesc(desc);
-  sizeToRead = MINOF(size,desc.getSize());
-  if (getRequest()->getBlackBoxLen() == -1)
+  NABoolean multipleChunks = FALSE;
+ 
+  if (getRequest()->getBlackBoxLen() == -1) // mxlobsrvr returned -1 indicating multiple chunks for this particular lob handle
     {
       multipleChunks = TRUE;
      
@@ -1613,10 +1625,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
     return LOB_DESC_READ_ERROR;
   }
 
- 
-
-
-  if (!fdData_ || (openFlags_ != O_RDONLY)) 
+  if (fdData_)// we may have a stale handle. close and open to refresh 
     {
       hdfsCloseFile(fs_, fdData_);
       fdData_=NULL;
@@ -1627,11 +1636,19 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
 	return LOB_DATA_FILE_OPEN_ERROR;
       }
     }
-     
+  else
+    {
+      fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+      if (!fdData_) {
+	openFlags_ = -1;
+	return LOB_DATA_FILE_OPEN_ERROR;
+      }
+    }
+  
   if (!multipleChunks)
     {
-      if ((bytesRead = hdfsPread(fs_, fdData_, desc.getOffset() + offset, 
-				 memAddr, sizeToRead)) == -1) {
+      if ((bytesRead = hdfsPread(fs_, fdData_, offset, 
+				 memAddr, size)) == -1) {
 	  
 	return LOB_DATA_READ_ERROR;
       }
@@ -1657,146 +1674,237 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
 }
 
  
-Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName,  Int64 offset, Int64 &writeOperLen, Int64 lobMaxChunkMemSize, Int32 fileflags)
+
+Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName,  Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemSize, Int32 fileflags)
 { 
     Ex_Lob_Error err;
     Int64 operLen = 0;
-    ExLobDesc desc;
-    err = getDesc(desc);
-    if (err != LOB_OPER_OK)
-      return err;
-
-    if (desc.getSize() <= 0)
-      {
-	return LOB_SOURCE_FILE_READ_ERROR;
-      }
-    Int64 srcLen = desc.getSize();
+   
+    Int64 srcLen = size;
     Int64 srcOffset = offset;
     Int64 tgtOffset = 0;
     char *lobData = 0;
     Int64 chunkSize = 0;
-    while ( srcLen > 0)
+
+    if (srcLen <=0)
+       return LOB_SOURCE_DATA_ALLOC_ERROR;
+    // open the targte file for writing 
+    int filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
+    int openFlags = O_RDWR ; // O_DIRECT needs mem alignment
+    if ((LobTgtFileFlags)fileflags == Lob_Append_Or_Error )
+      openFlags |= O_APPEND;
+    else
+      openFlags |= O_TRUNC;
+    int fdDestFile = open(fileName, openFlags, filePerms);
+	  
+    if (fdDestFile >0 )
       {
-	chunkSize = MINOF(srcLen, lobMaxChunkMemSize);
-	lobData = (char *) (getLobGlobalHeap())->allocateMemory(chunkSize);
+	if ((LobTgtFileFlags)fileflags == Lob_Error_Or_Create)
+	    return LOB_TARGET_FILE_EXISTS_ERROR;	  
+      }
+    if (fdDestFile == -1) 
+      {
+	if (((LobTgtFileFlags)fileflags == Lob_Append_Or_Error) ||
+	    ((LobTgtFileFlags)fileflags == Lob_Truncate_Or_Error))
+	  return LOB_TARGET_FILE_OPEN_ERROR;	  
+	else
+	  {
+	    openFlags = O_CREAT | O_RDWR ;
+	    fdDestFile = open(fileName, openFlags, filePerms);
+	    if (fdDestFile == -1)	  
+	      return LOB_TARGET_FILE_OPEN_ERROR; 
+	  }
+      }
+    if ((srcLen < lobMaxChunkMemSize) && (getRequest()->getBlackBoxLen() != -1)) // simple single I/O case
+      {
+	lobData = (char *) (getLobGlobalHeap())->allocateMemory(srcLen);
 
 	if (lobData == NULL) 
 	  {
 	    return LOB_SOURCE_DATA_ALLOC_ERROR;
 	  }
-
-	err = readToMem(lobData, chunkSize, srcOffset,operLen);
+	err = readDataToMem(lobData, srcOffset,srcLen,operLen);
 	if (err != LOB_OPER_OK)
 	  {
 	    getLobGlobalHeap()->deallocateMemory(lobData);
 	    return err;
 	  }
        
-
-	int filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
-	int openFlags = O_RDWR ; // O_DIRECT needs mem alignment
-	if (((LobTgtFileFlags)fileflags == Lob_Append_Or_Error ) ||
-	    (LobTgtFileFlags)fileflags == Lob_Append_Or_Create )
-	  openFlags |= O_APPEND;
-	else
-	  openFlags |= O_TRUNC;
-	int fdDestFile = open(fileName, openFlags, filePerms);
-	if (fdDestFile == -1) 
+	writeOperLen += pwrite(fdDestFile, lobData, srcLen, tgtOffset) ; 
+	if (writeOperLen <= 0)
 	  {
-	    if (((LobTgtFileFlags)fileflags == Lob_Append_Or_Error) ||
-		((LobTgtFileFlags)fileflags == Lob_Truncate_Or_Error))
-	      {		
+	    getLobGlobalHeap()->deallocateMemory(lobData);
+	    return LOB_TARGET_FILE_WRITE_ERROR;
+	  }
+	getLobGlobalHeap()->deallocateMemory(lobData);
+      }
+    else // multiple chunks to read
+      {
+	err = openCursor(getRequest()->getHandleIn(), 
+		     getRequest()->getHandleInLen());
+	if (err != LOB_OPER_OK)
+	  return err;
+	while ( srcLen > 0)
+	  {
+	    chunkSize = MINOF(srcLen, lobMaxChunkMemSize);
+	    lobData = (char *) (getLobGlobalHeap())->allocateMemory(chunkSize);
+
+	    if (lobData == NULL) 
+	      {
 		getLobGlobalHeap()->deallocateMemory(lobData);
-		return LOB_TARGET_FILE_OPEN_ERROR;
+		return LOB_SOURCE_DATA_ALLOC_ERROR;
 	      }
-	    else
+	    //handle reading the multiple chunks like a cursor
+	    err = readCursor(lobData,chunkSize, getRequest()->getHandleIn(),
+			     getRequest()->getHandleInLen(), operLen);
+
+	    if ((err != LOB_OPER_OK) || (operLen != chunkSize))
 	      {
-		openFlags = O_CREAT | O_RDWR ;
-	    	fdDestFile = open(fileName, openFlags, filePerms);
-		if (fdDestFile == -1)
-		  {
-		    getLobGlobalHeap()->deallocateMemory(lobData);
-		    return LOB_TARGET_FILE_OPEN_ERROR; 
-		  }
+		getLobGlobalHeap()->deallocateMemory(lobData);
+		return err;
 	      }
-	  }
-
-	writeOperLen += pwrite(fdDestFile, lobData, chunkSize, tgtOffset) ; 
-	if (writeOperLen <= 0)
-	  {
+       
+	    writeOperLen += pwrite(fdDestFile, lobData, chunkSize, tgtOffset) ; 
+	    if (writeOperLen <= 0)
+	      {
+		getLobGlobalHeap()->deallocateMemory(lobData);
+		return LOB_TARGET_FILE_WRITE_ERROR;
+	      }     
 	    getLobGlobalHeap()->deallocateMemory(lobData);
-	    return LOB_TARGET_FILE_WRITE_ERROR;
+	    srcLen -= chunkSize;
+	    tgtOffset += chunkSize;     
 	  }
-     
-	close(fdDestFile);
-	getLobGlobalHeap()->deallocateMemory(lobData);
-	srcLen -= chunkSize;
-	srcOffset += chunkSize;
-	tgtOffset += chunkSize;
+	closeCursor(getRequest()->getHandleIn(), 
+		    getRequest()->getHandleInLen());
       }
+    close(fdDestFile);
     return LOB_OPER_OK;
 }
 
-Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName,  Int64 offset, Int64 &writeOperLen, Int64 lobMaxChunkMemLen, Int32 fileflags)
+
+Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName,  Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemLen, Int32 fileflags)
 { 
   Ex_Lob_Error err;
   Int64 operLen = 0;
-  ExLobDesc desc;
-  err = getDesc(desc);
-  if (err != LOB_OPER_OK)
-    return err;
-
-  if (desc.getSize() <=0 )
-    {
-      return LOB_SOURCE_FILE_READ_ERROR;
-    }
-  Int64 srcLen = desc.getSize();
+ 
+  Int64 srcLen = size;
   Int64 srcOffset = offset;
   Int64 tgtOffset = 0;
   char *lobData = 0;
-  Int64 chunkSize = 0;
-  while (srcLen >0 )
+  Int64 chunkSize = 0;	
+  hdfsFile  fdTgtFile;
+  // open and write to the target file
+  int openFlags = O_WRONLY;
+  if ((LobTgtFileFlags)fileflags == Lob_Append_Or_Error )
+    openFlags |= O_APPEND;
+
+  //hdfsFile fdTgtFile = hdfsOpenFile(fs_,tgtFileName, openFlags, 0,0,0);
+  if (hdfsExists(fs_,tgtFileName) == 0)
     {
-      chunkSize = MINOF(srcLen,lobMaxChunkMemLen);
-      lobData = (char *) (getLobGlobalHeap())->allocateMemory(chunkSize);
-      if (lobData == NULL) {
-	return LOB_SOURCE_DATA_ALLOC_ERROR;
-      }
-      err = readToMem(lobData, chunkSize,offset,operLen);
+      if ((LobTgtFileFlags)fileflags == Lob_Error_Or_Create)
+	return LOB_TARGET_FILE_EXISTS_ERROR;
+      else
+	{
+	  openFlags =  O_WRONLY ;
+	  fdTgtFile = hdfsOpenFile(fs_, tgtFileName, openFlags, 0,0,0);
+	  if (fdTgtFile == NULL)
+	    return LOB_TARGET_FILE_OPEN_ERROR;
+	}
+    }
+  else
+    { 
+      if (((LobTgtFileFlags)fileflags == Lob_Append_Or_Error) ||
+	  ((LobTgtFileFlags)fileflags == Lob_Truncate_Or_Error))
+	return LOB_TARGET_FILE_OPEN_ERROR;	  
+      else
+	{ 
+	  openFlags =  O_WRONLY ;
+	  fdTgtFile = hdfsOpenFile(fs_, tgtFileName, openFlags, 0,0,0);
+	  if (fdTgtFile == NULL)
+	    return LOB_TARGET_FILE_OPEN_ERROR;
+	}
+    } 
+
+  if ((srcLen < lobMaxChunkMemLen) && (getRequest()->getBlackBoxLen() != -1)) // simple single I/O case
+    {
+      lobData = (char *) (getLobGlobalHeap())->allocateMemory(srcLen);
+
+      if (lobData == NULL) 
+	{
+	  return LOB_SOURCE_DATA_ALLOC_ERROR;
+	}
+      err = readDataToMem(lobData, srcOffset,srcLen,operLen);
       if (err != LOB_OPER_OK)
 	{
 	  getLobGlobalHeap()->deallocateMemory(lobData);
 	  return err;
 	}
-      // open and write to the target file
-      int openFlags = O_WRONLY|O_CREAT;
-      hdfsFile fdTgtFile = hdfsOpenFile(fs_,tgtFileName, openFlags, 0,0,0);
-      if (fdTgtFile == NULL) {
-	getLobGlobalHeap()->deallocateMemory(lobData);
-	return LOB_TARGET_FILE_OPEN_ERROR;
-      }
-      writeOperLen += hdfsWrite(fs_,fdTgtFile,lobData, chunkSize);
+     
+      writeOperLen += hdfsWrite(fs_,fdTgtFile,lobData, srcLen);
       if (writeOperLen <= 0)
-	return LOB_TARGET_FILE_WRITE_ERROR;
-	
+	{
+	  getLobGlobalHeap()->deallocateMemory(lobData);  
+	  return LOB_TARGET_FILE_WRITE_ERROR;
+	}
       if (hdfsFlush(fs_, fdTgtFile)) 
 	{
 	  getLobGlobalHeap()->deallocateMemory(lobData);
 	  return LOB_DATA_FLUSH_ERROR;
 	} 
-      hdfsCloseFile(fs_, fdTgtFile);
-    
       getLobGlobalHeap()->deallocateMemory(lobData);
-      fdTgtFile = NULL;
-      srcLen -= chunkSize;
-      srcOffset += chunkSize;
-      tgtOffset += chunkSize;
     }
-   
+  else
+    {// multiple chunks to read
+      err = openCursor(getRequest()->getHandleIn(), 
+		       getRequest()->getHandleInLen());
+      if (err != LOB_OPER_OK)
+	return err;
+      while ( srcLen > 0)
+	{
+	  chunkSize = MINOF(srcLen, lobMaxChunkMemLen);
+	  lobData = (char *) (getLobGlobalHeap())->allocateMemory(chunkSize);	      
+	  if (lobData == NULL) 
+	    {
+	      getLobGlobalHeap()->deallocateMemory(lobData);
+	      return LOB_SOURCE_DATA_ALLOC_ERROR;
+	    }
+	  //handle reading the multiple chunks like a cursor
+	  err = readCursor(lobData,chunkSize, getRequest()->getHandleIn(),
+			   getRequest()->getHandleInLen(), operLen);
+
+	  if ((err != LOB_OPER_OK) || (operLen != chunkSize))
+	    {
+	      getLobGlobalHeap()->deallocateMemory(lobData);
+	      return err;
+	    }
+	  writeOperLen += hdfsWrite(fs_,fdTgtFile,lobData, chunkSize);
+	  if (writeOperLen <= 0)
+	    {
+	      getLobGlobalHeap()->deallocateMemory(lobData);  
+	      return LOB_TARGET_FILE_WRITE_ERROR;
+	    }
+	  if (hdfsFlush(fs_, fdTgtFile)) 
+	    {
+	      getLobGlobalHeap()->deallocateMemory(lobData);
+	      return LOB_DATA_FLUSH_ERROR;
+	    } 
+	  getLobGlobalHeap()->deallocateMemory(lobData);
+	  srcLen -= chunkSize;
+
+	}
+      closeCursor(getRequest()->getHandleIn(), 
+		  getRequest()->getHandleInLen());	    
+    }
+  hdfsCloseFile(fs_, fdTgtFile);
+  hdfsCloseFile(fs_,fdData_);
+  
   return LOB_OPER_OK;
 }
 
-Ex_Lob_Error ExLob::readDataToExternalFile(char *tgtFileName,  Int64 offset, Int64 &operLen,Int64 lobMaxChunkMemLen,Int32 fileflags)
+
+
+
+Ex_Lob_Error ExLob::readDataToExternalFile(char *tgtFileName,  Int64 offset, Int64 size, Int64 &operLen,Int64 lobMaxChunkMemLen,Int32 fileflags)
 { 
   //TBD
   return LOB_OPER_OK;
@@ -2476,78 +2584,78 @@ Ex_Lob_Error ExLobGlobals::processPreOpens()
 #endif
 
 Ex_Lob_Error ExLobsOper (
-    char        *lobName,          // lob name
-    char        *handleIn,         // input handle (for cli calls)
-    Int64       handleInLen,       // input handle len
-    char        *hdfsServer,       // server where hdfs fs resides
-    Int64       hdfsPort,          // port number to access hdfs server
-    char        *handleOut,        // output handle (for cli calls)
-    Int64       &handleOutLen,     // output handle len
-    Int64       descNumIn,         // input desc Num (for flat files only)
-    Int64       &descNumOut,       // output desc Num (for flat files only)
-    Int64       &retOperLen,       // length of data involved in this operation
-    Int64       requestTagIn,      // only for checking status
-    Int64       &requestTagOut,    // returned with every request other than check status
-    Ex_Lob_Error  &requestStatus,  // returned req status
-    Int64       &cliError,         // err returned by cli call
-    char        *dir,              // directory in the storage
-    LobsStorage storage,           // storage type
-    char        *source,           // source (memory addr, filename, foreign lob etc)
-    Int64       sourceLen,         // source len (memory len, foreign desc offset etc)
-    Int64 cursorBytes,
-    char *cursorId,
-    LobsOper    operation,         // LOB operation
-    LobsSubOper subOperation,      // LOB sub operation
-    Int64       waited,            // waited or nowaited
-    void        *&globPtr,         // ptr to the Lob objects. 
-    Int64       transId,
-    void        *blackBox,         // black box to be sent to cli
-    Int64       blackBoxLen,       // length of black box
-    Int64       lobMaxSize,
-    Int64       lobMaxChunkMemSize,
-    int         bufferSize ,
-    short       replication ,
-    int         blockSize,
-    Lng32       openType)
+			 char        *lobName,          // lob name
+			 char        *handleIn,         // input handle (for cli calls)
+			 Int64       handleInLen,       // input handle len
+			 char        *hdfsServer,       // server where hdfs fs resides
+			 Int64       hdfsPort,          // port number to access hdfs server
+			 char        *handleOut,        // output handle (for cli calls)
+			 Int64       &handleOutLen,     // output handle len
+			 Int64       descNumIn,         // input desc Num (for flat files only)
+			 Int64       &descNumOut,       // output desc Num (for flat files only)
+			 Int64       &retOperLen,       // length of data involved in this operation
+			 Int64       requestTagIn,      // only for checking status
+			 Int64       &requestTagOut,    // returned with every request other than check status
+			 Ex_Lob_Error  &requestStatus,  // returned req status
+			 Int64       &cliError,         // err returned by cli call
+			 char        *dir,              // directory in the storage
+			 LobsStorage storage,           // storage type
+			 char        *source,           // source (memory addr, filename, foreign lob etc)
+			 Int64       sourceLen,         // source len (memory len, foreign desc offset etc)
+			 Int64 cursorBytes,
+			 char *cursorId,
+			 LobsOper    operation,         // LOB operation
+			 LobsSubOper subOperation,      // LOB sub operation
+			 Int64       waited,            // waited or nowaited
+			 void        *&globPtr,         // ptr to the Lob objects. 
+			 Int64       transId,
+			 void        *blackBox,         // black box to be sent to cli
+			 Int64       blackBoxLen,       // length of black box
+			 Int64       lobMaxSize,
+			 Int64       lobMaxChunkMemSize,
+			 int         bufferSize ,
+			 short       replication ,
+			 int         blockSize,
+			 Lng32       openType)
 { 
-    Ex_Lob_Error err = LOB_OPER_OK;
-    ExLob *lobPtr = NULL;
-    char fn[MAX_LOB_FILE_NAME_LEN];
-    struct timespec startTime;
-    struct timespec endTime;
-    Int64 secs, nsecs, totalnsecs;
-    ExLobPreOpen *preOpenObj;
-    ExLobGlobals *lobGlobals = NULL;
+  Ex_Lob_Error err = LOB_OPER_OK;
+  ExLob *lobPtr = NULL;
+  char fn[MAX_LOB_FILE_NAME_LEN];
+  struct timespec startTime;
+  struct timespec endTime;
+  Int64 secs, nsecs, totalnsecs;
+  ExLobPreOpen *preOpenObj;
+  ExLobGlobals *lobGlobals = NULL;
 
-    retOperLen = 0;
-    ExLobDesc desc;
+  retOperLen = 0;
+  ExLobDesc desc;
     
-    lobMap_t *lobMap = NULL;
-    lobMap_it it;
+  lobMap_t *lobMap = NULL;
+  lobMap_it it;
 
-    clock_gettime(CLOCK_MONOTONIC, &startTime);
+  clock_gettime(CLOCK_MONOTONIC, &startTime);
 
-    char *fileName = lobName;
+  char *fileName = lobName;
 
-    if (globPtr == NULL)
+  if (globPtr == NULL)
     {
       if (operation == Lob_Init)
-      {
-        globPtr = (void *) new ExLobGlobals();
-        if (globPtr == NULL) 
-          return LOB_INIT_ERROR;
+	{
+	  globPtr = (void *) new ExLobGlobals();
+	  if (globPtr == NULL) 
+	    return LOB_INIT_ERROR;
 
-        lobGlobals = (ExLobGlobals *)globPtr;
+	  lobGlobals = (ExLobGlobals *)globPtr;
 
-        err = lobGlobals->initialize(); 
-        return err;
-      }
+	  err = lobGlobals->initialize(); 
+	  return err;
+	}
       else
-      {
-        return LOB_GLOB_PTR_ERROR;
-      }
+	{
+	  return LOB_GLOB_PTR_ERROR;
+	}
     }
-    else
+  else
     {
       lobGlobals = (ExLobGlobals *)globPtr;
 
@@ -2556,224 +2664,224 @@ Ex_Lob_Error ExLobsOper (
       it = lobMap->find(string(fileName));
 
       if (it == lobMap->end())
-      {
-        //lobPtr = new (lobGlobals->getHeap())ExLob();
-	lobPtr = new ExLob();
-        if (lobPtr == NULL) 
-          return LOB_ALLOC_ERROR;
-
-        err = lobPtr->initialize(fileName, (operation == Lob_Create) ? EX_LOB_CREATE : EX_LOB_RW, 
-                                 dir, storage, hdfsServer, hdfsPort, bufferSize, replication, blockSize,
-                                 lobGlobals);
-        if (err != LOB_OPER_OK)
-          return err;
-
-        lobMap->insert(pair<string, ExLob*>(string(fileName), lobPtr));
-      }
+	{
+	  //lobPtr = new (lobGlobals->getHeap())ExLob();
+	  lobPtr = new ExLob();
+	  if (lobPtr == NULL) 
+	    return LOB_ALLOC_ERROR;
+
+	  err = lobPtr->initialize(fileName, (operation == Lob_Create) ? EX_LOB_CREATE : EX_LOB_RW, dir, storage, hdfsServer, hdfsPort, bufferSize, replication, blockSize,lobMaxSize,lobGlobals);
+	  if (err != LOB_OPER_OK)
+	    return err;
+
+	  lobMap->insert(pair<string, ExLob*>(string(fileName), lobPtr));
+	}
       else
-      {
-        lobPtr = it->second;
+	{
+	  lobPtr = it->second;
         
 #ifndef SQ_USE_HDFS
-        err = lobPtr->doSanityChecks(dir, storage, handleInLen, handleOutLen, blackBoxLen);
-        if (err != LOB_OPER_OK)
-          return err;
+	  err = lobPtr->doSanityChecks(dir, storage, handleInLen, handleOutLen, blackBoxLen);
+	  if (err != LOB_OPER_OK)
+	    return err;
 #endif
-      }
+	}
     }
    
-	MS_Mon_Transid_Type transIdBig;
-        MS_Mon_Transseq_Type transStartId;
- if (!lobGlobals->isHive())
-      {
-	// get current transaction
+  MS_Mon_Transid_Type transIdBig;
+  MS_Mon_Transseq_Type transStartId;
+  if (!lobGlobals->isHive())
+    {
+      // get current transaction
    
-	int transIdErr = ms_transid_get(false, false, &transIdBig, &transStartId);
-	// set the pass thru request object values in the lob
+      int transIdErr = ms_transid_get(false, false, &transIdBig, &transStartId);
+      // set the pass thru request object values in the lob
     
-	lobPtr->getRequest()->setValues(lobPtr->getDescFileName(),
-                                    descNumIn, handleInLen, handleIn, storage,
-                                    transId, transIdBig, transStartId,
-                                    (char *)blackBox, blackBoxLen);
-      }
-    switch(operation)
+      lobPtr->getRequest()->setValues(lobPtr->getDescFileName(),
+				      descNumIn, handleInLen, handleIn, storage,
+				      transId, transIdBig, transStartId,
+				      (char *)blackBox, blackBoxLen);
+    }
+  switch(operation)
     {
-      case Lob_Create:
-        break;
-
-      case Lob_InsertDesc:
-        err = lobPtr->writeDesc(sourceLen, source, subOperation, descNumOut, retOperLen, lobMaxSize);
-        break;
-
-      case Lob_InsertData:
-        err = lobPtr->writeLobData(source, sourceLen, subOperation, descNumIn, retOperLen, lobMaxChunkMemSize);
-        break;
-
-      case Lob_InsertDataSimple:
-        err = lobPtr->writeDataSimple(source, sourceLen, subOperation, retOperLen,
-                                        bufferSize , replication , blockSize);
-        break;
-
-      case Lob_Read:
-        if (subOperation == Lob_Memory)
-          err = lobPtr->readToMem(source, sourceLen, 0,retOperLen);
-        else if (subOperation == Lob_File)
-          err = lobPtr->readToFile(source, 0 /*offset */, retOperLen, lobMaxChunkMemSize,  openType);
-        else  
-          err = LOB_SUBOPER_ERROR;
-        break;
-
-      case Lob_ReadDesc: // read desc only. Needed for pass thru.
-        err = lobPtr->getDesc(desc);
-        retOperLen = 0;
-        break;
-	/*** ssss
-      case Lob_ReadData: // read data only. Lob data file is already opened. 
-        err = lobPtr->readDataFromFile(source, sourceLen, retOperLen);
-        break;
+    case Lob_Create:
+      break;
+
+    case Lob_InsertDesc:
+      err = lobPtr->writeDesc(sourceLen, source, subOperation, descNumOut, retOperLen, lobMaxSize);
+      break;
+
+    case Lob_InsertData:
+      err = lobPtr->insertData(source, sourceLen, subOperation, descNumIn, retOperLen, lobMaxSize,lobMaxChunkMemSize);
+      break;
+
+    case Lob_InsertDataSimple:
+      err = lobPtr->writeDataSimple(source, sourceLen, subOperation, retOperLen,
+				    bufferSize , replication , blockSize);
+      break;
+
+    case Lob_Read:
+      if (subOperation == Lob_Memory)
+	err = lobPtr->readToMem(source,sourceLen,retOperLen);
+      else if (subOperation == Lob_File)
+	err = lobPtr->readToFile(source, sourceLen, retOperLen, lobMaxChunkMemSize,  openType);
+      else  
+	err = LOB_SUBOPER_ERROR;
+      break;
+
+    case Lob_ReadDesc: // read desc only. Needed for pass thru.
+      err = lobPtr->getDesc(desc);
+      retOperLen = 0;
+      break;
+      /*** ssss
+	   case Lob_ReadData: // read data only. Lob data file is already opened. 
+	   err = lobPtr->readDataFromFile(source, sourceLen, retOperLen);
+	   break;
 	
-      case Lob_ReadDataSimple:
-        err = lobPtr->readDataFromFile(source, sourceLen, retOperLen);
-        break;
-	***/
-      case Lob_OpenCursor:
-        err = lobPtr->openCursor(handleIn, handleInLen);
-        break;
-
-      case Lob_OpenDataCursorSimple:  
-        if (openType == 1) { // preopen
-          sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
-          preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn, descNumIn, sourceLen, cursorBytes, waited);
-          lobGlobals->addToPreOpenList(preOpenObj);
-        } else if (openType == 2) { // must open
-          sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
-          fileName = fn;
-          err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals);
-        } else
-          err = LOB_SUBOPER_ERROR;
-        break;
-
-      case Lob_ReadCursor:
-        if (subOperation == Lob_Memory)
-          err = lobPtr->readCursor(source, sourceLen, handleIn, handleInLen, retOperLen);
-        else if (subOperation == Lob_File)
-          err = lobPtr->readCursor(source, -1, handleIn, handleInLen, retOperLen);
-        else  
-          err = LOB_SUBOPER_ERROR;
-        break;
-
-      case Lob_ReadDataCursorSimple:
-	 sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
-        fileName = fn;       
-        err = lobPtr->readDataCursorSimple(fileName, source, sourceLen, retOperLen, lobGlobals);
-        break;
-
-      case Lob_CloseFile:
-        if (lobPtr->hasNoOpenCursors()) {
-	  lobGlobals->traceMessage("Lob_CloseFile",NULL,__LINE__);
-	  err = lobPtr->closeFile();
-	  it = lobMap->find(string(lobName));
-	  lobMap->erase(it);
-	  delete lobPtr;
-          lobPtr = NULL;
-	}  
-        break;
-
-      case Lob_CloseCursor:
-        err = lobPtr->closeCursor(handleIn, handleInLen);
-        break;
-
-      case Lob_CloseDataCursorSimple:
+	   case Lob_ReadDataSimple:
+	   err = lobPtr->readDataFromFile(source, sourceLen, retOperLen);
+	   break;
+      ***/
+    case Lob_OpenCursor:
+      err = lobPtr->openCursor(handleIn, handleInLen);
+      break;
+
+    case Lob_OpenDataCursorSimple:  
+      if (openType == 1) { // preopen
+	sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
+	preOpenObj = new (lobGlobals->getHeap()) ExLobPreOpen(lobPtr, fn, descNumIn, sourceLen, cursorBytes, waited);
+	lobGlobals->addToPreOpenList(preOpenObj);
+      } else if (openType == 2) { // must open
 	sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
-        fileName = fn;
-        err = lobPtr->closeDataCursorSimple(fileName, lobGlobals);
-        break;
-
-      case Lob_Append:
-        if (subOperation == Lob_Memory)
-          err = lobPtr->append(source, sourceLen, descNumIn, retOperLen);
-        else if (subOperation == Lob_File)
-          err = lobPtr->append(source, -1, descNumIn, retOperLen);
-        else  
-          err = LOB_SUBOPER_ERROR;
-        break;
-
-       case Lob_Update:
-        if (subOperation == Lob_Memory)
-          err = lobPtr->update(source, sourceLen, descNumIn, retOperLen);
-        else  
-          err = LOB_SUBOPER_ERROR;
-        break;
-
-      case Lob_Delete:
-        err = lobPtr->delDesc();
-        break;
-
-      case Lob_Drop:
-        err = lobPtr->purgeLob();
-        it = lobMap->find(string(lobName));
-        lobMap->erase(it);
-        delete lobPtr;
-        lobPtr = NULL;
-        break;
-
-      case Lob_Purge:
-        err = lobPtr->purgeLob();
-        it = lobMap->find(string(lobName));
-        lobMap->erase(it);
-        delete lobPtr;
-        lobPtr = NULL;
-        break;
-
-      case Lob_Print:
-        err = lobPtr->print();
-        break;
-
-      case Lob_Stats:
+	fileName = fn;
+	err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals);
+      } else
+	err = LOB_SUBOPER_ERROR;
+      break;
+
+    case Lob_ReadCursor:
+      if (subOperation == Lob_Memory)
+	err = lobPtr->readCursor(source, sourceLen, handleIn, handleInLen, retOperLen);
+      else if (subOperation == Lob_File)
+	err = lobPtr->readCursor(source, -1, handleIn, handleInLen, retOperLen);
+      else  
+	err = LOB_SUBOPER_ERROR;
+      break;
+
+    case Lob_ReadDataCursorSimple:
+      sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
+      fileName = fn;       
+      err = lobPtr->readDataCursorSimple(fileName, source, sourceLen, retOperLen, lobGlobals);
+      break;
+
+    case Lob_CloseFile:
+      if (lobPtr->hasNoOpenCursors()) {
+	lobGlobals->traceMessage("Lob_CloseFile",NULL,__LINE__);
+	err = lobPtr->closeFile();
+	it = lobMap->find(string(lobName));
+	lobMap->erase(it);
+	delete lobPtr;
+	lobPtr = NULL;
+      }  
+      break;
+
+    case Lob_CloseCursor:
+      err = lobPtr->closeCursor(handleIn, handleInLen);
+      break;
+
+    case Lob_CloseDataCursorSimple:
+      sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
+      fileName = fn;
+      err = lobPtr->closeDataCursorSimple(fileName, lobGlobals);
+      break;
+
+    case Lob_Append:
+      if (subOperation == Lob_Memory)
+	err = lobPtr->append(source, sourceLen, subOperation, descNumIn, retOperLen,lobMaxSize, lobMaxChunkMemSize);
+      else if (subOperation == Lob_File)
+	err = lobPtr->append(source, -1, subOperation, descNumIn, retOperLen,lobMaxSize, lobMaxChunkMemSize);
+      else  
+	err = LOB_SUBOPER_ERROR;
+      break;
+
+    case Lob_Update:
+      if (subOperation == Lob_Memory)
+	err = lobPtr->update(source, sourceLen, subOperation, descNumIn, retOperLen, lobMaxSize, lobMaxChunkMemSize);
+      else if (subOperation == Lob_File)
+	err = lobPtr->update(source, -1, subOperation,descNumIn, retOperLen,lobMaxSize, lobMaxChunkMemSize); 
+      else
+	err = LOB_SUBOPER_ERROR;
+      break;
+
+    case Lob_Delete:
+      err = lobPtr->delDesc();
+      break;
+
+    case Lob_Drop:
+      err = lobPtr->purgeLob();
+      it = lobMap->find(string(lobName));
+      lobMap->erase(it);
+      delete lobPtr;
+      lobPtr = NULL;
+      break;
+
+    case Lob_Purge:
+      err = lobPtr->purgeLob();
+      it = lobMap->find(string(lobName));
+      lobMap->erase(it);
+      delete lobPtr;
+      lobPtr = NULL;
+      break;
+
+    case Lob_Print:
+      err = lobPtr->print();
+      break;
+
+    case Lob_Stats:
       err = lobPtr->readStats(source);
       lobPtr->initStats(); // because file may remain open across cursors
       break;
 
-      case Lob_Empty_Directory:
-        lobPtr->initialize(fileName, EX_LOB_RW,
-                          dir, storage, hdfsServer, hdfsPort, bufferSize, replication, blockSize);
-        err = lobPtr->emptyDirectory();
-        break;
+    case Lob_Empty_Directory:
+      lobPtr->initialize(fileName, EX_LOB_RW,
+			 dir, storage, hdfsServer, hdfsPort, bufferSize, replication, blockSize);
+      err = lobPtr->emptyDirectory();
+      break;
 
-      case Lob_Cleanup:
-        delete lobGlobals;
-        break;
+    case Lob_Cleanup:
+      delete lobGlobals;
+      break;
 
-      default:
-        err = LOB_OPER_ERROR;
-        break;
+    default:
+      err = LOB_OPER_ERROR;
+      break;
     }
 
-    if (!lobGlobals->isHive() )
-      {
-	if (lobPtr)
-	  // set the pass thru request object values from the lob
-	  lobPtr->getRequest()->getValues(descNumOut, handleOutLen, handleOut, 
-					  requestStatus, cliError,
-					  (char *)blackBox, blackBoxLen);    // reinstate the transaction
-	if (TRANSID_IS_VALID(transIdBig)) {
-	  ms_transid_reinstate(transIdBig, transStartId);
-	}
+  if (!lobGlobals->isHive() )
+    {
+      if (lobPtr)
+	// set the pass thru request object values from the lob
+	lobPtr->getRequest()->getValues(descNumOut, handleOutLen, handleOut, 
+					requestStatus, cliError,
+					(char *)blackBox, blackBoxLen);    // reinstate the transaction
+      if (TRANSID_IS_VALID(transIdBig)) {
+	ms_transid_reinstate(transIdBig, transStartId);
       }
+    }
 
-    clock_gettime(CLOCK_MONOTONIC, &endTime);
+  clock_gettime(CLOCK_MONOTONIC, &endTime);
 
-    secs = endTime.tv_sec - startTime.tv_sec;
-    nsecs = endTime.tv_nsec - startTime.tv_nsec;
-    if (nsecs < 0) {
-      secs--;
-      nsecs += NUM_NSECS_IN_SEC;
-    }
-    totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
-    if (lobPtr && lobPtr->getStats())
-      lobPtr->getStats()->hdfsAccessLayerTime += totalnsecs; 
+  secs = endTime.tv_sec - startTime.tv_sec;
+  nsecs = endTime.tv_nsec - startTime.tv_nsec;
+  if (nsecs < 0) {
+    secs--;
+    nsecs += NUM_NSECS_IN_SEC;
+  }
+  totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
+  if (lobPtr && lobPtr->getStats())
+    lobPtr->getStats()->hdfsAccessLayerTime += totalnsecs; 
        
-    return err;
+  return err;
 }
 
 void cleanupLOBDataDescFiles(const char *lobHdfsServer,int lobHdfsPort,const char *lobHdfsLoc)

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index 94da806..aeb6356 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -64,8 +64,6 @@
 
 using namespace std;
 
-// 2GB max Lob data file size 
-#define LOB_DATA_FILE_SIZE_MAX (1 << 31)
 
 #define MAX_LOB_FILE_NAME_LEN 256
 #define MAX_HANDLE_IN_LEN 1024
@@ -368,17 +366,17 @@ class ExLob
 
     Ex_Lob_Error initialize(char *lobFile, Ex_Lob_Mode mode, char *dir, 
                             LobsStorage storage, char *hdfsServer, Int64 hdfsPort,
-                            int bufferSize = 0, short replication =0, int blocksize=0, ExLobGlobals *lobGlobals = NULL);
+                            int bufferSize = 0, short replication =0, int blocksize=0, Int64 lobMaxSize = 0, ExLobGlobals *lobGlobals = NULL);
     Ex_Lob_Error initialize(char *lobFile);
     Ex_Lob_Error writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOperation, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize);
     Ex_Lob_Error writeLobData(char *source, Int64 sourceLen, 
 			      LobsSubOper subOperation, 
-			      Int64 descNumIn, Int64 &operLen, 
+			      Int64 tgtOffset,Int64 &operLen, 
 			      Int64 lobMaxMemChunkLen);
     Ex_Lob_Error writeDataSimple(char *data, Int64 size, LobsSubOper subOperation, Int64 &operLen,
                                  int bufferSize = 0, short replication =0, int blocksize=0);
-    Ex_Lob_Error readToMem(char *memAddr, Int64 size, Int64 offset, Int64 &operLen);
-    Ex_Lob_Error readToFile(char *fileName, Int64 descNum, Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags);
+    Ex_Lob_Error readToMem(char *memAddr, Int64 size,  Int64 &operLen);
+    Ex_Lob_Error readToFile(char *fileName, Int64 tgtLen,Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags);
     Ex_Lob_Error readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int64 handleInLen, Int64 &operLen);
     Ex_Lob_Error readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen);
     Ex_Lob_Error readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen);
@@ -391,9 +389,10 @@ class ExLob
     Ex_Lob_Error fetchCursor();
     Ex_Lob_Error selectCursorDesc(ExLobRequest *request);
     Ex_Lob_Error fetchCursorDesc(ExLobRequest *request);
-    Ex_Lob_Error append(char *data, Int64 size, Int64 headDescNum, Int64 &operLen);
+    Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize);
+    Ex_Lob_Error append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemLen);
     Ex_Lob_Error append(ExLobRequest *request);
-    Ex_Lob_Error update(char *data, Int64 size, Int64 headDescNum, Int64 &operLen);
+    Ex_Lob_Error update(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemLen);
     Ex_Lob_Error update(ExLobRequest *request);
     Ex_Lob_Error readSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
     Ex_Lob_Error readHdfsSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
@@ -411,7 +410,7 @@ class ExLob
     Ex_Lob_Error doSanityChecks(char *dir, LobsStorage storage,
                                 Int64 handleInLen, Int64 handleOutLen, 
                                 Int64 blackBoxLen);
-    Ex_Lob_Error allocateDesc(unsigned int size, Int64 &descNum, Int64 &dataOffset);
+    Ex_Lob_Error allocateDesc(unsigned int size, Int64 &descNum, Int64 &dataOffset,Int64 lobMaxSize);
     Ex_Lob_Error readStats(char *buffer);
     Ex_Lob_Error initStats();
 
@@ -437,9 +436,10 @@ class ExLob
     Ex_Lob_Error writeData(Int64 offset, char *data, Int32 size, Int64 &operLen);
     Ex_Lob_Error readDataToMem(char *memAddr, Int64 offset, Int64 size, Int64 &operLen);
    
-    Ex_Lob_Error readDataToLocalFile(char *fileName, Int64 offset, Int64 &operLen,Int64 lobMaxChunkMemLen ,Int32 fileFlags);
-    Ex_Lob_Error readDataToHdfsFile(char *fileName, Int64 offset, Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags);
-    Ex_Lob_Error readDataToExternalFile(char *tgtFileName,  Int64 offset, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags);
+    Ex_Lob_Error readDataToLocalFile(char *fileName, Int64 offset, Int64 size,Int64 &operLen,Int64 lobMaxChunkMemLen ,Int32 fileFlags);
+    Ex_Lob_Error readCursorDataToLocalFile(char *fileName,  Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemSize, Int32 fileflags);
+    Ex_Lob_Error readDataToHdfsFile(char *fileName, Int64 offset, Int64 size, Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags);
+    Ex_Lob_Error readDataToExternalFile(char *tgtFileName,  Int64 offset, Int64 size, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags);
     Ex_Lob_Error readDataFromFile(char *memAddr, Int64 len, Int64 &operLen);
 
     Ex_Lob_Error emptyDirectory();

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/exp/ExpLOBenums.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBenums.h b/core/sql/exp/ExpLOBenums.h
index a663eb4..f9c9568 100644
--- a/core/sql/exp/ExpLOBenums.h
+++ b/core/sql/exp/ExpLOBenums.h
@@ -88,6 +88,7 @@ typedef enum {
   LOB_OPER_CONTINUE,
   LOB_INVALID_ERROR_VAL,
   LOB_MAX_LIMIT_ERROR = 560,
+  LOB_TARGET_FILE_EXISTS_ERROR,
   LOB_MAX_ERROR_NUM     // keep this as the last element in enum list.
 } Ex_Lob_Error;
 
@@ -154,6 +155,7 @@ static const char * const lobErrorEnumStr[] =
   "LOB_OPER_CONTINUE",
   "LOB_INVALID_ERROR_VAL", 
   "LOB_MAX_LIMIT_ERROR", //560
+  "LOB_TGT_FILE_EXISTS_ERROR",
   "LOB_MAX_ERROR_NUM"     // keep this as the last element in enum list.
 };
 
@@ -272,7 +274,7 @@ typedef enum {
   Lob_Append_Or_Error =0,
   Lob_Truncate_Or_Error,
   Lob_Truncate_Or_Create,
-  Lob_Append_Or_Create
+  Lob_Error_Or_Create
      
 } LobTgtFileFlags;
 #endif

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/exp/ExpLOBexternal.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBexternal.h b/core/sql/exp/ExpLOBexternal.h
index ab3bf17..75ccaf5 100644
--- a/core/sql/exp/ExpLOBexternal.h
+++ b/core/sql/exp/ExpLOBexternal.h
@@ -142,7 +142,8 @@ Lng32 SQL_EXEC_LOBddlInterface
  /*IN*/     LOBcliQueryType qType,
  /*IN*/     short *lobNumList,
  /*IN*/     short *lobTypList,
- /*IN*/     char* *lobLocList
+ /*IN*/     char* *lobLocList,
+ /*IN*/     Int64 lobMaxSize
  );
 
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/exp/ExpLOBinterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.cpp b/core/sql/exp/ExpLOBinterface.cpp
index 850db34..4db9a71 100644
--- a/core/sql/exp/ExpLOBinterface.cpp
+++ b/core/sql/exp/ExpLOBinterface.cpp
@@ -31,7 +31,7 @@ using std::ofstream;
 #include "ExpLOBinterface.h"
 
 
-Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap,NABoolean isHive)
+Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap,NABoolean isHive, Int64  lobMaxSize)
 {
   Ex_Lob_Error err;
 
@@ -54,8 +54,8 @@ Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap,NABoolean isHive)
                    1, // waited op
 		   lobGlob,
 		   0,
-		   NULL, 0
-		   );
+		   NULL, 0,
+		   lobMaxSize);
   if (lobGlob)
     {
       ((ExLobGlobals *)lobGlob)->setIsHive(isHive);
@@ -108,6 +108,7 @@ Lng32 ExpLOBinterfaceCreate(
 			    void * lobGlob, char * lobName, char * lobLoc,
 			    Lng32 lobType,
 			    char * lobHdfsServer,
+			    Int64 lobMaxSize,
 			    Lng32 lobHdfsPort,
 	                    int    bufferSize ,
 	                    short  replication ,
@@ -132,9 +133,11 @@ Lng32 ExpLOBinterfaceCreate(
                    1, // waited op
 		   lobGlob,
 		   0, NULL, 0,
+		   lobMaxSize,
                    bufferSize ,
                    replication,
                    blockSize
+		   
 		   );
 
   if (err != LOB_OPER_OK)
@@ -474,7 +477,9 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
 				  short srcDescSchNameLen,
 				  char * srcDescSchName,
 				  Int64 srcDescKey, 
-				  Int64 srcDescTS
+				  Int64 srcDescTS,
+				  Int64 lobMaxSize,
+				  Int64 lobMaxChunkMemSize
 				  )
 {
   Ex_Lob_Error err;
@@ -498,7 +503,9 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
                    so,
                    1, 
                    lobGlob,
-                   0, NULL, 0
+                   xnId, NULL, 0,
+		   lobMaxSize,
+		   lobMaxChunkMemSize
                    );
 
   if ((err == LOB_OPER_OK) &&
@@ -549,7 +556,9 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
 			    short srcDescSchNameLen,
 			    char * srcDescSchName,
 			    Int64 srcDescKey, 
-			    Int64 srcDescTS)
+			    Int64 srcDescTS,
+			    Int64 lobMaxSize ,
+			    Int64 lobMaxChunkMemSize )
 {
   Ex_Lob_Error err;
 
@@ -573,7 +582,9 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
                    1, 
                    lobGlob,
                    xnId, 
-		   NULL, 0
+		   NULL, 0,
+		   lobMaxSize,
+		   lobMaxChunkMemSize
                    );
 
   if ((err == LOB_OPER_OK) &&

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/exp/ExpLOBinterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h
index 321be1c..ed4ec05 100644
--- a/core/sql/exp/ExpLOBinterface.h
+++ b/core/sql/exp/ExpLOBinterface.h
@@ -74,7 +74,7 @@ enum ExpLOBinterfaceInputFlags
     ERROR_IF_TGT_FILE_EXISTS_ =  0x0004
   };
 
-Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap, NABoolean isHive=FALSE);
+Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap, NABoolean isHive=FALSE, Int64   lobMaxSize =0);
 
 Lng32 ExpLOBinterfaceCleanup(void *& lobGlob, void * lobHeap);
 
@@ -83,6 +83,7 @@ Lng32 ExpLOBinterfaceCreate(void * lobGlob,
 			    char * lobLoc,
 			    Lng32 lobType = (Lng32)Lob_HDFS_File,
 			    char * lobHdfsServer = (char *)"default",
+			    Int64 lobMaxSize = 0,
 			    Lng32 lobHdfsPort = 0,
 	                    int    bufferSize = 0,
 	                    short  replication =0,
@@ -182,7 +183,9 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
 			    short srcDescSchNameLen,
 			    char * srcDescSchName,
 			    Int64 srcDescKey, 
-			    Int64 srcDescTS);
+			    Int64 srcDescTS,
+			    Int64 lobMaxSize = 0,
+			    Int64 lobMaxChunkMemSize = 0);
 
 Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob, 
 				  char * lobHdfsServer ,
@@ -209,7 +212,9 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
 				  short srcDescSchNameLen,
 				  char * srcDescSchName,
 				  Int64 srcDescKey, 
-				  Int64 srcDescTS
+				  Int64 srcDescTS,
+				  Int64 lobMaxSize = 0,
+				  Int64 lobMaxChunkMemSize = 0
 				  );
 
 Lng32 ExpLOBInterfaceDelete(void * lobGlob, 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/optimizer/BindItemExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindItemExpr.cpp b/core/sql/optimizer/BindItemExpr.cpp
index a68aea2..e2f2738 100644
--- a/core/sql/optimizer/BindItemExpr.cpp
+++ b/core/sql/optimizer/BindItemExpr.cpp
@@ -1,4 +1,6 @@
 /**********************************************************************
+ * b
+ *
 // @@@ START COPYRIGHT @@@
 //
 // (C) Copyright 1994-2015 Hewlett-Packard Development Company, L.P.

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/optimizer/ItemFunc.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h
index 0021bb6..0cc4d5e 100644
--- a/core/sql/optimizer/ItemFunc.h
+++ b/core/sql/optimizer/ItemFunc.h
@@ -2859,6 +2859,7 @@ class LOBupdate : public LOBoper
 	    NABoolean isAppend = FALSE)
     : LOBoper(ITM_LOBUPDATE, val1Ptr, val2Ptr, fromObj),
     objectUID_(-1),
+    lobSize_(0),
     append_(isAppend)
     {};
   
@@ -2879,7 +2880,7 @@ class LOBupdate : public LOBoper
   Int64 & updatedTableObjectUID() { return objectUID_; }
   
   NAString &updatedTableSchemaName() { return schName_; }
-
+  Lng32 & lobSize() { return lobSize_; }
  private:
   // ---------------------------------------------------------------//
   // ObjectUID of the table this blob is being inserted into
@@ -2891,6 +2892,7 @@ class LOBupdate : public LOBoper
 
 
   NABoolean append_;
+  Lng32 lobSize_;
 }; // class LOBupdate
 
 class LOBconvert : public LOBoper

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index e9e8d06..2dc3421 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -5324,7 +5324,7 @@ NATable::NATable(BindWA *bindWA,
 	 LOB_CLI_SELECT_CURSOR,
 	 lobNumList,
 	 lobTypList,
-	 lobLocList);
+	 lobLocList,0);
       
       if (cliRC == 0)
 	{

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/optimizer/RelExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelExeUtil.h b/core/sql/optimizer/RelExeUtil.h
index aa0766b..e4a6cde 100644
--- a/core/sql/optimizer/RelExeUtil.h
+++ b/core/sql/optimizer/RelExeUtil.h
@@ -1862,7 +1862,7 @@ public:
   };
   enum ExtractFileActionType
   {
-    ERROR_IF_NOT_EXISTS =1, TRUNCATE_EXISTING };
+    ERROR_IF_NOT_EXISTS =1, ERROR_IF_EXISTS, TRUNCATE_EXISTING };
   
   
  ExeUtilLobExtract(ItemExpr * handle, 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/parser/sqlparser.y
----------------------------------------------------------------------
diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y
index 5fc96a0..e56eb29 100755
--- a/core/sql/parser/sqlparser.y
+++ b/core/sql/parser/sqlparser.y
@@ -12679,15 +12679,15 @@ update_obj_to_lob_function :
 			        {
 				  $$ = new (PARSERHEAP()) LOBupdate( $3, NULL, LOBoper::STRING_, TRUE);
 				}
-                          | TOK_FILETOLOB '(' TOK_LOCATION character_literal_sbyte ',' TOK_FILE character_literal_sbyte ')'
+                          | TOK_FILETOLOB '('character_literal_sbyte ')'
 			        {
-				  YYERROR;
-				  $$ = new (PARSERHEAP()) LOBupdate( $4, $7, LOBoper::FILE_, FALSE);
+				 
+				  $$ = new (PARSERHEAP()) LOBupdate( $3, NULL,LOBoper::FILE_, FALSE);
 				}
-                          | TOK_FILETOLOB '(' TOK_LOCATION character_literal_sbyte ',' TOK_FILE character_literal_sbyte ',' TOK_APPEND ')'
+                          | TOK_FILETOLOB '('character_literal_sbyte ',' TOK_APPEND ')'
 			        {
-				  YYERROR;
-				  $$ = new (PARSERHEAP()) LOBupdate( $4, $7, LOBoper::FILE_, TRUE);
+				 
+				  $$ = new (PARSERHEAP()) LOBupdate( $3,NULL, LOBoper::FILE_, TRUE);
 				}
 			  | TOK_BUFFERTOLOB '(' TOK_LOCATION character_literal_sbyte ',' TOK_SIZE numeric_literal_exact ')'
 			        {
@@ -15661,7 +15661,7 @@ exe_util_lob_extract : TOK_EXTRACT TOK_LOBLENGTH '(' TOK_LOB QUOTED_STRING  ')'
               
             | TOK_EXTRACT TOK_LOBTOFILE '(' TOK_LOB QUOTED_STRING  ','  QUOTED_STRING ')'
                {
-                 // if file exists, truncate and replace contents. if file doesn't exist, error
+                 // if file exists, error . if file doesn't exist, create
                  // extract lobtofile (lob 'abc',  'file');
 
 		  ConstValue * handle = new(PARSERHEAP()) ConstValue(*$5);
@@ -15671,13 +15671,13 @@ exe_util_lob_extract : TOK_EXTRACT TOK_LOBLENGTH '(' TOK_LOB QUOTED_STRING  ')'
 		   (handle, 
 		    ExeUtilLobExtract::TO_FILE_,
 		    NULL, NULL, 
-		    ExeUtilLobExtract::ERROR_IF_NOT_EXISTS, 
-		    ExeUtilLobExtract::TRUNCATE_EXISTING,
+		    0, 
+		    0,
 		    (char*)$7->data());
 
 		 $$ = lle;
 	       }
-              | TOK_EXTRACT TOK_LOBTOFILE '(' TOK_LOB QUOTED_STRING ',' QUOTED_STRING ',' TOK_CREATE  ')'
+              | TOK_EXTRACT TOK_LOBTOFILE '(' TOK_LOB QUOTED_STRING ',' QUOTED_STRING ',' TOK_TRUNCATE  ')'
                {
                  // if file exists, truncate and replace contents. if file doesn't exist, create
                  // extract lobtofile (lob 'abc',  'file', create);
@@ -15707,7 +15707,7 @@ exe_util_lob_extract : TOK_EXTRACT TOK_LOBLENGTH '(' TOK_LOB QUOTED_STRING  ')'
 		   (handle, 
 		    ExeUtilLobExtract::TO_FILE_,
 		    NULL, NULL, 
-		    0,
+		    ExeUtilLobExtract::ERROR_IF_EXISTS,
 		    0,
 		    (char*)$7->data());
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
index 44825b1..0608a09 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
@@ -792,7 +792,7 @@ short CmpSeabaseMDcleanup::dropLOBs(ExeCliInterface *cliInterface)
                                    LOB_CLI_CLEANUP,
                                    lobNumList_,
                                    lobTypList_,
-                                   lobLocList_);
+                                   lobLocList_,0);
 
   return 0;
 }

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b3e9a310/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
index ef5d58b..78bc3e8 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
@@ -1832,7 +1832,7 @@ short CmpSeabaseDDL::createSeabaseTable2(
 	  j++;
 	}
     }
-  
+  Int64 lobMaxSize =  CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024; 
   if (j > 0)
      {
        Int64 objUID = getObjectUID(&cliInterface,
@@ -1852,7 +1852,8 @@ short CmpSeabaseDDL::createSeabaseTable2(
 					   LOB_CLI_CREATE,
 					   lobNumList,
 					   lobTypList,
-					   lobLocList);
+					   lobLocList,
+					   lobMaxSize);
        if (rc < 0)
 	 {
 	   //sss TBD need to retrive the cli diags here.
@@ -3172,7 +3173,7 @@ short CmpSeabaseDDL::dropSeabaseTable2(
 					  LOB_CLI_DROP,
 					  lobNumList,
 					  lobTypList,
-					  lobLocList);
+					  lobLocList,0);
       if (rc < 0)
 	{
 	  *CmpCommon::diags() << DgSqlCode(-CAT_UNABLE_TO_DROP_OBJECT)


Mime
View raw message