trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From se...@apache.org
Subject [2/3] trafodion git commit: [TRAFODION-2430] RMS enhancements
Date Wed, 25 Jul 2018 17:51:11 GMT
[TRAFODION-2430] RMS enhancements

RMS now displays 3 numeric values separated by '|' in "Details"
columns in the default output based on the type of stats entry.
These numeric values are VAL2, VAL3, VAL4 of the statistics virtual
table.

Stats Type          VAL2              VAL3             VAL4
ROOT_OPER_STATS    MessageBytes     MessageCount     MemoryAllocated
SE_STATS           SE_IO_KBytes     SE_IO_SumTime    ActRowsAccessed
BMO_STATS          Scr_IO_Count     bmoHeapAllocated ScrFileCount

VAL1 is operCpuTime and TEXT column contains the PID of the process

ExFastExtractTcb is now instrumented with ExHdfsScanStats.
Hdfs scan and access opearations are now properly instrumented with ExHdfsScanStats
counters.

Lob access is now instrumented and is part of the statistics
entry(ExHdfsScanStats or ExHbaseAccessStats) attached with tcb.

Additionally, the following changes are done
1. Removed obsolete ExLobStats
2. Optimized space usage in RMS shared segments for some of the operators
3. A new class ExStorageEngineStats is now created. This class is
   type-defed as ExHbaseAccessStats and ExHdfsScanStats.


Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/abda16ef
Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/abda16ef
Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/abda16ef

Branch: refs/heads/master
Commit: abda16efac2a0315cf74de16ec29db24f1280047
Parents: dcae57f
Author: selvaganesang <selva.govindarajan@esgyn.com>
Authored: Tue Jul 24 15:46:53 2018 +0000
Committer: selvaganesang <selva.govindarajan@esgyn.com>
Committed: Tue Jul 24 18:21:00 2018 +0000

----------------------------------------------------------------------
 core/sql/cli/sqlcli.h                   |   5 +-
 core/sql/executor/ExExeUtil.h           |   7 +
 core/sql/executor/ExExeUtilGetStats.cpp |   5 +-
 core/sql/executor/ExExeUtilLoad.cpp     |  94 +++-
 core/sql/executor/ExFastTransport.cpp   |  21 +-
 core/sql/executor/ExFastTransport.h     |   9 +
 core/sql/executor/ExHbaseAccess.h       |   1 -
 core/sql/executor/ExHbaseIUD.cpp        |  12 +-
 core/sql/executor/ExHdfsScan.cpp        |  28 +-
 core/sql/executor/ExHdfsScan.h          |   1 -
 core/sql/executor/ExStats.cpp           | 740 ++++++---------------------
 core/sql/executor/ExStats.h             | 160 ++----
 core/sql/executor/HBaseClient_JNI.h     |   6 +-
 core/sql/executor/HdfsClient_JNI.cpp    |   5 +
 core/sql/executor/OrcFileReader.h       |   1 +
 core/sql/executor/log4cpp.hdfs.config   |  37 --
 core/sql/executor/log4j.hdfs.config     |  50 --
 core/sql/exp/ExpHbaseInterface.h        |   1 -
 core/sql/exp/ExpLOB.cpp                 |   8 +
 core/sql/exp/ExpLOBaccess.cpp           |  42 +-
 core/sql/exp/ExpLOBaccess.h             |  17 +-
 core/sql/exp/ExpLOBenums.h              |   2 -
 core/sql/exp/ExpLOBinterface.cpp        |  70 ++-
 core/sql/exp/ExpLOBinterface.h          |  19 +-
 core/sql/exp/ExpLOBprocess.cpp          |   6 +-
 core/sql/exp/exp_clause.h               |  15 +-
 core/sql/exp/exp_fixup.cpp              |   2 +-
 core/sql/generator/GenExplain.cpp       |  26 +-
 core/sql/regress/core/FILTERRTS         |   1 +
 29 files changed, 401 insertions(+), 990 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/cli/sqlcli.h
----------------------------------------------------------------------
diff --git a/core/sql/cli/sqlcli.h b/core/sql/cli/sqlcli.h
index 2bfbfcb..00ae7b2 100644
--- a/core/sql/cli/sqlcli.h
+++ b/core/sql/cli/sqlcli.h
@@ -1028,9 +1028,10 @@ enum SQLSTATS_DESC_STATS_TYPE {
   SQLSTATS_DESC_REPLICATOR_STATS = 20,
   SQLSTATS_DESC_FAST_EXTRACT_STATS = 21,
   SQLSTATS_DESC_REORG_STATS = 22,
+  SQLSTATS_DESC_SE_STATS = 23,
   SQLSTATS_DESC_HDFSSCAN_STATS = 23,
-  SQLSTATS_DESC_HBASE_ACCESS_STATS = 24,
-  SQLSTATS_DESC_PROCESS_STATS = 25
+  SQLSTATS_DESC_HBASE_ACCESS_STATS = 23,
+  SQLSTATS_DESC_PROCESS_STATS = 25,
 };
 
 

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index 49261a6..435dfb5 100644
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -2128,6 +2128,7 @@ private:
     GET_PROCESS_STATS_ENTRY_,
     FORMAT_AND_RETURN_PROCESS_STATS_,
     GET_HBASE_STATS_ENTRY_,
+    GET_SE_STATS_ENTRY_ = GET_HBASE_STATS_ENTRY_,
     DISPLAY_HBASE_STATS_HEADING_,
     FORMAT_AND_RETURN_HBASE_STATS_,
     GET_HIVE_STATS_ENTRY_,
@@ -2812,6 +2813,8 @@ public:
   virtual short work();
   virtual ~ExExeUtilLobExtractTcb();
   virtual void freeResources();
+  virtual ExOperStats *doAllocateStatsEntry(CollHeap *heap,
+                                            ComTdb *tdb);
   
   ExExeUtilLobExtractTdb & lobTdb() const
   {
@@ -2921,6 +2924,8 @@ public:
   virtual short work();
   virtual ~ExExeUtilLobUpdateTcb();
   virtual void freeResources();
+  virtual ExOperStats *doAllocateStatsEntry(CollHeap *heap,
+                                            ComTdb *tdb);
   ExExeUtilLobUpdateTdb & lobTdb() const
   {
     return (ExExeUtilLobUpdateTdb &) tdb;
@@ -2997,6 +3002,8 @@ public:
 		       const ex_tcb * child_tcb, 
 		       ex_globals * glob = 0);
   
+  virtual ExOperStats *doAllocateStatsEntry(CollHeap *heap,
+                                            ComTdb *tdb);
   virtual short work();
 
  private:

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExExeUtilGetStats.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilGetStats.cpp b/core/sql/executor/ExExeUtilGetStats.cpp
index a8a128d..ead8cfd 100644
--- a/core/sql/executor/ExExeUtilGetStats.cpp
+++ b/core/sql/executor/ExExeUtilGetStats.cpp
@@ -1920,12 +1920,9 @@ short ExExeUtilGetRTSStatisticsTcb::work()
             case SQLSTATS_DESC_UDR_BASE_STATS:
               step_ = GET_UDR_BASE_STATS_ENTRY_;
               break;
-            case SQLSTATS_DESC_HBASE_ACCESS_STATS:
+            case SQLSTATS_DESC_SE_STATS:
               step_ = GET_HBASE_STATS_ENTRY_;
               break;
-            case SQLSTATS_DESC_HDFSSCAN_STATS:
-              step_ = GET_HIVE_STATS_ENTRY_;
-              break;
             default:
               step_ = GET_NEXT_STATS_DESC_ENTRY_;
               break;

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index 960c31f..7c051c4 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -2563,6 +2563,7 @@ void ExExeUtilLobExtractTcb::freeResources()
   if (lobHandle_ and lobName_)
     retcode = ExpLOBInterfaceSelectCursor
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,
@@ -2590,6 +2591,23 @@ ExExeUtilLobExtractTcb::~ExExeUtilLobExtractTcb()
   freeResources();
 }
 
+ExOperStats * ExExeUtilLobExtractTcb::doAllocateStatsEntry(CollHeap *heap,
+							    ComTdb *tdb)
+{
+  ExEspStmtGlobals *espGlobals = getGlobals()->castToExExeStmtGlobals()->castToExEspStmtGlobals();
+  StmtStats *ss; 
+  if (espGlobals != NULL)
+     ss = espGlobals->getStmtStats();
+  else
+     ss = getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()->getStatement()->getStmtStats(); 
+  ExHdfsScanStats *hdfsScanStats = new(heap) ExHdfsScanStats(heap,
+				   this,
+				   tdb);
+  if (ss != NULL) 
+     hdfsScanStats->setQueryId(ss->getQueryId(), ss->getQueryIdLen());
+  return hdfsScanStats;
+}
+
 short ExExeUtilLobExtractTcb::work()
 {
   Lng32 cliRC = 0;
@@ -3023,6 +3041,7 @@ short ExExeUtilLobExtractTcb::work()
 		if(lobTdb().appendOrCreate())
 		  tgtFlags = Lob_Append_Or_Create;
 		retcode = ExpLOBInterfaceSelect(lobGlobs, 
+                                                (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 						lobName_,
 						lobLoc_,
 						lobType_,
@@ -3067,6 +3086,7 @@ short ExExeUtilLobExtractTcb::work()
 	  {
 	    retcode = ExpLOBInterfaceSelectCursor
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,
@@ -3117,6 +3137,7 @@ short ExExeUtilLobExtractTcb::work()
                 
 	    retcode = ExpLOBInterfaceSelectCursor
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,
@@ -3186,6 +3207,7 @@ short ExExeUtilLobExtractTcb::work()
 	  {
 	    retcode = ExpLOBInterfaceSelectCursor
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,
@@ -3334,6 +3356,23 @@ void ExExeUtilLobUpdateTcb::freeResources()
  exLobGlobals_ = NULL;
 }
 
+ExOperStats * ExExeUtilLobUpdateTcb::doAllocateStatsEntry(CollHeap *heap,
+							    ComTdb *tdb)
+{
+  ExEspStmtGlobals *espGlobals = getGlobals()->castToExExeStmtGlobals()->castToExEspStmtGlobals();
+  StmtStats *ss; 
+  if (espGlobals != NULL)
+     ss = espGlobals->getStmtStats();
+  else
+     ss = getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()->getStatement()->getStmtStats(); 
+  ExHdfsScanStats *hdfsScanStats = new(heap) ExHdfsScanStats(heap,
+				   this,
+				   tdb);
+  if (ss != NULL) 
+     hdfsScanStats->setQueryId(ss->getQueryId(), ss->getQueryIdLen());
+  return hdfsScanStats;
+}
+
 short ExExeUtilLobUpdateTcb::work()
 {
   Lng32 cliRC = 0;
@@ -3467,6 +3506,7 @@ short ExExeUtilLobUpdateTcb::work()
                   }
               }
             retcode = ExpLOBInterfaceUpdate(lobGlobs,
+                                            (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
                                             lobTdb().getLobHdfsServer(),
                                             lobTdb().getLobHdfsPort(),
                                             lobName_,
@@ -3550,6 +3590,7 @@ short ExExeUtilLobUpdateTcb::work()
             Int32 outHandleLen;
             Int64 requestTag = 0;
             retcode = ExpLOBInterfaceUpdateAppend(lobGlobs,
+                                            (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
                                             lobTdb().getLobHdfsServer(),
                                             lobTdb().getLobHdfsPort(),
                                             lobName_,
@@ -3635,6 +3676,7 @@ short ExExeUtilLobUpdateTcb::work()
             Int64 requestTag = 0;
           
             retcode = ExpLOBInterfaceUpdate(lobGlobs,
+                                            (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
                                             lobTdb().getLobHdfsServer(),
                                             lobTdb().getLobHdfsPort(),
                                             lobName_,
@@ -3735,8 +3777,7 @@ NABoolean ExExeUtilFileExtractTcb::needStatsEntry()
     return FALSE;
 }
 
-ExOperStats * ExExeUtilFileExtractTcb::doAllocateStatsEntry(
-							    CollHeap *heap,
+ExOperStats * ExExeUtilFileExtractTcb::doAllocateStatsEntry(CollHeap *heap,
 							    ComTdb *tdb)
 {
   ExEspStmtGlobals *espGlobals = getGlobals()->castToExExeStmtGlobals()->castToExEspStmtGlobals();
@@ -3745,7 +3786,6 @@ ExOperStats * ExExeUtilFileExtractTcb::doAllocateStatsEntry(
      ss = espGlobals->getStmtStats();
   else
      ss = getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()->getStatement()->getStmtStats(); 
-  
   ExHdfsScanStats *hdfsScanStats = new(heap) ExHdfsScanStats(heap,
 				   this,
 				   tdb);
@@ -3811,6 +3851,7 @@ short ExExeUtilFileExtractTcb::work()
 
 	    retcode = ExpLOBInterfaceSelectCursor
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,
@@ -3857,6 +3898,7 @@ short ExExeUtilFileExtractTcb::work()
 
 	    retcode = ExpLOBInterfaceSelectCursor
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,
@@ -3914,6 +3956,7 @@ short ExExeUtilFileExtractTcb::work()
 	  {
 	    retcode = ExpLOBInterfaceSelectCursor
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,
@@ -3943,7 +3986,7 @@ short ExExeUtilFileExtractTcb::work()
 		break;
 	      }
 
-	    step_ = COLLECT_STATS_;
+	    step_ = DONE_;
 	  }
 	  break;
 
@@ -3957,30 +4000,6 @@ short ExExeUtilFileExtractTcb::work()
 	  }
 	  break;
 
-	case COLLECT_STATS_:
-	  {
-	    if (! getStatsEntry())
-	      {
-		step_ = DONE_;
-		break;
-	      }
-
-	    ExHdfsScanStats * stats =
-	      getStatsEntry()->castToExHdfsScanStats();
-
-	    retcode = ExpLOBinterfaceStats
-	      (lobGlobs,
-	       stats->lobStats(),
-	       lobName_, 
-	       lobLoc_,
-	       lobType_,
-	       lobTdb().getLobHdfsServer(),
-	       lobTdb().getLobHdfsPort());
-	    
-	    step_ = DONE_;
-	  }
-	  break;
-
 	case DONE_:
 	  {
 	    retcode = handleDone();
@@ -4006,6 +4025,23 @@ ExExeUtilFileLoadTcb::ExExeUtilFileLoadTcb
 {
 }
 
+ExOperStats * ExExeUtilFileLoadTcb::doAllocateStatsEntry(CollHeap *heap,
+							    ComTdb *tdb)
+{
+  ExEspStmtGlobals *espGlobals = getGlobals()->castToExExeStmtGlobals()->castToExEspStmtGlobals();
+  StmtStats *ss; 
+  if (espGlobals != NULL)
+     ss = espGlobals->getStmtStats();
+  else
+     ss = getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()->getStatement()->getStmtStats(); 
+  ExHdfsScanStats *hdfsScanStats = new(heap) ExHdfsScanStats(heap,
+				   this,
+				   tdb);
+  if (ss != NULL) 
+     hdfsScanStats->setQueryId(ss->getQueryId(), ss->getQueryIdLen());
+  return hdfsScanStats;
+}
+
 short ExExeUtilFileLoadTcb::work()
 {
   Lng32 cliRC = 0;
@@ -4162,6 +4198,7 @@ short ExExeUtilFileLoadTcb::work()
 	    Int64 dummy;
 	    retcode = ExpLOBInterfaceInsert
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,
@@ -4204,6 +4241,7 @@ short ExExeUtilFileLoadTcb::work()
 	  {
 	    retcode = ExpLOBinterfaceCloseFile
 	      (lobGlobs,
+               (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
 	       lobName_, 
 	       lobLoc_,
 	       lobType_,

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExFastTransport.cpp b/core/sql/executor/ExFastTransport.cpp
index 7849746..a2eaa94 100644
--- a/core/sql/executor/ExFastTransport.cpp
+++ b/core/sql/executor/ExFastTransport.cpp
@@ -422,7 +422,16 @@ ExOperStats * ExFastExtractTcb::doAllocateStatsEntry(
 
   if (statsType == ComTdb::OPERATOR_STATS)
   {
-    return ex_tcb::doAllocateStatsEntry(heap, tdb);;
+     ExEspStmtGlobals *espGlobals = getGlobals()->castToExExeStmtGlobals()->castToExEspStmtGlobals();
+     StmtStats *ss; 
+     if (espGlobals != NULL)
+        ss = espGlobals->getStmtStats();
+     else
+        ss = getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()->getStatement()->getStmtStats(); 
+     ExHdfsScanStats *hdfsScanStats = new (heap) ExHdfsScanStats(heap, this, tdb);
+     if (ss != NULL)
+        hdfsScanStats->setQueryId(ss->getQueryId(), ss->getQueryIdLen());
+     return hdfsScanStats;
   }
   else
   {
@@ -622,6 +631,7 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
 
   ExOperStats *stats = NULL;
   ExFastExtractStats *feStats = getFastExtractStats();
+  ExHdfsScanStats *hdfsStats = getHdfsScanStats();
 
   while (TRUE)
   {
@@ -790,7 +800,7 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
           }
           else if (!isSequenceFile() && hdfsClient_ == NULL)
           {
-             hdfsClient_ = HdfsClient::newInstance((NAHeap *)getHeap(), NULL, hdfsClientRetCode);
+             hdfsClient_ = HdfsClient::newInstance((NAHeap *)getHeap(), (ExHdfsScanStats *)getHdfsScanStats(), hdfsClientRetCode);
              if (hdfsClientRetCode != HDFS_CLIENT_OK)
              {
                 createHdfsClientFileError(hdfsClientRetCode);
@@ -978,6 +988,10 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
           {
             feStats->incProcessedRowsCount();
           }
+              if (hdfsStats != NULL) {
+                 hdfsStats->incUsedRows();
+                 hdfsStats->incAccessedRows();
+              }
           pstate.successRowCount_ ++;
         }
         else
@@ -986,6 +1000,8 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
           {
             feStats->incErrorRowsCount();
           }
+              if (hdfsStats != NULL) 
+                 hdfsStats->incAccessedRows();
           pstate.errorRowCount_ ++;
         }
         if (currBuffer_->bytesLeft_ < (Int32) maxExtractRowLength_)
@@ -1243,7 +1259,6 @@ void ExHdfsFastExtractTcb::insertUpQueueEntry(ex_queue::up_status status, ComDia
     g->setRowsAffected(privateState.matchCount_);
   }
 
-
   //
   // Insert into up queue
   qParent_.up->insert();

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExFastTransport.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExFastTransport.h b/core/sql/executor/ExFastTransport.h
index 5bf1219..37d7875 100644
--- a/core/sql/executor/ExFastTransport.h
+++ b/core/sql/executor/ExFastTransport.h
@@ -327,6 +327,14 @@ public:
         return NULL;
     }
 
+  ExHdfsScanStats * getHdfsScanStats()
+    {
+      if (getStatsEntry())
+        return getStatsEntry()->castToExHdfsScanStats();
+      else
+        return NULL;
+    }
+
 
 
 protected:
@@ -359,6 +367,7 @@ protected:
   NABoolean          endOfData_;
   CollHeap           *heap_;
   ExFastExtractStats *feStats_;
+  ExHdfsScanStats    *hdfsStats_;
 
   time_t              tstart_;
 

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.h b/core/sql/executor/ExHbaseAccess.h
index 7be1551..071b8c0 100644
--- a/core/sql/executor/ExHbaseAccess.h
+++ b/core/sql/executor/ExHbaseAccess.h
@@ -47,7 +47,6 @@ class ExHbaseAccessTcb;
 // Classes referenced in this file
 // -----------------------------------------------------------------------
 class ex_tcb;
-class ExHbaseAccessStats;
 class ExpHbaseInterface;
 class ExHbaseAccessSelectTcb;
 class ExHbaseAccessUMDTcb;

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExHbaseIUD.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseIUD.cpp b/core/sql/executor/ExHbaseIUD.cpp
index 6a52457..bad439d 100644
--- a/core/sql/executor/ExHbaseIUD.cpp
+++ b/core/sql/executor/ExHbaseIUD.cpp
@@ -34,6 +34,7 @@
 #include "ExHdfsScan.h"
 #include "Context.h"
 #include "HdfsClient_JNI.h"
+#include "ExStats.h"
 
 ExHbaseAccessInsertTcb::ExHbaseAccessInsertTcb(
           const ExHbaseAccessTdb &hbaseAccessTdb, 
@@ -1012,8 +1013,7 @@ ExWorkProcRetcode ExHbaseAccessUpsertVsbbSQTcb::work()
               break;
 	    }
 	    if (getHbaseAccessStats()) {
-              getHbaseAccessStats()->lobStats()->numReadReqs++;
-              getHbaseAccessStats()->incUsedRows(numRowsInVsbbBuffer_);
+              getHbaseAccessStats()->incUsedRows((Int64)numRowsInVsbbBuffer_);
 	    }
             rowsInserted_ += numRowsInVsbbBuffer_; 
             if (asyncOperation_) {
@@ -1611,7 +1611,6 @@ ExWorkProcRetcode ExHbaseAccessBulkLoadPrepSQTcb::work()
 
         if (getHbaseAccessStats())
         {
-          getHbaseAccessStats()->lobStats()->numReadReqs++;
           getHbaseAccessStats()->incUsedRows(numRowsInVsbbBuffer_);
         }
 
@@ -4247,7 +4246,6 @@ ExWorkProcRetcode ExHbaseAccessSQRowsetTcb::work()
                break;
             }
             if (getHbaseAccessStats()) {
-	      getHbaseAccessStats()->lobStats()->numReadReqs++;
 	      getHbaseAccessStats()->incUsedRows(numRowsInVsbbBuffer_);
 	    }
 	    step_ = RS_CLOSE;
@@ -4271,10 +4269,6 @@ ExWorkProcRetcode ExHbaseAccessSQRowsetTcb::work()
 	      }
               step_ = NEXT_ROW;
 
-	      if (getHbaseAccessStats())
-	      {
-	          getHbaseAccessStats()->lobStats()->numReadReqs++;
-	      }
            }
            else
                step_ = SETUP_SELECT;
@@ -4345,7 +4339,6 @@ ExWorkProcRetcode ExHbaseAccessSQRowsetTcb::work()
                break;
             }
             if (getHbaseAccessStats()) {
-	      getHbaseAccessStats()->lobStats()->numReadReqs++;
 	      getHbaseAccessStats()->incUsedRows(numRowsInVsbbBuffer_);
 	    }
 	    step_ = RS_CLOSE;
@@ -4385,7 +4378,6 @@ ExWorkProcRetcode ExHbaseAccessSQRowsetTcb::work()
             if (step_ == HANDLE_ERROR)
                break;
             if (getHbaseAccessStats()) {
-	      getHbaseAccessStats()->lobStats()->numReadReqs++;
 	      getHbaseAccessStats()->incUsedRows(numRowsInVsbbBuffer_);
             }
             step_ = RS_CLOSE;

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index 4db5a1d..b09cffd 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -771,6 +771,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
                 openType = 2; // must open
                 retcode = ExpLOBInterfaceSelectCursor
                   (lobGlob_,
+                   (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
                    hdfsFileName_, //hdfsScanTdb().hdfsFileName_,
                    NULL, //(char*)"",
                    (Lng32)Lob_External_HDFS_File,
@@ -830,6 +831,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
                 
                     retcode = ExpLOBInterfaceSelectCursor
                       (lobGlob_,
+                       (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
                        hdfsFileName_, //hdfsScanTdb().hdfsFileName_,
                        NULL, //(char*)"",
                        (Lng32)Lob_External_HDFS_File,
@@ -940,6 +942,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
                 Int32 hdfsErrorDetail = 0;///this is the errno returned from the underlying hdfs call.
                 retcode = ExpLOBInterfaceSelectCursor
                   (lobGlob_,
+                   (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
                    hdfsFileName_,
                    NULL, 
                    (Lng32)Lob_External_HDFS_File,
@@ -1511,6 +1514,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
 	      {
                 retcode = ExpLOBInterfaceSelectCursor
                   (lobGlob_,
+                  (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
                    hdfsFileName_, 
                    NULL,
                    (Lng32)Lob_External_HDFS_File,
@@ -1640,29 +1644,6 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
 	case CLOSE_FILE:
 	case ERROR_CLOSE_FILE:
 	  {
-	    if (getStatsEntry())
-	      {
-		ExHdfsScanStats * stats =
-		  getStatsEntry()->castToExHdfsScanStats();
-		
-		if (stats)
-		  {
-		    ExLobStats s;
-		    s.init();
-
-		    retcode = ExpLOBinterfaceStats
-		      (lobGlob_,
-		       &s, 
-		       hdfsFileName_, //hdfsScanTdb().hdfsFileName_,
-		       NULL, //(char*)"",
-		       (Lng32)Lob_External_HDFS_File,
-		       hdfsScanTdb().hostName_,
-		       hdfsScanTdb().port_);
-
-		    *stats->lobStats() = *stats->lobStats() + s;
-		  }
-	      }
-
             // if next file is not same as current file, then close the current file. 
             bool closeFile = true;
 
@@ -1679,6 +1660,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
             {
                 retcode = ExpLOBinterfaceCloseFile
                   (lobGlob_,
+                   (getStatsEntry() != NULL ? getStatsEntry()->castToExHdfsScanStats() : NULL),
                    hdfsFileName_,
                    NULL, 
                    (Lng32)Lob_External_HDFS_File,

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExHdfsScan.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.h b/core/sql/executor/ExHdfsScan.h
index 371b6b5..a9bd86e 100644
--- a/core/sql/executor/ExHdfsScan.h
+++ b/core/sql/executor/ExHdfsScan.h
@@ -50,7 +50,6 @@ class HdfsClient;
 // Classes referenced in this file
 // -----------------------------------------------------------------------
 class ex_tcb;
-class ExHdfsScanStats;
 class SequenceFileReader;
 class ExpORCinterface;
 

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExStats.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExStats.cpp b/core/sql/executor/ExStats.cpp
index 663d2bf..d5aaf9f 100644
--- a/core/sql/executor/ExStats.cpp
+++ b/core/sql/executor/ExStats.cpp
@@ -62,6 +62,7 @@
 #include "ComTdbExeUtil.h"
 #include "ComTdbHdfsScan.h"
 #include "ComTdbHbaseAccess.h"
+#include "ComTdbFastTransport.h"
 #include "ex_exe_stmt_globals.h"
 #include "exp_clause_derived.h"
 #include "Int64.h"
@@ -1314,10 +1315,6 @@ Lng32 ExOperStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
   case SQLSTATS_DOP:
     sqlStats_item->int64_value = dop_;
     break;
-  case SQLSTATS_DETAIL:
-    if (sqlStats_item->str_value != NULL)
-       sqlStats_item->str_ret_len = 0;
-    break;
   default:
     sqlStats_item->error_code = -EXE_STAT_NOT_FOUND;
     break;
@@ -1761,11 +1758,8 @@ void ExFragRootOperStats::merge(ExOperStats * other)
     case BMO_STATS:
       merge((ExBMOStats *)other);
       break;
-    case HBASE_ACCESS_STATS:
-      merge((ExHbaseAccessStats *)other);
-      break;
-    case HDFSSCAN_STATS:
-      merge((ExHdfsScanStats *)other);
+    case SE_STATS:
+      merge((ExStorageEngineStats *)other);
       break;
     default:
       // do nothing - This type of stat has no merge data
@@ -1815,11 +1809,11 @@ const char * ExFragRootOperStats::getNumValTxt(Int32 i) const
     case 1:
       return "OperCpuTime";
     case 2:
-      return "CpuTime";
+      return "messageBytes";
     case 3:
-      return "waitTime";
+      return "messageCount";
     case 4:
-      return "Timestamp";
+      return "memoryAllocated";
     }
   return NULL;
 }
@@ -1831,11 +1825,11 @@ Int64 ExFragRootOperStats::getNumVal(Int32 i) const
     case 1:
       return ExOperStats::getNumVal(i);
     case 2:
-         return cpuTime_ + espCpuTime_;
+         return reqMsgBytes_ + replyMsgBytes_;
     case 3:
-         return ((ExFragRootOperStats *)this)->getAvgWaitTime();
+         return reqMsgCnt_ + replyMsgCnt_;
     case 4:
-      return timestamp_;
+      return heapAlloc_+espHeapAlloc_+spaceAlloc_+espSpaceAlloc_;
     }
   return 0;
 }
@@ -1921,24 +1915,7 @@ Lng32 ExFragRootOperStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
 {
   char tmpBuf[100];
   Int32 len;
-  if (sqlStats_item->statsItem_id == SQLSTATS_DETAIL)
-  {
-     if (sqlStats_item->str_value != NULL)
-     {
-         sprintf(tmpBuf, "%ld",
-             cpuTime_ + espCpuTime_);
-         len = str_len(tmpBuf);
-         if (len > sqlStats_item->str_max_len)
-            sqlStats_item->error_code = EXE_ERROR_IN_STAT_ITEM;
-         else
-            str_cpy(sqlStats_item->str_value, tmpBuf, len);
-         sqlStats_item->str_ret_len = len;
-     }
-     return 0;
-  }
-  ExOperStats::getStatsItem(sqlStats_item);
-  if(sqlStats_item -> error_code == -EXE_STAT_NOT_FOUND)
-  {   
+  Lng32 retcode = 0;
     sqlStats_item->error_code = 0;
     switch (sqlStats_item->statsItem_id)
     {
@@ -2020,12 +1997,25 @@ Lng32 ExFragRootOperStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
     case SQLSTATS_SQL_MAX_WAIT_TIME:
       sqlStats_item->int64_value = maxWaitTime_;
       break;
+    case SQLSTATS_DETAIL:
+      if (sqlStats_item->str_value != NULL)
+      {
+         sprintf(tmpBuf, "%ld|%ld|%ld|%ld|",
+             cpuTime_ + espCpuTime_, getNumVal(2), getNumVal(3), getNumVal(4));
+         len =  strlen(tmpBuf);
+
+         if (len > sqlStats_item->str_max_len)
+            sqlStats_item->error_code = EXE_ERROR_IN_STAT_ITEM;
+         else
+            str_cpy(sqlStats_item->str_value, tmpBuf, len);
+         sqlStats_item->str_ret_len = len;
+      }
+      break;
     default:
-        sqlStats_item->error_code = -EXE_STAT_NOT_FOUND;
-        break;
+      retcode = ExOperStats::getStatsItem(sqlStats_item);
+      break;
     }
-  }
-  return 0;
+  return retcode;
 }
 
 NABoolean ExFragRootOperStats::filterForCpuStats()
@@ -2045,10 +2035,10 @@ NABoolean ExFragRootOperStats::filterForCpuStats()
 }
 
 //////////////////////////////////////////////////////////////////
-// class ExHdfsScanStats
+// class ExStorageEngineStats
 //////////////////////////////////////////////////////////////////
 
-ExHdfsScanStats::ExHdfsScanStats(NAMemory * heap,
+ExStorageEngineStats::ExStorageEngineStats(NAMemory * heap,
                           ex_tcb *tcb,
                           ComTdb * tdb)
   : ExOperStats(heap,
@@ -2057,10 +2047,32 @@ ExHdfsScanStats::ExHdfsScanStats(NAMemory * heap,
                 tdb)
   ,  timer_(CLOCK_MONOTONIC)
 {
-  ComTdbHdfsScan *hdfsTdb = (ComTdbHdfsScan *) tdb;
-
+  const char * name;
+  switch (tdb->getNodeType())
+  {
+    case ComTdb::ex_HBASE_ACCESS:
+    {
+       ComTdbHbaseAccess *hbaseTdb = (ComTdbHbaseAccess *) tdb;
+       name = hbaseTdb->getTableName();
+       break;
+    }
+    case ComTdb::ex_HDFS_SCAN:
+    {
+       ComTdbHdfsScan *hdfsTdb = (ComTdbHdfsScan *) tdb;
+       name = hdfsTdb->tableName();
+       break;
+    }
+    case ComTdb::ex_FAST_EXTRACT:
+    {
+       ComTdbFastExtract *feTdb = (ComTdbFastExtract *)tdb;
+       name = feTdb->getTargetName();
+       break;
+    }
+    default:
+       name = "";
+  }
+     
   // allocate memory and copy the ansi name into the stats entry
-  const char * name = hdfsTdb->tableName();
   Lng32 len = (Lng32)str_len(name); 
   tableName_ = (char *)heap_->allocateMemory(len + 1);
   sprintf(tableName_, "%s", name);
@@ -2070,9 +2082,9 @@ ExHdfsScanStats::ExHdfsScanStats(NAMemory * heap,
   init(FALSE);
 }
 
-ExHdfsScanStats::ExHdfsScanStats(NAMemory * heap)
+ExStorageEngineStats::ExStorageEngineStats(NAMemory * heap)
   : ExOperStats(heap,
-                HDFSSCAN_STATS)
+                SE_STATS)
   , tableName_(NULL)
   , timer_(CLOCK_MONOTONIC)
 {
@@ -2081,22 +2093,20 @@ ExHdfsScanStats::ExHdfsScanStats(NAMemory * heap)
   init(FALSE);
 }
 
-void ExHdfsScanStats::init(NABoolean resetDop)
+void ExStorageEngineStats::init(NABoolean resetDop)
 {
   ExOperStats::init(resetDop);
   timer_.reset();
 
-  lobStats_.init();
-
   numBytesRead_ = 0;
   accessedRows_ = 0;
   usedRows_     = 0;
-  numHdfsCalls_ = 0;
-  maxHdfsIOTime_ = 0;
+  numIOCalls_ = 0;
+  maxIOTime_ = 0;
   blockTime_ = 0;
 }
 
-ExHdfsScanStats::~ExHdfsScanStats()
+ExStorageEngineStats::~ExStorageEngineStats()
 {
   if (tableName_ != NULL)
   {
@@ -2110,19 +2120,18 @@ ExHdfsScanStats::~ExHdfsScanStats()
   }
 }
 
-UInt32 ExHdfsScanStats::packedLength()
+UInt32 ExStorageEngineStats::packedLength()
 {
   UInt32 size = ExOperStats::packedLength();
   size += sizeof(timer_);
-  size += sizeof(lobStats_);
 
   advanceSize2(size, tableName_);
 
   size += sizeof(numBytesRead_);
   size += sizeof(accessedRows_);
   size += sizeof(usedRows_);
-  size += sizeof(numHdfsCalls_);
-  size += sizeof(maxHdfsIOTime_);
+  size += sizeof(numIOCalls_);
+  size += sizeof(maxIOTime_);
   if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
   {
     size += sizeof(blockTime_);
@@ -2132,20 +2141,19 @@ UInt32 ExHdfsScanStats::packedLength()
   return size;
 }
 
-UInt32 ExHdfsScanStats::pack(char *buffer)
+UInt32 ExStorageEngineStats::pack(char *buffer)
 {
   UInt32 size = ExOperStats::pack(buffer);
   buffer += size;
   size += packIntoBuffer(buffer, timer_);
-  size += packIntoBuffer(buffer, lobStats_);
 
   size += packCharStarIntoBuffer(buffer, tableName_);
 
   size += packIntoBuffer(buffer, numBytesRead_);
   size += packIntoBuffer(buffer, accessedRows_);
   size += packIntoBuffer(buffer, usedRows_);
-  size += packIntoBuffer(buffer, numHdfsCalls_);
-  size += packIntoBuffer(buffer, maxHdfsIOTime_);
+  size += packIntoBuffer(buffer, numIOCalls_);
+  size += packIntoBuffer(buffer, maxIOTime_);
   if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
   {
     size += packIntoBuffer(buffer, blockTime_);
@@ -2157,20 +2165,19 @@ UInt32 ExHdfsScanStats::pack(char *buffer)
   return size;
 }
 
-void ExHdfsScanStats::unpack(const char* &buffer)
+void ExStorageEngineStats::unpack(const char* &buffer)
 {
   ExOperStats::unpack(buffer);
 
   unpackBuffer(buffer, timer_);
-  unpackBuffer(buffer, lobStats_);
 
   unpackBuffer(buffer, tableName_, heap_);
 
   unpackBuffer(buffer, numBytesRead_);
   unpackBuffer(buffer, accessedRows_);
   unpackBuffer(buffer, usedRows_);
-  unpackBuffer(buffer, numHdfsCalls_);
-  unpackBuffer(buffer, maxHdfsIOTime_);
+  unpackBuffer(buffer, numIOCalls_);
+  unpackBuffer(buffer, maxIOTime_);
   if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
   {
     unpackBuffer(buffer, blockTime_);
@@ -2184,20 +2191,19 @@ void ExHdfsScanStats::unpack(const char* &buffer)
   }
 }
 
-void ExHdfsScanStats::merge(ExHdfsScanStats *other)
+void ExStorageEngineStats::merge(ExStorageEngineStats *other)
 {
   ExOperStats::merge(other);
   timer_ = timer_ + other->timer_;
-  lobStats_ = lobStats_ + other->lobStats_;
   numBytesRead_ += other->numBytesRead_;
   accessedRows_ += other->accessedRows_;
   usedRows_     += other->usedRows_;
-  numHdfsCalls_    += other->numHdfsCalls_;
-  if (maxHdfsIOTime_ < other->maxHdfsIOTime_) // take the larger value
-    maxHdfsIOTime_ = other->maxHdfsIOTime_;
+  numIOCalls_    += other->numIOCalls_;
+  if (maxIOTime_ < other->maxIOTime_) // take the larger value
+    maxIOTime_ = other->maxIOTime_;
 }
 
-void ExHdfsScanStats::copyContents(ExHdfsScanStats *other)
+void ExStorageEngineStats::copyContents(ExStorageEngineStats *other)
 {
   ExOperStats::copyContents(other);
 
@@ -2211,12 +2217,11 @@ void ExHdfsScanStats::copyContents(ExHdfsScanStats *other)
   }
 
   timer_ = other->timer_;
-  lobStats_ = other->lobStats_;
   numBytesRead_ = other->numBytesRead_;
   accessedRows_ = other->accessedRows_;
   usedRows_     = other->usedRows_;
-  numHdfsCalls_  = other->numHdfsCalls_;
-  maxHdfsIOTime_ = other->maxHdfsIOTime_;
+  numIOCalls_  = other->numIOCalls_;
+  maxIOTime_ = other->maxIOTime_;
   if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
   {
     blockTime_ = other->blockTime_;
@@ -2237,9 +2242,9 @@ void ExHdfsScanStats::copyContents(ExHdfsScanStats *other)
   }
 }
 
-ExOperStats * ExHdfsScanStats::copyOper(NAMemory * heap)
+ExOperStats * ExStorageEngineStats::copyOper(NAMemory * heap)
 {
-  ExHdfsScanStats *stat = new(heap) ExHdfsScanStats(heap);
+  ExStorageEngineStats *stat = new(heap) ExStorageEngineStats(heap);
   stat->copyContents(this);
   return stat;
 }
@@ -2250,410 +2255,29 @@ ExHdfsScanStats
   return this;
 }
 
-const char *ExHdfsScanStats::getNumValTxt(Int32 i) const
-{
-  switch (i)
-    {
-    case 1:
-      return "OperCpuTime";
-    case 2:
-      return "BytesRead";
-    case 3:
-      return "TimeWaitingOnHdfs";
-    case 4:
-      return "AccessedRows";
-    }
-  return NULL;
-}
-
-Int64 ExHdfsScanStats::getNumVal(Int32 i) const
-{
-  switch (i)
-    {
-    case 1:
-      return ExOperStats::getNumVal(i);
-    case 2:
-      return numBytesRead_;
-    case 3:
-      return timer_.getTime();
-    case 4:
-      return accessedRows_;
-    }
-  return 0;
-}
-
-NABoolean ExHdfsScanStats::filterForSEstats(struct timespec currTimespec, Lng32 filter)
-{
-   Int64 sumIOTime;
-
-   if (filter > 0) {
-      blockTime_ = timer_.filterForSEstats(currTimespec);
-      if (blockTime_ >= filter)
-         return TRUE;
-   }
-   else
-   if (queryId_ != NULL && (sumIOTime = timer_.getTime()) > 0 && (sumIOTime = sumIOTime /(1000000LL)) >= -filter) {
-      blockTime_ = sumIOTime;
-      return TRUE;
-   }
-   return FALSE;
-}
-
-void ExHdfsScanStats::getVariableStatsInfo(char * dataBuffer,
-						   char * dataLen,
-						   Lng32 maxLen)
-{
-  char *buf = dataBuffer;
-  if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
-  {
-     sprintf(buf, "statsRowType: %d Qid: %s blockedFor: %d ",
-        statType(),
-        ((queryId_ != NULL) ? queryId_ : "NULL"), blockTime_);
-     buf += str_len(buf);
-  }
-  else 
-  {
-     ExOperStats::getVariableStatsInfo(dataBuffer, dataLen, maxLen);
-     buf += *((short *) dataLen);
-
-     lobStats()->getVariableStatsInfo(buf, dataLen, maxLen);
-     buf += *((short *) dataLen);
-  }
-  sprintf (buf, 
-	   "AnsiName: %s  MessagesBytes: %ld AccessedRows: %ld UsedRows: %ld HiveIOCalls: %ld HiveSumIOTime: %ld HdfsMaxIOTime: %ld",
-	       (char*)tableName_,
-	       numBytesRead(), 
-	       rowsAccessed(),
-	       rowsUsed(),
-               numHdfsCalls_,
-	       timer_.getTime(),
-	       maxHdfsIOTime_
-	       );
-  buf += str_len(buf);
-  
- *(short*)dataLen = (short) (buf - dataBuffer);
-}
-
-Lng32 ExHdfsScanStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
-{
-  sqlStats_item->error_code = 0;
-  Int32 len;
-  Int32 len1;
-  const char *tableName;
-  char tmpBuf[100];
-
-  switch (sqlStats_item->statsItem_id)
-  {
-  case SQLSTATS_TABLE_ANSI_NAME:
-    if (sqlStats_item->str_value != NULL)
-    {
-      if (tableName_ != NULL)
-        tableName = tableName_;
-      else
-        tableName = "NO_NAME_YET";
-      len = str_len(tableName);
-      if (len > sqlStats_item->str_max_len)
-      {
-        len = sqlStats_item->str_max_len;
-        sqlStats_item->error_code = EXE_ERROR_IN_STAT_ITEM;
-      }
-      str_cpy(sqlStats_item->str_value, tableName, len);
-      sqlStats_item->str_ret_len = len;
-    }
-    break;
-  case SQLSTATS_EST_ROWS_ACCESSED:
-    sqlStats_item->double_value = getEstRowsAccessed();
-    break;
-  case SQLSTATS_EST_ROWS_USED:
-    sqlStats_item->double_value = getEstRowsUsed();
-    break;
-  case SQLSTATS_ACT_ROWS_ACCESSED:
-    sqlStats_item->int64_value = accessedRows_;
-    break;
-  case SQLSTATS_ACT_ROWS_USED:
-    sqlStats_item->int64_value = usedRows_;
-    break;
-  case SQLSTATS_HIVE_IOS:
-    sqlStats_item->int64_value = numHdfsCalls_;
-    break;
-  case SQLSTATS_HIVE_IO_BYTES:
-    sqlStats_item->int64_value = numBytesRead_;
-    break;
-  case SQLSTATS_HIVE_IO_ELAPSED_TIME:
-    sqlStats_item->int64_value = timer_.getTime();
-    break;
-  case SQLSTATS_HIVE_IO_MAX_TIME:
-    sqlStats_item->int64_value = maxHdfsIOTime();
-    break;
-  case SQLSTATS_DETAIL:
-    if (sqlStats_item->str_value != NULL)
-    {
-      if (tableName_ != NULL)
-      {
-        len = str_len(tableName_);
-        if (len > sqlStats_item->str_max_len)
-        {
-           sqlStats_item->error_code = EXE_ERROR_IN_STAT_ITEM;
-           str_cpy_all(sqlStats_item->str_value, tableName_, sqlStats_item->str_max_len);
-        }
-        else
-           str_cpy_all(sqlStats_item->str_value, tableName_, len);
-      }
-      else 
-        len = 0; 
-      sprintf(tmpBuf, "|%ld|%ld", accessedRows_,
-               numBytesRead_);
-      len1 = str_len(tmpBuf);
-      if ((len+len1) > sqlStats_item->str_max_len)
-        sqlStats_item->error_code = EXE_ERROR_IN_STAT_ITEM;
-      else
-        str_cpy(sqlStats_item->str_value+len, tmpBuf, len1);
-      sqlStats_item->str_ret_len = len+len1;
-    }
-    break;
-  default:
-    ExOperStats::getStatsItem(sqlStats_item);
-    break;
-  }
-  return 0;
-}
-  
-
-//////////////////////////////////////////////////////////////////
-// class ExHbaseAccessStats
-//////////////////////////////////////////////////////////////////
-
-ExHbaseAccessStats::ExHbaseAccessStats(NAMemory * heap,
-                          ex_tcb *tcb,
-                          ComTdb * tdb)
-  : ExOperStats(heap,
-                HBASE_ACCESS_STATS,
-                tcb, 
-                tdb)
-  ,  timer_(CLOCK_MONOTONIC)
-{
-  ComTdbHbaseAccess *hbaseTdb = (ComTdbHbaseAccess *) tdb;
-
-  if (hbaseTdb->getTableName())
-    {
-      // allocate memory and copy the ansi name into the stats entry
-      const char * name = hbaseTdb->getTableName();
-      Lng32 len = (Lng32)str_len(name); 
-      tableName_ = (char *)heap_->allocateMemory(len + 1);
-      sprintf(tableName_, "%s", name);
-    }
-  else
-    {
-      tableName_ = NULL;
-    }
-  queryId_ = NULL;
-  queryIdLen_ = 0;
-  init(FALSE);
-}
-
-ExHbaseAccessStats::ExHbaseAccessStats(NAMemory * heap)
-  : ExOperStats(heap,
-                HBASE_ACCESS_STATS)
-  , tableName_(NULL)
-  , timer_(CLOCK_MONOTONIC)
-{
-  queryId_ = NULL;
-  queryIdLen_ = 0;
-  init(FALSE);
-}
-
-void ExHbaseAccessStats::init(NABoolean resetDop)
-{
-  ExOperStats::init(resetDop);
-  timer_.reset();
-
-  lobStats_.init();
-
-  numBytesRead_ = 0;
-  accessedRows_ = 0;
-  usedRows_     = 0;
-  numHbaseCalls_ = 0;
-  maxHbaseIOTime_ = 0;
-  blockTime_ = 0;
-}
-
-ExHbaseAccessStats::~ExHbaseAccessStats()
-{
-  if (tableName_ != NULL)
-  {
-     NADELETEBASIC(tableName_,getHeap());
-     tableName_ = NULL;
-  }
-  if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS && queryId_ != NULL)
-  {
-    NADELETEBASIC(queryId_, getHeap());
-    queryId_ = NULL;
-  }
-}
-
-UInt32 ExHbaseAccessStats::packedLength()
-{
-  UInt32 size = ExOperStats::packedLength();
-  size += sizeof(timer_);
-  size += sizeof(lobStats_);
-
-  advanceSize2(size, tableName_);
-
-  size += sizeof(numBytesRead_);
-  size += sizeof(accessedRows_);
-  size += sizeof(usedRows_);
-  size += sizeof(numHbaseCalls_);
-  size += sizeof(maxHbaseIOTime_);
-  if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
-  {
-    size += sizeof(blockTime_);
-    size += sizeof(queryIdLen_);
-    size += queryIdLen_;
-  }
-  return size;
-}
-
-UInt32 ExHbaseAccessStats::pack(char *buffer)
-{
-  UInt32 size = ExOperStats::pack(buffer);
-  buffer += size;
-  size += packIntoBuffer(buffer, timer_);
-  size += packIntoBuffer(buffer, lobStats_);
-
-  size += packCharStarIntoBuffer(buffer, tableName_);
-
-  size += packIntoBuffer(buffer, numBytesRead_);
-  size += packIntoBuffer(buffer, accessedRows_);
-  size += packIntoBuffer(buffer, usedRows_);
-  size += packIntoBuffer(buffer, numHbaseCalls_);
-  size += packIntoBuffer(buffer, maxHbaseIOTime_);
-  if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
-  {
-    size += packIntoBuffer(buffer, blockTime_);
-    size += packIntoBuffer(buffer, queryIdLen_);
-    if (queryIdLen_ != 0 && queryId_ != NULL)
-      size += packStrIntoBuffer(buffer, queryId_, queryIdLen_);
-  }
-
-  return size;
-}
-
-void ExHbaseAccessStats::unpack(const char* &buffer)
-{
-  ExOperStats::unpack(buffer);
-
-  unpackBuffer(buffer, timer_);
-  unpackBuffer(buffer, lobStats_);
-
-  unpackBuffer(buffer, tableName_, heap_);
-
-  unpackBuffer(buffer, numBytesRead_);
-  unpackBuffer(buffer, accessedRows_);
-  unpackBuffer(buffer, usedRows_);
-  unpackBuffer(buffer, numHbaseCalls_);
-  unpackBuffer(buffer, maxHbaseIOTime_);
-  if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
-  {
-    unpackBuffer(buffer, blockTime_);
-    unpackBuffer(buffer, queryIdLen_);
-    if (queryIdLen_ != 0)
-    {
-      queryId_ = new ((NAHeap *)(getHeap())) char[queryIdLen_+1];
-      unpackStrFromBuffer(buffer, queryId_, queryIdLen_);
-      queryId_[queryIdLen_] = '\0';
-    }
-  }
-}
-
-void ExHbaseAccessStats::merge(ExHbaseAccessStats *other)
-{
-  ExOperStats::merge(other);
-  timer_ = timer_ + other->timer_;
-  lobStats_ = lobStats_ + other->lobStats_;
-  numBytesRead_ += other->numBytesRead_;
-  accessedRows_ += other->accessedRows_;
-  usedRows_ += other->usedRows_;
-  numHbaseCalls_ += other->numHbaseCalls_;
-  if (maxHbaseIOTime_ < other->maxHbaseIOTime_) // take the larger value
-    maxHbaseIOTime_ = other->maxHbaseIOTime_;
-}
-
-void ExHbaseAccessStats::copyContents(ExHbaseAccessStats *other)
-{
-  ExOperStats::copyContents(other);
-
-  // copy names only if we don't have one
-  if (tableName_ == NULL && other->tableName_) 
-  {
-    Lng32 len = (Lng32)str_len(other->tableName_);
-    tableName_ = (char *)heap_->allocateMemory(len + 1);
-    str_cpy_all(tableName_, other->tableName_, len);
-    tableName_[len] = 0;
-  }
-
-  timer_ = other->timer_;
-  lobStats_ = other->lobStats_;
-  numBytesRead_ = other->numBytesRead_;
-  accessedRows_ = other->accessedRows_;
-  usedRows_ = other->usedRows_;
-  numHbaseCalls_ = other->numHbaseCalls_;
-  maxHbaseIOTime_ = other->maxHbaseIOTime_;
-  if ((Int32)getCollectStatsType() == SQLCLI_SE_OFFENDER_STATS)
-  {
-    blockTime_ = other->blockTime_;
-    queryIdLen_ = other->queryIdLen_;
-    if (queryIdLen_ != 0)
-    {
-      queryId_ = new ((NAHeap *)(getHeap())) char[queryIdLen_+1];
-      str_cpy_all(queryId_, other->queryId_, queryIdLen_);
-      queryId_[queryIdLen_] = '\0';
-    }
-    else
-      queryId_ = NULL;
-  }
-  else
-  {
-    queryId_ = other->queryId_;
-    queryIdLen_ = other->queryIdLen_;
-  }
-}
-
-ExOperStats * ExHbaseAccessStats::copyOper(NAMemory * heap)
-{
-  ExHbaseAccessStats *stat = new(heap) ExHbaseAccessStats(heap);
-  stat->copyContents(this);
-  return stat;
-}
-
 ExHbaseAccessStats 
-*ExHbaseAccessStats::castToExHbaseAccessStats()
+*ExHdfsScanStats::castToExHbaseAccessStats()
 {
   return this;
 }
 
-const char *ExHbaseAccessStats::getNumValTxt(Int32 i) const
+const char *ExStorageEngineStats::getNumValTxt(Int32 i) const
 {
   switch (i)
     {
     case 1:
       return "OperCpuTime";
     case 2:
-      return "BytesRead";
+      return "SE_IO_KBytes";
     case 3:
-      return "TimeWaitingOnHbase";
+      return "SE_IO_SumTime";
     case 4:
-      return "AccessedRows";
-    case 5:
-      return "UsedRows";
-    case 6:
-      return "NumHbaseCalls";
-    case 7:
-      return "MaxHbaseIOTime";
+      return "ActRowsAccessed";
     }
   return NULL;
 }
 
-Int64 ExHbaseAccessStats::getNumVal(Int32 i) const
+Int64 ExStorageEngineStats::getNumVal(Int32 i) const
 {
   switch (i)
     {
@@ -2665,19 +2289,14 @@ Int64 ExHbaseAccessStats::getNumVal(Int32 i) const
       return timer_.getTime();
     case 4:
       return accessedRows_;
-    case 5:
-      return usedRows_;
-    case 6:
-      return numHbaseCalls_;
-    case 7:
-      return maxHbaseIOTime_;
     }
   return 0;
 }
 
-NABoolean ExHbaseAccessStats::filterForSEstats(struct timespec currTimespec, Lng32 filter)
+NABoolean ExStorageEngineStats::filterForSEstats(struct timespec currTimespec, Lng32 filter)
 {
    Int64 sumIOTime;
+
    if (filter > 0) {
       blockTime_ = timer_.filterForSEstats(currTimespec);
       if (blockTime_ >= filter)
@@ -2691,8 +2310,7 @@ NABoolean ExHbaseAccessStats::filterForSEstats(struct timespec currTimespec, Lng
    return FALSE;
 }
 
-
-void ExHbaseAccessStats::getVariableStatsInfo(char * dataBuffer,
+void ExStorageEngineStats::getVariableStatsInfo(char * dataBuffer,
 						   char * dataLen,
 						   Lng32 maxLen)
 {
@@ -2708,26 +2326,28 @@ void ExHbaseAccessStats::getVariableStatsInfo(char * dataBuffer,
   {
      ExOperStats::getVariableStatsInfo(dataBuffer, dataLen, maxLen);
      buf += *((short *) dataLen);
-
-     lobStats()->getVariableStatsInfo(buf, dataLen, maxLen);
-     buf += *((short *) dataLen);
   }
   sprintf (buf, 
-	   "AnsiName: %s  MessagesBytes: %ld AccessedRows: %ld UsedRows: %ld HbaseSumIOCalls: %ld HbaseSumIOTime: %ld HbaseMaxIOTime: %ld",
+	   "AnsiName: %s  MessagesBytes: %ld AccessedRows: %ld UsedRows: %ld HiveIOCalls: %ld HiveSumIOTime: %ld HdfsMaxIOTime: %ld "
+           "HbaseSumIOCalls: %ld HbaseSumIOTime: %ld HbaseMaxIOTime: %ld ",
+           
 	       (char*)tableName_,
 	       numBytesRead(), 
 	       rowsAccessed(),
 	       rowsUsed(),
-	       hbaseCalls(), 
+               numIOCalls_,
 	       timer_.getTime(),
-	       maxHbaseIOTime()
+	       maxIOTime_,
+               hbaseCalls(),
+               timer_.getTime(),
+               maxHbaseIOTime() 
 	       );
   buf += str_len(buf);
   
  *(short*)dataLen = (short) (buf - dataBuffer);
 }
 
-Lng32 ExHbaseAccessStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
+Lng32 ExStorageEngineStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
 {
   sqlStats_item->error_code = 0;
   Int32 len;
@@ -2766,17 +2386,21 @@ Lng32 ExHbaseAccessStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
   case SQLSTATS_ACT_ROWS_USED:
     sqlStats_item->int64_value = usedRows_;
     break;
+  case SQLSTATS_HIVE_IOS:
   case SQLSTATS_HBASE_IOS:
-    sqlStats_item->int64_value = numHbaseCalls_;
+    sqlStats_item->int64_value = numIOCalls_;
     break;
+  case SQLSTATS_HIVE_IO_BYTES:
   case SQLSTATS_HBASE_IO_BYTES:
     sqlStats_item->int64_value = numBytesRead_;
     break;
+  case SQLSTATS_HIVE_IO_ELAPSED_TIME:
   case SQLSTATS_HBASE_IO_ELAPSED_TIME:
     sqlStats_item->int64_value = timer_.getTime();
     break;
+  case SQLSTATS_HIVE_IO_MAX_TIME:
   case SQLSTATS_HBASE_IO_MAX_TIME:
-    sqlStats_item->int64_value = maxHbaseIOTime_;
+    sqlStats_item->int64_value = maxIOTime_;
     break;
   case SQLSTATS_DETAIL:
     if (sqlStats_item->str_value != NULL)
@@ -2794,8 +2418,8 @@ Lng32 ExHbaseAccessStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
       }
       else 
         len = 0; 
-      sprintf(tmpBuf, "|%ld|%ld", accessedRows_,
-               numBytesRead_);
+
+      sprintf(tmpBuf, "|%ld|%ld|%ld", getNumVal(2), getNumVal(3), getNumVal(4));
       len1 = str_len(tmpBuf);
       if ((len+len1) > sqlStats_item->str_max_len)
         sqlStats_item->error_code = EXE_ERROR_IN_STAT_ITEM;
@@ -2810,7 +2434,7 @@ Lng32 ExHbaseAccessStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
   }
   return 0;
 }
-
+  
 //////////////////////////////////////////////////////////////////
 // class ExProbeCacheStats
 //////////////////////////////////////////////////////////////////
@@ -4783,7 +4407,7 @@ void ExMeasStats::merge(ExBMOStats *other)
       topN_ = other->topN_;
 }
 
-void ExMeasStats::merge(ExHdfsScanStats* other)
+void ExMeasStats::merge(ExStorageEngineStats* other)
 {
   exeSEStats()->incAccessedRows(other->rowsAccessed());
   exeSEStats()->incUsedRows(other->rowsUsed());
@@ -4792,15 +4416,6 @@ void ExMeasStats::merge(ExHdfsScanStats* other)
   exeSEStats()->incMaxIOTime(other->maxHdfsIOTime());
 }
 
-void ExMeasStats::merge(ExHbaseAccessStats* other)
-{
-  exeSEStats()->incAccessedRows(other->rowsAccessed());
-  exeSEStats()->incUsedRows(other->rowsUsed());
-  exeSEStats()->incNumIOCalls(other->hbaseCalls());
-  exeSEStats()->incNumIOBytes(other->numBytesRead());
-  exeSEStats()->incMaxIOTime(other->maxHbaseIOTime());
-}
-
 void ExMeasStats::merge(ExMeasStats* other)
 {
   ExMeasBaseStats::merge(other);
@@ -4866,11 +4481,9 @@ void ExMeasStats::merge(ExOperStats * other)
     case BMO_STATS:
       merge((ExBMOStats *)other);
       break;
-    case HDFSSCAN_STATS:
-      merge((ExHdfsScanStats *)other);
+    case SE_STATS:
+      merge((ExStorageEngineStats *)other);
       break;
-    case HBASE_ACCESS_STATS:
-      merge((ExHbaseAccessStats *)other);
       break;
     default:
       // do nothing - This type of stat has no merge data
@@ -5475,11 +5088,8 @@ void ExStatisticsArea::removeEntries()
       case ExOperStats::UDR_BASE_STATS:
         NADELETE((ExUDRBaseStats *)stat, ExUDRBaseStats, heap_);
         break;
-      case ExOperStats::HDFSSCAN_STATS:
-        NADELETE((ExHdfsScanStats *)stat, ExHdfsScanStats, heap_);
-        break;
-     case ExOperStats::HBASE_ACCESS_STATS:
-        NADELETE((ExHbaseAccessStats *)stat, ExHbaseAccessStats, heap_);
+      case ExOperStats::SE_STATS:
+        NADELETE((ExStorageEngineStats *)stat, ExStorageEngineStats, heap_);
         break;
       default:
         NADELETE(stat, ExOperStats, heap_);
@@ -5588,19 +5198,11 @@ NABoolean ExStatisticsArea::merge(ExOperStats * other, UInt16 statsMergeType)
             return TRUE;
           }
           break;
-        case ExOperStats::HDFSSCAN_STATS:
-          if (stat->statType() == ExOperStats::HDFSSCAN_STATS
+        case ExOperStats::SE_STATS:
+          if (stat->statType() == ExOperStats::SE_STATS
             && stat->getPertableStatsId() == other->getPertableStatsId())
           {
-            ((ExHdfsScanStats *)stat)->merge((ExHdfsScanStats *)other);
-            return TRUE;
-          }
-          break;
-        case ExOperStats::HBASE_ACCESS_STATS:
-          if (stat->statType() == ExOperStats::HBASE_ACCESS_STATS
-            && stat->getPertableStatsId() == other->getPertableStatsId())
-          {
-            ((ExHbaseAccessStats *)stat)->merge((ExHbaseAccessStats *)other);
+            ((ExStorageEngineStats *)stat)->merge((ExStorageEngineStats *)other);
             return TRUE;
           }
           break;
@@ -5644,11 +5246,8 @@ NABoolean ExStatisticsArea::merge(ExOperStats * other, UInt16 statsMergeType)
           case ExOperStats::UDR_BASE_STATS:
             ((ExUDRBaseStats *)stat)->merge((ExUDRBaseStats *)other);
             break;
-         case ExOperStats::HDFSSCAN_STATS:
-            ((ExHdfsScanStats*)stat)->merge((ExHdfsScanStats*)other);
-            break;
-         case ExOperStats::HBASE_ACCESS_STATS:
-            ((ExHbaseAccessStats*)stat)->merge((ExHbaseAccessStats*)other);
+         case ExOperStats::SE_STATS:
+            ((ExStorageEngineStats*)stat)->merge((ExStorageEngineStats*)other);
             break;
           default:
               ex_assert(FALSE, "Merging unknown operator statistics type");
@@ -5764,22 +5363,11 @@ NABoolean ExStatisticsArea::merge(ExStatisticsArea * otherStatsArea, UInt16 stat
 		      insert(newStat);
 		    }
 		  break;
-		  
-		case ExOperStats::HDFSSCAN_STATS:
-		  {
-		    newStat = new(heap_) ExHdfsScanStats(heap_);
-		    newStat->setCollectStatsType(tempStatsMergeType);
-		    ((ExHdfsScanStats *)newStat)->copyContents((ExHdfsScanStats *)stat);
-		    newStat->setCollectStatsType(tempStatsMergeType);
-		    insert(newStat);
-		  }
-		  break;
-		  
-		case ExOperStats::HBASE_ACCESS_STATS:
+		case ExOperStats::SE_STATS:
 		  {
-		    newStat = new(heap_) ExHbaseAccessStats(heap_);
+		    newStat = new(heap_) ExStorageEngineStats(heap_);
 		    newStat->setCollectStatsType(tempStatsMergeType);
-		    ((ExHbaseAccessStats *)newStat)->copyContents((ExHbaseAccessStats *)stat);
+		    ((ExStorageEngineStats *)newStat)->copyContents((ExStorageEngineStats *)stat);
 		    newStat->setCollectStatsType(tempStatsMergeType);
 		    insert(newStat);
 		  }
@@ -5827,16 +5415,9 @@ NABoolean ExStatisticsArea::merge(ExStatisticsArea * otherStatsArea, UInt16 stat
 		    insert(newStat);
 		    break;
 
-		  case ExOperStats::HDFSSCAN_STATS:
-		    newStat = new(heap_) ExHdfsScanStats(heap_);
-		    ((ExHdfsScanStats *)newStat)->copyContents((ExHdfsScanStats *)stat);
-		    newStat->setCollectStatsType(tempStatsMergeType);
-		    insert(newStat);
-		    break;
-
-		  case ExOperStats::HBASE_ACCESS_STATS:
-		    newStat = new(heap_) ExHbaseAccessStats(heap_);
-		    ((ExHbaseAccessStats *)newStat)->copyContents((ExHbaseAccessStats *)stat);
+		  case ExOperStats::SE_STATS:
+		    newStat = new(heap_) ExStorageEngineStats(heap_);
+		    ((ExStorageEngineStats *)newStat)->copyContents((ExStorageEngineStats *)stat);
 		    newStat->setCollectStatsType(tempStatsMergeType);
 		    insert(newStat);
 		    break;
@@ -5950,18 +5531,10 @@ NABoolean ExStatisticsArea::merge(ExStatisticsArea * otherStatsArea, UInt16 stat
 			    insert(newStat);
 			    break;
 
-			  case ExOperStats::HDFSSCAN_STATS:
-			    newStat = new(heap_)ExHdfsScanStats(heap_);
-			    ((ExHdfsScanStats *)newStat)->
-			      copyContents((ExHdfsScanStats *)stat);
-			    newStat->setCollectStatsType(tempStatsMergeType);
-			    insert(newStat);
-			    break;
-
-			  case ExOperStats::HBASE_ACCESS_STATS:
-			    newStat = new(heap_)ExHbaseAccessStats(heap_);
-			    ((ExHbaseAccessStats *)newStat)->
-			      copyContents((ExHbaseAccessStats *)stat);
+			  case ExOperStats::SE_STATS:
+			    newStat = new(heap_)ExStorageEngineStats(heap_);
+			    ((ExStorageEngineStats *)newStat)->
+			      copyContents((ExStorageEngineStats *)stat);
 			    newStat->setCollectStatsType(tempStatsMergeType);
 			    insert(newStat);
 			    break;
@@ -6398,12 +5971,9 @@ void ExStatisticsArea::unpackThisClass(const char* &buffer, ExOperStats *parentS
     case ExOperStats::PROCESS_STATS:
       stat = new(heap_) ExProcessStats((NAHeap *)heap_);
       break;
-	case ExOperStats::HDFSSCAN_STATS:
-     stat = new(heap_) ExHdfsScanStats((NAHeap *)heap_);
+    case ExOperStats::SE_STATS:
+     stat = new(heap_) ExStorageEngineStats((NAHeap *)heap_);
      break;
-    case ExOperStats::HBASE_ACCESS_STATS:
-     stat = new(heap_) ExHbaseAccessStats((NAHeap *)heap_);
-      break;
     default:
       FAILURE ;
     }
@@ -6536,6 +6106,7 @@ Lng32 ExStatisticsArea::getStatsItems(Lng32 no_of_stats_items,
   ExPartitionAccessStats* partitionAccessStats = NULL;
   ExProbeCacheStats* probeCacheStats = NULL;
   ExFastExtractStats* fastExtractStats = NULL;
+  ExStorageEngineStats *seStats = NULL;
   ExHdfsScanStats* hdfsScanStats = NULL;
   ExHbaseAccessStats* hbaseAccessStats = NULL;
   ExSortStats* sortStats = NULL;
@@ -6669,19 +6240,11 @@ Lng32 ExStatisticsArea::getStatsItems(Lng32 no_of_stats_items,
       else
         tempRetcode = -EXE_STAT_NOT_FOUND;
       break;
-    case ExOperStats::HDFSSCAN_STATS:
-      if (hdfsScanStats == NULL)
-        hdfsScanStats = (ExHdfsScanStats*)get(ExOperStats::HDFSSCAN_STATS, sqlStats_items[i].tdb_id);
-      if (hdfsScanStats != NULL)
-        tempRetcode = hdfsScanStats->getStatsItem(&sqlStats_items[i]);
-      else
-        tempRetcode = -EXE_STAT_NOT_FOUND;
-      break;
-    case ExOperStats::HBASE_ACCESS_STATS:
-      if (hbaseAccessStats == NULL)
-        hbaseAccessStats = (ExHbaseAccessStats*)get(ExOperStats::HBASE_ACCESS_STATS, sqlStats_items[i].tdb_id);
-      if (hbaseAccessStats != NULL)
-        tempRetcode = hbaseAccessStats->getStatsItem(&sqlStats_items[i]);
+    case ExOperStats::SE_STATS:
+      if (seStats == NULL)
+        seStats = (ExStorageEngineStats*)get(ExOperStats::HDFSSCAN_STATS, sqlStats_items[i].tdb_id);
+      if (seStats != NULL)
+        tempRetcode = seStats->getStatsItem(&sqlStats_items[i]);
       else
         tempRetcode = -EXE_STAT_NOT_FOUND;
       break;
@@ -7026,8 +6589,9 @@ NABoolean ExStatisticsArea::appendCpuStats(ExStatisticsArea *other,
   ExUDRBaseStats *udrBaseStats;
   ExMasterStats *masterStats;
   ExProcessStats *processStats;
-  ExHbaseAccessStats *hbaseAccessStats;
+  ExStorageEngineStats *seStats;
   ExHdfsScanStats *hdfsScanStats;
+  ExHbaseAccessStats *hbaseAccessStats;
   NABoolean retcode = FALSE;
   ExOperStats::StatType statType;
   ExOperStats *stat1;
@@ -7172,26 +6736,16 @@ NABoolean ExStatisticsArea::appendCpuStats(ExStatisticsArea *other,
             retcode = TRUE;
           }
           break;
-        case ExOperStats::HDFSSCAN_STATS:
+        case ExOperStats::SE_STATS:
           if (detailLevel_ == stat->getTdbId())
           {
-            hdfsScanStats = new (getHeap()) ExHdfsScanStats(getHeap());
-            hdfsScanStats->setCollectStatsType(getCollectStatsType());
-            hdfsScanStats->copyContents((ExHdfsScanStats *)stat);
-            insert(hdfsScanStats);
+            seStats = new (getHeap()) ExStorageEngineStats(getHeap());
+            seStats->setCollectStatsType(getCollectStatsType());
+            seStats->copyContents((ExHdfsScanStats *)stat);
+            insert(seStats);
             retcode = TRUE;
           }
           break;
-        case ExOperStats::HBASE_ACCESS_STATS:
-          if (detailLevel_ == stat->getTdbId())
-          {
-            hbaseAccessStats = new (getHeap()) ExHbaseAccessStats(getHeap());
-            hbaseAccessStats->setCollectStatsType(getCollectStatsType());
-            hbaseAccessStats->copyContents((ExHbaseAccessStats *)stat);
-            insert(hbaseAccessStats);
-            retcode = TRUE;
-          }
-          break; 
         default:
           break;
         } // StatType case
@@ -10529,9 +10083,9 @@ const char * ExBMOStats::getNumValTxt(Int32 i) const
     case 1:
       return "OperCpuTime";
     case 2:
-      return "scrIORead";
+      return "scrIOCount";
     case 3:
-      return "scrIOWritten";
+      return "bmoHeapAllocated";
     case 4:
       return "scrFileCount";
   }
@@ -10545,9 +10099,9 @@ Int64 ExBMOStats::getNumVal(Int32 i) const
      case 1:
         return ExOperStats::getNumVal(i);
      case 2:
-        return scratchReadCount_;
+        return scratchReadCount_+scratchWriteCount_;
      case 3:
-        return scratchWriteCount_;
+        return bmoHeapAlloc_;
      case 4:
         return scratchFileCount_;
   }
@@ -10718,10 +10272,8 @@ Lng32 ExBMOStats::getStatsItem(SQLSTATS_ITEM* sqlStats_item)
   case SQLSTATS_DETAIL:
    if (sqlStats_item->str_value != NULL)
     {
-      sprintf(tmpBuf, "%d|%d|%d", 
-              scratchBufferBlockRead_,
-              scratchBufferBlockWritten_,
-              scratchFileCount_);
+      char* buf = tmpBuf;
+      sprintf(buf, "%ld|%ld|%ld|", getNumVal(2), getNumVal(3), getNumVal(4)); 
       len = str_len(tmpBuf);
       if (len > sqlStats_item->str_max_len)
         sqlStats_item->error_code = EXE_ERROR_IN_STAT_ITEM;

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/ExStats.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExStats.h b/core/sql/executor/ExStats.h
index e244b4d..6fd10e0 100644
--- a/core/sql/executor/ExStats.h
+++ b/core/sql/executor/ExStats.h
@@ -42,6 +42,7 @@
 
 #include "Int64.h"
 #include "ComTdb.h"
+#include "ex_stdh.h"
 #include "ExScheduler.h"
 #include "ComTdbStats.h"
 #include "ComTdbUdr.h"
@@ -85,8 +86,10 @@ class ExRMSStats;
 class ExBMOStats;
 class ExUDRBaseStats;
 class ExFastExtractStats;
-class ExHdfsScanStats;
-class ExHbaseAccessStats;
+class ExStorageEngineStats;
+
+typedef ExStorageEngineStats ExHbaseAccessStats; 
+typedef ExStorageEngineStats ExHdfsScanStats; 
 
 //////////////////////////////////////////////////////////////////
 // forward classes
@@ -434,8 +437,7 @@ public:
   UInt32 pack(char * buffer);
 
   void merge(ExeSEStats * other);
-  void merge(ExHbaseAccessStats * other);
-  void merge(ExHdfsScanStats * other);
+  void merge(ExStorageEngineStats * other);
 
   void unpack(const char* &buffer);
 
@@ -566,8 +568,9 @@ public:
     REPLICATOR_STATS        = SQLSTATS_DESC_REPLICATOR_STATS,
     FAST_EXTRACT_STATS      = SQLSTATS_DESC_FAST_EXTRACT_STATS,
     REORG_STATS             = SQLSTATS_DESC_REORG_STATS,
-    HDFSSCAN_STATS    = SQLSTATS_DESC_HDFSSCAN_STATS,
-    HBASE_ACCESS_STATS    = SQLSTATS_DESC_HBASE_ACCESS_STATS,
+    SE_STATS                = SQLSTATS_DESC_SE_STATS,
+    HDFSSCAN_STATS          = SQLSTATS_DESC_SE_STATS,
+    HBASE_ACCESS_STATS      = SQLSTATS_DESC_SE_STATS,
     PROCESS_STATS           = SQLSTATS_DESC_PROCESS_STATS
   };
 
@@ -1795,24 +1798,24 @@ private:
 };
 
 /////////////////////////////////////////////////////////////////
-//// class ExHdfsScanStats
+//// class ExStorageEngineStats
 ///////////////////////////////////////////////////////////////////
 
-class ExHdfsScanStats : public ExOperStats {
+class ExStorageEngineStats : public ExOperStats {
 public:
-  ExHdfsScanStats(NAMemory * heap,
+  ExStorageEngineStats(NAMemory * heap,
                           ex_tcb *tcb,
                           ComTdb * tdb);  // tbd - other, specific params?
 
-  ExHdfsScanStats(NAMemory * heap);
+  ExStorageEngineStats(NAMemory * heap);
 
-  ~ExHdfsScanStats();
+  ~ExStorageEngineStats();
 
   void init(NABoolean resetDop);
 
-  void merge(ExHdfsScanStats* other);
+  void merge(ExStorageEngineStats* other);
 
-  void copyContents(ExHdfsScanStats* other);
+  void copyContents(ExStorageEngineStats* other);
 
   ExOperStats * copyOper(NAMemory * heap);
   
@@ -1831,24 +1834,32 @@ public:
 char * tableName() const {return tableName_;}
 
   ExTimeStats &getHdfsTimer() { return timer_; }
+  ExTimeStats &getHbaseTimer() { return timer_; }
+
   inline void incBytesRead(Int64 bytesRead) {numBytesRead_ += bytesRead;}
 
   inline void incAccessedRows() {++accessedRows_;}
+  inline void incAccessedRows(Int64 v) {accessedRows_ += v;}
 
   inline void incUsedRows() {++usedRows_;}
+  inline void incUsedRows(Int64 v) {usedRows_ += v;}
 
-  inline void incMaxHdfsIOTime(Int64 v) {maxHdfsIOTime_ += v;}
-  Int64 numBytesRead() const {return numBytesRead_;}
+  inline void incMaxHdfsIOTime(Int64 v) {maxIOTime_ += v;}
+  inline void incMaxHbaseIOTime(Int64 v) {maxIOTime_ += v;}
 
+  Int64 numBytesRead() const {return numBytesRead_;}
   Int64 rowsAccessed() const {return accessedRows_;}
-
   Int64 rowsUsed() const {return usedRows_;}
 
   NABoolean filterForSEstats(struct timespec currTimespec, Lng32 filter);
-  Int64 maxHdfsIOTime() const {return maxHdfsIOTime_;}
+
+  Int64 maxHdfsIOTime() const {return maxIOTime_;}
+  Int64 maxHbaseIOTime() const {return maxIOTime_;}
 
   ExHdfsScanStats * castToExHdfsScanStats();
 
+  ExHbaseAccessStats * castToExHbaseAccessStats();
+
   virtual const char * getNumValTxt(Int32 i) const;
 
   virtual Int64 getNumVal(Int32 i) const;
@@ -1859,125 +1870,27 @@ char * tableName() const {return tableName_;}
 
   Lng32 getStatsItem(SQLSTATS_ITEM* sqlStats_item);
   
-  ExLobStats * lobStats() { return &lobStats_;}
   void setQueryId(char *queryId, Lng32 queryIdLen)
     {queryId_ = queryId;
      queryIdLen_ = queryIdLen;} 
   char *getQueryId() { return queryId_; }
   Lng32 getQueryIdLen() { return queryIdLen_; }
-  inline void incHdfsCalls() {++numHdfsCalls_;}
-  Int64 hdfsCalls() const {return numHdfsCalls_;}
-
-private:
-
-  ExTimeStats timer_;
-  ExLobStats lobStats_;
-
-  char * tableName_;
-
-  Int64  numBytesRead_;
-  Int64  accessedRows_;
-  Int64  usedRows_;
-  Int64  numHdfsCalls_;
-  Int64  maxHdfsIOTime_;
-  char *queryId_;
-  Lng32 queryIdLen_;
-  Lng32 blockTime_;
-};
-
-/////////////////////////////////////////////////////////////////
-//// class ExHbaseAccessStats
-///////////////////////////////////////////////////////////////////
-
-class ExHbaseAccessStats : public ExOperStats {
- public:
-    ExHbaseAccessStats(NAMemory * heap,
-		       ex_tcb *tcb,
-		       ComTdb * tdb);  // tbd - other, specific params?
-  
-    ExHbaseAccessStats(NAMemory * heap);
-  
-    ~ExHbaseAccessStats();
-  
-    void init(NABoolean resetDop);
-  
-    void merge(ExHbaseAccessStats* other);
-  
-    void copyContents(ExHbaseAccessStats* other);
-  
-    ExOperStats * copyOper(NAMemory * heap);
-  
-    UInt32 packedLength();
-  
-  //////////////////////////////////////////////////////////////////
-  //// packs 'this' into a message. Converts pointers to offsets.
-  ////////////////////////////////////////////////////////////////////
-    UInt32 pack(char * buffer);
-  
-    void unpack(const char* &buffer);
-  
-  /////////////////////////////////////////////////////////////////
-  //// accessors, mutators
-  ///////////////////////////////////////////////////////////////////
-  char * tableName() const {return tableName_;}
-  
-    ExTimeStats &getHbaseTimer() { return timer_; }
-    inline void incBytesRead(Int64 bytesRead) {numBytesRead_ += bytesRead;}
-  
-    inline void incAccessedRows() {++accessedRows_;}
-    inline void incAccessedRows(Int64 v) {accessedRows_ += v;}
-  
-    inline void incUsedRows() {++usedRows_;}
-    inline void incUsedRows(Int64 v) {usedRows_ += v;}
- 
-    inline void incHbaseCalls() {++numHbaseCalls_;}
 
-    inline void incMaxHbaseIOTime(Int64 v) {maxHbaseIOTime_ += v;}
- 
-    Int64 numBytesRead() const {return numBytesRead_;}
-  
-    Int64 rowsAccessed() const {return accessedRows_;}
-  
-    Int64 rowsUsed() const {return usedRows_;}
-  
-    Int64 hbaseCalls() const {return numHbaseCalls_;}
+  inline void incHdfsCalls() {++numIOCalls_;}
+  inline void incHbaseCalls() {++numIOCalls_;}
 
-    Int64 maxHbaseIOTime() const {return maxHbaseIOTime_;}
-   
-    NABoolean filterForSEstats(struct timespec currTimespec, Lng32 filter);
+  Int64 hdfsCalls() const {return numIOCalls_;}
+  Int64 hbaseCalls() const {return numIOCalls_;}
 
-    ExHbaseAccessStats * castToExHbaseAccessStats();
-  
-    virtual const char * getNumValTxt(Int32 i) const;
-  
-    virtual Int64 getNumVal(Int32 i) const;
-    
-    virtual void getVariableStatsInfo(char * dataBuffer,
-				      char * datalen,
-				      Lng32 maxLen);
-  Lng32 getStatsItem(SQLSTATS_ITEM* sqlStats_item);
-  
-  ExLobStats * lobStats() { return &lobStats_;}
+private:
 
-  //  ExHbaseStats * hbaseStats() { return &hbaseStats_;}
-  void setQueryId(char *queryId, Lng32 queryIdLen)
-    {queryId_ = queryId;
-     queryIdLen_ = queryIdLen;} 
-  char *getQueryId() { return queryId_; }
-  Lng32 getQueryIdLen() { return queryIdLen_; }
-  
- private:
-  
   ExTimeStats timer_;
-  ExLobStats lobStats_;
-  
   char * tableName_;
-  
   Int64  numBytesRead_;
   Int64  accessedRows_;
   Int64  usedRows_;
-  Int64  numHbaseCalls_;
-  Int64  maxHbaseIOTime_;
+  Int64  numIOCalls_;
+  Int64  maxIOTime_;
   char *queryId_;
   Lng32 queryIdLen_;
   Lng32 blockTime_;
@@ -2087,8 +2000,7 @@ public:
   void merge(ExFragRootOperStats* other);
   void merge(ExUDRBaseStats * other);
   void merge(ExBMOStats * other);
-  void merge(ExHdfsScanStats * other);
-  void merge(ExHbaseAccessStats * other);
+  void merge(ExStorageEngineStats * other);
 
   ExOperStats * copyOper(NAMemory * heap);
 

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/HBaseClient_JNI.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.h b/core/sql/executor/HBaseClient_JNI.h
index c89146a..eb339aa 100644
--- a/core/sql/executor/HBaseClient_JNI.h
+++ b/core/sql/executor/HBaseClient_JNI.h
@@ -29,23 +29,19 @@
 #include "Platform.h"
 #include "Collections.h"
 #include "NABasicObject.h"
-
+#include "ExStats.h"
 #include "JavaObjectInterface.h"
 #include "Hbase_types.h"
 #include "ExpHbaseDefs.h"
 #include "NAMemory.h"
 #include "org_trafodion_sql_HTableClient.h"
 
-// forward declare
-class ExHbaseAccessStats;
-
 using namespace apache::hadoop::hbase::thrift;
 
 namespace {
   typedef std::vector<Text> TextVec;
 }
 
-
 class ContextCli;
 
 class HBulkLoadClient_JNI;

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/HdfsClient_JNI.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/HdfsClient_JNI.cpp b/core/sql/executor/HdfsClient_JNI.cpp
index d5b5043..65c83cd 100644
--- a/core/sql/executor/HdfsClient_JNI.cpp
+++ b/core/sql/executor/HdfsClient_JNI.cpp
@@ -288,6 +288,8 @@ HDFS_Scan_RetCode HdfsScan::trafHdfsRead(int retArray[], short arrayLen)
    short retArrayLen = jenv_->GetArrayLength(j_retArray);
    ex_assert(retArrayLen == arrayLen, "HdfsScan::trafHdfsRead() InternalError: retArrayLen != arrayLen");
    jenv_->GetIntArrayRegion(j_retArray, 0, 4, retArray);
+   if (hdfsStats_ != NULL)
+      hdfsStats_->incBytesRead(retArray[ExHdfsScanTcb::BYTES_COMPLETED]);
    return HDFS_SCAN_OK;
 }
 
@@ -666,6 +668,7 @@ Int32 HdfsClient::hdfsWrite(const char* data, Int64 len, HDFS_Client_RetCode &hd
      if (hdfsStats_ != NULL) {
          hdfsStats_->incMaxHdfsIOTime(hdfsStats_->getHdfsTimer().stop());
          hdfsStats_->incHdfsCalls();
+         hdfsStats_->incBytesRead(writeLen);
      }
      if (jenv_->ExceptionCheck())
      {
@@ -724,6 +727,7 @@ Int64 HdfsClient::hdfsWriteImmediate(const char* data, Int64 len, HDFS_Client_Re
      if (hdfsStats_ != NULL) {
          hdfsStats_->incMaxHdfsIOTime(hdfsStats_->getHdfsTimer().stop());
          hdfsStats_->incHdfsCalls();
+         hdfsStats_->incBytesRead(writeLen);
      }
      if (jenv_->ExceptionCheck())
      {
@@ -765,6 +769,7 @@ Int32 HdfsClient::hdfsRead(Int64 pos, const char* data, Int64 len, HDFS_Client_R
   if (hdfsStats_ != NULL) {
       hdfsStats_->incMaxHdfsIOTime(hdfsStats_->getHdfsTimer().stop());
       hdfsStats_->incHdfsCalls();
+      hdfsStats_->incBytesRead(bytesRead);
   }
   if (jenv_->ExceptionCheck())
   {

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/OrcFileReader.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/OrcFileReader.h b/core/sql/executor/OrcFileReader.h
index 536235a..2c7d7b8 100644
--- a/core/sql/executor/OrcFileReader.h
+++ b/core/sql/executor/OrcFileReader.h
@@ -24,6 +24,7 @@
 #define ORC_FILE_READER_H
 
 #include "JavaObjectInterface.h"
+#include "ExStats.h"
 
 // ===========================================================================
 // ===== The OrcFileReader class implements access to th Java 

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/log4cpp.hdfs.config
----------------------------------------------------------------------
diff --git a/core/sql/executor/log4cpp.hdfs.config b/core/sql/executor/log4cpp.hdfs.config
deleted file mode 100644
index d27650b..0000000
--- a/core/sql/executor/log4cpp.hdfs.config
+++ /dev/null
@@ -1,37 +0,0 @@
-# log4cxx configuration for HDFS access via JNI
-
-# @@@ START COPYRIGHT @@@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# @@@ END COPYRIGHT @@@
-
-log4j.rootLogger=INFO, rootAppender
-
-log4j.appender.rootAppender=org.apache.log4j.RollingFileAppender
-log4j.appender.rootAppender.fileName=hdfs.log
-log4j.appender.rootAppender.maxFileSize=100000000
-log4j.appender.rootAppender.maxBackupIndex=1
-log4j.appender.rootAppender.addPid=false
-log4j.appender.rootAppender.layout=org.apache.log4j.PatternLayout
-log4j.appender.rootAppender.layout.ConversionPattern=%d, %p, %c, %m%n
-
-log4j.logger.JniTop=ERROR
-log4j.logger.SeqFileReader=ERROR
-log4j.logger.SeqFileWriter=ERROR
-log4j.logger.HBase=ERROR

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/executor/log4j.hdfs.config
----------------------------------------------------------------------
diff --git a/core/sql/executor/log4j.hdfs.config b/core/sql/executor/log4j.hdfs.config
deleted file mode 100644
index 6b8f9fe..0000000
--- a/core/sql/executor/log4j.hdfs.config
+++ /dev/null
@@ -1,50 +0,0 @@
-# Define some default values that can be overridden by system properties
-
-# @@@ START COPYRIGHT @@@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# @@@ END COPYRIGHT @@@
-
-hbase.root.logger=INFO,hbaseclient
-hbase.log.dir=.
-hbase.log.file=hdfs.java.log
-
-# Define the root logger to the system property "hbase.root.logger".
-log4j.rootLogger=${hbase.root.logger}
-
-# Logging Threshold
-log4j.threshhold=ALL
-
-#
-# Daily Rolling File Appender
-#
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.hbaseclient=org.apache.log4j.RollingFileAppender
-log4j.appender.hbaseclient.file=hdfs.java.log
-log4j.appender.hbaseclient.layout=org.apache.log4j.PatternLayout
-log4j.appender.hbaseclient.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.appender.hbaseclient.immediateFlush=true
-
-# Custom Logging levels
-
-log4j.logger.org.apache.zookeeper=ERROR
-log4j.logger.org.apache.hadoop.hbase=ERROR
-log4j.logger.com.tandem.sqlmx=ERROR
-
-#log4j.logger.org.apache.hadoop.hbase=DEBUG

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/exp/ExpHbaseInterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.h b/core/sql/exp/ExpHbaseInterface.h
index c9583e5..a7eee97 100644
--- a/core/sql/exp/ExpHbaseInterface.h
+++ b/core/sql/exp/ExpHbaseInterface.h
@@ -60,7 +60,6 @@
 
 class ex_globals;
 class CliGlobals;
-class ExHbaseAccessStats;
 
 Int64 getTransactionIDFromContext();
 

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/exp/ExpLOB.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOB.cpp b/core/sql/exp/ExpLOB.cpp
index 5fb0fc2..914861b 100644
--- a/core/sql/exp/ExpLOB.cpp
+++ b/core/sql/exp/ExpLOB.cpp
@@ -833,6 +833,7 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[],
       
       rc = ExpLOBInterfaceInsertSelect
         (getExeGlobals()->getExLobGlobal(), 
+         (getTcb()->getStatsEntry() != NULL ? getTcb()->getStatsEntry()->castToExHdfsScanStats() : NULL),
          getLobHdfsServer(), getLobHdfsPort(),
          tgtLobName, 
          so,
@@ -845,6 +846,7 @@ ex_expr::exp_return_type ExpLOBiud::insertDesc(char *op_data[],
 else
   rc = ExpLOBInterfaceInsert
     (getExeGlobals()->getExLobGlobal(), 
+     (getTcb()->getStatsEntry() != NULL ? getTcb()->getStatsEntry()->castToExHdfsScanStats() : NULL),
      tgtLobName, 
      lobStorageLocation(),
      lobStorageType(),
@@ -992,6 +994,7 @@ ex_expr::exp_return_type ExpLOBiud::insertData(Lng32 handleLen,
 
   
       rc = ExpLOBInterfaceInsert(getExeGlobals()->getExLobGlobal(),
+                                 (getTcb()->getStatsEntry() != NULL ? getTcb()->getStatsEntry()->castToExHdfsScanStats() : NULL),
 				 tgtLobName, 
 				 lobStorageLocation(),
 				 lobType,
@@ -1160,6 +1163,7 @@ ex_expr::exp_return_type ExpLOBdelete::eval(char *op_data[],
   rc = ExpLOBInterfaceDelete
     (
      getExeGlobals()->getExLobGlobal(),
+     (getTcb()->getStatsEntry() != NULL ? getTcb()->getStatsEntry()->castToExHdfsScanStats() : NULL),
      getLobHdfsServer(),
      getLobHdfsPort(),
      lobName, 
@@ -1439,6 +1443,7 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
     {
       rc = ExpLOBInterfaceUpdateAppend
 	(getExeGlobals()->getExLobGlobal(), 
+         (getTcb()->getStatsEntry() != NULL ? getTcb()->getStatsEntry()->castToExHdfsScanStats() : NULL),
 	 getLobHdfsServer(),
 	 getLobHdfsPort(),
 	 tgtLobName, 
@@ -1462,6 +1467,7 @@ ex_expr::exp_return_type ExpLOBupdate::eval(char *op_data[],
     {
       rc = ExpLOBInterfaceUpdate
 	(getExeGlobals()->getExLobGlobal(), 
+         (getTcb()->getStatsEntry() != NULL ? getTcb()->getStatsEntry()->castToExHdfsScanStats() : NULL),
 	 getLobHdfsServer(),
 	 getLobHdfsPort(),
 	 tgtLobName, 
@@ -1636,6 +1642,7 @@ ex_expr::exp_return_type ExpLOBconvert::eval(char *op_data[],
       so = Lob_File;
       tgtFileName = tgtFileName_;
       rc = ExpLOBInterfaceSelect(getExeGlobals()->getExLobGlobal(), 
+                                 (getTcb()->getStatsEntry() != NULL ? getTcb()->getStatsEntry()->castToExHdfsScanStats() : NULL),
 				 lobName, 
 				 lobStorageLocation(),
 				 lobType,
@@ -1661,6 +1668,7 @@ ex_expr::exp_return_type ExpLOBconvert::eval(char *op_data[],
       lobLen = getConvertSize(); 
       lobData = new(h) char[(Lng32)lobLen];
       rc = ExpLOBInterfaceSelect(getExeGlobals()->getExLobGlobal(), 
+                                 (getTcb()->getStatsEntry() != NULL ? getTcb()->getStatsEntry()->castToExHdfsScanStats() : NULL),
 				 lobName, 
 				 lobStorageLocation(),
 				 lobType,

http://git-wip-us.apache.org/repos/asf/trafodion/blob/abda16ef/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index a24018a..bb9b580 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -77,7 +77,7 @@ extern int ms_transid_reinstate(MS_Mon_Transid_Type, MS_Mon_Transseq_Type);
 // short LobServerFNum;
 SB_Phandle_Type serverPhandle;
 
-ExLob::ExLob(NAHeap * heap) :
+ExLob::ExLob(NAHeap * heap, ExHdfsScanStats *hdfsAccessStats) :
     lobDataFile_(heap),
     storage_(Lob_Invalid_Storage),
     lobStorageLocation_(string()),
@@ -85,6 +85,7 @@ ExLob::ExLob(NAHeap * heap) :
     fs_(NULL),
     fdData_(NULL),
     openFlags_(0),
+    stats_(hdfsAccessStats),
     lobTrace_(FALSE),
     useLibHdfs_(FALSE),
     hdfsClient_(NULL)    
@@ -149,8 +150,6 @@ Ex_Lob_Error ExLob::initialize(const char *lobFile, Ex_Lob_Mode mode,
       storage_ = storage;
     }
 
-  stats_.init(); 
-
   hdfsServer_ = hdfsServer;
   hdfsPort_ = hdfsPort;
   // lobLocation_ = lobLocation;
@@ -165,7 +164,7 @@ Ex_Lob_Error ExLob::initialize(const char *lobFile, Ex_Lob_Mode mode,
      hdfsClient_ = NULL;
   }
   else {
-     hdfsClient_ = HdfsClient::newInstance(lobGlobalHeap_, NULL, hdfsClientRetcode);
+     hdfsClient_ = HdfsClient::newInstance(lobGlobalHeap_, stats_, hdfsClientRetcode);
      fs_ = NULL;
      if (hdfsClient_ == NULL)
         return LOB_HDFS_CONNECT_ERROR;
@@ -181,7 +180,6 @@ Ex_Lob_Error ExLob::initialize(const char *lobFile, Ex_Lob_Mode mode,
       nsecs += NUM_NSECS_IN_SEC;
     }
   totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
-  stats_.hdfsConnectionTime += totalnsecs;
    
   if (! useLibHdfs_) {
      if (mode == EX_LOB_CREATE) {
@@ -2192,7 +2190,6 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
         nsecs += NUM_NSECS_IN_SEC;
       }
       Int64 totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
-      stats_.CumulativeReadTime += totalnsecs;
       } // useLibHdfs
       if (bytesRead == -1) {
          return LOB_DATA_READ_ERROR;
@@ -2603,20 +2600,19 @@ Ex_Lob_Error ExLob::closeFile()
 
 Ex_Lob_Error ExLob::readStats(char *statsBuffer)
 {
-    memcpy(statsBuffer, (char *)&stats_, sizeof(stats_));
+    stats_ = (ExHdfsScanStats *)statsBuffer;
     return LOB_OPER_OK;
 }
 
 Ex_Lob_Error ExLob::initStats()
 {
-    stats_.init();
     return LOB_OPER_OK;
 }
 //Main driver of any LOB related operation 
 
 Ex_Lob_Error ExLobsOper (
 			 char        *lobName,          // lob name
-
+                         ExHdfsScanStats *hdfsAccessStats,
 			 char        *handleIn,         // input handle (for cli calls)
 			 Int32       handleInLen,       // input handle len
 
@@ -2709,7 +2705,7 @@ Ex_Lob_Error ExLobsOper (
 
       if (it == lobMap->end())
 	{
-	  lobPtr = new (lobGlobals->getHeap())ExLob(lobGlobals->getHeap());
+	  lobPtr = new (lobGlobals->getHeap())ExLob(lobGlobals->getHeap(), hdfsAccessStats);
 	  if (lobPtr == NULL) 
 	    return LOB_ALLOC_ERROR;
 
@@ -2802,7 +2798,7 @@ Ex_Lob_Error ExLobsOper (
 
         if (it2 == lobMap->end())
 	{
-	  srcLobPtr = new (lobGlobals->getHeap())ExLob(lobGlobals->getHeap());
+	  srcLobPtr = new (lobGlobals->getHeap())ExLob(lobGlobals->getHeap(), hdfsAccessStats);
 	  if (srcLobPtr == NULL) 
 	    return LOB_ALLOC_ERROR;
 
@@ -3019,11 +3015,6 @@ Ex_Lob_Error ExLobsOper (
         lobDebugInfo("purgeLob failed ",err,__LINE__,lobGlobals->lobTrace_);
       break;
 
-    case Lob_Stats:
-      err = lobPtr->readStats(source);
-      lobPtr->initStats(); // because file may remain open across cursors
-      break;
-
     case Lob_Empty_Directory:    
       err = lobPtr->emptyDirectory(lobStorageLocation, lobGlobals);
 
@@ -3090,7 +3081,6 @@ if (!lobGlobals->isHive() )
 
   */
   clock_gettime(CLOCK_MONOTONIC, &endTime);
-
   secs = endTime.tv_sec - startTime.tv_sec;
   nsecs = endTime.tv_nsec - startTime.tv_nsec;
   if (nsecs < 0) {
@@ -3098,9 +3088,10 @@ if (!lobGlobals->isHive() )
     nsecs += NUM_NSECS_IN_SEC;
   }
   totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
+/*
   if (lobPtr && lobPtr->getStats())
     lobPtr->getStats()->hdfsAccessLayerTime += totalnsecs; 
-       
+*/
   return err;
 }
 
@@ -3209,15 +3200,15 @@ Ex_Lob_Error ExLob::readDataCursorSimple(const char *file, char *tgt, Int64 tgtS
         buf->bytesUsed_ += bytesToCopy;
         buf->bytesRemaining_ -= bytesToCopy;
       }
-      stats_.bytesPrefetched += bytesToCopy;
+      //stats_.bytesPrefetched += bytesToCopy;
       operLen += bytesToCopy;
     } 
-
+/*
     // update stats
     stats_.bytesRead += operLen;
     stats_.bytesToRead += tgtSize;
     stats_.numReadReqs++;
-
+*/
     return LOB_OPER_OK;
 }
 
@@ -3262,7 +3253,7 @@ Ex_Lob_Error ExLob::closeDataCursorSimple(const char *fileName, ExLobGlobals *lo
       nsecs += NUM_NSECS_IN_SEC;
     }
     Int64 totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
-    stats_.cursorElapsedTime += totalnsecs;
+    //stats_.cursorElapsedTime += totalnsecs;
 
     return LOB_OPER_OK;
 }
@@ -3317,7 +3308,7 @@ Ex_Lob_Error ExLobGlobals::performRequest(ExLobHdfsRequest *request)
           cursor->lock_.unlock();
           buf = new (getHeap()) ExLobCursorBuffer();
           buf->data_ = (char *) (getHeap())->allocateMemory( cursor->bufMaxSize_);
-          lobPtr->stats_.buffersUsed++;
+          //lobPtr->stats_.buffersUsed++;
         }
         size = min(cursor->bufMaxSize_, (cursor->maxBytes_ - cursor->bytesRead_));
         if (buf->data_) {
@@ -3418,7 +3409,7 @@ Ex_Lob_Error ExLob::readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cur
       // bytesRead = hdfsPread(fs_, fdData_, offset, tgt, bytesToCopy);
       bytesRead = hdfsRead(fs_, fdData_, tgt, bytesToCopy);
 
-      stats_.numHdfsReqs++;
+      //stats_.numHdfsReqs++;
 
       if (bytesRead == -1) {
          return LOB_DATA_READ_ERROR;
@@ -3441,10 +3432,11 @@ Ex_Lob_Error ExLob::readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cur
    }
 
    Int64 totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
-   stats_.CumulativeReadTime += totalnsecs;
+   //stats_.CumulativeReadTime += totalnsecs;
 
    return LOB_OPER_OK;
 }
+
 void ExLobCursor::emptyPrefetchList(ExLobGlobals *lobGlobals)
 {
     ExLobCursor::bufferList_t::iterator c_it;


Mime
View raw message