trafodion-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dbirds...@apache.org
Subject [1/6] incubator-trafodion git commit: [TRAFODION-2617] Optionally push estimate row count logic into HBase coproc
Date Thu, 17 Aug 2017 16:45:19 GMT
Repository: incubator-trafodion
Updated Branches:
  refs/heads/master baa034f89 -> 548ff71b4


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto
----------------------------------------------------------------------
diff --git a/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto b/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto
index a69e6b8..b3edbfa 100755
--- a/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto
+++ b/core/sqf/src/seatrans/hbase-trx/src/main/protobuf/TrxRegion.proto
@@ -404,6 +404,23 @@ message TlogTransactionStatesFromIntervalResponse {
   optional bool hasException = 4;
 }
 
+message TrafEstimateRowCountRequest {
+  // numCols = 1 for aligned tables; = # columns for non-aligned tables
+  required int32 numCols = 1;  
+}
+
+message TrafEstimateRowCountResponse {
+  required int64 totalEntries = 1;
+  required int64 totalSizeBytes = 2;
+  // the next three values will be non-zero only for the first region,
+  // and only when the format is non-aligned
+  required int32 putKVsSampled = 3;
+  required int32 nonPutKVsSampled = 4;
+  required int32 missingKVsCount = 5;
+  optional string exception = 6;
+  optional bool hasException = 7;
+}
+
 message TransactionalAggregateRequest {
   /** The request passed to the TransactionalAggregateService consists of three parts
    *  (1) the (canonical) classname of the ColumnInterpreter implementation
@@ -510,6 +527,8 @@ service TrxRegionService {
     returns(TlogWriteResponse);
   rpc getTransactionStatesPriorToAsn(TlogTransactionStatesFromIntervalRequest)
     returns(TlogTransactionStatesFromIntervalResponse);
+  rpc trafEstimateRowCount(TrafEstimateRowCountRequest)
+    returns(TrafEstimateRowCountResponse);
   rpc GetMax (TransactionalAggregateRequest)
     returns (TransactionalAggregateResponse);
   rpc GetMin (TransactionalAggregateRequest) 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sql/executor/HBaseClient_JNI.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.cpp b/core/sql/executor/HBaseClient_JNI.cpp
index f5ca420..da05b6c 100644
--- a/core/sql/executor/HBaseClient_JNI.cpp
+++ b/core/sql/executor/HBaseClient_JNI.cpp
@@ -250,6 +250,8 @@ HBC_RetCode HBaseClient_JNI::init()
     JavaMethods_[JM_GET_HBLC   ].jm_signature = "()Lorg/trafodion/sql/HBulkLoadClient;";
     JavaMethods_[JM_EST_RC     ].jm_name      = "estimateRowCount";
     JavaMethods_[JM_EST_RC     ].jm_signature = "(Ljava/lang/String;III[J)Z";
+    JavaMethods_[JM_EST_RC_COPROC     ].jm_name      = "estimateRowCountViaCoprocessor";
+    JavaMethods_[JM_EST_RC_COPROC     ].jm_signature = "(Ljava/lang/String;III[J)Z";
     JavaMethods_[JM_REL_HBLC   ].jm_name      = "releaseHBulkLoadClient";
     JavaMethods_[JM_REL_HBLC   ].jm_signature = "(Lorg/trafodion/sql/HBulkLoadClient;)V";
     JavaMethods_[JM_GET_CAC_FRC].jm_name      = "getBlockCacheFraction";
@@ -1339,6 +1341,7 @@ HBC_RetCode HBaseClient_JNI::estimateRowCount(const char* tblName,
                                               Int32 partialRowSize,
                                               Int32 numCols,
                                               Int32 retryLimitMilliSeconds,
+                                              NABoolean useCoprocessor,
                                               Int64& rowCount,
                                               Int32& breadCrumb)
 {
@@ -1364,8 +1367,9 @@ HBC_RetCode HBaseClient_JNI::estimateRowCount(const char* tblName,
   jint jNumCols = numCols;
   jint jRetryLimitMilliSeconds = retryLimitMilliSeconds;
   jlongArray jRowCount = jenv_->NewLongArray(1);
-  tsRecentJMFromJNI = JavaMethods_[JM_EST_RC].jm_full_name;
-  jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_EST_RC].methodID,
+  enum JAVA_METHODS method = (useCoprocessor ? JM_EST_RC_COPROC : JM_EST_RC);
+  tsRecentJMFromJNI = JavaMethods_[method].jm_full_name;
+  jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[method].methodID,
                                               js_tblName, jPartialRowSize,
                                               jNumCols, jRetryLimitMilliSeconds, jRowCount);
   jboolean isCopy;

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sql/executor/HBaseClient_JNI.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.h b/core/sql/executor/HBaseClient_JNI.h
index ea957e7..09e495a 100644
--- a/core/sql/executor/HBaseClient_JNI.h
+++ b/core/sql/executor/HBaseClient_JNI.h
@@ -458,7 +458,8 @@ public:
   HBC_RetCode grant(const Text& user, const Text& tableName, const TextVec& actionCodes);

   HBC_RetCode revoke(const Text& user, const Text& tableName, const TextVec&
actionCodes);
   HBC_RetCode estimateRowCount(const char* tblName, Int32 partialRowSize,
-                               Int32 numCols, Int32 retryLimitMilliSeconds, Int64& rowCount,
Int32 & breadCrumb);
+                               Int32 numCols, Int32 retryLimitMilliSeconds, NABoolean useCoprocessor,
+                               Int64& rowCount, Int32 & breadCrumb);
   HBC_RetCode getLatestSnapshot(const char * tabname, char *& snapshotName, NAHeap *
heap);
   HBC_RetCode cleanSnpTmpLocation(const char * path);
   HBC_RetCode setArchivePermissions(const char * path);
@@ -556,6 +557,7 @@ private:
    ,JM_REVOKE
    ,JM_GET_HBLC
    ,JM_EST_RC
+   ,JM_EST_RC_COPROC
    ,JM_REL_HBLC
    ,JM_GET_CAC_FRC
    ,JM_GET_LATEST_SNP

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sql/exp/ExpHbaseInterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.cpp b/core/sql/exp/ExpHbaseInterface.cpp
index 248e089..22b3c50 100644
--- a/core/sql/exp/ExpHbaseInterface.cpp
+++ b/core/sql/exp/ExpHbaseInterface.cpp
@@ -1490,6 +1490,7 @@ Lng32 ExpHbaseInterface_JNI::estimateRowCount(HbaseStr& tblName,
                                               Int32 partialRowSize,
                                               Int32 numCols,
                                               Int32 retryLimitMilliSeconds,
+                                              NABoolean useCoprocessor,
                                               Int64& estRC,
                                               Int32& breadCrumb)
 {
@@ -1503,7 +1504,8 @@ Lng32 ExpHbaseInterface_JNI::estimateRowCount(HbaseStr& tblName,
 
   estRC = 0;
   retCode_ = client_->estimateRowCount(tblName.val, partialRowSize, numCols, 
-                                       retryLimitMilliSeconds, estRC, breadCrumb /* out */);
+                                       retryLimitMilliSeconds, useCoprocessor,
+                                       estRC, breadCrumb /* out */);
   return retCode_;
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sql/exp/ExpHbaseInterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.h b/core/sql/exp/ExpHbaseInterface.h
index 5f44e9c..daba401 100644
--- a/core/sql/exp/ExpHbaseInterface.h
+++ b/core/sql/exp/ExpHbaseInterface.h
@@ -356,6 +356,7 @@ class ExpHbaseInterface : public NABasicObject
                                  Int32 partialRowSize,
                                  Int32 numCols,
                                  Int32 retryLimitMilliSeconds,
+                                 NABoolean useCoprocessor,
                                  Int64& estRC,
                                  Int32& breadCrumb) = 0;
   virtual Lng32 getLatestSnapshot(const char * tableName, char *& snapshotName, NAHeap
* heap) = 0;
@@ -670,6 +671,7 @@ virtual Lng32 initHFileParams(HbaseStr &tblName,
                                  Int32 partialRowSize,
                                  Int32 numCols,
                                  Int32 retryLimitMilliSeconds,
+                                 NABoolean useCoprocessor,
                                  Int64& estRC,
                                  Int32& breadCrumb);
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index 4b65ef4..0da775e 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -7883,11 +7883,21 @@ Int64 NATable::estimateHBaseRowCount(Int32 retryLimitMilliSeconds,
Int32& errorC
       fqTblName.val[fqTblName.len] = '\0';
 
       Int32 partialRowSize = computeHBaseRowSizeFromMetaData();
+      NABoolean useCoprocessor = 
+        (CmpCommon::getDefault(HBASE_ESTIMATE_ROW_COUNT_VIA_COPROCESSOR) == DF_ON);
+
+      // Note: If in the future we support a hybrid parially aligned +
+      // partially non-aligned format, the following has to be revised.
+      CollIndex keyValueCountPerRow = 1;  // assume aligned format
+      if (!isAlignedFormat(NULL))
+        keyValueCountPerRow = colcount_;  // non-aligned; each column is a separate KV
+
       errorCode = ehi->estimateRowCount(fqTblName,
                                       partialRowSize,
-                                      colcount_,
+                                      keyValueCountPerRow,
                                       retryLimitMilliSeconds,
-                                      estRowCount,
+                                      useCoprocessor,
+                                      estRowCount /* out */,
                                       breadCrumb /* out */);
       NADELETEBASIC(fqTblName.val, STMTHEAP);
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h
index 34832fb..71176a6 100644
--- a/core/sql/sqlcomp/DefaultConstants.h
+++ b/core/sql/sqlcomp/DefaultConstants.h
@@ -3413,6 +3413,10 @@ enum DefaultConstants
   // if ON, hbase coprocessors could be used, if the query allows it.
   HBASE_COPROCESSORS,
 
+  // if ON, use an HBase coprocessor when estimating row counts; if OFF use
+  // client-side code (the latter does not work if HBase encryption is in use)
+  HBASE_ESTIMATE_ROW_COUNT_VIA_COPROCESSOR,
+
   // if OFF or '0' is disabled, ON or '1' is simple pushdown, '2' is for advance pushdown
   // It will depends on the query on which predicates or sub-predicates could be pushed.
   HBASE_FILTER_PREDS,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 431447b..5603ca3 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -1732,6 +1732,10 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS,		"OFF"),
   // HBASE_CATALOG:                Catalog of "_ROW_" and "_CELL_" schemas
   // HBASE_COPROCESSORS:           Enable use of co-processors for aggregates.
   //                               need to set the coprocessor in HBase config file
+  // HBASE_ESTIMATE_ROW_COUNT_VIA_COPROCESSOR:  If ON, use a coprocessor when
+  //                               estimating row counts; if OFF, use client side
+  //                               code (the latter doesn't work if HBase encryption
+  //                               is being used)
   // HBASE_INTERFACE:              JNI or JNI_TRX (transactional interface)
   // HBASE_MAX_COLUMN_xxx_LENGTH:  Max length of some
   //                               string columns in the "_ROW_" and "_CELL_" schemas
@@ -1759,6 +1763,7 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS,		"OFF"),
  // exposure.
  DDkwd__(HBASE_DELETE_COSTING,		             "ON"),
  DDflt0_(HBASE_DOP_PARALLEL_SCANNER,             "0."),
+ DDkwd__(HBASE_ESTIMATE_ROW_COUNT_VIA_COPROCESSOR,   "OFF"),
  DDkwd__(HBASE_FILTER_PREDS,		             "OFF"),
  DDkwd__(HBASE_HASH2_PARTITIONING,                   "ON"),
  DDui___(HBASE_INDEX_LEVEL,                          "0"),

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8b93ad50/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
index f23dd70..72fa0af 100644
--- a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
+++ b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
@@ -41,6 +41,10 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hbase.client.ConnectionFactory;
 import org.apache.hadoop.hbase.client.Connection;
 import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.KeyValue;
@@ -100,8 +104,22 @@ import org.apache.hadoop.hbase.client.DtmConst;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.hadoop.hbase.util.CompressionTest;
 
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+import com.google.protobuf.RpcCallback;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.Service;
 import com.google.protobuf.ServiceException;
 
+import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
+import org.apache.hadoop.hbase.ipc.ServerRpcController;
+
+import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos;
+import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.TrafEstimateRowCountRequest;
+import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.TrafEstimateRowCountResponse;
+import org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.TrxRegionService;
+
+
 public class HBaseClient {
 
     static Logger logger = Logger.getLogger(HBaseClient.class.getName());
@@ -1051,6 +1069,8 @@ public class HBaseClient {
     // columns in the table) to the size of the HFile.
     private long estimateMemStoreRows(String tblName, int rowSize)
                  throws MasterNotRunningException, IOException {
+      if (logger.isDebugEnabled()) logger.debug("estimateMemStoreRows called for " + tblName
+ " with row size " + rowSize);
+
       if (rowSize == 0)
         return 0;
 
@@ -1086,6 +1106,12 @@ public class HBaseClient {
           }
         }
       }
+      catch (IOException e) {
+        if (logger.isDebugEnabled()) logger.debug("IOException caught in estimateMemStoreRows:
" + e);
+      }
+      catch (Throwable e) {
+        if (logger.isDebugEnabled()) logger.debug("Throwable caught in estimateMemStoreRows:
" + e);
+      }
       finally {
         admin.close();
       }
@@ -1255,6 +1281,7 @@ public class HBaseClient {
                                tblName + "/" + REGION_NAME_PATTERN +
                                "/#1/" + HFILE_NAME_PATTERN));
       for (FileStatus fs : fsArr) {
+        if (logger.isDebugEnabled()) logger.debug("Estimate row count is processing file
" + fs.getPath());
         // Make sure the file name conforms to HFile name pattern.
         if (!StoreFileInfo.isHFile(fs.getPath())) {
           if (logger.isDebugEnabled()) logger.debug("Skipped file " + fs.getPath() + " --
not a valid HFile name.");
@@ -1395,6 +1422,169 @@ public class HBaseClient {
       return true;
     }
 
+    // Similar to estimateRowCount, except that the implementation
+    // uses a coprocessor. This is necessary when HBase encryption is
+    // in use, because the Trafodion ID does not have the proper 
+    // authorization to the KeyStore file used by HBase.
+    public boolean estimateRowCountViaCoprocessor(String tblName, int partialRowSize,
+                                    int numCols, int retryLimitMilliSeconds, long[] rc)
+                   throws ServiceException, IOException {
+      if (logger.isDebugEnabled()) {
+        logger.debug("HBaseClient.estimateRowCountViaCoprocessor(" + tblName + ") called.");
+        logger.debug("numCols = " + numCols + ", partialRowSize = " + partialRowSize);
+      }
+
+      boolean retcode = true; 
+      rc[0] = 0;
+
+      HConnection connection = null;
+      HTableInterface table = null; 
+      connection = HConnectionManager.createConnection(config);
+      table = connection.getTable(tblName);
+
+      int putKVsSampled = 0;
+      int nonPutKVsSampled = 0;
+      int missingKVsCount = 0;
+      long totalEntries = 0;   // KeyValues in all HFiles for table
+      long totalSizeBytes = 0; // Size of all HFiles for table 
+
+      final int finalNumCols = numCols;
+
+      Batch.Call<TrxRegionService, TrafEstimateRowCountResponse> callable = 
+        new Batch.Call<TrxRegionService, TrafEstimateRowCountResponse>() {
+          ServerRpcController controller = new ServerRpcController();
+          BlockingRpcCallback<TrafEstimateRowCountResponse> rpcCallback = 
+            new BlockingRpcCallback<TrafEstimateRowCountResponse>();         
+
+          @Override
+          public TrafEstimateRowCountResponse call(TrxRegionService instance) throws IOException
{    
+            if (logger.isDebugEnabled()) logger.debug("call method for TrxRegionService was
called");
+            
+            // one of these God-awful long type identifiers common in Java/Maven environments...
+            org.apache.hadoop.hbase.coprocessor.transactional.generated.TrxRegionProtos.TrafEstimateRowCountRequest.Builder
+              builder = TrafEstimateRowCountRequest.newBuilder();        
+            builder.setNumCols(finalNumCols);
+        
+            instance.trafEstimateRowCount(controller, builder.build(), rpcCallback);
+            TrafEstimateRowCountResponse response = rpcCallback.get();
+            if (logger.isDebugEnabled()) {
+              if (response == null)
+                logger.debug("response was null");
+              else
+                logger.debug("response was non-null");
+              if (controller.failed())
+                logger.debug("controller.failed() is true");
+              else
+                logger.debug("controller.failed() is false");
+              if (controller.errorText() != null)
+                logger.debug("controller.errorText() is " + controller.errorText());
+              else
+                logger.debug("controller.errorText() is null");
+              IOException ioe = controller.getFailedOn();
+              if (ioe != null)
+                logger.debug("controller.getFailedOn() returned " + ioe.getMessage());
+              else
+                logger.debug("controller.getFailedOn() returned null");
+            }
+            return response;        
+          }
+      };
+    
+      Map<byte[], TrafEstimateRowCountResponse> result = null;
+      try {
+        result = table.coprocessorService(TrxRegionService.class, null, null, callable);
+      } catch (Throwable e) {
+        throw new IOException("Exception from coprocessorService caught in estimateRowCountViaCoprocessor",e);
+      }      
+
+      for (TrafEstimateRowCountResponse response : result.values()) {
+        boolean hasException = response.getHasException();
+        String exception = response.getException();
+        if (hasException) {
+          if (logger.isDebugEnabled()) logger.debug("HBaseClient.estimateRowCountViaCoprocessor
exception " + exception);
+          throw new IOException(exception);
+        }
+        totalEntries = totalEntries + response.getTotalEntries();
+        totalSizeBytes = totalSizeBytes + response.getTotalSizeBytes();
+        putKVsSampled = putKVsSampled + response.getPutKVsSampled();
+        nonPutKVsSampled = nonPutKVsSampled + response.getNonPutKVsSampled();
+        missingKVsCount = missingKVsCount + response.getMissingKVsCount();
+      }
+
+      if (logger.isDebugEnabled()) { 
+        logger.debug("The coprocessor service for estimating row count returned " + result.size()
+ " messages.");
+        logger.debug("totalEntries = " + totalEntries + ", totalSizeBytes = " + totalSizeBytes);
+        logger.debug("putKVsSampled = " + putKVsSampled + ", nonPutKVsSampled = " + nonPutKVsSampled
+
+                     ", missingKVsCount = " + missingKVsCount);
+      }
+
+      final int ROWS_TO_SAMPLE = 500;
+      long estimatedEntries = ((ROWS_TO_SAMPLE > 0) && (numCols > 1)
+                                 ? 0               // get from sample data, below
+                                 : totalEntries);  // no sampling, use stored value
+      if (putKVsSampled > 0) // avoid div by 0 if no Put KVs in sample
+        {
+          long estimatedTotalPuts = (putKVsSampled * totalEntries) / 
+                               (putKVsSampled + nonPutKVsSampled);
+          estimatedEntries = ((putKVsSampled + missingKVsCount) * estimatedTotalPuts)
+                                   / putKVsSampled;
+        }
+
+      if (logger.isDebugEnabled()) { 
+        logger.debug("estimatedEntries = " + estimatedEntries + ", numCols = " + numCols);
+      } 
+
+      // Calculate estimate of rows in all HFiles of table.
+      rc[0] = (estimatedEntries + (numCols/2)) / numCols; // round instead of truncate
+
+      if (logger.isDebugEnabled()) { 
+        logger.debug("rc[0] = " + rc[0]);
+      }       
+
+      // Estimate # of rows in MemStores of all regions of table. Pass
+      // a value to divide the size of the MemStore by. Base this on the
+      // ratio of bytes-to-rows in the HFiles, or the actual row size if
+      // the HFiles were empty.
+      int rowSize;
+
+      if (rc[0] > 0)
+        rowSize = (int)(totalSizeBytes / rc[0]);
+      else {
+        // From Traf metadata we have calculated and passed in part of the row
+        // size, including size of column qualifiers (col names), which are not
+        // known to HBase.  Add to this the length of the fixed part of the
+        // KeyValue format, times the number of columns.
+        int fixedSizePartOfKV = KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE // key len + value
len
+                              + KeyValue.KEY_INFRASTRUCTURE_SIZE;     // rowkey & col
family len, timestamp, key type
+        rowSize = partialRowSize   // for all cols: row key + col qualifiers + values
+                      + (fixedSizePartOfKV * numCols);
+
+
+        // Trafodion tables have a single col family at present, so we only look
+        // at the first family name, and multiply its length times the number of
+        // columns. Even if more than one family is used in the future, presumably
+        // they will all be the same short size.
+        Table htbl = getConnection().getTable(TableName.valueOf(tblName));
+        //HTable htbl = new HTable(config, tblName);
+        HTableDescriptor htblDesc = htbl.getTableDescriptor();
+        HColumnDescriptor[] families = htblDesc.getColumnFamilies();
+        rowSize += (families[0].getName().length * numCols);
+      }
+
+      // Get the estimate of MemStore rows
+      long memStoreRows = estimateMemStoreRows(tblName, rowSize);
+
+      if (logger.isDebugEnabled()) {
+        logger.debug("Estimated row count from HFiles = " + rc[0]);
+        logger.debug("Estimated row count from MemStores = " + memStoreRows);
+      }
+
+      rc[0] += memStoreRows;  // Add memstore estimate to total
+      if (logger.isDebugEnabled()) logger.debug("Total estimated row count for " + tblName
+ " = " + rc[0]);
+
+      return retcode;
+    }
+
 
     /**
     This method returns node names where Hbase Table regions reside


Mime
View raw message