quickstep-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jianq...@apache.org
Subject [01/27] incubator-quickstep git commit: Measure memory consumption of a running query [Forced Update!]
Date Wed, 12 Apr 2017 19:34:04 GMT
Repository: incubator-quickstep
Updated Branches:
  refs/heads/new-op d70a6397d -> da9baf7e6 (forced update)


Measure memory consumption of a running query

- Methods to get memory footprints of join hash tables (for all of its
  implementations)
- Method to get the memory consumed by the active temporary relations
  during the query execution.
- AggregationOperationState can provide total memory used for a single
  aggregation node.
- QueryContext can aggregate the memory consumed by all temporary data
  structures (e.g. join hash tables, aggregation hash tables).
- QueryManagerBase provides an API to get the total memory footprint of
  a query (only implemented for single node version).
- Mutex protection for hash join tables' list, insert destinations' list and
  aggregation operation states' list in the QueryContext class.


Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/4b04fb84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/4b04fb84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/4b04fb84

Branch: refs/heads/new-op
Commit: 4b04fb841d7b6763b6b2de0a42e20c75aebf6999
Parents: 9d2e0b6
Author: Harshad Deshmukh <hbdeshmukh@apache.org>
Authored: Tue Mar 21 14:53:38 2017 -0500
Committer: Harshad Deshmukh <hbdeshmukh@apache.org>
Committed: Fri Mar 31 22:34:55 2017 -0500

----------------------------------------------------------------------
 catalog/CatalogRelation.hpp                     | 14 ++++
 query_execution/CMakeLists.txt                  |  2 +
 query_execution/QueryContext.cpp                | 42 ++++++++++++
 query_execution/QueryContext.hpp                | 67 +++++++++++++++++++-
 query_execution/QueryManagerBase.hpp            |  9 +++
 query_execution/QueryManagerSingleNode.cpp      | 24 ++++++-
 query_execution/QueryManagerSingleNode.hpp      | 11 ++++
 storage/AggregationOperationState.cpp           | 21 ++++++
 storage/AggregationOperationState.hpp           |  9 +++
 storage/CollisionFreeVectorTable.hpp            |  4 ++
 storage/HashTable.hpp                           |  5 ++
 storage/HashTableBase.hpp                       |  2 +
 storage/HashTablePool.hpp                       | 13 ++++
 storage/LinearOpenAddressingHashTable.hpp       |  4 ++
 storage/PackedPayloadHashTable.hpp              |  4 ++
 storage/PartitionedHashTablePool.hpp            | 13 ++++
 storage/SeparateChainingHashTable.hpp           |  4 ++
 .../SimpleScalarSeparateChainingHashTable.hpp   |  4 ++
 18 files changed, 249 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/catalog/CatalogRelation.hpp
----------------------------------------------------------------------
diff --git a/catalog/CatalogRelation.hpp b/catalog/CatalogRelation.hpp
index e1fd79a..af7f273 100644
--- a/catalog/CatalogRelation.hpp
+++ b/catalog/CatalogRelation.hpp
@@ -396,6 +396,20 @@ class CatalogRelation : public CatalogRelationSchema {
     return statistics_.get();
   }
 
+  /**
+   * @brief Get the size of this relation in bytes.
+   *
+   * @note The output signifies the amount of memory allocated for this
+   *       relation. The true memory footprint of this relation could be lower
+   *       than the output of this method.
+   **/
+  inline std::size_t getRelationSizeBytes() const {
+    SpinSharedMutexSharedLock<false> lock(blocks_mutex_);
+    return blocks_.size() *
+           getDefaultStorageBlockLayout().getDescription().num_slots() *
+           kSlotSizeBytes;
+  }
+
  private:
   // A list of blocks belonged to the relation.
   std::vector<block_id> blocks_;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/query_execution/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt
index d81ab44..eeed791 100644
--- a/query_execution/CMakeLists.txt
+++ b/query_execution/CMakeLists.txt
@@ -216,6 +216,7 @@ target_link_libraries(quickstep_queryexecution_QueryContext
                       quickstep_storage_InsertDestination
                       quickstep_storage_InsertDestination_proto
                       quickstep_storage_WindowAggregationOperationState
+                      quickstep_threading_SpinSharedMutex
                       quickstep_types_TypedValue
                       quickstep_types_containers_Tuple
                       quickstep_utility_Macros
@@ -281,6 +282,7 @@ if (ENABLE_DISTRIBUTED)
                         tmb)
 endif(ENABLE_DISTRIBUTED)
 target_link_libraries(quickstep_queryexecution_QueryManagerSingleNode
+                      quickstep_catalog_CatalogDatabase
                       quickstep_catalog_CatalogTypedefs
                       quickstep_queryexecution_QueryContext
                       quickstep_queryexecution_QueryExecutionState

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/query_execution/QueryContext.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.cpp b/query_execution/QueryContext.cpp
index 71839a7..8ba77ab 100644
--- a/query_execution/QueryContext.cpp
+++ b/query_execution/QueryContext.cpp
@@ -269,4 +269,46 @@ bool QueryContext::ProtoIsValid(const serialization::QueryContext &proto,
   return proto.IsInitialized();
 }
 
+std::size_t QueryContext::getJoinHashTablesMemoryBytes() const {
+  SpinSharedMutexSharedLock<false> lock(hash_tables_mutex_);
+  std::size_t memory = 0;
+  for (std::size_t hashtable_id = 0;
+       hashtable_id < join_hash_tables_.size();
+       ++hashtable_id) {
+    for (std::size_t partition_num = 0;
+         partition_num < join_hash_tables_[hashtable_id].size();
+         ++partition_num) {
+      if (join_hash_tables_[hashtable_id][partition_num] != nullptr) {
+        memory += join_hash_tables_[hashtable_id][partition_num]
+                      ->getHashTableMemorySizeBytes();
+      }
+    }
+  }
+  return memory;
+}
+
+std::size_t QueryContext::getAggregationStatesMemoryBytes() const {
+  SpinSharedMutexSharedLock<false> lock(aggregation_states_mutex_);
+  std::size_t memory = 0;
+  for (std::size_t agg_state_id = 0;
+       agg_state_id < aggregation_states_.size();
+       ++agg_state_id) {
+    if (aggregation_states_[agg_state_id] != nullptr) {
+      memory += aggregation_states_[agg_state_id]->getMemoryConsumptionBytes();
+    }
+  }
+  return memory;
+}
+
+void QueryContext::getTempRelationIDs(
+    std::vector<relation_id> *temp_relation_ids) const {
+  SpinSharedMutexSharedLock<false> lock(insert_destinations_mutex_);
+  DCHECK(temp_relation_ids != nullptr);
+  for (std::size_t id = 0; id < insert_destinations_.size(); ++id) {
+    InsertDestination *curr_insert_dest = insert_destinations_[id].get();
+    DCHECK(curr_insert_dest != nullptr);
+    temp_relation_ids->emplace_back(curr_insert_dest->getRelation().getID());
+  }
+}
+
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/query_execution/QueryContext.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryContext.hpp b/query_execution/QueryContext.hpp
index ed0f99c..ebc9506 100644
--- a/query_execution/QueryContext.hpp
+++ b/query_execution/QueryContext.hpp
@@ -34,6 +34,7 @@
 #include "storage/HashTable.hpp"
 #include "storage/InsertDestination.hpp"
 #include "storage/WindowAggregationOperationState.hpp"
+#include "threading/SpinSharedMutex.hpp"
 #include "types/containers/Tuple.hpp"
 #include "utility/Macros.hpp"
 #include "utility/SortConfiguration.hpp"
@@ -172,6 +173,7 @@ class QueryContext {
    * @return True if valid, otherwise false.
    **/
   bool isValidAggregationStateId(const aggregation_state_id id) const {
+    SpinSharedMutexSharedLock<false> lock(aggregation_states_mutex_);
     return id < aggregation_states_.size();
   }
 
@@ -183,6 +185,7 @@ class QueryContext {
    * @return The AggregationOperationState, alreadly created in the constructor.
    **/
   inline AggregationOperationState* getAggregationState(const aggregation_state_id id) {
+    SpinSharedMutexSharedLock<false> lock(aggregation_states_mutex_);
     DCHECK_LT(id, aggregation_states_.size());
     DCHECK(aggregation_states_[id]);
     return aggregation_states_[id].get();
@@ -194,6 +197,7 @@ class QueryContext {
    * @param id The ID of the AggregationOperationState to destroy.
    **/
   inline void destroyAggregationState(const aggregation_state_id id) {
+    SpinSharedMutexExclusiveLock<false> lock(aggregation_states_mutex_);
     DCHECK_LT(id, aggregation_states_.size());
     DCHECK(aggregation_states_[id]);
     aggregation_states_[id].reset(nullptr);
@@ -231,6 +235,7 @@ class QueryContext {
    * @return True if valid, otherwise false.
    **/
   bool isValidInsertDestinationId(const insert_destination_id id) const {
+    SpinSharedMutexSharedLock<false> lock(insert_destinations_mutex_);
     return id != kInvalidInsertDestinationId
         && id >= 0
         && static_cast<std::size_t>(id) < insert_destinations_.size();
@@ -244,6 +249,7 @@ class QueryContext {
    * @return The InsertDestination, alreadly created in the constructor.
    **/
   inline InsertDestination* getInsertDestination(const insert_destination_id id) {
+    SpinSharedMutexSharedLock<false> lock(insert_destinations_mutex_);
     DCHECK_GE(id, 0);
     DCHECK_LT(static_cast<std::size_t>(id), insert_destinations_.size());
     return insert_destinations_[id].get();
@@ -255,6 +261,7 @@ class QueryContext {
    * @param id The id of the InsertDestination to destroy.
    **/
   inline void destroyInsertDestination(const insert_destination_id id) {
+    SpinSharedMutexExclusiveLock<false> lock(insert_destinations_mutex_);
     DCHECK_GE(id, 0);
     DCHECK_LT(static_cast<std::size_t>(id), insert_destinations_.size());
     insert_destinations_[id].reset();
@@ -263,12 +270,16 @@ class QueryContext {
   /**
    * @brief Whether the given JoinHashTable id is valid.
    *
+   * @note This is a thread-safe function. Check isValidJoinHashTableIdUnsafe
+   *       for the the unsafe version.
+   *
    * @param id The JoinHashTable id.
    * @param part_id The partition id.
    *
    * @return True if valid, otherwise false.
    **/
   bool isValidJoinHashTableId(const join_hash_table_id id, const partition_id part_id) const
{
+    SpinSharedMutexSharedLock<false> lock(hash_tables_mutex_);
     return id < join_hash_tables_.size() &&
            part_id < join_hash_tables_[id].size();
   }
@@ -282,7 +293,8 @@ class QueryContext {
    * @return The JoinHashTable, already created in the constructor.
    **/
   inline JoinHashTable* getJoinHashTable(const join_hash_table_id id, const partition_id
part_id) {
-    DCHECK(isValidJoinHashTableId(id, part_id));
+    SpinSharedMutexSharedLock<false> lock(hash_tables_mutex_);
+    DCHECK(isValidJoinHashTableIdUnsafe(id, part_id));
     return join_hash_tables_[id][part_id].get();
   }
 
@@ -293,7 +305,8 @@ class QueryContext {
    * @param part_id The partition id.
    **/
   inline void destroyJoinHashTable(const join_hash_table_id id, const partition_id part_id)
{
-    DCHECK(isValidJoinHashTableId(id, part_id));
+    SpinSharedMutexExclusiveLock<false> lock(hash_tables_mutex_);
+    DCHECK(isValidJoinHashTableIdUnsafe(id, part_id));
     join_hash_tables_[id][part_id].reset();
   }
 
@@ -551,7 +564,53 @@ class QueryContext {
     return window_aggregation_states_[id].release();
   }
 
+  /**
+   * @brief Get the total memory footprint of the temporary data structures
+   *        used for query execution (e.g. join hash tables, aggregation hash
+   *        tables) in bytes.
+   **/
+  std::size_t getTempStructuresMemoryBytes() const {
+    return getJoinHashTablesMemoryBytes() + getAggregationStatesMemoryBytes();
+  }
+
+  /**
+   * @brief Get the total memory footprint in bytes of the join hash tables
+   *        used for query execution.
+   **/
+  std::size_t getJoinHashTablesMemoryBytes() const;
+
+  /**
+   * @brief Get the total memory footprint in bytes of the aggregation hash
+   *        tables used for query execution.
+   **/
+  std::size_t getAggregationStatesMemoryBytes() const;
+
+  /**
+   * @brief Get the list of IDs of temporary relations in this query.
+   *
+   * @param temp_relation_ids A pointer to the vector that will store the
+   *        relation IDs.
+   **/
+  void getTempRelationIDs(std::vector<relation_id> *temp_relation_ids) const;
+
  private:
+  /**
+   * @brief Whether the given JoinHashTable id is valid.
+   *
+   * @note This is a thread-unsafe function. Check isValidJoinHashTableId
+   *       for the the thread-safe version.
+   *
+   * @param id The JoinHashTable id.
+   * @param part_id The partition id.
+   *
+   * @return True if valid, otherwise false.
+   **/
+  bool isValidJoinHashTableIdUnsafe(const join_hash_table_id id,
+                                    const partition_id part_id) const {
+    return id < join_hash_tables_.size() &&
+           part_id < join_hash_tables_[id].size();
+  }
+
   // Per hash join, the index is the partition id.
   typedef std::vector<std::unique_ptr<JoinHashTable>> PartitionedJoinHashTables;
 
@@ -568,6 +627,10 @@ class QueryContext {
   std::vector<std::unordered_map<attribute_id, std::unique_ptr<const Scalar>>>
update_groups_;
   std::vector<std::unique_ptr<WindowAggregationOperationState>> window_aggregation_states_;
 
+  mutable SpinSharedMutex<false> hash_tables_mutex_;
+  mutable SpinSharedMutex<false> aggregation_states_mutex_;
+  mutable SpinSharedMutex<false> insert_destinations_mutex_;
+
   DISALLOW_COPY_AND_ASSIGN(QueryContext);
 };
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/query_execution/QueryManagerBase.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp
index 27fa6dc..b6ed247 100644
--- a/query_execution/QueryManagerBase.hpp
+++ b/query_execution/QueryManagerBase.hpp
@@ -271,6 +271,15 @@ class QueryManagerBase {
     return query_exec_state_->hasRebuildInitiated(index);
   }
 
+  /**
+   * @brief Get the query's current memory consumption in bytes.
+   *
+   * @note This method returns a best guess consumption, at the time of the call.
+   **/
+  virtual std::size_t getQueryMemoryConsumptionBytes() const {
+    return 0;
+  }
+
   std::unique_ptr<QueryHandle> query_handle_;
 
   const std::size_t query_id_;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/query_execution/QueryManagerSingleNode.cpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerSingleNode.cpp b/query_execution/QueryManagerSingleNode.cpp
index 237796f..e3f349f 100644
--- a/query_execution/QueryManagerSingleNode.cpp
+++ b/query_execution/QueryManagerSingleNode.cpp
@@ -24,6 +24,7 @@
 #include <utility>
 #include <vector>
 
+#include "catalog/CatalogDatabase.hpp"
 #include "catalog/CatalogTypedefs.hpp"
 #include "query_execution/WorkerMessage.hpp"
 #include "query_optimizer/QueryHandle.hpp"
@@ -58,7 +59,8 @@ QueryManagerSingleNode::QueryManagerSingleNode(
                                       foreman_client_id_,
                                       bus_)),
       workorders_container_(
-          new WorkOrdersContainer(num_operators_in_dag_, num_numa_nodes)) {
+          new WorkOrdersContainer(num_operators_in_dag_, num_numa_nodes)),
+      database_(static_cast<const CatalogDatabase&>(*catalog_database)) {
   // Collect all the workorders from all the relational operators in the DAG.
   for (dag_node_index index = 0; index < num_operators_in_dag_; ++index) {
     if (checkAllBlockingDependenciesMet(index)) {
@@ -193,4 +195,24 @@ void QueryManagerSingleNode::getRebuildWorkOrders(const dag_node_index
index,
   }
 }
 
+std::size_t QueryManagerSingleNode::getQueryMemoryConsumptionBytes() const {
+  const std::size_t temp_relations_memory =
+      getTotalTempRelationMemoryInBytes();
+  const std::size_t temp_data_structures_memory =
+      query_context_->getTempStructuresMemoryBytes();
+  return temp_relations_memory + temp_data_structures_memory;
+}
+
+std::size_t QueryManagerSingleNode::getTotalTempRelationMemoryInBytes() const {
+  std::vector<relation_id> temp_relation_ids;
+  query_context_->getTempRelationIDs(&temp_relation_ids);
+  std::size_t memory = 0;
+  for (std::size_t rel_id : temp_relation_ids) {
+    if (database_.hasRelationWithId(rel_id)) {
+      memory += database_.getRelationById(rel_id)->getRelationSizeBytes();
+    }
+  }
+  return memory;
+}
+
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/query_execution/QueryManagerSingleNode.hpp
----------------------------------------------------------------------
diff --git a/query_execution/QueryManagerSingleNode.hpp b/query_execution/QueryManagerSingleNode.hpp
index dd044a5..6c5e38e 100644
--- a/query_execution/QueryManagerSingleNode.hpp
+++ b/query_execution/QueryManagerSingleNode.hpp
@@ -36,6 +36,7 @@ namespace tmb { class MessageBus; }
 
 namespace quickstep {
 
+class CatalogDatabase;
 class CatalogDatabaseLite;
 class QueryHandle;
 class StorageManager;
@@ -94,6 +95,8 @@ class QueryManagerSingleNode final : public QueryManagerBase {
     return query_context_.get();
   }
 
+  std::size_t getQueryMemoryConsumptionBytes() const override;
+
  private:
   bool checkNormalExecutionOver(const dag_node_index index) const override {
     return (checkAllDependenciesMet(index) &&
@@ -123,6 +126,12 @@ class QueryManagerSingleNode final : public QueryManagerBase {
   void getRebuildWorkOrders(const dag_node_index index,
                             WorkOrdersContainer *container);
 
+  /**
+   * @brief Get the total memory (in bytes) occupied by temporary relations
+   *        created during the query execution.
+   **/
+  std::size_t getTotalTempRelationMemoryInBytes() const;
+
   const tmb::client_id foreman_client_id_;
 
   StorageManager *storage_manager_;
@@ -132,6 +141,8 @@ class QueryManagerSingleNode final : public QueryManagerBase {
 
   std::unique_ptr<WorkOrdersContainer> workorders_container_;
 
+  const CatalogDatabase &database_;
+
   DISALLOW_COPY_AND_ASSIGN(QueryManagerSingleNode);
 };
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/AggregationOperationState.cpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.cpp b/storage/AggregationOperationState.cpp
index aea5bf5..9388bdb 100644
--- a/storage/AggregationOperationState.cpp
+++ b/storage/AggregationOperationState.cpp
@@ -946,4 +946,25 @@ void AggregationOperationState::finalizeHashTableImplThreadPrivate(
   output_destination->bulkInsertTuples(&complete_result);
 }
 
+std::size_t AggregationOperationState::getMemoryConsumptionBytes() const {
+  std::size_t memory = getMemoryConsumptionBytesHelper(distinctify_hashtables_);
+  memory += getMemoryConsumptionBytesHelper(group_by_hashtables_);
+  memory += collision_free_hashtable_->getMemoryConsumptionBytes();
+  memory += group_by_hashtable_pool_->getMemoryConsumptionPoolBytes();
+  memory += partitioned_group_by_hashtable_pool_->getMemoryConsumptionPoolBytes();
+  return memory;
+}
+
+std::size_t AggregationOperationState::getMemoryConsumptionBytesHelper(
+    const std::vector<std::unique_ptr<AggregationStateHashTableBase>>
+        &hashtables) const {
+  std::size_t memory = 0;
+  for (std::size_t ht_id = 0; ht_id < hashtables.size(); ++ht_id) {
+    if (hashtables[ht_id] != nullptr) {
+      memory += hashtables[ht_id]->getMemoryConsumptionBytes();
+    }
+  }
+  return memory;
+}
+
 }  // namespace quickstep

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/AggregationOperationState.hpp
----------------------------------------------------------------------
diff --git a/storage/AggregationOperationState.hpp b/storage/AggregationOperationState.hpp
index 6c9690a..e6af494 100644
--- a/storage/AggregationOperationState.hpp
+++ b/storage/AggregationOperationState.hpp
@@ -207,6 +207,11 @@ class AggregationOperationState {
    */
   CollisionFreeVectorTable* getCollisionFreeVectorTable() const;
 
+  /**
+   * @brief Get the memory footprint of the AggregationOperationState.
+   **/
+  std::size_t getMemoryConsumptionBytes() const;
+
  private:
   // Check whether partitioned aggregation can be applied.
   bool checkAggregatePartitioned(
@@ -253,6 +258,10 @@ class AggregationOperationState {
 
   void finalizeHashTableImplThreadPrivate(InsertDestination *output_destination);
 
+  std::size_t getMemoryConsumptionBytesHelper(
+      const std::vector<std::unique_ptr<AggregationStateHashTableBase>>
+          &hashtables) const;
+
   // Common state for all aggregates in this operation: the input relation, the
   // filter predicate (if any), and the list of GROUP BY expressions (if any).
   const CatalogRelationSchema &input_relation_;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/CollisionFreeVectorTable.hpp
----------------------------------------------------------------------
diff --git a/storage/CollisionFreeVectorTable.hpp b/storage/CollisionFreeVectorTable.hpp
index 772d47d..490a5cc 100644
--- a/storage/CollisionFreeVectorTable.hpp
+++ b/storage/CollisionFreeVectorTable.hpp
@@ -169,6 +169,10 @@ class CollisionFreeVectorTable : public AggregationStateHashTableBase
{
                      const std::size_t handle_id,
                      NativeColumnVector *output_cv) const;
 
+  std::size_t getMemoryConsumptionBytes() const override {
+    return memory_size_;
+  }
+
  private:
   inline static std::size_t CacheLineAlignedBytes(const std::size_t actual_bytes) {
     return (actual_bytes + kCacheLineBytes - 1) / kCacheLineBytes * kCacheLineBytes;

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/HashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTable.hpp b/storage/HashTable.hpp
index 786a9bb..3bf8709 100644
--- a/storage/HashTable.hpp
+++ b/storage/HashTable.hpp
@@ -541,6 +541,11 @@ class HashTable : public HashTableBase<resizable,
       FunctorT *functor);
 
   /**
+   * @brief Get the size of the hash table at the time this function is called.
+   **/
+  virtual std::size_t getHashTableMemorySizeBytes() const = 0;
+
+  /**
    * @brief Determine the number of entries (key-value pairs) contained in this
    *        HashTable.
    * @note For some HashTable implementations, this is O(1), but for others it

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/HashTableBase.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTableBase.hpp b/storage/HashTableBase.hpp
index b4b6918..8be388a 100644
--- a/storage/HashTableBase.hpp
+++ b/storage/HashTableBase.hpp
@@ -115,6 +115,8 @@ class AggregationStateHashTableBase {
 
   virtual void destroyPayload() = 0;
 
+  virtual std::size_t getMemoryConsumptionBytes() const = 0;
+
  protected:
   AggregationStateHashTableBase() {}
 

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/HashTablePool.hpp
----------------------------------------------------------------------
diff --git a/storage/HashTablePool.hpp b/storage/HashTablePool.hpp
index df527b8..6dbd7f9 100644
--- a/storage/HashTablePool.hpp
+++ b/storage/HashTablePool.hpp
@@ -123,6 +123,19 @@ class HashTablePool {
     return &hash_tables_;
   }
 
+  /**
+   * @brief Get the total memory consumed by the hash tables in this pool.
+   **/
+  std::size_t getMemoryConsumptionPoolBytes() const {
+    std::size_t memory = 0;
+    for (std::size_t ht_id = 0; ht_id <  hash_tables_.size(); ++ht_id) {
+      if (hash_tables_[ht_id] != nullptr) {
+        memory += hash_tables_[ht_id]->getMemoryConsumptionBytes();
+      }
+    }
+    return memory;
+  }
+
  private:
   AggregationStateHashTableBase* createNewHashTable() {
     return AggregationStateHashTableFactory::CreateResizable(

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/LinearOpenAddressingHashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/LinearOpenAddressingHashTable.hpp b/storage/LinearOpenAddressingHashTable.hpp
index 4953b4d..044ec6f 100644
--- a/storage/LinearOpenAddressingHashTable.hpp
+++ b/storage/LinearOpenAddressingHashTable.hpp
@@ -120,6 +120,10 @@ class LinearOpenAddressingHashTable : public HashTable<ValueT,
   void getAllCompositeKey(const std::vector<TypedValue> &key,
                           std::vector<const ValueT*> *values) const override;
 
+  std::size_t getHashTableMemorySizeBytes() const override {
+     return sizeof(Header) + numEntries() * bucket_size_;
+  }
+
  protected:
   HashTablePutResult putInternal(const TypedValue &key,
                                  const std::size_t variable_key_size,

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/PackedPayloadHashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/PackedPayloadHashTable.hpp b/storage/PackedPayloadHashTable.hpp
index b06200d..960d5a7 100644
--- a/storage/PackedPayloadHashTable.hpp
+++ b/storage/PackedPayloadHashTable.hpp
@@ -314,6 +314,10 @@ class PackedPayloadHashTable : public AggregationStateHashTableBase {
   inline std::size_t forEachCompositeKey(FunctorT *functor,
                                          const std::size_t index) const;
 
+  std::size_t getMemoryConsumptionBytes() const override {
+    return sizeof(Header) + numEntries() * bucket_size_;
+  }
+
  private:
   void resize(const std::size_t extra_buckets,
               const std::size_t extra_variable_storage,

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/PartitionedHashTablePool.hpp
----------------------------------------------------------------------
diff --git a/storage/PartitionedHashTablePool.hpp b/storage/PartitionedHashTablePool.hpp
index 0e62511..bfe8b8b 100644
--- a/storage/PartitionedHashTablePool.hpp
+++ b/storage/PartitionedHashTablePool.hpp
@@ -113,6 +113,19 @@ class PartitionedHashTablePool {
     return num_partitions_;
   }
 
+  /**
+   * @brief Get the total memory consumed by the hash tables in this pool.
+   **/
+  std::size_t getMemoryConsumptionPoolBytes() const {
+    std::size_t memory = 0;
+    for (std::size_t ht_id = 0; ht_id <  hash_tables_.size(); ++ht_id) {
+      if (hash_tables_[ht_id] != nullptr) {
+        memory += hash_tables_[ht_id]->getMemoryConsumptionBytes();
+      }
+    }
+    return memory;
+  }
+
  private:
   void initializeAllHashTables() {
     for (std::size_t part_num = 0; part_num < num_partitions_; ++part_num) {

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/SeparateChainingHashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/SeparateChainingHashTable.hpp b/storage/SeparateChainingHashTable.hpp
index 3ca060d..2403623 100644
--- a/storage/SeparateChainingHashTable.hpp
+++ b/storage/SeparateChainingHashTable.hpp
@@ -114,6 +114,10 @@ class SeparateChainingHashTable : public HashTable<ValueT,
   void getAllCompositeKey(const std::vector<TypedValue> &key,
                           std::vector<const ValueT*> *values) const override;
 
+  std::size_t getHashTableMemorySizeBytes() const override {
+    return sizeof(Header) + numEntries() * bucket_size_;
+  }
+
  protected:
   HashTablePutResult putInternal(const TypedValue &key,
                                  const std::size_t variable_key_size,

http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/4b04fb84/storage/SimpleScalarSeparateChainingHashTable.hpp
----------------------------------------------------------------------
diff --git a/storage/SimpleScalarSeparateChainingHashTable.hpp b/storage/SimpleScalarSeparateChainingHashTable.hpp
index 8448896..81f2044 100644
--- a/storage/SimpleScalarSeparateChainingHashTable.hpp
+++ b/storage/SimpleScalarSeparateChainingHashTable.hpp
@@ -135,6 +135,10 @@ class SimpleScalarSeparateChainingHashTable : public HashTable<ValueT,
     return getAll(key.front(), values);
   }
 
+  std::size_t getHashTableMemorySizeBytes() const override {
+    return sizeof(Header) + numEntries() * sizeof(Bucket);
+  }
+
  protected:
   HashTablePutResult putInternal(const TypedValue &key,
                                  const std::size_t variable_key_size,


Mime
View raw message