kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [kudu] 02/05: cfile: track memory for CFileReaders and BloomFileReaders
Date Wed, 02 Oct 2019 04:11:19 GMT
This is an automated email from the ASF dual-hosted git repository.

adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 62c2fddcdc65f2092ee335ac1a35288dd3f7e4a5
Author: Andrew Wong <awong@apache.org>
AuthorDate: Tue Oct 1 13:30:05 2019 -0700

    cfile: track memory for CFileReaders and BloomFileReaders
    
    This lumps a tablet's CFileReader tracked memory and BloomFileReader
    tracked memory into the ids "CFileReaders" and "BloomFileReaders"
    respectively.
    
    I test this locally and verified via the web UI that I could see
    CFileReader and BloomFileReader entries once they had been flushed.
    
    Change-Id: Ic62c89079edffb41d1f5b40d7b699c525fa447a7
    Reviewed-on: http://gerrit.cloudera.org:8080/14322
    Reviewed-by: Adar Dembo <adar@cloudera.com>
    Tested-by: Kudu Jenkins
---
 src/kudu/tablet/cfile_set-test.cc     | 15 ++++++++++-----
 src/kudu/tablet/cfile_set.cc          | 22 +++++++++++++---------
 src/kudu/tablet/cfile_set.h           |  9 ++++++---
 src/kudu/tablet/diskrowset.cc         |  6 ++++--
 src/kudu/tablet/tablet_mem_trackers.h |  8 +++++++-
 5 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/src/kudu/tablet/cfile_set-test.cc b/src/kudu/tablet/cfile_set-test.cc
index e3fd517..c074e9d 100644
--- a/src/kudu/tablet/cfile_set-test.cc
+++ b/src/kudu/tablet/cfile_set-test.cc
@@ -278,7 +278,8 @@ TEST_F(TestCFileSet, TestPartiallyMaterialize) {
   WriteTestRowSet(kNumRows);
 
   shared_ptr<CFileSet> fileset;
-  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+                           nullptr, &fileset));
 
   unique_ptr<CFileSet::Iterator> iter(fileset->NewIterator(&schema_, nullptr));
   ASSERT_OK(iter->Init(nullptr));
@@ -358,7 +359,8 @@ TEST_F(TestCFileSet, TestIteratePartialSchema) {
   WriteTestRowSet(kNumRows);
 
   shared_ptr<CFileSet> fileset;
-  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+                           nullptr, &fileset));
 
   Schema new_schema;
   ASSERT_OK(schema_.CreateProjectionByNames({ "c0", "c2" }, &new_schema));
@@ -391,7 +393,8 @@ TEST_F(TestCFileSet, TestRangeScan) {
   WriteTestRowSet(kNumRows);
 
   shared_ptr<CFileSet> fileset;
-  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+                           nullptr, &fileset));
 
   // Create iterator.
   unique_ptr<CFileSet::Iterator> cfile_iter(fileset->NewIterator(&schema_, nullptr));
@@ -445,7 +448,8 @@ TEST_F(TestCFileSet, TestRangePredicates2) {
   WriteTestRowSet(kNumRows);
 
   shared_ptr<CFileSet> fileset;
-  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+                           nullptr, &fileset));
 
   // Range scan where rows match on both ends
   DoTestRangeScan(fileset, 2000, 2010);
@@ -501,7 +505,8 @@ TEST_F(TestCFileSet, TestBloomFilterPredicates) {
                        &ret1_contain, &ret1_exclude, &ret2_contain, &ret2_exclude);
 
   shared_ptr<CFileSet> fileset;
-  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), nullptr, &fileset));
+  ASSERT_OK(CFileSet::Open(rowset_meta_, MemTracker::GetRootTracker(), MemTracker::GetRootTracker(),
+                           nullptr, &fileset));
 
   vector<ColumnPredicate::BloomFilterInner> bfs;
   // BloomFilter of column 0 contain.
diff --git a/src/kudu/tablet/cfile_set.cc b/src/kudu/tablet/cfile_set.cc
index 52a892b..e1b7881 100644
--- a/src/kudu/tablet/cfile_set.cc
+++ b/src/kudu/tablet/cfile_set.cc
@@ -88,7 +88,7 @@ using strings::Substitute;
 ////////////////////////////////////////////////////////////
 
 static Status OpenReader(FsManager* fs,
-                         shared_ptr<MemTracker> parent_mem_tracker,
+                         shared_ptr<MemTracker> cfile_reader_tracker,
                          const BlockId& block_id,
                          const IOContext* io_context,
                          unique_ptr<CFileReader>* new_reader) {
@@ -96,7 +96,7 @@ static Status OpenReader(FsManager* fs,
   RETURN_NOT_OK(fs->OpenBlock(block_id, &block));
 
   ReaderOptions opts;
-  opts.parent_mem_tracker = std::move(parent_mem_tracker);
+  opts.parent_mem_tracker = std::move(cfile_reader_tracker);
   opts.io_context = io_context;
   return CFileReader::OpenNoInit(std::move(block),
                                  std::move(opts),
@@ -108,20 +108,24 @@ static Status OpenReader(FsManager* fs,
 ////////////////////////////////////////////////////////////
 
 CFileSet::CFileSet(shared_ptr<RowSetMetadata> rowset_metadata,
-                   shared_ptr<MemTracker> parent_mem_tracker)
+                   shared_ptr<MemTracker> bloomfile_tracker,
+                   shared_ptr<MemTracker> cfile_reader_tracker)
     : rowset_metadata_(std::move(rowset_metadata)),
-      parent_mem_tracker_(std::move(parent_mem_tracker)) {
+      bloomfile_tracker_(std::move(bloomfile_tracker)),
+      cfile_reader_tracker_(std::move(cfile_reader_tracker)) {
 }
 
 CFileSet::~CFileSet() {
 }
 
 Status CFileSet::Open(shared_ptr<RowSetMetadata> rowset_metadata,
-                      shared_ptr<MemTracker> parent_mem_tracker,
+                      shared_ptr<MemTracker> bloomfile_tracker,
+                      shared_ptr<MemTracker> cfile_reader_tracker,
                       const IOContext* io_context,
                       shared_ptr<CFileSet>* cfile_set) {
   shared_ptr<CFileSet> cfs(new CFileSet(std::move(rowset_metadata),
-                                        std::move(parent_mem_tracker)));
+                                        std::move(bloomfile_tracker),
+                                        std::move(cfile_reader_tracker)));
   RETURN_NOT_OK(cfs->DoOpen(io_context));
 
   cfile_set->swap(cfs);
@@ -140,7 +144,7 @@ Status CFileSet::DoOpen(const IOContext* io_context) {
 
     unique_ptr<CFileReader> reader;
     RETURN_NOT_OK(OpenReader(rowset_metadata_->fs_manager(),
-                             parent_mem_tracker_,
+                             cfile_reader_tracker_,
                              rowset_metadata_->column_data_block_for_col_id(col_id),
                              io_context,
                              &reader));
@@ -152,7 +156,7 @@ Status CFileSet::DoOpen(const IOContext* io_context) {
 
   if (rowset_metadata_->has_adhoc_index_block()) {
     RETURN_NOT_OK(OpenReader(rowset_metadata_->fs_manager(),
-                             parent_mem_tracker_,
+                             cfile_reader_tracker_,
                              rowset_metadata_->adhoc_index_block(),
                              io_context,
                              &ad_hoc_idx_reader_));
@@ -183,8 +187,8 @@ Status CFileSet::OpenBloomReader(const IOContext* io_context) {
   RETURN_NOT_OK(fs->OpenBlock(rowset_metadata_->bloom_block(), &block));
 
   ReaderOptions opts;
-  opts.parent_mem_tracker = parent_mem_tracker_;
   opts.io_context = io_context;
+  opts.parent_mem_tracker = bloomfile_tracker_;
   Status s = BloomFileReader::OpenNoInit(std::move(block),
                                          std::move(opts),
                                          &bloom_reader_);
diff --git a/src/kudu/tablet/cfile_set.h b/src/kudu/tablet/cfile_set.h
index 4721e95..a34dd21 100644
--- a/src/kudu/tablet/cfile_set.h
+++ b/src/kudu/tablet/cfile_set.h
@@ -74,7 +74,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> {
   class Iterator;
 
   static Status Open(std::shared_ptr<RowSetMetadata> rowset_metadata,
-                     std::shared_ptr<MemTracker> parent_mem_tracker,
+                     std::shared_ptr<MemTracker> bloomfile_tracker,
+                     std::shared_ptr<MemTracker> cfile_reader_tracker,
                      const fs::IOContext* io_context,
                      std::shared_ptr<CFileSet>* cfile_set);
 
@@ -133,7 +134,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> {
   DISALLOW_COPY_AND_ASSIGN(CFileSet);
 
   CFileSet(std::shared_ptr<RowSetMetadata> rowset_metadata,
-           std::shared_ptr<MemTracker> parent_mem_tracker);
+           std::shared_ptr<MemTracker> bloomfile_tracker,
+           std::shared_ptr<MemTracker> cfile_reader_tracker);
 
   Status DoOpen(const fs::IOContext* io_context);
   Status OpenBloomReader(const fs::IOContext* io_context);
@@ -153,7 +155,8 @@ class CFileSet : public std::enable_shared_from_this<CFileSet> {
   const Schema &tablet_schema() const { return rowset_metadata_->tablet_schema();
}
 
   std::shared_ptr<RowSetMetadata> rowset_metadata_;
-  std::shared_ptr<MemTracker> parent_mem_tracker_;
+  std::shared_ptr<MemTracker> bloomfile_tracker_;
+  std::shared_ptr<MemTracker> cfile_reader_tracker_;
 
   std::string min_encoded_key_;
   std::string max_encoded_key_;
diff --git a/src/kudu/tablet/diskrowset.cc b/src/kudu/tablet/diskrowset.cc
index b2395a9..f4dd37d 100644
--- a/src/kudu/tablet/diskrowset.cc
+++ b/src/kudu/tablet/diskrowset.cc
@@ -535,7 +535,8 @@ DiskRowSet::DiskRowSet(shared_ptr<RowSetMetadata> rowset_metadata,
 Status DiskRowSet::Open(const IOContext* io_context) {
   TRACE_EVENT0("tablet", "DiskRowSet::Open");
   RETURN_NOT_OK(CFileSet::Open(rowset_metadata_,
-                               mem_trackers_.tablet_tracker,
+                               mem_trackers_.bloomfile_tracker,
+                               mem_trackers_.cfile_reader_tracker,
                                io_context,
                                &base_data_));
 
@@ -607,7 +608,8 @@ Status DiskRowSet::MajorCompactDeltaStoresWithColumnIds(const vector<ColumnId>&
   // appropriate blocks to match the update.
   shared_ptr<CFileSet> new_base;
   RETURN_NOT_OK(CFileSet::Open(rowset_metadata_,
-                               mem_trackers_.tablet_tracker,
+                               mem_trackers_.bloomfile_tracker,
+                               mem_trackers_.cfile_reader_tracker,
                                io_context,
                                &new_base));
   {
diff --git a/src/kudu/tablet/tablet_mem_trackers.h b/src/kudu/tablet/tablet_mem_trackers.h
index 4882318..405b4cb 100644
--- a/src/kudu/tablet/tablet_mem_trackers.h
+++ b/src/kudu/tablet/tablet_mem_trackers.h
@@ -30,6 +30,8 @@ struct TabletMemTrackers {
   // Intended for unit tests where the tracker hierarchy doesn't matter.
   TabletMemTrackers()
     : tablet_tracker(MemTracker::GetRootTracker()),
+      bloomfile_tracker(MemTracker::GetRootTracker()),
+      cfile_reader_tracker(MemTracker::GetRootTracker()),
       dms_tracker(MemTracker::GetRootTracker()) {
   }
 
@@ -38,13 +40,17 @@ struct TabletMemTrackers {
     : tablet_tracker(MemTracker::CreateTracker(
         -1,
         strings::Substitute("tablet-$0", tablet_id),
-        parent_mem_tracker)),
+        std::move(parent_mem_tracker))),
+      bloomfile_tracker(MemTracker::CreateTracker(-1, "BloomFileReaders", tablet_tracker)),
+      cfile_reader_tracker(MemTracker::CreateTracker(-1, "CFileReaders", tablet_tracker)),
       dms_tracker(MemTracker::CreateTracker(-1, "DeltaMemStores", tablet_tracker)) {
   }
 
   std::shared_ptr<MemTracker> tablet_tracker;
 
   // All of the below are children of tablet_tracker;
+  std::shared_ptr<MemTracker> bloomfile_tracker;
+  std::shared_ptr<MemTracker> cfile_reader_tracker;
   std::shared_ptr<MemTracker> dms_tracker;
 };
 


Mime
View raw message