iotdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qiaojia...@apache.org
Subject [iotdb] branch rel/0.10 updated: [To rel/0.10] [IOTDB-962] Improve the chunk metadata cache process (#1873)
Date Sun, 01 Nov 2020 03:03:54 GMT
This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch rel/0.10
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/rel/0.10 by this push:
     new b96bf61  [To rel/0.10] [IOTDB-962] Improve the chunk metadata cache process (#1873)
b96bf61 is described below

commit b96bf61f49613f7ac9730ff5c1cf70fc27f57585
Author: Jackie Tien <JackieTien@foxmail.com>
AuthorDate: Sun Nov 1 11:03:41 2020 +0800

    [To rel/0.10] [IOTDB-962] Improve the chunk metadata cache process (#1873)
---
 .../iotdb/db/engine/cache/ChunkMetadataCache.java  | 30 +++++++---------------
 .../iotdb/db/engine/merge/task/MergeFileTask.java  |  9 +++++++
 .../engine/storagegroup/StorageGroupProcessor.java |  9 +++++++
 .../chunk/metadata/DiskChunkMetadataLoader.java    |  9 ++++---
 .../chunk/metadata/MemChunkMetadataLoader.java     |  3 ++-
 .../db/engine/cache/ChunkMetadataCacheTest.java    |  4 +--
 .../apache/iotdb/db/engine/merge/MergeTest.java    |  4 +++
 .../query/reader/series/SeriesReaderTestUtil.java  |  4 +++
 .../apache/iotdb/db/utils/EnvironmentUtils.java    |  4 +++
 .../tsfile/file/metadata/TimeseriesMetadata.java   |  2 +-
 .../read/controller/IChunkMetadataLoader.java      |  3 ++-
 11 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
index 4ff58e7..5ca9010 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCache.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.cache;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -30,12 +31,11 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.query.control.FileReaderManager;
-import org.apache.iotdb.db.utils.FileLoaderUtils;
 import org.apache.iotdb.db.utils.TestOnly;
 import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
 import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
 import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.utils.BloomFilter;
 import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -106,22 +106,16 @@ public class ChunkMetadataCache {
   /**
    * get {@link ChunkMetadata}. THREAD SAFE.
    */
-  public List<ChunkMetadata> get(String filePath, Path seriesPath)
-      throws IOException {
+  public List<ChunkMetadata> get(String filePath, Path seriesPath,
+      TimeseriesMetadata timeSeriesMetadata) throws IOException {
+    if (timeSeriesMetadata == null) {
+      return Collections.emptyList();
+    }
     if (!CACHE_ENABLE) {
       // bloom filter part
       TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
-      BloomFilter bloomFilter = tsFileReader.readBloomFilter();
-      if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath()))
{
-        if (logger.isDebugEnabled()) {
-          logger.debug(String
-              .format("path not found by bloom filter, file is: %s, path is: %s", filePath,
-                  seriesPath));
-        }
-        return new ArrayList<>();
-      }
       // If timeseries isn't included in the tsfile, empty list is returned.
-      return tsFileReader.getChunkMetadataList(seriesPath);
+      return tsFileReader.readChunkMetaDataList(timeSeriesMetadata);
     }
 
     AccountableString key = new AccountableString(filePath + IoTDBConstant.PATH_SEPARATOR
@@ -137,19 +131,13 @@ public class ChunkMetadataCache {
       lock.readLock().unlock();
     }
 
-
     if (chunkMetadataList != null) {
       printCacheLog(true);
       cacheHitNum.incrementAndGet();
     } else {
       printCacheLog(false);
-      // bloom filter part
       TsFileSequenceReader tsFileReader = FileReaderManager.getInstance().get(filePath, true);
-      BloomFilter bloomFilter = tsFileReader.readBloomFilter();
-      if (bloomFilter != null && !bloomFilter.contains(seriesPath.getFullPath()))
{
-        return new ArrayList<>();
-      }
-      chunkMetadataList = FileLoaderUtils.getChunkMetadataList(seriesPath, filePath);
+      chunkMetadataList = tsFileReader.readChunkMetaDataList(timeSeriesMetadata);
       lock.writeLock().lock();
       try {
         lruCache.put(key, chunkMetadataList);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
index 006ec5c..378e394 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/task/MergeFileTask.java
@@ -29,7 +29,10 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 import org.apache.iotdb.db.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
 import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
 import org.apache.iotdb.db.engine.merge.manage.MergeContext;
 import org.apache.iotdb.db.engine.merge.manage.MergeResource;
 import org.apache.iotdb.db.engine.merge.recover.MergeLogger;
@@ -274,6 +277,12 @@ class MergeFileTask {
     } catch (Exception e) {
       logger.error(e.getMessage(), e);
     } finally {
+      // clean cache
+      if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) {
+        ChunkCache.getInstance().clear();
+        ChunkMetadataCache.getInstance().clear();
+        TimeSeriesMetadataCache.getInstance().clear();
+      }
       seqFile.getWriteQueryLock().writeLock().unlock();
     }
   }
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 70c9064..15213dd 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -45,6 +45,9 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.conf.directories.DirectoryManager;
 import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
+import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
 import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
 import org.apache.iotdb.db.engine.flush.TsFileFlushPolicy;
 import org.apache.iotdb.db.engine.merge.manage.MergeManager;
@@ -1612,6 +1615,12 @@ public class StorageGroupProcessor {
       try {
         unseqFile.remove();
       } finally {
+        // clean cache
+        if (IoTDBDescriptor.getInstance().getConfig().isMetaDataCacheEnable()) {
+          ChunkCache.getInstance().clear();
+          ChunkMetadataCache.getInstance().clear();
+          TimeSeriesMetadataCache.getInstance().clear();
+        }
         unseqFile.getWriteQueryLock().writeLock().unlock();
       }
     }
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
index 877647b..6b2bfa9 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/DiskChunkMetadataLoader.java
@@ -18,6 +18,8 @@
  */
 package org.apache.iotdb.db.query.reader.chunk.metadata;
 
+import java.io.IOException;
+import java.util.List;
 import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
 import org.apache.iotdb.db.engine.modification.Modification;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
@@ -25,11 +27,10 @@ import org.apache.iotdb.db.query.context.QueryContext;
 import org.apache.iotdb.db.query.reader.chunk.DiskChunkLoader;
 import org.apache.iotdb.db.utils.QueryUtils;
 import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
 import org.apache.iotdb.tsfile.read.common.Path;
 import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
 import org.apache.iotdb.tsfile.read.filter.basic.Filter;
-import java.io.IOException;
-import java.util.List;
 
 public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
 
@@ -47,9 +48,9 @@ public class DiskChunkMetadataLoader implements IChunkMetadataLoader {
   }
 
   @Override
-  public List<ChunkMetadata> loadChunkMetadataList() throws IOException {
+  public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata)
throws IOException {
     List<ChunkMetadata> chunkMetadataList = ChunkMetadataCache
-        .getInstance().get(resource.getPath(), seriesPath);
+        .getInstance().get(resource.getPath(), seriesPath, timeseriesMetadata);
 
     setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
 
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
index 8ab268c..aa1e7cb 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/metadata/MemChunkMetadataLoader.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.query.context.QueryContext;
 import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
 import org.apache.iotdb.tsfile.read.common.Path;
 import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
 import org.apache.iotdb.tsfile.read.filter.basic.Filter;
@@ -44,7 +45,7 @@ public class MemChunkMetadataLoader implements IChunkMetadataLoader {
   }
 
   @Override
-  public List<ChunkMetadata> loadChunkMetadataList() {
+  public List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata)
{
     List<ChunkMetadata> chunkMetadataList = resource.getChunkMetadataList();
 
     DiskChunkMetadataLoader.setDiskChunkLoader(chunkMetadataList, resource, seriesPath, context);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
index 2926973..0766fed 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/cache/ChunkMetadataCacheTest.java
@@ -135,7 +135,7 @@ public class ChunkMetadataCacheTest {
     Assert.assertFalse(unseqResources.get(3).isClosed());
 
     List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance()
-        .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5));
+        .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5), null);
     Assert.assertEquals(0, metaDataList.size());
   }
 
@@ -157,7 +157,7 @@ public class ChunkMetadataCacheTest {
     Assert.assertFalse(unseqResources.get(3).isClosed());
 
     List<ChunkMetadata> metaDataList = ChunkMetadataCache.getInstance()
-        .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5));
+        .get(seqResources.get(0).getPath(), new Path(storageGroup, measurementId5), null);
     Assert.assertEquals(0, metaDataList.size());
   }
 
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index 55cee97..691c02f 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -29,7 +29,9 @@ import java.util.List;
 import org.apache.iotdb.db.conf.IoTDBConstant;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.constant.TestConstant;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
 import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
 import org.apache.iotdb.db.engine.merge.manage.MergeManager;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -86,7 +88,9 @@ abstract class MergeTest {
     seqResources.clear();
     unseqResources.clear();
     IoTDBDescriptor.getInstance().getConfig().setChunkMergePointThreshold(prevMergeChunkThreshold);
+    ChunkCache.getInstance().clear();
     ChunkMetadataCache.getInstance().clear();
+    TimeSeriesMetadataCache.getInstance().clear();
     MManager.getInstance().clear();
     EnvironmentUtils.cleanAllDir();
     MergeManager.getINSTANCE().stop();
diff --git a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
index 58177fd..5466366 100644
--- a/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
+++ b/server/src/test/java/org/apache/iotdb/db/query/reader/series/SeriesReaderTestUtil.java
@@ -30,7 +30,9 @@ import java.util.Map;
 
 import org.apache.iotdb.db.conf.IoTDBConstant;
 import org.apache.iotdb.db.constant.TestConstant;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
 import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
 import org.apache.iotdb.db.engine.merge.manage.MergeManager;
 import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.exception.metadata.MetadataException;
@@ -71,7 +73,9 @@ public class SeriesReaderTestUtil {
     removeFiles(seqResources, unseqResources);
     seqResources.clear();
     unseqResources.clear();
+    ChunkCache.getInstance().clear();
     ChunkMetadataCache.getInstance().clear();
+    TimeSeriesMetadataCache.getInstance().clear();
     MManager.getInstance().clear();
     EnvironmentUtils.cleanAllDir();
     MergeManager.getINSTANCE().stop();
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 6e1ff67..1e06653 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -36,7 +36,9 @@ import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
 import org.apache.iotdb.db.conf.directories.DirectoryManager;
 import org.apache.iotdb.db.constant.TestConstant;
 import org.apache.iotdb.db.engine.StorageEngine;
+import org.apache.iotdb.db.engine.cache.ChunkCache;
 import org.apache.iotdb.db.engine.cache.ChunkMetadataCache;
+import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
 import org.apache.iotdb.db.exception.StorageEngineException;
 import org.apache.iotdb.db.metadata.MManager;
 import org.apache.iotdb.db.query.context.QueryContext;
@@ -132,7 +134,9 @@ public class EnvironmentUtils {
 
     // clean cache
     if (config.isMetaDataCacheEnable()) {
+      ChunkCache.getInstance().clear();
       ChunkMetadataCache.getInstance().clear();
+      TimeSeriesMetadataCache.getInstance().clear();
     }
     // close metadata
     MManager.getInstance().clear();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index 4893563..60dce34 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -130,7 +130,7 @@ public class TimeseriesMetadata implements Accountable {
   }
 
   public List<ChunkMetadata> loadChunkMetadataList() throws IOException {
-    return chunkMetadataLoader.loadChunkMetadataList();
+    return chunkMetadataLoader.loadChunkMetadataList(this);
   }
 
   public boolean isModified() {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
index f80c42f..9badcda 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/controller/IChunkMetadataLoader.java
@@ -22,13 +22,14 @@ import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
 
 import java.io.IOException;
 import java.util.List;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
 
 public interface IChunkMetadataLoader {
 
   /**
    * read all chunk metadata of one time series in one file.
    */
-  List<ChunkMetadata> loadChunkMetadataList() throws IOException;
+  List<ChunkMetadata> loadChunkMetadataList(TimeseriesMetadata timeseriesMetadata)
throws IOException;
 
   /**
    * For query 0.9/v1 tsfile only


Mime
View raw message