From commits-return-11408-archive-asf-public=cust-asf.ponee.io@iotdb.apache.org Thu Nov 19 00:59:32 2020 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mxout1-he-de.apache.org (mxout1-he-de.apache.org [95.216.194.37]) by mx-eu-01.ponee.io (Postfix) with ESMTPS id 5AD5C18063D for ; Thu, 19 Nov 2020 01:59:32 +0100 (CET) Received: from mail.apache.org (mailroute1-lw-us.apache.org [207.244.88.153]) by mxout1-he-de.apache.org (ASF Mail Server at mxout1-he-de.apache.org) with SMTP id C3927672B7 for ; Thu, 19 Nov 2020 00:59:31 +0000 (UTC) Received: (qmail 58273 invoked by uid 500); 19 Nov 2020 00:59:30 -0000 Mailing-List: contact commits-help@iotdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@iotdb.apache.org Delivered-To: mailing list commits@iotdb.apache.org Received: (qmail 58263 invoked by uid 99); 19 Nov 2020 00:59:30 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 Nov 2020 00:59:30 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 81D5B8087B; Thu, 19 Nov 2020 00:59:30 +0000 (UTC) Date: Thu, 19 Nov 2020 00:59:29 +0000 To: "commits@iotdb.apache.org" Subject: [iotdb] branch master updated: Update level compaction delete and change list lock (#2070) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <160574756920.21324.1060019542665518657@gitbox.apache.org> From: qiaojialin@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: iotdb X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 5f287aeb415985a6bcec80812f5c42973d0845d5 X-Git-Newrev: 344c3875c4325c07bb14c0b2a95a9073714eb8cc X-Git-Rev: 344c3875c4325c07bb14c0b2a95a9073714eb8cc X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. qiaojialin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/iotdb.git The following commit(s) were added to refs/heads/master by this push: new 344c387 Update level compaction delete and change list lock (#2070) 344c387 is described below commit 344c3875c4325c07bb14c0b2a95a9073714eb8cc Author: zhanglingzhe0820 <445073309@qq.com> AuthorDate: Thu Nov 19 08:56:04 2020 +0800 Update level compaction delete and change list lock (#2070) --- .../level/LevelCompactionTsFileManagement.java | 23 +++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java index 3f36d0b..515ef6b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/level/LevelCompactionTsFileManagement.java @@ -83,13 +83,17 @@ public class LevelCompactionTsFileManagement extends TsFileManagement { clear(); } - private void deleteLevelFiles(long timePartitionId, Collection mergeTsFiles) { - logger.debug("{} [compaction] merge starts to delete file", storageGroupName); + private void deleteLevelFilesInDisk(Collection mergeTsFiles) { + logger.debug("{} [compaction] merge starts to delete real file", storageGroupName); for (TsFileResource mergeTsFile : mergeTsFiles) { deleteLevelFile(mergeTsFile); logger .info("{} [Compaction] delete TsFile {}", storageGroupName, mergeTsFile.getTsFilePath()); } + } + + private void deleteLevelFilesInList(long timePartitionId, Collection mergeTsFiles) { + logger.debug("{} [compaction] merge starts to delete file list", storageGroupName); for (int i = 0; i < seqLevelNum; i++) { if (sequenceTsFileResources.containsKey(timePartitionId)) { if (sequenceTsFileResources.get(timePartitionId).size() > i) { @@ -338,12 +342,14 @@ public class LevelCompactionTsFileManagement extends TsFileManagement { if (isSeq) { for (TreeSet currMergeFile : sequenceTsFileResources .get(timePartition)) { - deleteLevelFiles(timePartition, currMergeFile); + deleteLevelFilesInDisk(currMergeFile); + deleteLevelFilesInList(timePartition, currMergeFile); } } else { for (List currMergeFile : unSequenceTsFileResources .get(timePartition)) { - deleteLevelFiles(timePartition, currMergeFile); + deleteLevelFilesInDisk(currMergeFile); + deleteLevelFilesInList(timePartition, currMergeFile); } } } @@ -369,13 +375,15 @@ public class LevelCompactionTsFileManagement extends TsFileManagement { CompactionUtils .merge(targetResource, sourceTsFileResources, storageGroupName, new CompactionLogger(storageGroupDir, storageGroupName), deviceSet, true); - deleteLevelFiles(timePartition, sourceTsFileResources); + deleteLevelFilesInDisk(sourceTsFileResources); + deleteLevelFilesInList(timePartition, sourceTsFileResources); sequenceTsFileResources.get(timePartition).get(level + 1).add(targetResource); } else { CompactionUtils .merge(targetResource, sourceTsFileResources, storageGroupName, new CompactionLogger(storageGroupDir, storageGroupName), deviceSet, false); - deleteLevelFiles(timePartition, sourceTsFileResources); + deleteLevelFilesInDisk(sourceTsFileResources); + deleteLevelFilesInList(timePartition, sourceTsFileResources); unSequenceTsFileResources.get(timePartition).get(level + 1).add(targetResource); } } @@ -491,19 +499,20 @@ public class LevelCompactionTsFileManagement extends TsFileManagement { storageGroupName, i, toMergeTsFiles.size()); writeLock(); try { - deleteLevelFiles(timePartition, toMergeTsFiles); compactionLogger.logMergeFinish(); if (sequence) { sequenceTsFileResources.get(timePartition).get(i + 1).add(newResource); } else { unSequenceTsFileResources.get(timePartition).get(i + 1).add(newResource); } + deleteLevelFilesInList(timePartition, toMergeTsFiles); if (mergeResources.size() > i + 1) { mergeResources.get(i + 1).add(newResource); } } finally { writeUnlock(); } + deleteLevelFilesInDisk(toMergeTsFiles); compactionLogger.close(); File logFile = FSFactoryProducer.getFSFactory() .getFile(storageGroupDir, storageGroupName + COMPACTION_LOG_NAME);