accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dlmar...@apache.org
Subject [accumulo] branch main updated: Remove redundant information in tablets external compaction entry (#2136)
Date Mon, 07 Jun 2021 12:26:42 GMT
This is an automated email from the ASF dual-hosted git repository.

dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/main by this push:
     new 5d3344d  Remove redundant information in tablets external compaction entry (#2136)
5d3344d is described below

commit 5d3344dbb488d602025ce8f2acb5f807969f45ec
Author: Dave Marion <dlmarion@apache.org>
AuthorDate: Mon Jun 7 08:26:34 2021 -0400

    Remove redundant information in tablets external compaction entry (#2136)
    
    Remove redundant information in tablets external compaction entry
    
    Closes #2110
---
 .../schema/ExternalCompactionMetadata.java         | 21 ++++--------
 .../accumulo/tserver/tablet/CompactableImpl.java   | 18 +++++-----
 .../accumulo/tserver/tablet/CompactableUtils.java  | 18 ++++++++--
 .../accumulo/tserver/tablet/DatafileManager.java   |  7 ++--
 .../org/apache/accumulo/tserver/tablet/Tablet.java |  5 +++
 .../tserver/compaction/CompactableUtilsTest.java   | 39 ++++++++++++++++++++++
 6 files changed, 77 insertions(+), 31 deletions(-)

diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
index e2efc63..eada2c7 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionMetadata.java
@@ -42,7 +42,6 @@ public class ExternalCompactionMetadata {
   private final Set<StoredTabletFile> jobFiles;
   private final Set<StoredTabletFile> nextFiles;
   private final TabletFile compactTmpName;
-  private final TabletFile newFile;
   private final String compactorId;
   private final CompactionKind kind;
   private final short priority;
@@ -52,13 +51,12 @@ public class ExternalCompactionMetadata {
   private final Long compactionId;
 
   public ExternalCompactionMetadata(Set<StoredTabletFile> jobFiles, Set<StoredTabletFile>
nextFiles,
-      TabletFile compactTmpName, TabletFile newFile, String compactorId, CompactionKind kind,
-      short priority, CompactionExecutorId ceid, boolean propagateDeletes,
-      boolean initiallySelectedAll, Long compactionId) {
+      TabletFile compactTmpName, String compactorId, CompactionKind kind, short priority,
+      CompactionExecutorId ceid, boolean propagateDeletes, boolean initiallySelectedAll,
+      Long compactionId) {
     this.jobFiles = Objects.requireNonNull(jobFiles);
     this.nextFiles = Objects.requireNonNull(nextFiles);
     this.compactTmpName = Objects.requireNonNull(compactTmpName);
-    this.newFile = Objects.requireNonNull(newFile);
     this.compactorId = Objects.requireNonNull(compactorId);
     this.kind = Objects.requireNonNull(kind);
     this.priority = priority;
@@ -80,10 +78,6 @@ public class ExternalCompactionMetadata {
     return compactTmpName;
   }
 
-  public TabletFile getNewFile() {
-    return newFile;
-  }
-
   public String getCompactorId() {
     return compactorId;
   }
@@ -118,7 +112,6 @@ public class ExternalCompactionMetadata {
     List<String> inputs;
     List<String> nextFiles;
     String tmp;
-    String dest;
     String compactor;
     String kind;
     String executorId;
@@ -135,7 +128,6 @@ public class ExternalCompactionMetadata {
     jData.nextFiles =
         nextFiles.stream().map(StoredTabletFile::getMetaUpdateDelete).collect(toList());
     jData.tmp = compactTmpName.getMetaInsert();
-    jData.dest = newFile.getMetaInsert();
     jData.compactor = compactorId;
     jData.kind = kind.name();
     jData.executorId = ((CompactionExecutorIdImpl) ceid).getExternalName();
@@ -152,10 +144,9 @@ public class ExternalCompactionMetadata {
     return new ExternalCompactionMetadata(
         jData.inputs.stream().map(StoredTabletFile::new).collect(toSet()),
         jData.nextFiles.stream().map(StoredTabletFile::new).collect(toSet()),
-        new TabletFile(new Path(jData.tmp)), new TabletFile(new Path(jData.dest)), jData.compactor,
-        CompactionKind.valueOf(jData.kind), jData.priority,
-        CompactionExecutorIdImpl.externalId(jData.executorId), jData.propDels, jData.selectedAll,
-        jData.compactionId);
+        new TabletFile(new Path(jData.tmp)), jData.compactor, CompactionKind.valueOf(jData.kind),
+        jData.priority, CompactionExecutorIdImpl.externalId(jData.executorId), jData.propDels,
+        jData.selectedAll, jData.compactionId);
   }
 
   @Override
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index a7b3ded..ece9f44 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -74,7 +74,6 @@ import org.apache.accumulo.tserver.compactions.Compactable;
 import org.apache.accumulo.tserver.compactions.CompactionManager;
 import org.apache.accumulo.tserver.compactions.ExternalCompactionJob;
 import org.apache.accumulo.tserver.managermessage.TabletStatusMessage;
-import org.apache.hadoop.fs.Path;
 import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -973,14 +972,13 @@ public class CompactableImpl implements Compactable {
       Map<String,String> overrides =
           CompactableUtils.getOverrides(job.getKind(), tablet, cInfo.localHelper, job.getFiles());
 
-      var newFile = tablet.getNextMapFilename(!cInfo.propagateDeletes ? "A" : "C");
-      var compactTmpName = new TabletFile(new Path(newFile.getMetaInsert() + "_tmp"));
+      TabletFile compactTmpName = tablet.getNextMapFilenameForMajc(cInfo.propagateDeletes);
 
       ExternalCompactionInfo ecInfo = new ExternalCompactionInfo();
 
       ecInfo.meta = new ExternalCompactionMetadata(cInfo.jobFiles,
-          Sets.difference(cInfo.selectedFiles, cInfo.jobFiles), compactTmpName, newFile,
-          compactorId, job.getKind(), job.getPriority(), job.getExecutor(), cInfo.propagateDeletes,
+          Sets.difference(cInfo.selectedFiles, cInfo.jobFiles), compactTmpName, compactorId,
+          job.getKind(), job.getPriority(), job.getExecutor(), cInfo.propagateDeletes,
           cInfo.initiallySelectedAll, cInfo.checkCompactionId);
 
       tablet.getContext().getAmple().mutateTablet(getExtent())
@@ -1029,11 +1027,11 @@ public class CompactableImpl implements Compactable {
         log.debug("Attempting to commit external compaction {}", extCompactionId);
         StoredTabletFile metaFile = null;
         try {
-          metaFile = tablet.getDatafileManager().bringMajorCompactionOnline(
-              ecInfo.meta.getJobFiles(), ecInfo.meta.getCompactTmpName(), ecInfo.meta.getNewFile(),
-              ecInfo.meta.getCompactionId(),
-              Sets.union(ecInfo.meta.getJobFiles(), ecInfo.meta.getNextFiles()),
-              new DataFileValue(fileSize, entries), Optional.of(extCompactionId));
+          metaFile =
+              tablet.getDatafileManager().bringMajorCompactionOnline(ecInfo.meta.getJobFiles(),
+                  ecInfo.meta.getCompactTmpName(), ecInfo.meta.getCompactionId(),
+                  Sets.union(ecInfo.meta.getJobFiles(), ecInfo.meta.getNextFiles()),
+                  new DataFileValue(fileSize, entries), Optional.of(extCompactionId));
           TabletLogger.compacted(getExtent(), ecInfo.job, metaFile);
         } catch (Exception e) {
           metaFile = null;
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
index 3e3ca65..49884cd 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
@@ -613,8 +613,7 @@ public class CompactableUtils {
     HashMap<StoredTabletFile,DataFileValue> compactFiles = new HashMap<>();
     jobFiles.forEach(file -> compactFiles.put(file, allFiles.get(file)));
 
-    TabletFile newFile = tablet.getNextMapFilename(!propagateDeletes ? "A" : "C");
-    TabletFile compactTmpName = new TabletFile(new Path(newFile.getMetaInsert() + "_tmp"));
+    TabletFile compactTmpName = tablet.getNextMapFilenameForMajc(propagateDeletes);
 
     FileCompactor compactor = new FileCompactor(tablet.getContext(), tablet.getExtent(),
         compactFiles, compactTmpName, propagateDeletes, cenv, iters, compactionConfig);
@@ -632,7 +631,7 @@ public class CompactableUtils {
     stats.add(mcs);
 
     metaFile = tablet.getDatafileManager().bringMajorCompactionOnline(compactFiles.keySet(),
-        compactTmpName, newFile, compactionId, selectedFiles,
+        compactTmpName, compactionId, selectedFiles,
         new DataFileValue(mcs.getFileSize(), mcs.getEntriesWritten()), Optional.empty());
     return metaFile;
   }
@@ -650,4 +649,17 @@ public class CompactableUtils {
         throw new IllegalArgumentException("Unknown kind " + ck);
     }
   }
+
+  public static TabletFile computeCompactionFileDest(TabletFile tmpFile) {
+    String newFilePath = tmpFile.getMetaInsert();
+    int idx = newFilePath.indexOf("_tmp");
+    if (idx > 0) {
+      newFilePath = newFilePath.substring(0, idx);
+    } else {
+      throw new IllegalArgumentException(
+          "Expected compaction tmp file " + tmpFile.getMetaInsert() + " to have suffix '_tmp'");
+    }
+    return new TabletFile(new Path(newFilePath));
+  }
+
 }
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index b0fb60d..e2ebc39 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -398,13 +398,14 @@ class DatafileManager {
   }
 
   StoredTabletFile bringMajorCompactionOnline(Set<StoredTabletFile> oldDatafiles,
-      TabletFile tmpDatafile, TabletFile newDatafile, Long compactionId,
-      Set<StoredTabletFile> selectedFiles, DataFileValue dfv, Optional<ExternalCompactionId>
ecid)
-      throws IOException {
+      TabletFile tmpDatafile, Long compactionId, Set<StoredTabletFile> selectedFiles,
+      DataFileValue dfv, Optional<ExternalCompactionId> ecid) throws IOException {
     final KeyExtent extent = tablet.getExtent();
     VolumeManager vm = tablet.getTabletServer().getContext().getVolumeManager();
     long t1, t2;
 
+    TabletFile newDatafile = CompactableUtils.computeCompactionFileDest(tmpDatafile);
+
     if (vm.exists(newDatafile.getPath())) {
       log.error("Target map file already exist " + newDatafile, new Exception());
       throw new IllegalStateException("Target map file already exist " + newDatafile);
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index f096e4a..3a417bd 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -273,6 +273,11 @@ public class Tablet {
         + context.getUniqueNameAllocator().getNextName() + "." + extension));
   }
 
+  TabletFile getNextMapFilenameForMajc(boolean propagateDeletes) throws IOException {
+    String tmpFileName = getNextMapFilename(!propagateDeletes ? "A" : "C").getMetaInsert()
+ "_tmp";
+    return new TabletFile(new Path(tmpFileName));
+  }
+
   private void checkTabletDir(Path path) throws IOException {
     if (!checkedTabletDirs.contains(path)) {
       FileStatus[] files = null;
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/CompactableUtilsTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/CompactableUtilsTest.java
new file mode 100644
index 0000000..d77b2c4
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/CompactableUtilsTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.tserver.compaction;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.tserver.tablet.CompactableUtils;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+public class CompactableUtilsTest {
+
+  @Test
+  public void testEquivalence() {
+    TabletFile expected = new TabletFile(
+        new Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F0000070.rf"));
+    TabletFile tmpFile = new TabletFile(new Path(expected.getMetaInsert() + "_tmp"));
+    TabletFile dest = CompactableUtils.computeCompactionFileDest(tmpFile);
+    assertEquals(expected, dest);
+  }
+
+}

Mime
View raw message