accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmil...@apache.org
Subject [accumulo] branch main updated: Handle old sorted map files in Upgrade (#2185)
Date Fri, 30 Jul 2021 16:43:35 GMT
This is an automated email from the ASF dual-hosted git repository.

mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git


The following commit(s) were added to refs/heads/main by this push:
     new 09e3d77  Handle old sorted map files in Upgrade (#2185)
09e3d77 is described below

commit 09e3d771ef11fcc2c2a8eb1d1b29b024d7da900f
Author: Mike Miller <mmiller@apache.org>
AuthorDate: Fri Jul 30 12:43:30 2021 -0400

    Handle old sorted map files in Upgrade (#2185)
    
    * Add dropSortedMapWALFiles to Upgrader9to10 for resolving sorted
    map files that may still be around during upgrade. Follow up for #2117
    * Closes #2179
---
 .../accumulo/manager/upgrade/Upgrader9to10.java    | 41 +++++++++++++++++++++
 .../manager/upgrade/Upgrader9to10Test.java         | 42 ++++++++++++++++++++++
 2 files changed, 83 insertions(+)

diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
index 5e874c5..597f88c 100644
--- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/Upgrader9to10.java
@@ -105,6 +105,12 @@ import com.google.common.base.Preconditions;
  * <a href="https://github.com/apache/accumulo/issues/1642">#1642</a>, and
  * <a href="https://github.com/apache/accumulo/issues/1643">#1643</a> as well.</li>
  * </ul>
+ *
+ * Sorted recovery was updated to use RFiles instead of map files. So to prevent issues during
+ * tablet recovery, remove the old temporary map files and resort using RFiles. This is done
in
+ * {@link #dropSortedMapWALFiles(VolumeManager)}. For more information see the following
issues:
+ * <a href="https://github.com/apache/accumulo/issues/2117">#2117</a> and
+ * <a href="https://github.com/apache/accumulo/issues/2179">#2179</a>
  */
 public class Upgrader9to10 implements Upgrader {
 
@@ -142,6 +148,8 @@ public class Upgrader9to10 implements Upgrader {
     upgradeRelativePaths(ctx, Ample.DataLevel.USER);
     upgradeDirColumns(ctx, Ample.DataLevel.USER);
     upgradeFileDeletes(ctx, Ample.DataLevel.USER);
+    // special case where old files need to be deleted
+    dropSortedMapWALFiles(ctx.getVolumeManager());
   }
 
   private void setMetaTableProps(ServerContext ctx) {
@@ -726,4 +734,37 @@ public class Upgrader9to10 implements Upgrader {
     }
     return new Path(upgradeProperty, VolumeManager.FileType.TABLE.getDirectory() + oldDelete);
   }
+
+  /**
+   * Remove old temporary map files to prevent problems during recovery.
+   */
+  static void dropSortedMapWALFiles(VolumeManager vm) {
+    Path recoveryDir = new Path("/accumulo/recovery");
+    try {
+      if (!vm.exists(recoveryDir)) {
+        log.info("There are no recovery files in /accumulo/recovery");
+        return;
+      }
+      List<Path> directoriesToDrop = new ArrayList<>();
+      for (FileStatus walDir : vm.listStatus(recoveryDir)) {
+        // map files will be in a directory starting with "part"
+        Path walDirPath = walDir.getPath();
+        for (FileStatus dirOrFile : vm.listStatus(walDirPath)) {
+          if (dirOrFile.isDirectory()) {
+            directoriesToDrop.add(walDirPath);
+            break;
+          }
+        }
+      }
+      if (!directoriesToDrop.isEmpty()) {
+        log.info("Found {} old sorted map directories to delete.", directoriesToDrop.size());
+        for (Path dir : directoriesToDrop) {
+          log.info("Deleting everything in old sorted map directory: {}", dir);
+          vm.deleteRecursively(dir);
+        }
+      }
+    } catch (IOException ioe) {
+      throw new UncheckedIOException(ioe);
+    }
+  }
 }
diff --git a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
index a5752b9..25800da 100644
--- a/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
+++ b/server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader9to10Test.java
@@ -24,11 +24,13 @@ import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -49,6 +51,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Da
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.server.fs.VolumeManager;
 import org.apache.accumulo.server.gc.GcVolumeUtil;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.junit.Test;
@@ -324,4 +327,43 @@ public class Upgrader9to10Test {
 
     assertEquals("Replacements should have update for every delete", deleteCount, updateCount);
   }
+
+  @Test
+  public void testDropSortedMapWALs() throws IOException {
+    Path recoveryDir = new Path("/accumulo/recovery");
+    VolumeManager fs = createMock(VolumeManager.class);
+    FileStatus[] dirs = new FileStatus[2];
+    dirs[0] = createMock(FileStatus.class);
+    Path dir0 = new Path("/accumulo/recovery/A123456789");
+    FileStatus[] dir0Files = new FileStatus[1];
+    dir0Files[0] = createMock(FileStatus.class);
+    dirs[1] = createMock(FileStatus.class);
+    Path dir1 = new Path("/accumulo/recovery/B123456789");
+    FileStatus[] dir1Files = new FileStatus[1];
+    dir1Files[0] = createMock(FileStatus.class);
+    Path part1Dir = new Path("/accumulo/recovery/B123456789/part-r-0000");
+
+    expect(fs.exists(recoveryDir)).andReturn(true).once();
+    expect(fs.listStatus(recoveryDir)).andReturn(dirs).once();
+    expect(dirs[0].getPath()).andReturn(dir0).once();
+    expect(fs.listStatus(dir0)).andReturn(dir0Files).once();
+    expect(dir0Files[0].isDirectory()).andReturn(false).once();
+
+    expect(dirs[1].getPath()).andReturn(dir1).once();
+    expect(fs.listStatus(dir1)).andReturn(dir1Files).once();
+    expect(dir1Files[0].isDirectory()).andReturn(true).once();
+    expect(dir1Files[0].getPath()).andReturn(part1Dir).once();
+
+    expect(fs.deleteRecursively(dir1)).andReturn(true).once();
+
+    replay(fs, dirs[0], dirs[1], dir0Files[0], dir1Files[0]);
+    Upgrader9to10.dropSortedMapWALFiles(fs);
+
+    reset(fs);
+
+    // test case where there is no recovery
+    expect(fs.exists(recoveryDir)).andReturn(false).once();
+    replay(fs);
+    Upgrader9to10.dropSortedMapWALFiles(fs);
+  }
 }

Mime
View raw message